diff --git a/MAINTAINERS b/MAINTAINERS index a7815fd1072f5948ca318c7d0b952befb9a9c7f2..7fabeed7f3b97fae036cbddc519e5f7bc0e5d50f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -19504,6 +19504,12 @@ S: Maintained F: Documentation/input/devices/yealink.rst F: drivers/input/misc/yealink.* +YUNSILICON XSC DRIVERS +M: Weihonggang +S: Maintained +F: drivers/infiniband/hw/xsc +F: drivers/net/ethernet/yunsilicon/xsc + Z8530 DRIVER FOR AX.25 M: Joerg Reuter L: linux-hams@vger.kernel.org diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index b0425025151f926e6985182ed96e503841934307..4422ac761861ac649041194b7cab58429d150345 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -2907,6 +2907,9 @@ CONFIG_SMSC9420=m # CONFIG_NET_VENDOR_XILINX is not set CONFIG_NET_VENDOR_NEBULA_MATRIX=y CONFIG_M1600=m +CONFIG_NET_VENDOR_YUNSILICON=y +CONFIG_YUNSILICON_XSC_ETH=m +CONFIG_YUNSILICON_XSC_PCI=m # CONFIG_FDDI is not set # CONFIG_HIPPI is not set CONFIG_UBL=y @@ -5601,6 +5604,7 @@ CONFIG_INFINIBAND_HNS=m CONFIG_INFINIBAND_HNS_HIP08=y CONFIG_INFINIBAND_BNXT_RE=m CONFIG_INFINIBAND_QEDR=m +CONFIG_INFINIBAND_XSC=m CONFIG_RDMA_RXE=m # CONFIG_RDMA_SIW is not set CONFIG_INFINIBAND_IPOIB=m diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index 5171aa50736b9a73943511beec15b38004f64c1a..2edb79666c8d83e275eff28c4320020ce817ac5b 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -2878,6 +2878,9 @@ CONFIG_SFC_MCDI_LOGGING=y # CONFIG_NET_VENDOR_XILINX is not set CONFIG_NET_VENDOR_NEBULA_MATRIX=y CONFIG_M1600=m +CONFIG_NET_VENDOR_YUNSILICON=y +CONFIG_YUNSILICON_XSC_ETH=m +CONFIG_YUNSILICON_XSC_PCI=m # CONFIG_FDDI is not set # CONFIG_HIPPI is not set # CONFIG_NET_SB1000 is not set @@ -6248,6 +6251,7 @@ CONFIG_INFINIBAND_HFI1=m # CONFIG_HFI1_DEBUG_SDMA_ORDER is not set # CONFIG_SDMA_VERBOSITY is not set CONFIG_INFINIBAND_QEDR=m +CONFIG_INFINIBAND_XSC=m CONFIG_INFINIBAND_RDMAVT=m CONFIG_RDMA_RXE=m # CONFIG_RDMA_SIW is not set diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index 9325e189a2153615ae36b131a89d349e08d4cc58..ec00b4f72fb7345a068a97470f96c0e4fd4b23b8 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -91,6 +91,7 @@ source "drivers/infiniband/hw/hns/Kconfig" source "drivers/infiniband/hw/bnxt_re/Kconfig" source "drivers/infiniband/hw/hfi1/Kconfig" source "drivers/infiniband/hw/qedr/Kconfig" +source "drivers/infiniband/hw/xsc/Kconfig" source "drivers/infiniband/sw/rdmavt/Kconfig" source "drivers/infiniband/sw/rxe/Kconfig" source "drivers/infiniband/sw/siw/Kconfig" diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile index 0aeccd9848892ab10877cee6740aaff96348ea73..478dc420a612e92eca74c5d7c95c525fca813f9e 100644 --- a/drivers/infiniband/hw/Makefile +++ b/drivers/infiniband/hw/Makefile @@ -13,3 +13,4 @@ obj-$(CONFIG_INFINIBAND_HFI1) += hfi1/ obj-$(CONFIG_INFINIBAND_HNS) += hns/ obj-$(CONFIG_INFINIBAND_QEDR) += qedr/ obj-$(CONFIG_INFINIBAND_BNXT_RE) += bnxt_re/ +obj-$(CONFIG_INFINIBAND_XSC) += xsc/ diff --git a/drivers/infiniband/hw/xsc/Kconfig b/drivers/infiniband/hw/xsc/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..dfac1fa6f2b45a2309e9462a146665226f89206f --- /dev/null +++ b/drivers/infiniband/hw/xsc/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. +# All rights reserved. + +config INFINIBAND_XSC + tristate "Yunsilicon XSC RDMA driver" + default n + depends on NETDEVICES && ETHERNET && PCI && INET + depends on YUNSILICON_XSC_PCI && YUNSILICON_XSC_ETH + help + This driver provides RDMA support for + Yunsilicon XSC devices. diff --git a/drivers/infiniband/hw/xsc/Makefile b/drivers/infiniband/hw/xsc/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..ac6243ef217ac928f90e45ffd5ed81438d8974ef --- /dev/null +++ b/drivers/infiniband/hw/xsc/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. +# All rights reserved. + +ccflags-y := -I $(srctree)/drivers/net/ethernet/yunsilicon/xsc/ +ccflags-y += -Wno-implicit-fallthrough + +obj-$(CONFIG_INFINIBAND_XSC) += xsc_ib.o + +xsc_ib-y := main.o xsc_rdma_ctrl.o cq.o qp.o mem.o mr.o ah.o mad.o counters.o devx.o global.o private_dev.o ib_umem_ex.o rtt.o +xsc_ib-$(CONFIG_MLX_PEER_SUPPORT) := peer_mem.o diff --git a/drivers/infiniband/hw/xsc/ah.c b/drivers/infiniband/hw/xsc/ah.c new file mode 100644 index 0000000000000000000000000000000000000000..720adc93bb7f097053643610136217702cc32655 --- /dev/null +++ b/drivers/infiniband/hw/xsc/ah.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include + +#include "xsc_ib.h" +#include "user.h" + +static u32 xsc_calc_roce_udp_flow_label(void) +{ + u32 factor = 0; + u32 hash = 0; + u32 flow_label = 0; + + /*This function will generate a 20 bit flow_label*/ + factor = (IB_GRH_FLOWLABEL_MASK - IB_ROCE_UDP_ENCAP_VALID_PORT_MIN + 1); + hash = get_random_u32() % factor; + flow_label = hash & IB_GRH_FLOWLABEL_MASK; + + return flow_label; +} + +static u16 xsc_ah_get_udp_sport(const struct xsc_ib_dev *dev, + struct rdma_ah_attr *ah_attr) +{ + enum ib_gid_type gid_type = ah_attr->grh.sgid_attr->gid_type; + u16 sport = 0; + u32 fl = 0; + + if ((gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) && + (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) && + (ah_attr->grh.flow_label & IB_GRH_FLOWLABEL_MASK)) { + fl = ah_attr->grh.flow_label; + } else { + /*generate a 20bit flow_label and output to user layer*/ + fl = xsc_calc_roce_udp_flow_label(); + ah_attr->grh.flow_label = fl; + } + + sport = xsc_flow_label_to_udp_sport(fl); + xsc_ib_dbg(dev, "fl=0x%x,sport=0x%x\n", fl, sport); + return sport; +} + +static struct ib_ah *create_ib_ah(struct xsc_ib_dev *dev, + struct xsc_ib_ah *ah, + struct rdma_ah_attr *ah_attr) +{ + enum ib_gid_type gid_type; + + if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { + const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); + + memcpy(ah->av.rgid, &grh->dgid, 16); + ah->av.grh_gid_fl = cpu_to_be32(grh->flow_label | + (1 << 30) | + grh->sgid_index << 20); + ah->av.hop_limit = grh->hop_limit; + ah->av.tclass = grh->traffic_class; + } + + ah->av.stat_rate_sl = (rdma_ah_get_static_rate(ah_attr) << 4); + + if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) { + gid_type = ah_attr->grh.sgid_attr->gid_type; + memcpy(ah->av.rmac, ah_attr->roce.dmac, + sizeof(ah_attr->roce.dmac)); + ah->av.udp_sport = xsc_ah_get_udp_sport(dev, ah_attr); + ah->av.stat_rate_sl |= (rdma_ah_get_sl(ah_attr) & 0x7) << 1; + if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) +#define XSC_ECN_ENABLED BIT(1) + ah->av.tclass |= XSC_ECN_ENABLED; + } else { + ah->av.rlid = cpu_to_be16(rdma_ah_get_dlid(ah_attr)); + ah->av.fl_mlid = rdma_ah_get_path_bits(ah_attr) & 0x7f; + ah->av.stat_rate_sl |= (rdma_ah_get_sl(ah_attr) & 0xf); + } + + return &ah->ibah; +} + +xsc_ib_create_ah_def() +{ + struct xsc_ib_ah *ah = to_mah(ibah); + struct xsc_ib_dev *dev = to_mdev(ibah->device); + + struct rdma_ah_attr *ah_attr = init_attr->ah_attr; + enum rdma_ah_attr_type ah_type = ah_attr->type; + + if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) && + !(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) + return RET_VALUE(-EINVAL); + + if (ah_type == RDMA_AH_ATTR_TYPE_ROCE && udata) { + int err; + struct xsc_ib_create_ah_resp resp = {}; + u32 min_resp_len = offsetof(typeof(resp), dmac) + + sizeof(resp.dmac); + + if (udata->outlen < min_resp_len) + return RET_VALUE(-EINVAL); + + resp.response_length = min_resp_len; + + memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN); + err = ib_copy_to_udata(udata, &resp, resp.response_length); + if (err) + return RET_VALUE(err); + } + + create_ib_ah(dev, ah, ah_attr); /* never fails */ + return 0; +} + +int xsc_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) +{ +// struct xsc_ib_ah *ah = to_mah(ibah); +// u32 tmp; + +// pr_err("[%s:%d]", __func__, __LINE__); + +// memset(ah_attr, 0, sizeof(*ah_attr)); + +// tmp = be32_to_cpu(ah->av.grh_gid_fl); +// if (tmp & (1 << 30)) { +// ah_attr->ah_flags = IB_AH_GRH; +// ah_attr->grh.sgid_index = (tmp >> 20) & 0xff; +// ah_attr->grh.flow_label = tmp & 0xfffff; +// memcpy(&ah_attr->grh.dgid, ah->av.rgid, 16); +// ah_attr->grh.hop_limit = ah->av.hop_limit; +// ah_attr->grh.traffic_class = ah->av.tclass; +// } +// ah_attr->dlid = be16_to_cpu(ah->av.rlid); +// ah_attr->static_rate = ah->av.stat_rate_sl >> 4; +// ah_attr->sl = ah->av.stat_rate_sl & 0xf; + + return 0; +} + +xsc_ib_destroy_ah_def() +{ + return 0; +} diff --git a/drivers/infiniband/hw/xsc/counters.c b/drivers/infiniband/hw/xsc/counters.c new file mode 100644 index 0000000000000000000000000000000000000000..3b5330c233d394ed7be7952855451303c575624b --- /dev/null +++ b/drivers/infiniband/hw/xsc/counters.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include "counters.h" + +#define COUNTERS_FILE_NAME "counters" +#define COUNTERS_NAMES_FILE_NAME "counters_names" +#define COUNTERS_VALUE_FILE_NAME "counters_value" +#define COUNTERS_ATTER_GROUP_NAME "counters" + +static const struct counter_desc hw_stats_desc[] = { + /*by mac port*/ + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_tx_pkts) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_tx_bytes) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_rx_pkts) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_rx_bytes) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, np_cnp_sent) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rp_cnp_handled) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, np_ecn_marked_roce_packets) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rp_cnp_ignored) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, tx_pause) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_pause) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_fcs_errors) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_discards) }, + /*by function*/ + { XSC_DECLARE_STAT(struct xsc_hw_stats, out_of_sequence) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, packet_seq_err) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, out_of_buffer) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rnr_nak_retry_err) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, local_ack_timeout_err) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_read_requests) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_write_requests) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, duplicate_requests) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_tx_pkts_func) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_tx_payload_bytes) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_rx_pkts_func) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_rx_payload_bytes) }, + /*by global*/ + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_loopback_pkts) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_loopback_bytes) }, +}; + +static const struct counter_desc vf_hw_stats_desc[] = { + /*by function*/ + { XSC_DECLARE_STAT(struct xsc_hw_stats, out_of_sequence) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, packet_seq_err) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, out_of_buffer) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rnr_nak_retry_err) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, local_ack_timeout_err) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_read_requests) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_write_requests) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, duplicate_requests) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_tx_pkts_func) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_tx_payload_bytes) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_rx_pkts_func) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_rx_payload_bytes) }, +}; + +static ssize_t counters_names_show(struct kobject *kobjs, + struct attribute *attr, char *buf) +{ + int i; + ssize_t count = 0; + struct xsc_counters_attribute *xsc_counters_name_attr; + const struct counter_desc *desc; + + xsc_counters_name_attr = container_of(attr, + struct xsc_counters_attribute, attr); + + if (!xsc_counters_name_attr->dev || + !xsc_counters_name_attr->desc || + xsc_counters_name_attr->desc_size == 0) + return 0; + + for (i = 0; i < xsc_counters_name_attr->desc_size; ++i) { + desc = xsc_counters_name_attr->desc + i; + count += sprintf(&buf[count], "%s\n", desc->format); + } + + return count; +} + +static ssize_t counters_show(struct kobject *kobjs, + struct attribute *attr, char *buf) +{ + int i; + ssize_t count = 0; + struct xsc_counters_attribute *xsc_counters_attr; + struct xsc_hw_stats_mbox_in *in; + struct xsc_hw_stats_mbox_out out; + struct xsc_core_device *dev; + int ret; + u64 counter; + const struct counter_desc *desc; + int inlen; + struct xsc_lag *ldev; + + xsc_counters_attr = container_of(attr, + struct xsc_counters_attribute, attr); + + if (!xsc_counters_attr->dev || + !xsc_counters_attr->desc || + xsc_counters_attr->desc_size == 0) + return 0; + + dev = xsc_counters_attr->dev; + ldev = xsc_lag_dev_get(dev); + if (ldev && __xsc_lag_is_roce(ldev)) + inlen = sizeof(struct xsc_hw_stats_mbox_in) + XSC_MAX_PORTS; + else + inlen = sizeof(struct xsc_hw_stats_mbox_in); + + in = kzalloc(inlen, GFP_KERNEL); + if (!in) + return 0; + + memset(&out, 0, sizeof(out)); + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_HW_STATS); + + if (ldev && __xsc_lag_is_roce(ldev)) { + in->is_lag = 1; + in->lag_member_num = XSC_MAX_PORTS; + for (i = 0; i < XSC_MAX_PORTS; i++) + in->member_port[i] = ldev->pf[i].xdev->mac_port; + } else { + in->is_lag = 0; + in->mac_port = dev->mac_port; + } + ret = xsc_cmd_exec(dev, (void *)in, inlen, (void *)&out, sizeof(out)); + if (ret || out.hdr.status) { + kfree(in); + return 0; + } + + for (i = 0 ; i < xsc_counters_attr->desc_size; ++i) { + desc = xsc_counters_attr->desc + i; + counter = *(u64 *)((char *)&out.hw_stats + desc->offset); + counter = be64_to_cpu(counter); + count += sprintf(&buf[count], "%-26s %-20llu\n", + desc->format, counter); + } + + kfree(in); + return count; +} + +static ssize_t counters_value_read(struct file *file, + struct kobject *kob, + struct bin_attribute *bin_attr, + char *buf, loff_t loff, size_t size) +{ + int i; + struct xsc_counters_bin_attribute *xsc_counters_bin_attr; + u64 *tmp_value; + struct xsc_hw_stats_mbox_in in; + struct xsc_hw_stats_mbox_out out; + struct xsc_core_device *dev; + int ret; + const struct counter_desc *desc; + + xsc_counters_bin_attr = container_of(&bin_attr->attr, + struct xsc_counters_bin_attribute, attr); + + if (!xsc_counters_bin_attr->dev || + !xsc_counters_bin_attr->desc || + xsc_counters_bin_attr->desc_size == 0 || + xsc_counters_bin_attr->size == 0) + return 0; + + dev = xsc_counters_bin_attr->dev; + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_HW_STATS); + in.mac_port = dev->mac_port; + ret = xsc_cmd_exec(dev, (void *)&in, sizeof(in), (void *)&out, sizeof(out)); + if (ret || out.hdr.status) + return 0; + + tmp_value = kmalloc(xsc_counters_bin_attr->size, GFP_KERNEL); + if (!tmp_value) + return 0; + + for (i = 0; i < xsc_counters_bin_attr->desc_size; ++i) { + desc = xsc_counters_bin_attr->desc + i; + tmp_value[i] = *(u64 *)((char *)&out.hw_stats + desc->offset); + tmp_value[i] = be64_to_cpu(tmp_value[i]); + } + + memcpy(buf, tmp_value, xsc_counters_bin_attr->size); + kfree(tmp_value); + + return xsc_counters_bin_attr->size; +} + +int xsc_counters_init(struct ib_device *ib_dev, struct xsc_core_device *dev) +{ + struct xsc_counters_attribute *xsc_counters_name, *xsc_counters; + struct xsc_counters_bin_attribute *xsc_counters_bin; + struct attribute_group *counters_attr_g; + struct bin_attribute **counters_bin_attrs; + struct attribute **counters_attrs; + int id, is_pf; + int ret = -ENOMEM; + + xsc_counters_name = kzalloc(sizeof(*xsc_counters_name), GFP_KERNEL); + if (!xsc_counters_name) + return -ENOMEM; + + xsc_counters = kzalloc(sizeof(*xsc_counters), GFP_KERNEL); + if (!xsc_counters) + goto err_xsc_counters; + + xsc_counters_bin = kzalloc(sizeof(*xsc_counters_bin), GFP_KERNEL); + if (!xsc_counters_bin) + goto err_xsc_counters_bin; + + id = dev->mac_port; + + is_pf = xsc_core_is_pf(dev); + xsc_core_dbg(dev, "get xscale id = %d, is_pf = %d.\n", id, is_pf); + + counters_bin_attrs = kzalloc(sizeof(*counters_bin_attrs) * 2, GFP_KERNEL); + if (!counters_bin_attrs) + goto err_counters_bin_attrs; + + counters_attrs = kzalloc(sizeof(*counters_attrs) * 3, GFP_KERNEL); + if (!counters_attrs) + goto err_counters_attrs; + + counters_attr_g = kzalloc(sizeof(*counters_attr_g), GFP_KERNEL); + if (!counters_attr_g) + goto err_counters_attr_g; + + sysfs_attr_init(&xsc_counters_name->attr); + xsc_counters_name->attr.name = COUNTERS_NAMES_FILE_NAME; + xsc_counters_name->attr.mode = 0444; + xsc_counters_name->show = counters_names_show; + + sysfs_attr_init(&xsc_counters->attr); + xsc_counters->attr.name = COUNTERS_FILE_NAME; + xsc_counters->attr.mode = 0444; + xsc_counters->show = counters_show; + xsc_counters->id = id; + xsc_counters->dev = dev; + + sysfs_attr_init(&xsc_counters_bin->attr); + xsc_counters_bin->attr.name = COUNTERS_VALUE_FILE_NAME; + xsc_counters_bin->attr.mode = 0444; + xsc_counters_bin->read = counters_value_read; + xsc_counters_bin->id = id; + xsc_counters_bin->dev = dev; + + if (is_pf) { + xsc_counters_name->desc = xsc_counters->desc = + xsc_counters_bin->desc = &hw_stats_desc[0]; + xsc_counters_name->desc_size = xsc_counters->desc_size = + xsc_counters_bin->desc_size = ARRAY_SIZE(hw_stats_desc); + xsc_counters_bin->size = xsc_counters_bin->desc_size * sizeof(u64); + } else { + xsc_counters_name->desc = xsc_counters->desc = + xsc_counters_bin->desc = &vf_hw_stats_desc[0]; + xsc_counters_name->desc_size = xsc_counters->desc_size = + xsc_counters_bin->desc_size = ARRAY_SIZE(vf_hw_stats_desc); + xsc_counters_bin->size = xsc_counters_bin->desc_size * sizeof(u64); + } + + counters_bin_attrs[0] = (struct bin_attribute *)xsc_counters_bin; + counters_attrs[0] = (struct attribute *)xsc_counters_name; + counters_attrs[1] = (struct attribute *)xsc_counters; + + counters_attr_g->name = COUNTERS_ATTER_GROUP_NAME; + counters_attr_g->attrs = counters_attrs; + counters_attr_g->bin_attrs = counters_bin_attrs; + + dev->counters_priv = counters_attr_g; + + ret = sysfs_create_group(&(ib_dev->dev.kobj), counters_attr_g); + if (ret) + goto err_counters_create_group; + + return 0; + +err_counters_create_group: + kfree(counters_attr_g); + counters_attr_g = NULL; + +err_counters_attr_g: + kfree(counters_attrs); + counters_attrs = NULL; + +err_counters_attrs: + kfree(counters_bin_attrs); + counters_bin_attrs = NULL; + +err_counters_bin_attrs: + kfree(xsc_counters_bin); + xsc_counters_bin = NULL; + +err_xsc_counters_bin: + kfree(xsc_counters); + xsc_counters = NULL; + +err_xsc_counters: + kfree(xsc_counters_name); + xsc_counters_name = NULL; + + return ret; +} + +void xsc_counters_fini(struct ib_device *ib_dev, struct xsc_core_device *dev) +{ + struct xsc_counters_attribute *xsc_counters_name, *xsc_counters; + struct xsc_counters_bin_attribute *xsc_counters_bin; + struct bin_attribute **counters_bin_attrs; + struct attribute **counters_attrs; + struct attribute_group *counters_attr_g; + + counters_attr_g = dev->counters_priv; + counters_attrs = counters_attr_g->attrs; + counters_bin_attrs = counters_attr_g->bin_attrs; + + xsc_counters_bin = (struct xsc_counters_bin_attribute *)counters_bin_attrs[0]; + xsc_counters_name = (struct xsc_counters_attribute *)counters_attrs[0]; + xsc_counters = (struct xsc_counters_attribute *)counters_attrs[1]; + + if (counters_attr_g) { + sysfs_remove_group(&(ib_dev->dev.kobj), counters_attr_g); + kfree(counters_attr_g); + counters_attr_g = NULL; + } + + kfree(counters_attrs); + counters_attrs = NULL; + + kfree(counters_bin_attrs); + counters_bin_attrs = NULL; + + kfree(xsc_counters_bin); + xsc_counters_bin = NULL; + + kfree(xsc_counters_name); + xsc_counters_name = NULL; + + kfree(xsc_counters); + xsc_counters = NULL; + + xsc_core_dbg(dev, "ok\n"); +} + diff --git a/drivers/infiniband/hw/xsc/counters.h b/drivers/infiniband/hw/xsc/counters.h new file mode 100644 index 0000000000000000000000000000000000000000..d9a41cb06c8765c33d63df0ab4c9543cd61878c5 --- /dev/null +++ b/drivers/infiniband/hw/xsc/counters.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef __COUNTERS_H__ +#define __COUNTERS_H__ + +#define STRING_LEN 32 +#define XSC_DECLARE_STAT(type, fld) ""#fld, offsetof(type, fld) + +struct counter_desc { + char format[STRING_LEN]; + size_t offset; /* Byte offset */ +}; + +struct xsc_counters_attribute { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, + struct attribute *attr, char *buf); + ssize_t (*store)(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count); + int id; + struct xsc_core_device *dev; + const struct counter_desc *desc; + int desc_size; +}; + +struct xsc_counters_bin_attribute { + struct attribute attr; + ssize_t (*read)(struct file *file, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t offset, size_t size); + ssize_t (*write)(struct file *file, + struct kobject *kobj, + struct bin_attribute *attr, + char *buf, loff_t offset, size_t size); + int (*mmap)(struct file *file, + struct kobject *kobj, + struct bin_attribute *attr, + struct vm_area_struct *vma); + int id; + struct xsc_core_device *dev; + const struct counter_desc *desc; + int desc_size; + size_t size; +}; + +ssize_t counters_vf_names_show(struct kobject *kobjs, + struct attribute *attr, char *buf); + +ssize_t counters_vf_value_read(struct file *file, + struct kobject *kob, + struct bin_attribute *bin_attr, + char *buf, loff_t loff, size_t size); + +ssize_t counters_vf_show(struct kobject *kobjs, + struct attribute *attr, char *buf); + +#endif diff --git a/drivers/infiniband/hw/xsc/cq.c b/drivers/infiniband/hw/xsc/cq.c new file mode 100644 index 0000000000000000000000000000000000000000..ceaa9c20500cad0b72a03a5c0cbec0683868da48 --- /dev/null +++ b/drivers/infiniband/hw/xsc/cq.c @@ -0,0 +1,695 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include "xsc_ib.h" +#include "user.h" +#include +#include + +enum { + CQ_OK = 0, + CQ_EMPTY = -1, + CQ_POLL_ERR = -2 +}; + +enum { + XSC_CQE_APP_TAG_MATCHING = 1, +}; + +enum { + XSC_CQE_APP_OP_TM_CONSUMED = 0x1, + XSC_CQE_APP_OP_TM_EXPECTED = 0x2, + XSC_CQE_APP_OP_TM_UNEXPECTED = 0x3, + XSC_CQE_APP_OP_TM_NO_TAG = 0x4, + XSC_CQE_APP_OP_TM_APPEND = 0x5, + XSC_CQE_APP_OP_TM_REMOVE = 0x6, + XSC_CQE_APP_OP_TM_NOOP = 0x7, + XSC_CQE_APP_OP_TM_CONSUMED_SW_RDNV = 0x9, + XSC_CQE_APP_OP_TM_CONSUMED_MSG = 0xA, + XSC_CQE_APP_OP_TM_CONSUMED_MSG_SW_RDNV = 0xB, + XSC_CQE_APP_OP_TM_MSG_COMPLETION_CANCELED = 0xC, +}; + +static const u32 xsc_msg_opcode[][2][2] = { + [XSC_MSG_OPCODE_SEND][XSC_REQ][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_REQ_SEND, + [XSC_MSG_OPCODE_SEND][XSC_REQ][XSC_WITH_IMMDT] = XSC_OPCODE_RDMA_REQ_SEND_IMMDT, + [XSC_MSG_OPCODE_SEND][XSC_RSP][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_RSP_RECV, + [XSC_MSG_OPCODE_SEND][XSC_RSP][XSC_WITH_IMMDT] = XSC_OPCODE_RDMA_RSP_RECV_IMMDT, + [XSC_MSG_OPCODE_RDMA_WRITE][XSC_REQ][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_REQ_WRITE, + [XSC_MSG_OPCODE_RDMA_WRITE][XSC_REQ][XSC_WITH_IMMDT] = XSC_OPCODE_RDMA_REQ_WRITE_IMMDT, + [XSC_MSG_OPCODE_RDMA_WRITE][XSC_RSP][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_WRITE][XSC_RSP][XSC_WITH_IMMDT] = XSC_OPCODE_RDMA_RSP_WRITE_IMMDT, + [XSC_MSG_OPCODE_RDMA_READ][XSC_REQ][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_REQ_READ, + [XSC_MSG_OPCODE_RDMA_READ][XSC_REQ][XSC_WITH_IMMDT] = XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_READ][XSC_RSP][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_RDMA_READ][XSC_RSP][XSC_WITH_IMMDT] = XSC_OPCODE_RDMA_CQE_ERROR, + [XSC_MSG_OPCODE_MAD][XSC_REQ][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_MAD_REQ_SEND, + [XSC_MSG_OPCODE_MAD][XSC_RSP][XSC_WITHOUT_IMMDT] = XSC_OPCODE_RDMA_MAD_RSP_RECV, +}; + +static const u32 xsc_cqe_opcode[] = { + [XSC_OPCODE_RDMA_REQ_SEND] = IB_WC_SEND, + [XSC_OPCODE_RDMA_REQ_SEND_IMMDT] = IB_WC_SEND, + [XSC_OPCODE_RDMA_RSP_RECV] = IB_WC_RECV, + [XSC_OPCODE_RDMA_RSP_RECV_IMMDT] = IB_WC_RECV, + [XSC_OPCODE_RDMA_REQ_WRITE] = IB_WC_RDMA_WRITE, + [XSC_OPCODE_RDMA_REQ_WRITE_IMMDT] = IB_WC_RDMA_WRITE, + [XSC_OPCODE_RDMA_RSP_WRITE_IMMDT] = IB_WC_RECV_RDMA_WITH_IMM, + [XSC_OPCODE_RDMA_REQ_READ] = IB_WC_RDMA_READ, + [XSC_OPCODE_RDMA_MAD_REQ_SEND] = IB_WC_SEND, + [XSC_OPCODE_RDMA_MAD_RSP_RECV] = IB_WC_RECV, +}; + +int xsc_stall_num_loop = 60; +int xsc_stall_cq_poll_min = 60; +int xsc_stall_cq_poll_max = 100000; +int xsc_stall_cq_inc_step = 100; +int xsc_stall_cq_dec_step = 10; + +static inline u8 xsc_get_cqe_opcode(struct xsc_cqe *cqe) +{ + if (cqe->is_error) + return cqe->type ? XSC_OPCODE_RDMA_RSP_ERROR : XSC_OPCODE_RDMA_REQ_ERROR; + if (cqe->msg_opcode > XSC_MSG_OPCODE_MAD) + return XSC_OPCODE_RDMA_CQE_ERROR; + return xsc_msg_opcode[cqe->msg_opcode][cqe->type][cqe->with_immdt]; +} + +static void xsc_ib_cq_comp(struct xsc_core_cq *cq) +{ + struct ib_cq *ibcq = &to_xibcq(cq)->ibcq; + + ibcq->comp_handler(ibcq, ibcq->cq_context); +} + +static void xsc_ib_cq_event(struct xsc_core_cq *xcq, enum xsc_event type) +{ + struct xsc_ib_cq *cq = container_of(xcq, struct xsc_ib_cq, xcq); + struct xsc_ib_dev *dev = to_mdev(cq->ibcq.device); + struct ib_cq *ibcq = &cq->ibcq; + struct ib_event event; + + if (type != XSC_EVENT_TYPE_CQ_ERROR) { + xsc_ib_err(dev, "Unexpected event type %d on CQ %06x\n", + type, xcq->cqn); + return; + } + + if (ibcq->event_handler) { + event.device = &dev->ib_dev; + event.event = IB_EVENT_CQ_ERR; + event.element.cq = ibcq; + ibcq->event_handler(&event, ibcq->cq_context); + } +} + +static void *get_cqe_from_buf(struct xsc_ib_cq_buf *buf, int n, int size) +{ + return xsc_buf_offset(&buf->buf, n * size); +} + +static void *get_cqe(struct xsc_ib_cq *cq, int n) +{ + return get_cqe_from_buf(&cq->buf, n, cq->xcq.cqe_sz); +} + +static void *get_sw_cqe(struct xsc_ib_cq *cq, int n) +{ + struct xsc_cqe64 *cqe64; + + cqe64 = (struct xsc_cqe64 *)get_cqe(cq, n & cq->ibcq.cqe); + + return ((cqe64->owner & XSC_CQE_OWNER_MASK) ^ + !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe64; +} + +static inline void handle_good_req(struct ib_wc *wc, + struct xsc_cqe *cqe, u8 opcode) +{ + wc->opcode = xsc_cqe_opcode[opcode]; + if (opcode == XSC_OPCODE_RDMA_REQ_READ) + wc->byte_len = RD_LE_32(cqe->msg_len); + wc->status = IB_WC_SUCCESS; +} + +static void handle_responder(struct ib_wc *wc, struct xsc_cqe *cqe, + struct xsc_ib_qp *qp, u8 opcode) +{ + struct xsc_ib_wq *wq = &qp->rq; + u16 idx; + + wc->byte_len = RD_LE_32(cqe->msg_len); + wc->opcode = xsc_cqe_opcode[opcode]; + wc->status = IB_WC_SUCCESS; + + idx = wq->tail & (wq->wqe_cnt - 1); + wc->wr_id = wq->wrid[idx]; + ++wq->tail; + +} + +struct ib_mad_list_head { + struct list_head list; + struct ib_cqe cqe; + struct ib_mad_queue *mad_queue; +}; + +struct ib_mad_private_header { + struct ib_mad_list_head mad_list; + struct ib_mad_recv_wc recv_wc; + struct ib_wc wc; + u64 mapping; +} __packed; + +struct ib_mad_private { + struct ib_mad_private_header header; + size_t mad_size; + struct ib_grh grh; + u8 mad[0]; +} __packed; + +static void *get_wqe(struct xsc_ib_qp *qp, int offset) +{ + return xsc_buf_offset(&qp->buf, offset); +} + +static void *get_recv_wqe(struct xsc_ib_qp *qp, int n) +{ + return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); +} + +static void *get_seg_wqe(void *first, int n) +{ + return first + (n << XSC_BASE_WQE_SHIFT); +} + +static void xsc_handle_rdma_mad_resp_recv(struct xsc_ib_cq *cq, + struct xsc_ib_qp **cur_qp, + struct ib_wc *wc, + struct xsc_cqe *cqe, + u8 opcode) +{ + struct xsc_ib_dev *dev = to_mdev(cq->ibcq.device); + void *recv; + struct xsc_wqe_data_seg *data_seg; + struct iphdr *ip4h = NULL; + struct udphdr *udph; + struct ib_unpacked_eth *eth; + struct ib_unpacked_vlan *vlan; + struct ib_grh *grh; + struct ib_mad *mad; + struct rxe_bth *bth; + struct rxe_deth *deth; + unsigned int pading_sz = 0; + struct xsc_ib_wq *wq; + int idx; + + wq = &(*cur_qp)->rq; + idx = wq->tail & (wq->wqe_cnt - 1); + + handle_responder(wc, cqe, *cur_qp, opcode); + + data_seg = get_seg_wqe(get_recv_wqe(*cur_qp, idx), 0); + recv = phys_to_virt(dma_to_phys(dev->ib_dev.dma_device, data_seg->va)); + + eth = (struct ib_unpacked_eth *)recv; + grh = (struct ib_grh *)recv; + if (eth->type == htons(ETH_P_8021Q)) { + vlan = (struct ib_unpacked_vlan *)(eth + 1); + ip4h = (struct iphdr *)(vlan + 1); + + wc->vlan_id = ntohs(vlan->tag) & 0x0fff; + wc->sl = (ntohs(vlan->tag) >> 13) & 0x7; + wc->wc_flags |= IB_WC_WITH_VLAN; + } else { + ip4h = (struct iphdr *)(eth + 1); + } + udph = (struct udphdr *)(ip4h + 1); + bth = (struct rxe_bth *)(udph + 1); + deth = (struct rxe_deth *)(bth + 1); + mad = (struct ib_mad *)(deth + 1); + + memcpy(grh + 1, mad, sizeof(*mad)); + pading_sz = sizeof(*grh) - sizeof(*ip4h); + memmove((u8 *)(grh + 1) - sizeof(*ip4h), ip4h, sizeof(*ip4h)); + memset(grh, 0, pading_sz); + + wc->wc_flags |= IB_WC_GRH; + + xsc_ib_dbg(dev, "recv cqe idx:%u, len:%u\n", wq->tail, wc->byte_len); + wc->status = IB_WC_SUCCESS; +} + +static int xsc_poll_one(struct xsc_ib_cq *cq, + struct xsc_ib_qp **cur_qp, + struct ib_wc *wc) +{ + struct xsc_ib_dev *dev = to_mdev(cq->ibcq.device); + struct xsc_core_qp *xqp; + struct xsc_ib_wq *wq; + u8 opcode; + u32 qpn; + int idx; + struct xsc_cqe *cqe; + u32 *p = NULL; + + cqe = get_sw_cqe(cq, cq->xcq.cons_index); + if (!cqe) + return -EAGAIN; + + ++cq->xcq.cons_index; + + /* Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rmb(); + + p = (u32 *)cqe; + + qpn = cqe->qp_id; + qpn = le32_to_cpu(qpn); + if (!*cur_qp || (qpn != (*cur_qp)->ibqp.qp_num)) { + /* We do not have to take the QP table lock here, + * because CQs will be locked while QPs are removed + * from the table. + */ + xqp = __xsc_qp_lookup(dev->xdev, qpn); + if (unlikely(!xqp)) { + xsc_ib_warn(dev, "CQE@CQ %d for unknown QPN %d\n", + cq->xcq.cqn, qpn); + return -EINVAL; + } + + *cur_qp = to_xibqp(xqp); + } + + memset(wc, 0, sizeof(*wc)); + wc->qp = &(*cur_qp)->ibqp; + opcode = xsc_get_cqe_opcode(cqe); + switch (opcode) { + case XSC_OPCODE_RDMA_REQ_SEND: + case XSC_OPCODE_RDMA_REQ_WRITE: + case XSC_OPCODE_RDMA_REQ_READ: + case XSC_OPCODE_RDMA_MAD_REQ_SEND: + wq = &(*cur_qp)->sq; + idx = cqe->wqe_id >> (wq->wqe_shift - XSC_BASE_WQE_SHIFT); + idx &= (wq->wqe_cnt - 1); + handle_good_req(wc, cqe, opcode); + wc->wr_id = wq->wrid[idx]; + wq->tail = wq->wqe_head[idx] + 1; + xsc_ib_dbg(dev, "wqeid:%u, wq tail:%u qpn:%u\n", idx, wq->tail, qpn); + wc->status = IB_WC_SUCCESS; + break; + case XSC_OPCODE_RDMA_RSP_RECV: + wq = &(*cur_qp)->rq; + handle_responder(wc, cqe, *cur_qp, opcode); + xsc_ib_dbg(dev, "recv cqe idx:%u, len:%u, qpn:%u\n", wq->tail, wc->byte_len, qpn); + wc->status = IB_WC_SUCCESS; + break; + + case XSC_OPCODE_RDMA_MAD_RSP_RECV: + xsc_ib_dbg(dev, "recv MAD, qpn:%u\n", qpn); + xsc_handle_rdma_mad_resp_recv(cq, cur_qp, wc, cqe, opcode); + break; + + default: + xsc_ib_err(dev, "completion error\n%08x %08x %08x %08x %08x %08x\n", + p[0], p[1], p[2], p[3], p[5], p[6]); + wc->status = IB_WC_GENERAL_ERR; + wc->wr_id = 0; + break; + } + + return 0; +} + +int xsc_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct xsc_ib_cq *cq = to_xcq(ibcq); + struct xsc_core_cq *xcq = &cq->xcq; + struct xsc_ib_qp *cur_qp = NULL; + int npolled = 0; + int err = 0; + unsigned long flags; + u32 next_cid; + + spin_lock_irqsave(&cq->lock, flags); + next_cid = xcq->cons_index; + + for (npolled = 0; npolled < num_entries; npolled++) { + err = xsc_poll_one(cq, &cur_qp, wc + npolled); + if (err) + break; + } + + /* make sure cqe read out before update ci */ + rmb(); + + if (next_cid != xcq->cons_index) + xsc_cq_set_ci(xcq); + + spin_unlock_irqrestore(&cq->lock, flags); + + return npolled; + +} + +int xsc_cqe_is_empty(struct xsc_ib_cq *cq) +{ + struct xsc_cqe *cqe = get_sw_cqe(cq, cq->xcq.cons_index); + + if (!cqe) + return 1; + + return 0; +} + +int xsc_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ +#ifdef MSIX_SUPPORT + union xsc_cq_doorbell db; + struct xsc_ib_cq *xcq = to_xcq(ibcq); + struct xsc_core_cq *cq = &xcq->xcq; + int ret = 0; + unsigned long irq_flags; + + spin_lock_irqsave(&xcq->lock, irq_flags); + db.val = 0; + db.cq_next_cid = cq->cons_index; + db.cq_id = cq->cqn; + if (flags & IB_CQ_NEXT_COMP) + db.arm = 0; + else if (flags & IB_CQ_SOLICITED) + db.arm = 1;/* arm next:0 arm solicited:1 */ + + if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && (!xsc_cqe_is_empty(xcq))) { + ret = 1; + goto out; + } + + /* make sure val write to memory done */ + wmb(); + writel(db.val, REG_ADDR(cq->dev, cq->arm_db)); +out: + spin_unlock_irqrestore(&xcq->lock, irq_flags); + return ret; +#else + if ((flags & IB_CQ_REPORT_MISSED_EVENTS)) + return 1; + return 0; +#endif +} + +static int alloc_cq_buf(struct xsc_ib_dev *dev, struct xsc_ib_cq_buf *buf, + int nent, int cqe_size) +{ + int err; + + err = xsc_buf_alloc(dev->xdev, nent * cqe_size, + PAGE_SIZE, &buf->buf); + if (err) + return err; + + buf->cqe_size = cqe_size; + + return 0; +} + +static void free_cq_buf(struct xsc_ib_dev *dev, struct xsc_ib_cq_buf *buf) +{ + xsc_buf_free(dev->xdev, &buf->buf); +} + +static int create_cq_user(struct xsc_ib_dev *dev, struct ib_udata *udata, + struct ib_ucontext *context, struct xsc_ib_cq *cq, + int entries, struct xsc_create_cq_mbox_in **cqb, + int *cqe_size, int *index, int *inlen) +{ + struct xsc_ib_create_cq ucmd; + int page_shift; + int npages; + int ncont; + int err; + int log_cq_sz; + int hw_npages; + + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) + return -EFAULT; + + *cqe_size = ucmd.cqe_size; + + cq->buf.umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, + entries * ucmd.cqe_size, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(cq->buf.umem)) { + err = PTR_ERR(cq->buf.umem); + return err; + } + + xsc_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, &npages, &page_shift, + &ncont, NULL); + if (ncont != npages) { + xsc_ib_warn(dev, "bad page_shift:%d, ncont:%d\n", page_shift, ncont); + /* amber doesn't support compound pages */ + page_shift = PAGE_SHIFT; + ncont = npages; + xsc_ib_warn(dev, "overwrite to page_shift:%d, ncont:%d\n", + page_shift, ncont); + } + log_cq_sz = ilog2(entries); + hw_npages = DIV_ROUND_UP((1 << log_cq_sz) * sizeof(struct xsc_cqe), PAGE_SIZE_4K); + xsc_ib_dbg(dev, "addr 0x%llx, entries %d, size %u, npages %d, page_shift %d, ncont %d, hw_npages %d\n", + ucmd.buf_addr, entries, ucmd.cqe_size, npages, page_shift, ncont, hw_npages); + + *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * hw_npages; + *cqb = xsc_vzalloc(*inlen); + if (!*cqb) { + err = -ENOMEM; + goto err_umem; + } + xsc_ib_populate_pas(dev, cq->buf.umem, page_shift, (*cqb)->pas, hw_npages, true); + (*cqb)->ctx.pa_num = cpu_to_be16(hw_npages); + + return 0; + +err_umem: + ib_umem_release(cq->buf.umem); + return err; +} + +static void destroy_cq_user(struct xsc_ib_cq *cq, struct ib_udata *udata) +{ + ib_umem_release(cq->buf.umem); +} + +static int create_cq_kernel(struct xsc_ib_dev *dev, struct xsc_ib_cq *cq, + int entries, int cqe_size, + struct xsc_create_cq_mbox_in **cqb, + int *index, int *inlen) +{ + int err; + int i = 0; + struct xsc_cqe *cqe = NULL; + int hw_npages; + + cq->xcq.cqe_sz = cqe_size; + + err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size); + if (err) + return err; + + for (i = 0; i < entries; i++) { + cqe = (struct xsc_cqe *)get_cqe(cq, i); + cqe->owner = 1; + } + + hw_npages = DIV_ROUND_UP(entries * cqe_size, PAGE_SIZE_4K); + *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * hw_npages; + *cqb = xsc_vzalloc(*inlen); + if (!*cqb) { + err = -ENOMEM; + goto err_buf; + } + xsc_fill_page_array(&cq->buf.buf, (*cqb)->pas, hw_npages); + (*cqb)->ctx.pa_num = cpu_to_be16(hw_npages); + + return 0; + +err_buf: + free_cq_buf(dev, &cq->buf); + return err; +} + +static void destroy_cq_kernel(struct xsc_ib_dev *dev, struct xsc_ib_cq *cq) +{ + free_cq_buf(dev, &cq->buf); +} + +xsc_ib_create_cq_def() +{ + struct ib_device *ibdev = ibcq->device; + int entries = attr->cqe; + int vector = attr->comp_vector; + struct xsc_create_cq_mbox_in *cqb = NULL; + struct xsc_ib_dev *dev = to_mdev(ibdev); + struct xsc_ib_cq *cq; + int index; + int inlen; + int cqe_size; + int irqn; + int err; + unsigned int eqn; + + entries = roundup_pow_of_two(entries + 1); + + xsc_ib_dbg(dev, "entries:%d, vector:%d, max_cqes:%d\n", entries, vector, + dev->xdev->caps.max_cqes); + + if (entries > dev->xdev->caps.max_cqes) + entries = dev->xdev->caps.max_cqes; + cq = to_xcq(ibcq); + cq->ibcq.cqe = entries - 1; + mutex_init(&cq->resize_mutex); + spin_lock_init(&cq->lock); + cq->resize_buf = NULL; + cq->resize_umem = NULL; + + if (udata) { + err = create_cq_user(dev, udata, NULL, cq, entries, + &cqb, &cqe_size, &index, &inlen); + if (err) + goto err_create; + } else { + cqe_size = sizeof(struct xsc_cqe); + err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, &index, &inlen); + if (err) + goto err_create; + } + + cq->cqe_size = cqe_size; + cqb->ctx.log_cq_sz = ilog2(entries); + cqb->ctx.glb_func_id = cpu_to_be16(dev->xdev->glb_func_id); + + err = xsc_vector2eqn(dev->xdev, vector, &eqn, &irqn); + if (err) + goto err_cqb; + + cqb->ctx.eqn = eqn; + cqb->ctx.eqn = cpu_to_be16(cqb->ctx.eqn); + + err = xsc_core_create_cq(dev->xdev, &cq->xcq, cqb, inlen); + if (err) + goto err_cqb; + + xsc_ib_dbg(dev, "cqn 0x%x\n", cq->xcq.cqn); + cq->xcq.irqn = irqn; + cq->xcq.comp = xsc_ib_cq_comp; + cq->xcq.event = xsc_ib_cq_event; + + if (udata) { + if (ib_copy_to_udata(udata, &cq->xcq.cqn, sizeof(__u32))) { + err = -EFAULT; + goto err_cmd; + } + } + + xsc_vfree(cqb); + + return 0; + +err_cmd: + xsc_core_destroy_cq(dev->xdev, &cq->xcq); + +err_cqb: + xsc_vfree(cqb); + if (udata) + destroy_cq_user(cq, udata); + else + destroy_cq_kernel(dev, cq); + +err_create: + kfree(cq); + return err; +} + +xsc_ib_destroy_cq_def() +{ + struct xsc_ib_dev *dev = to_mdev(cq->device); + struct xsc_ib_cq *xcq = to_xcq(cq); + + xsc_core_destroy_cq(dev->xdev, &xcq->xcq); + if (udata) + destroy_cq_user(xcq, udata); + else + destroy_cq_kernel(dev, xcq); + + return 0; +} + +static int is_equal_rsn(struct xsc_cqe64 *cqe, u32 rsn) +{ + u32 qpn = le32_to_cpu(cqe->qp_id); + return rsn == qpn; +} + +void __xsc_ib_cq_clean(struct xsc_ib_cq *cq, u32 rsn) +{ + struct xsc_cqe64 *cqe, *dest; + u32 prod_index; + int nfreed = 0; + u8 owner_bit; + + if (!cq) + return; + + /* First we need to find the current producer index, so we + * know where to start cleaning from. It doesn't matter if HW + * adds new entries after this loop -- the QP we're worried + * about is already in RESET, so the new entries won't come + * from our QP and therefore don't need to be checked. + */ + for (prod_index = cq->xcq.cons_index; get_sw_cqe(cq, prod_index); prod_index++) + if (prod_index == cq->xcq.cons_index + cq->ibcq.cqe) + break; + + /* Now sweep backwards through the CQ, removing CQ entries + * that match our QP by copying older entries on top of them. + */ + while ((int) --prod_index - (int) cq->xcq.cons_index >= 0) { + cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); + if (is_equal_rsn(cqe, rsn)) { + ++nfreed; + } else if (nfreed) { + dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); + owner_bit = dest->owner & XSC_CQE_OWNER_MASK; + memcpy(dest, cqe, cq->xcq.cqe_sz); + dest->owner = owner_bit | + (dest->owner & ~XSC_CQE_OWNER_MASK); + } + } + + if (nfreed) { + cq->xcq.cons_index += nfreed; + /* Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + xsc_cq_set_ci(&cq->xcq); + } +} + +void xsc_ib_cq_clean(struct xsc_ib_cq *cq, u32 qpn) +{ + if (!cq) + return; + + spin_lock_irq(&cq->lock); + __xsc_ib_cq_clean(cq, qpn); + spin_unlock_irq(&cq->lock); +} + diff --git a/drivers/infiniband/hw/xsc/devx.c b/drivers/infiniband/hw/xsc/devx.c new file mode 100644 index 0000000000000000000000000000000000000000..f6ab45d9ad0727f71346406f6d8a220b72fb596e --- /dev/null +++ b/drivers/infiniband/hw/xsc/devx.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include "xsc_ib.h" +#define UVERBS_MODULE_NAME xsc_ib +#include +#include "user.h" + +static struct xsc_ib_ucontext *devx_uattrs2uctx(struct uverbs_attr_bundle *attrs) +{ + return to_xucontext(ib_uverbs_get_ucontext(attrs)); +} + +static bool devx_is_general_cmd(void *in) +{ + struct xsc_inbox_hdr *hdr = + (struct xsc_inbox_hdr *)in; + u16 opcode = be16_to_cpu(hdr->opcode); + + switch (opcode) { + case XSC_CMD_OP_QUERY_HCA_CAP: + return true; + default: + return false; + } +} + +static int UVERBS_HANDLER(XSC_IB_METHOD_DEVX_OTHER)(struct uverbs_attr_bundle *attrs) +{ + struct xsc_ib_ucontext *c; + struct xsc_ib_dev *dev; + void *cmd_in = uverbs_attr_get_alloced_ptr( + attrs, XSC_IB_ATTR_DEVX_OTHER_CMD_IN); + int cmd_out_len = uverbs_attr_get_len(attrs, + XSC_IB_ATTR_DEVX_OTHER_CMD_OUT); + void *cmd_out; + int err; + + c = devx_uattrs2uctx(attrs); + if (IS_ERR(c)) + return PTR_ERR(c); + dev = to_mdev(c->ibucontext.device); + + if (!devx_is_general_cmd(cmd_in)) + return -EINVAL; + + cmd_out = uverbs_zalloc(attrs, cmd_out_len); + if (IS_ERR(cmd_out)) + return PTR_ERR(cmd_out); + + err = xsc_cmd_exec(dev->xdev, cmd_in, + uverbs_attr_get_len(attrs, XSC_IB_ATTR_DEVX_OTHER_CMD_IN), + cmd_out, cmd_out_len); + if (err) + return err; + + return uverbs_copy_to(attrs, XSC_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out, cmd_out_len); +} + +DECLARE_UVERBS_NAMED_METHOD( + XSC_IB_METHOD_DEVX_OTHER, + UVERBS_ATTR_PTR_IN( + XSC_IB_ATTR_DEVX_OTHER_CMD_IN, + UVERBS_ATTR_MIN_SIZE(sizeof(struct xsc_inbox_hdr)), + UA_MANDATORY, + UA_ALLOC_AND_COPY), + UVERBS_ATTR_PTR_OUT( + XSC_IB_ATTR_DEVX_OTHER_CMD_OUT, + UVERBS_ATTR_MIN_SIZE(sizeof(struct xsc_outbox_hdr)), + UA_MANDATORY)); + +DECLARE_UVERBS_GLOBAL_METHODS(XSC_IB_OBJECT_DEVX, + &UVERBS_METHOD(XSC_IB_METHOD_DEVX_OTHER)); + +const struct uverbs_object_tree_def *xsc_ib_get_devx_tree(void) +{ + return NULL; +} diff --git a/drivers/infiniband/hw/xsc/global.c b/drivers/infiniband/hw/xsc/global.c new file mode 100644 index 0000000000000000000000000000000000000000..e46c1665803236c26b6022163052f6f78c02defc --- /dev/null +++ b/drivers/infiniband/hw/xsc/global.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include "global.h" +#include + +#define QOS_DSCP_MAX 63 +#define QOS_PCP_MAX 7 + +// default value 255 for no global setting +int g_xsc_force_pcp = GLOBAL_UNSET_FORCE_VALUE; +int g_xsc_force_dscp = GLOBAL_UNSET_FORCE_VALUE; + +static int is_valid_pcp(int pcp) +{ + if ((pcp >= 0 && pcp <= QOS_PCP_MAX) || pcp == GLOBAL_UNSET_FORCE_VALUE) + return 0; + return -1; +} +static int is_valid_dscp(int dscp) +{ + if ((dscp >= 0 && dscp <= QOS_DSCP_MAX) || dscp == GLOBAL_UNSET_FORCE_VALUE) + return 0; + return -1; +} + +int get_global_force_pcp(void) +{ + return g_xsc_force_pcp; +} + +int set_global_force_pcp(int force_pcp) +{ + if (is_valid_pcp(force_pcp) == 0) { + g_xsc_force_pcp = force_pcp; + return 0; + } else { + return -1; + } +} + +int get_global_force_dscp(void) +{ + return g_xsc_force_dscp; +} + +int set_global_force_dscp(int force_dscp) +{ + if (is_valid_dscp(force_dscp) == 0) { + g_xsc_force_dscp = force_dscp; + return 0; + } else { + return -1; + } +} diff --git a/drivers/infiniband/hw/xsc/global.h b/drivers/infiniband/hw/xsc/global.h new file mode 100644 index 0000000000000000000000000000000000000000..bdca76559a083db0cc44685e26997f4aa8cd845b --- /dev/null +++ b/drivers/infiniband/hw/xsc/global.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_GLOBAL_H +#define XSC_GLOBAL_H + +#define GLOBAL_UNSET_FORCE_VALUE 255 + +int get_global_force_pcp(void); +int set_global_force_pcp(int force_pcp); + +int get_global_force_dscp(void); +int set_global_force_dscp(int force_dscp); + +#endif diff --git a/drivers/infiniband/hw/xsc/ib_peer_mem.h b/drivers/infiniband/hw/xsc/ib_peer_mem.h new file mode 100644 index 0000000000000000000000000000000000000000..17da6169f97177bab140202bbc4d09a7cbd0d4e4 --- /dev/null +++ b/drivers/infiniband/hw/xsc/ib_peer_mem.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#if !defined(IB_PEER_MEM_H) +#define IB_PEER_MEM_H + +#include "peer_mem.h" + +struct ib_peer_memory_statistics { + atomic64_t num_alloc_mrs; + atomic64_t num_dealloc_mrs; + atomic64_t num_reg_pages; + atomic64_t num_dereg_pages; + atomic64_t num_reg_bytes; + atomic64_t num_dereg_bytes; + unsigned long num_free_callbacks; +}; + +struct ib_ucontext; +struct ib_umem_ex; +struct invalidation_ctx; + +struct ib_peer_memory_client { + const struct peer_memory_client *peer_mem; + struct list_head core_peer_list; + int invalidation_required; + struct kref ref; + struct completion unload_comp; + /* lock is used via the invalidation flow */ + struct mutex lock; + struct list_head core_ticket_list; + u64 last_ticket; + struct kobject *kobj; + struct attribute_group peer_mem_attr_group; + struct ib_peer_memory_statistics stats; +}; + +enum ib_peer_mem_flags { + IB_PEER_MEM_ALLOW = 1, + IB_PEER_MEM_INVAL_SUPP = (1<<1), +}; + +struct core_ticket { + unsigned long key; + void *context; + struct list_head ticket_list; +}; + +struct ib_peer_memory_client *ib_get_peer_client(struct ib_ucontext *context, unsigned long addr, + size_t size, unsigned long peer_mem_flags, + void **peer_client_context); + +void ib_put_peer_client(struct ib_peer_memory_client *ib_peer_client, + void *peer_client_context); + +int ib_peer_create_invalidation_ctx(struct ib_peer_memory_client *ib_peer_mem, + struct ib_umem_ex *umem, + struct invalidation_ctx **invalidation_ctx); + +void ib_peer_destroy_invalidation_ctx(struct ib_peer_memory_client *ib_peer_mem, + struct invalidation_ctx *invalidation_ctx); + +int ib_get_peer_private_data(struct ib_ucontext *context, __u64 peer_id, + char *peer_name); +void ib_put_peer_private_data(struct ib_ucontext *context); + +#endif diff --git a/drivers/infiniband/hw/xsc/ib_umem_ex.c b/drivers/infiniband/hw/xsc/ib_umem_ex.c new file mode 100644 index 0000000000000000000000000000000000000000..cc0566f8c956cc7bdc31e90be92c50f862285465 --- /dev/null +++ b/drivers/infiniband/hw/xsc/ib_umem_ex.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include + +#include "ib_peer_mem.h" + +#include +#include "ib_umem_ex.h" + +struct ib_umem_ex *ib_umem_ex(struct ib_umem *umem) +{ + struct ib_umem_ex *ret_umem; + + if (!umem) + return ERR_PTR(-EINVAL); + + ret_umem = kzalloc(sizeof(struct ib_umem_ex), GFP_KERNEL); + if (!ret_umem) + return ERR_PTR(-ENOMEM); + + ret_umem->umem = *umem; + kfree(umem); + return ret_umem; +} + +struct ib_umem_ex *ib_client_umem_get(struct ib_ucontext *context, + unsigned long addr, + size_t size, int access, + int dmasync, u8 *peer_exists) +{ + return NULL; +} + +void ib_umem_ex_release(struct ib_umem_ex *umem_ex) +{ + struct ib_umem *umem = (struct ib_umem *)umem_ex; + + ib_umem_release(umem); +} + +int ib_client_umem_activate_invalidation_notifier(struct ib_umem_ex *umem_ex, + umem_invalidate_func_t func, + void *cookie) +{ + return 0; +} + diff --git a/drivers/infiniband/hw/xsc/ib_umem_ex.h b/drivers/infiniband/hw/xsc/ib_umem_ex.h new file mode 100644 index 0000000000000000000000000000000000000000..62f8673247dcaadac224fbc3d698307e59d79318 --- /dev/null +++ b/drivers/infiniband/hw/xsc/ib_umem_ex.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_IB_UMEM_EX_H +#define XSC_IB_UMEM_EX_H + +#include + +struct ib_umem_ex; +struct invalidation_ctx; + +// ib umem ex ib_umem add peer memory support +struct ib_umem_ex { + struct ib_umem umem; + struct ib_peer_memory_client *ib_peer_mem; + struct invalidation_ctx *invalidation_ctx; + void *peer_mem_client_context; +}; + +// expand ib_umem to ib_umem_ex by reallocate +struct ib_umem_ex *ib_umem_ex(struct ib_umem *umem); + +typedef void (*umem_invalidate_func_t)(void *invalidation_cookie, + struct ib_umem_ex *umem_ex, unsigned long addr, size_t size); + +struct invalidation_ctx { + struct ib_umem_ex *umem_ex; + u64 context_ticket; + umem_invalidate_func_t func; + void *cookie; + int peer_callback; + int inflight_invalidation; + int peer_invalidated; + struct completion comp; +}; + +struct ib_umem_ex *ib_client_umem_get(struct ib_ucontext *context, + unsigned long addr, + size_t size, int access, + int dmasync, u8 *peer_exists); + +void ib_umem_ex_release(struct ib_umem_ex *umem_ex); + +int ib_client_umem_activate_invalidation_notifier(struct ib_umem_ex *umem_ex, + umem_invalidate_func_t func, + void *cookie); +#endif diff --git a/drivers/infiniband/hw/xsc/mad.c b/drivers/infiniband/hw/xsc/mad.c new file mode 100644 index 0000000000000000000000000000000000000000..8d93cb820c37dc5c59474382fc7df500a215751b --- /dev/null +++ b/drivers/infiniband/hw/xsc/mad.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include "xsc_ib.h" + +enum { + XSC_IB_VENDOR_CLASS1 = 0x9, + XSC_IB_VENDOR_CLASS2 = 0xa +}; + +int xsc_MAD_IFC(struct xsc_ib_dev *dev, int ignore_mkey, int ignore_bkey, + int port, struct ib_wc *in_wc, struct ib_grh *in_grh, + void *in_mad, void *response_mad) +{ + u8 op_modifier = 0; + + /* Key check traps can't be generated unless we have in_wc to + * tell us where to send the trap. + */ + if (ignore_mkey || !in_wc) + op_modifier |= 0x1; + if (ignore_bkey || !in_wc) + op_modifier |= 0x2; + + return xsc_core_mad_ifc(dev->xdev, in_mad, response_mad, op_modifier, port); +} + +int xsc_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + struct ib_wc *in_wc, struct ib_grh *in_grh, + struct ib_mad *in_mad, struct ib_mad *out_mad) +{ + u16 slid; + int err; + + slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); + + if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; + + if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || + in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { + if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && + in_mad->mad_hdr.method != IB_MGMT_METHOD_SET && + in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS) + return IB_MAD_RESULT_SUCCESS; + + /* Don't process SMInfo queries -- the SMA can't handle them. + */ + if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO) + return IB_MAD_RESULT_SUCCESS; + } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || + in_mad->mad_hdr.mgmt_class == XSC_IB_VENDOR_CLASS1 || + in_mad->mad_hdr.mgmt_class == XSC_IB_VENDOR_CLASS2 || + in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_CONG_MGMT) { + if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && + in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) + return IB_MAD_RESULT_SUCCESS; + } else { + return IB_MAD_RESULT_SUCCESS; + } + + err = xsc_MAD_IFC(to_mdev(ibdev), + mad_flags & IB_MAD_IGNORE_MKEY, + mad_flags & IB_MAD_IGNORE_BKEY, + port_num, in_wc, in_grh, in_mad, out_mad); + if (err) + return IB_MAD_RESULT_FAILURE; + + /* set return bit in status of directed route responses */ + if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) + out_mad->mad_hdr.status |= cpu_to_be16(1 << 15); + + if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) + /* no response for trap repress */ + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; + + return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +} + diff --git a/drivers/infiniband/hw/xsc/main.c b/drivers/infiniband/hw/xsc/main.c new file mode 100644 index 0000000000000000000000000000000000000000..e7ac738af739b111c1f2d0cab2b65a6a30d40ab0 --- /dev/null +++ b/drivers/infiniband/hw/xsc/main.c @@ -0,0 +1,1095 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "user.h" +#include "xsc_ib.h" +#include "xsc_rdma_ctrl.h" + +#define DRIVER_NAME "xsc_ib" +#define DRIVER_VERSION "1.0.0" +#define DRIVER_RELDATE "Jan 2022" + +MODULE_DESCRIPTION("Yunsilicon XSC RDMA driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRIVER_VERSION); + +static int prof_sel = 2; +module_param_named(prof_sel, prof_sel, int, 0444); +MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); + +static char xsc_version[] = + DRIVER_NAME ": Yunsilicon Amber Infiniband driver" + DRIVER_VERSION " (" DRIVER_RELDATE ")\n"; + +struct xsc_profile profile[] = { + [0] = { + .mask = 0, + }, + [1] = { + .mask = XSC_PROF_MASK_QP_SIZE, + .log_max_qp = 12, + }, + [2] = { + .mask = XSC_PROF_MASK_QP_SIZE | + XSC_PROF_MASK_MR_CACHE, + .log_max_qp = 17, + .mr_cache[0] = { + .size = 500, + .limit = 250 + }, + .mr_cache[1] = { + .size = 500, + .limit = 250 + }, + .mr_cache[2] = { + .size = 500, + .limit = 250 + }, + .mr_cache[3] = { + .size = 500, + .limit = 250 + }, + .mr_cache[4] = { + .size = 500, + .limit = 250 + }, + .mr_cache[5] = { + .size = 500, + .limit = 250 + }, + .mr_cache[6] = { + .size = 500, + .limit = 250 + }, + .mr_cache[7] = { + .size = 500, + .limit = 250 + }, + .mr_cache[8] = { + .size = 500, + .limit = 250 + }, + .mr_cache[9] = { + .size = 500, + .limit = 250 + }, + .mr_cache[10] = { + .size = 500, + .limit = 250 + }, + .mr_cache[11] = { + .size = 500, + .limit = 250 + }, + .mr_cache[12] = { + .size = 64, + .limit = 32 + }, + .mr_cache[13] = { + .size = 32, + .limit = 16 + }, + .mr_cache[14] = { + .size = 16, + .limit = 8 + }, + .mr_cache[15] = { + .size = 8, + .limit = 4 + }, + }, +}; + +__be64 sys_image_guid; +void xsc_get_sys_image_guid(u8 *dev_addr, u8 *guid) +{ + static bool is_set; + u8 mac[ETH_ALEN]; + + if (is_set) + return; + + memcpy(mac, dev_addr, ETH_ALEN); + guid[0] = mac[0]; + guid[1] = mac[1]; + guid[2] = mac[2]; + guid[3] = 0x30; + guid[4] = 0x0; + guid[5] = mac[3]; + guid[6] = mac[4]; + guid[7] = mac[5]; + + is_set = 1; +} + +static int xsc_ib_query_device(struct ib_device *ibdev, + struct ib_device_attr *props, + struct ib_udata *udata) +{ + struct xsc_ib_dev *dev = to_mdev(ibdev); + int max_rq_sg; + int max_sq_sg; + u64 flags; + struct xsc_ib_query_device_resp resp; + size_t resp_len; + u64 max_tso; + int err = -ENOMEM; + union xsc_ib_fw_ver fw_ver; + + memset(&resp, 0, sizeof(resp)); + memset(props, 0, sizeof(*props)); + + resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length); + /*check param*/ + if (udata->outlen && udata->outlen < resp_len) + return -EINVAL; + + if (udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen)) + return -EINVAL; + + resp.response_length = resp_len; + + fw_ver.data = 0; + fw_ver.s.chip_ver_h = dev->xdev->chip_ver_h & 0xffff; + fw_ver.s.hotfix_num = dev->xdev->hotfix_num & 0xffff; + fw_ver.s.chip_ver_l = dev->xdev->chip_ver_l & 0xffff; + fw_ver.s.feature_flag = dev->xdev->feature_flag & 0xffff; + props->fw_ver = fw_ver.data; + + props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT | + IB_DEVICE_PORT_ACTIVE_EVENT | + IB_DEVICE_SYS_IMAGE_GUID | + IB_DEVICE_RC_RNR_NAK_GEN | + IB_DEVICE_BLOCK_MULTICAST_LOOPBACK; + flags = dev->xdev->caps.flags; + if (flags & XSC_DEV_CAP_FLAG_BAD_PKEY_CNTR) + props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR; + if (flags & XSC_DEV_CAP_FLAG_BAD_QKEY_CNTR) + props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR; + if (flags & XSC_DEV_CAP_FLAG_APM) + props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; + props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; + if (flags & XSC_DEV_CAP_FLAG_XRC) + props->device_cap_flags |= IB_DEVICE_XRC; + props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; + + props->page_size_cap = dev->xdev->caps.min_page_sz; + props->max_mr_size = (1 << dev->xdev->caps.log_max_mtt)*PAGE_SIZE; + props->max_qp = 1 << dev->xdev->caps.log_max_qp; + props->max_qp_wr = dev->xdev->caps.max_wqes; + max_rq_sg = dev->xdev->caps.max_rq_desc_sz / sizeof(struct xsc_wqe_data_seg); + max_sq_sg = (dev->xdev->caps.max_sq_desc_sz - sizeof(struct xsc_wqe_ctrl_seg_2)) / + sizeof(struct xsc_wqe_data_seg_2); + + props->max_send_sge = dev->xdev->caps.send_ds_num; + props->max_recv_sge = dev->xdev->caps.recv_ds_num; + props->max_sge_rd = 0;/*xsc unsupported RD server type*/ + props->max_cq = 1 << dev->xdev->caps.log_max_cq; + props->max_cqe = dev->xdev->caps.max_cqes - 1; + props->max_mr = 1 << dev->xdev->caps.log_max_mkey; + props->max_pd = 1 << dev->xdev->caps.log_max_pd; + props->max_qp_rd_atom = dev->xdev->caps.max_ra_req_qp; + props->max_qp_init_rd_atom = dev->xdev->caps.max_ra_res_qp; + props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; + props->max_srq = + dev->xdev->caps.log_max_srq?(1 << dev->xdev->caps.log_max_srq):0; + props->max_srq_wr = dev->xdev->caps.max_srq_wqes - 1; + props->max_srq_sge = dev->xdev->caps.log_max_srq?(max_rq_sg - 1):0; + props->max_fast_reg_page_list_len = (unsigned int)-1; + props->local_ca_ack_delay = dev->xdev->caps.local_ca_ack_delay; + props->atomic_cap = dev->xdev->caps.flags & XSC_DEV_CAP_FLAG_ATOMIC ? + IB_ATOMIC_HCA : IB_ATOMIC_NONE; + props->masked_atomic_cap = IB_ATOMIC_HCA; + props->max_mcast_grp = + dev->xdev->caps.log_max_mcg?(1 << dev->xdev->caps.log_max_mcg):0; + props->max_mcast_qp_attach = dev->xdev->caps.max_qp_mcg; + props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * + props->max_mcast_grp; + + props->sys_image_guid = sys_image_guid; + props->vendor_id = dev->xdev->pdev->vendor; + props->vendor_part_id = dev->xdev->pdev->device; + props->hw_ver = 0x0; + props->max_pkeys = 0x80; + props->max_wq_type_rq = 1 << dev->xdev->caps.log_max_qp; + + props->hca_core_clock = dev->xdev->caps.hca_core_clock; + props->rss_caps.max_rwq_indirection_tables = + dev->xdev->caps.max_rwq_indirection_tables; + props->rss_caps.max_rwq_indirection_table_size = + dev->xdev->caps.max_rwq_indirection_table_size; + props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET; + + /*response tso_caps extend param*/ + if (field_avail(typeof(resp), tso_caps, udata->outlen)) { + max_tso = dev->xdev->caps.log_max_tso?(1 << dev->xdev->caps.log_max_tso):0; + if (max_tso) { + resp.tso_caps.max_tso = max_tso; + resp.tso_caps.supported_qpts |= 1 << IB_QPT_RAW_PACKET; + resp.response_length += sizeof(resp.tso_caps); + } + } + + /*response rss_caps extend param*/ + if (field_avail(typeof(resp), rss_caps, udata->outlen)) { + resp.rss_caps.rx_hash_function = XSC_RX_HASH_FUNC_TOEPLITZ; + resp.rss_caps.rx_hash_fields_mask = + XSC_RX_HASH_SRC_IPV4 | + XSC_RX_HASH_DST_IPV4 | + XSC_RX_HASH_SRC_IPV6 | + XSC_RX_HASH_DST_IPV6 | + XSC_RX_HASH_SRC_PORT_TCP | + XSC_RX_HASH_DST_PORT_TCP | + XSC_RX_HASH_SRC_PORT_UDP | + XSC_RX_HASH_DST_PORT_UDP | + XSC_RX_HASH_INNER; + resp.response_length += sizeof(resp.rss_caps); + } + + /*response packet pacing caps*/ + if (field_avail(typeof(resp), packet_pacing_caps, udata->outlen)) { + resp.packet_pacing_caps.qp_rate_limit_max = + dev->xdev->caps.qp_rate_limit_max; + resp.packet_pacing_caps.qp_rate_limit_min = + dev->xdev->caps.qp_rate_limit_min; + resp.packet_pacing_caps.supported_qpts |= 1 << IB_QPT_RAW_PACKET; + + resp.response_length += sizeof(resp.packet_pacing_caps); + } + + /*copy response data to user*/ + if (udata->outlen) { + err = ib_copy_to_udata(udata, &resp, resp.response_length); + if (err) { + xsc_ib_err(dev, "copy response info to udata fail,err=%d\n", err); + return err; + } + } + + return 0; +} + +static enum rdma_link_layer xsc_ib_port_link_layer(struct ib_device *ibdev, u8 port) +{ + return IB_LINK_LAYER_ETHERNET; +} + +int xsc_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props) +{ + struct xsc_ib_dev *dev = to_mdev(ibdev); + struct net_device *ndev = dev->netdev; + + if (port < 1 || port > dev->xdev->caps.num_ports) { + xsc_ib_warn(dev, "invalid port number %d\n", port); + return -EINVAL; + } + + memset(props, 0, sizeof(*props)); + + props->state = IB_PORT_ACTIVE; + props->max_mtu = IB_MTU_4096; + props->active_mtu = min(props->max_mtu, xsc_net_to_ib_mtu(ndev->mtu)); + props->gid_tbl_len = 256; + props->port_cap_flags = 0x4010000; + props->max_msg_sz = 0x40000000; + props->bad_pkey_cntr = 0; + props->qkey_viol_cntr = 0; + props->pkey_tbl_len = 1; + props->lid = 0; + props->sm_lid = 0; + props->lmc = 0; + props->max_vl_num = 0; + props->sm_sl = 0; + props->subnet_timeout = 0; + props->init_type_reply = 0; + props->active_width = 1; + props->active_speed = 32; + props->phys_state = 5; + + return 0; +} +const struct xsc_gid xsc_gid_zero; + +static int xsc_ib_query_gid(struct ib_device *ibdev, u8 port_num, + int index, union ib_gid *gid) +{ + struct xsc_ib_dev *dev = to_mdev(ibdev); + struct xsc_sgid_tbl *sgid_tbl = &dev->ib_res.sgid_tbl; + + /* Ignore port_num */ + memset(gid, 0, sizeof(*gid)); + if (index >= sgid_tbl->max) + return -EINVAL; + + memcpy(gid, &sgid_tbl->tbl[index], sizeof(*gid)); + + return 0; +} + +static int xsc_ib_del_gid(const struct ib_gid_attr *attr, void **context) +{ + int index = 0; + struct xsc_ib_dev *dev = to_mdev(attr->device); + struct xsc_sgid_tbl *sgid_tbl = &dev->ib_res.sgid_tbl; + struct xsc_gid *gid = (struct xsc_gid *)&attr->gid; + + if (!sgid_tbl) + return -EINVAL; + + if (!sgid_tbl->count) + return -ENOMEM; + + for (index = 0; index < sgid_tbl->max; index++) { + if (!memcmp(&sgid_tbl->tbl[index], gid, sizeof(*gid))) + break; + } + + if (index == sgid_tbl->max) + return 0; + + memcpy(&sgid_tbl->tbl[index], &xsc_gid_zero, sizeof(xsc_gid_zero)); + sgid_tbl->count--; + xsc_ib_dbg(dev, "Del gid from index:%u, count:%u\n", index, sgid_tbl->count); + + return 0; +} + +int xsc_ib_add_gid(const struct ib_gid_attr *attr, void **context) +{ + int i = 0; + u32 free_idx = 0; + struct xsc_ib_dev *dev = to_mdev(attr->device); + struct xsc_sgid_tbl *sgid_tbl = &dev->ib_res.sgid_tbl; + struct xsc_gid *gid = (struct xsc_gid *)&attr->gid; + + if (!sgid_tbl) + return -EINVAL; + + if (sgid_tbl->count == sgid_tbl->max) + return -ENOMEM; + + free_idx = sgid_tbl->max; + for (i = 0; i < sgid_tbl->max; i++) { + if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid))) { + return 0; + } else if (!memcmp(&sgid_tbl->tbl[i], &xsc_gid_zero, sizeof(xsc_gid_zero)) && + free_idx == sgid_tbl->max) { + free_idx = i; + } + } + + if (free_idx == sgid_tbl->max) + return -ENOMEM; + + memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid)); + sgid_tbl->count++; + xsc_ib_dbg(dev, "Add gid to index:%u, count:%u, max:%u\n", free_idx, sgid_tbl->count, + sgid_tbl->max); + + return 0; +} +static int xsc_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, + u16 *pkey) +{ + *pkey = 0xffff; + return 0; +} + +struct xsc_reg_node_desc { + u8 desc[64]; +}; + +static int xsc_ib_modify_device(struct ib_device *ibdev, int mask, + struct ib_device_modify *props) +{ + struct xsc_ib_dev *dev = to_mdev(ibdev); + struct xsc_reg_node_desc in; + struct xsc_reg_node_desc out; + int err; + + return 0; + + if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) + return -EOPNOTSUPP; + + if (!(mask & IB_DEVICE_MODIFY_NODE_DESC)) + return 0; + + /* If possible, pass node desc to FW, so it can generate + * a 144 trap. If cmd fails, just ignore. + */ + memcpy(&in, props->node_desc, 64); + err = xsc_core_access_reg(dev->xdev, &in, sizeof(in), &out, + sizeof(out), XSC_REG_NODE_DESC, 0, 1); + if (err) + return err; + + memcpy(ibdev->node_desc, props->node_desc, 64); + + return err; +} +static int xsc_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, + struct ib_port_modify *props) +{ + struct xsc_ib_dev *dev = to_mdev(ibdev); + struct ib_port_attr attr; + u32 tmp; + int err; + + return 0; + + mutex_lock(&dev->cap_mask_mutex); + + err = xsc_ib_query_port(ibdev, port, &attr); + if (err) + goto out; + + tmp = (attr.port_cap_flags | props->set_port_cap_mask) & + ~props->clr_port_cap_mask; + + err = xsc_set_port_caps(dev->xdev, port, tmp); + +out: + mutex_unlock(&dev->cap_mask_mutex); + return err; +} + +xsc_ib_alloc_ucontext_def() +{ + struct ib_device *ibdev = uctx->device; + struct xsc_ib_dev *dev = to_mdev(ibdev); + struct xsc_ib_alloc_ucontext_req req; + struct xsc_ib_alloc_ucontext_resp resp; + struct xsc_ib_ucontext *context; + int err; + + pr_err("[%s:%d]", __func__, __LINE__); + + if (!dev->ib_active) + return RET_VALUE(-EAGAIN); + + err = ib_copy_from_udata(&req, udata, sizeof(req)); + if (err) + return RET_VALUE(err); + + resp.qp_tab_size = 1 << dev->xdev->caps.log_max_qp; + resp.cache_line_size = L1_CACHE_BYTES; + resp.max_sq_desc_sz = dev->xdev->caps.max_sq_desc_sz; + resp.max_rq_desc_sz = dev->xdev->caps.max_rq_desc_sz; + resp.max_send_wqebb = dev->xdev->caps.max_wqes; + resp.max_recv_wr = dev->xdev->caps.max_wqes; + resp.qpm_tx_db = dev->xdev->regs.tx_db; + resp.qpm_rx_db = dev->xdev->regs.rx_db; + resp.cqm_next_cid_reg = dev->xdev->regs.complete_reg; + resp.cqm_armdb = dev->xdev->regs.complete_db; + resp.send_ds_num = dev->xdev->caps.send_ds_num; + resp.recv_ds_num = dev->xdev->caps.recv_ds_num; + resp.cmds_supp_uhw |= XSC_USER_CMDS_SUPP_UHW_QUERY_DEVICE; + + context = to_xucontext(uctx); + + INIT_LIST_HEAD(&context->db_page_list); + mutex_init(&context->db_page_mutex); + + resp.num_ports = dev->xdev->caps.num_ports; + err = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (err) + goto out_ctx; + + return 0; + +out_ctx: + return err; +} + +xsc_ib_dealloc_ucontext_def() +{ +} + +static int xsc_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) +{ + struct xsc_ib_dev *dev = to_mdev(ibcontext->device); + struct xsc_core_device *xdev = dev->xdev; + unsigned long offset = vma->vm_pgoff << PAGE_SHIFT; + resource_size_t reg_base; + resource_size_t reg_size = vma->vm_end - vma->vm_start; + + xsc_core_dbg(xdev, "offset:0x%lx", offset); + + if (offset == (xdev->regs.tx_db & PAGE_MASK)) + reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) + + (xdev->regs.tx_db & PAGE_MASK); + else if (offset == (xdev->regs.rx_db & PAGE_MASK)) + reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) + + (xdev->regs.rx_db & PAGE_MASK); + else if (offset == (xdev->regs.complete_reg & PAGE_MASK)) + reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) + + (xdev->regs.complete_reg & PAGE_MASK); + else if (offset == (xdev->regs.complete_db & PAGE_MASK)) + reg_base = pci_resource_start(xdev->pdev, xdev->bar_num) + + (xdev->regs.complete_db & PAGE_MASK); + else + return -EINVAL; + + xsc_core_dbg(xdev, "regbase:0x%llx", reg_base); + + reg_base = xsc_core_is_pf(xdev) ? reg_base - 0xA0000000 : reg_base; + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + return remap_pfn_range(vma, vma->vm_start, reg_base >> PAGE_SHIFT, + reg_size, vma->vm_page_prot); + + return 0; +} + +xsc_ib_alloc_pd_def() +{ + struct ib_device *ibdev = ibpd->device; + struct xsc_ib_alloc_pd_resp resp; + struct xsc_ib_pd *pd; + int err; + + pd = to_mpd(ibpd); + + err = xsc_core_alloc_pd(to_mdev(ibdev)->xdev, &pd->pdn); + if (err) { + kfree(pd); + return RET_VALUE(err); + } + + if (udata) { + resp.pdn = pd->pdn; + if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { + xsc_core_dealloc_pd(to_mdev(ibdev)->xdev, pd->pdn); + return -EFAULT; + } + } else { + pd->pa_lkey = 0; + } + + return 0; +} + +xsc_ib_dealloc_pd_def() +{ + struct xsc_ib_dev *mdev = to_mdev(pd->device); + struct xsc_ib_pd *mpd = to_mpd(pd); + + xsc_core_dealloc_pd(mdev->xdev, mpd->pdn); + + return 0; +} + +static int xsc_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int err; + + err = ib_query_port(ibdev, port_num, &attr); + if (err) + return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE + | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; + immutable->max_mad_size = IB_MGMT_MAD_SIZE * 2; + + return 0; +} + +static void _xsc_get_netdev(struct xsc_ib_dev *dev) +{ + struct net_device *netdev = (struct net_device *)(dev->xdev->netdev); + + dev->netdev = netdev; +} + +static struct net_device *xsc_get_netdev(struct ib_device *ibdev, u8 port_num) +{ + struct xsc_ib_dev *dev = to_mdev(ibdev); + struct net_device *netdev = dev->netdev; + + rcu_read_lock(); + if (netdev) + dev_hold(netdev); + rcu_read_unlock(); + return netdev; +} + +void xsc_get_guid(u8 *dev_addr, u8 *guid) +{ + u8 mac[ETH_ALEN]; + + /* MAC-48 to EUI-64 mapping */ + memcpy(mac, dev_addr, ETH_ALEN); + guid[0] = mac[0] ^ 2; + guid[1] = mac[1]; + guid[2] = mac[2]; + guid[3] = 0xff; + guid[4] = 0xfe; + guid[5] = mac[3]; + guid[6] = mac[4]; + guid[7] = mac[5]; +} + +static int init_node_data(struct xsc_ib_dev *dev) +{ + int err = -ENOMEM; + + strlcpy(dev->ib_dev.node_desc, "xsc_node_desc", sizeof(dev->ib_dev.node_desc)); + + if (unlikely(!dev->netdev->dev_addr)) + _xsc_get_netdev(dev); + xsc_get_guid(dev->netdev->dev_addr, (u8 *)&dev->ib_dev.node_guid); + err = 0; + return err; +} + +void xsc_core_event(struct xsc_core_device *xdev, enum xsc_dev_event event, + unsigned long param) +{ + struct xsc_priv *priv = &xdev->priv; + struct xsc_device_context *dev_ctx; + unsigned long flags; + + spin_lock_irqsave(&priv->ctx_lock, flags); + + /* After xsc_detach_device, the dev_ctx->intf is still set and dev_ctx is + * still in priv->ctx_list. In this case, only notify the dev_ctx if its + * ADDED or ATTACHED bit are set. + */ + list_for_each_entry(dev_ctx, &priv->ctx_list, list) { + if (dev_ctx->intf->event) + dev_ctx->intf->event(xdev, dev_ctx->context, 0, param); + } + spin_unlock_irqrestore(&priv->ctx_lock, flags); +} + +static void xsc_ib_event(struct xsc_core_device *dev, void *context, + enum xsc_dev_event event, unsigned long data) +{ + struct xsc_ib_dev *ibdev = (struct xsc_ib_dev *)context; + struct ib_event ibev; + u8 port = 0; + + switch (event) { + case XSC_DEV_EVENT_SYS_ERROR: + ibdev->ib_active = false; + ibev.event = IB_EVENT_DEVICE_FATAL; + break; + + case XSC_DEV_EVENT_PORT_UP: + ibev.event = IB_EVENT_PORT_ACTIVE; + port = *(u8 *)data; + break; + + case XSC_DEV_EVENT_PORT_DOWN: + ibev.event = IB_EVENT_PORT_ERR; + port = *(u8 *)data; + break; + + case XSC_DEV_EVENT_PORT_INITIALIZED: + /* not used by ULPs */ + return; + + case XSC_DEV_EVENT_LID_CHANGE: + ibev.event = IB_EVENT_LID_CHANGE; + port = *(u8 *)data; + break; + + case XSC_DEV_EVENT_PKEY_CHANGE: + ibev.event = IB_EVENT_PKEY_CHANGE; + port = *(u8 *)data; + break; + + case XSC_DEV_EVENT_GUID_CHANGE: + ibev.event = IB_EVENT_GID_CHANGE; + port = *(u8 *)data; + break; + + case XSC_DEV_EVENT_CLIENT_REREG: + ibev.event = IB_EVENT_CLIENT_REREGISTER; + port = *(u8 *)data; + break; + } + + ibev.device = &ibdev->ib_dev; + ibev.element.port_num = port; + + if (ibdev->ib_active) + ib_dispatch_event(&ibev); +} + +static int get_port_caps(struct xsc_ib_dev *dev) +{ + struct ib_device_attr *dprops = NULL; + struct ib_port_attr *pprops = NULL; + int err = -ENOMEM; + u32 port; + /*used to prevent coredump when insmod xsc*/ + struct ib_udata uhw = {.inlen = 0, .outlen = 0}; + + pprops = kmalloc(sizeof(*pprops), GFP_KERNEL); + if (!pprops) + goto out; + + dprops = kmalloc(sizeof(*dprops), GFP_KERNEL); + if (!dprops) + goto out; + + err = xsc_ib_query_device(&dev->ib_dev, dprops, &uhw); + if (err) { + xsc_ib_warn(dev, "query_device failed %d\n", err); + goto out; + } + + for (port = 1; port <= dev->xdev->caps.num_ports; port++) { + err = xsc_ib_query_port(&dev->ib_dev, port, pprops); + if (err) { + xsc_ib_warn(dev, "query_port %d failed %d\n", port, err); + break; + } + dev->xdev->caps.port[port - 1].pkey_table_len = dprops->max_pkeys; + dev->xdev->caps.port[port - 1].gid_table_len = pprops->gid_tbl_len; + xsc_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n", + dprops->max_pkeys, pprops->gid_tbl_len); + } + +out: + kfree(pprops); + kfree(dprops); + + return err; +} + +static int xsc_create_dev_res(struct xsc_ib_res *ib_res) +{ + struct xsc_ib_dev *dev; + + dev = container_of(ib_res, struct xsc_ib_dev, ib_res); + ib_res->sgid_tbl.max = dev->xdev->caps.port[0].gid_table_len; + + ib_res->sgid_tbl.tbl = kcalloc(ib_res->sgid_tbl.max, sizeof(struct xsc_gid), + GFP_KERNEL); + + if (!ib_res->sgid_tbl.tbl) + return -ENOMEM; + + return 0; +} + +static void xsc_destroy_dev_res(struct xsc_ib_res *ib_res) +{ + kfree(ib_res->sgid_tbl.tbl); +} + +static int populate_specs_root(struct xsc_ib_dev *dev) +{ + const struct uverbs_object_tree_def **trees = + (const struct uverbs_object_tree_def **)dev->driver_trees; + size_t num_trees = 0; + + trees[num_trees++] = xsc_ib_get_devx_tree(); + + WARN_ON(num_trees >= ARRAY_SIZE(dev->driver_trees)); + trees[num_trees] = NULL; + + return 0; +} + +static void crc_table_init(struct xsc_ib_dev *dev) +{ + u32 c, i, j; + + for (i = 0; i < 256; i++) { + c = i; + for (j = 0; j < 8; j++) { + if (c & 1) + c = 0xedb88320L ^ (c >> 1); + else + c = c >> 1; + } + dev->crc_32_table[i] = c; + } +} + +static void xsc_ib_dev_setting(struct xsc_ib_dev *dev) +{ + dev->ib_dev.ops.owner = THIS_MODULE; + dev->ib_dev.ops.uverbs_abi_ver = XSC_IB_UVERBS_ABI_VERSION; + dev->ib_dev.ops.driver_id = RDMA_DRIVER_XSC5; + dev->ib_dev.ops.uverbs_no_driver_id_binding = 1; + + dev->ib_dev.ops.query_device = xsc_ib_query_device; + dev->ib_dev.ops.query_port = xsc_ib_query_port; + dev->ib_dev.ops.query_gid = xsc_ib_query_gid; + dev->ib_dev.ops.add_gid = xsc_ib_add_gid; + dev->ib_dev.ops.del_gid = xsc_ib_del_gid; + dev->ib_dev.ops.query_pkey = xsc_ib_query_pkey; + + dev->ib_dev.ops.modify_device = xsc_ib_modify_device; + dev->ib_dev.ops.modify_port = xsc_ib_modify_port; + dev->ib_dev.ops.alloc_ucontext = xsc_ib_alloc_ucontext; + dev->ib_dev.ops.dealloc_ucontext = xsc_ib_dealloc_ucontext; + dev->ib_dev.ops.mmap = xsc_ib_mmap; + + dev->ib_dev.ops.alloc_pd = xsc_ib_alloc_pd; + dev->ib_dev.ops.dealloc_pd = xsc_ib_dealloc_pd; + dev->ib_dev.ops.create_ah = xsc_ib_create_ah; + dev->ib_dev.ops.query_ah = xsc_ib_query_ah; + dev->ib_dev.ops.destroy_ah = xsc_ib_destroy_ah; + + dev->ib_dev.ops.get_link_layer = xsc_ib_port_link_layer; + dev->ib_dev.ops.get_netdev = xsc_get_netdev; + + dev->ib_dev.ops.create_qp = xsc_ib_create_qp; + dev->ib_dev.ops.modify_qp = xsc_ib_modify_qp; + dev->ib_dev.ops.query_qp = xsc_ib_query_qp; + dev->ib_dev.ops.destroy_qp = xsc_ib_destroy_qp; + dev->ib_dev.ops.post_send = xsc_ib_post_send; + dev->ib_dev.ops.post_recv = xsc_ib_post_recv; + dev->ib_dev.ops.create_cq = xsc_ib_create_cq; + dev->ib_dev.ops.destroy_cq = xsc_ib_destroy_cq; + dev->ib_dev.ops.poll_cq = xsc_ib_poll_cq; + dev->ib_dev.ops.req_notify_cq = xsc_ib_arm_cq; + dev->ib_dev.ops.get_dma_mr = xsc_ib_get_dma_mr; + dev->ib_dev.ops.reg_user_mr = xsc_ib_reg_user_mr;//optional + dev->ib_dev.ops.dereg_mr = xsc_ib_dereg_mr; + dev->ib_dev.ops.alloc_mr = xsc_ib_alloc_mr; + dev->ib_dev.ops.map_mr_sg = xsc_ib_map_mr_sg; + + dev->ib_dev.ops.get_port_immutable = xsc_port_immutable; + + dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_ah, xsc_ib_ah, ibah); + dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_cq, xsc_ib_cq, ibcq); + dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_pd, xsc_ib_pd, ibpd); + dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_ucontext, xsc_ib_ucontext, ibucontext); +} + +static int init_one(struct xsc_core_device *xdev, + struct xsc_ib_dev **m_ibdev) +{ + struct xsc_ib_dev *dev; + int err; + + printk_once(KERN_INFO "%s", xsc_version); + + pr_err("[%s:%d]", __func__, __LINE__); + + dev = (struct xsc_ib_dev *)ib_alloc_device(xsc_ib_dev, ib_dev); + if (!dev) + return -ENOMEM; + + pr_err("[%s:%d]", __func__, __LINE__); + + dev->xdev = xdev; + xdev->event = xsc_core_event; + _xsc_get_netdev(dev); +// if (prof_sel >= ARRAY_SIZE(profile)) { +// pr_warn("selected pofile out of range, selceting default\n"); +// prof_sel = 0; +// } +// mdev->profile = &profile[prof_sel]; + + _xsc_get_netdev(dev); + xsc_get_sys_image_guid(dev->netdev->dev_addr, (u8 *)&sys_image_guid); + err = get_port_caps(dev); + if (err) + goto err_free; +// +// get_ext_port_caps(dev); +// +// err = alloc_comp_eqs(dev); +// if (err) +// goto err_cleanup; +// +// XSC_INIT_DOORBELL_LOCK(&dev->uar_lock); + if (!xdev->caps.msix_enable) + dev->num_comp_vectors = 1; + else + dev->num_comp_vectors = xdev->dev_res->eq_table.num_comp_vectors; + + pr_err("[%s:%d]", __func__, __LINE__); + + strlcpy(dev->ib_dev.name, "xscale_%d", IB_DEVICE_NAME_MAX); + dev->ib_dev.node_type = RDMA_NODE_IB_CA; + dev->ib_dev.local_dma_lkey = 0xFF; + dev->num_ports = xdev->caps.num_ports; + dev->ib_dev.phys_port_cnt = dev->num_ports; + dev->ib_dev.num_comp_vectors = dev->num_comp_vectors; + dev->ib_dev.dev.parent = &xdev->pdev->dev; + xsc_ib_dev_setting(dev); + dev->cm_dscp = DSCP_PCP_UNSET; + dev->cm_pcp = DSCP_PCP_UNSET; + + dev->ib_dev.uverbs_cmd_mask = + (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | + (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | + (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | + (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_CREATE_AH) | + (1ull << IB_USER_VERBS_CMD_DESTROY_AH) | + (1ull << IB_USER_VERBS_CMD_REG_MR) | + (1ull << IB_USER_VERBS_CMD_REREG_MR) | + (1ull << IB_USER_VERBS_CMD_DEREG_MR) | + (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | + (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | + (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_QP) | + (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | + (1ull << IB_USER_VERBS_CMD_QUERY_QP) | + (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | + (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | + (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) | + (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | + (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | + (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) | + (1ull << IB_USER_VERBS_CMD_OPEN_QP); + dev->ib_dev.uverbs_ex_cmd_mask = + (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE) | + (1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ) | + (1ull << IB_USER_VERBS_EX_CMD_CREATE_QP) | + (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP) | + (1ull << IB_USER_VERBS_EX_CMD_MODIFY_CQ); + + init_node_data(dev); + + mutex_init(&dev->cap_mask_mutex); + spin_lock_init(&dev->mr_lock); + + err = xsc_create_dev_res(&dev->ib_res); + if (err) + goto err_free; + + pr_err("[%s:%d]", __func__, __LINE__); + crc_table_init(dev); + + populate_specs_root(dev); + + xsc_reg_local_dma_mr(xdev); + if (ib_register_device(&dev->ib_dev, dev->ib_dev.name, dev->xdev->device)) + goto err_rsrc; + + pr_err("[%s:%d]", __func__, __LINE__); + dev->ib_active = true; + *m_ibdev = dev; + + xdev->xsc_ib_dev = dev; + xsc_counters_init(&dev->ib_dev, xdev); + + xsc_priv_dev_init(&dev->ib_dev, xdev); + + xsc_rtt_sysfs_init(&dev->ib_dev, xdev); + + return 0; + +err_rsrc: + xsc_destroy_dev_res(&dev->ib_res); + +err_free: + ib_dealloc_device((struct ib_device *)dev); + + return err; +} + +static void remove_one(struct xsc_core_device *xdev, void *intf_ctx) +{ + struct xsc_ib_dev *dev = (struct xsc_ib_dev *)intf_ctx; + + xsc_rtt_sysfs_fini(xdev); + xsc_priv_dev_fini(&dev->ib_dev, xdev); + xsc_counters_fini(&dev->ib_dev, xdev); + ib_unregister_device(&dev->ib_dev); + ib_dealloc_device(&dev->ib_dev); +} + +static void *xsc_add(struct xsc_core_device *xpdev) +{ + struct xsc_ib_dev *m_ibdev = NULL; + int ret = -1; + + pr_err("===> %s: enter\n", __func__); +#ifdef USE_VIRTIO + pr_err("pcidev:%p bar0:%x\n", xpdev, xpdev->bar0); +#endif /* USE_VIRTIO */ + pr_err("enter virt_rdma_probe\n"); + + ret = init_one(xpdev, &m_ibdev); + if (ret) { + pr_err("%s fail, ret = %d\n", __func__, ret); + return NULL; + } + + xpdev->rdma_ready = true; + return m_ibdev; +} + +static void xsc_remove(struct xsc_core_device *xpdev, void *context) +{ + pr_err("<=== %s: enter\n", __func__); + remove_one(xpdev, context); + xpdev->rdma_ready = false; +} + +static struct xsc_interface xsc_interface = { + .add = xsc_add, + .remove = xsc_remove, + .event = xsc_ib_event, + .protocol = XSC_INTERFACE_PROTOCOL_IB, +}; + +static int __init xsc_ib_init(void) +{ + int ret; + + ret = xsc_register_interface(&xsc_interface); + if (ret) + goto out; + + ret = xsc_rdma_ctrl_init(); + if (ret != 0) { + pr_err("failed to register port control node\n"); + xsc_unregister_interface(&xsc_interface); + goto out; + } + + return 0; +out: + return ret; +} + +static void __exit xsc_ib_cleanup(void) +{ + xsc_rdma_ctrl_fini(); + xsc_unregister_interface(&xsc_interface); +} + +module_init(xsc_ib_init); +module_exit(xsc_ib_cleanup); diff --git a/drivers/infiniband/hw/xsc/mem.c b/drivers/infiniband/hw/xsc/mem.c new file mode 100644 index 0000000000000000000000000000000000000000..f78c715d4ceff8ca0e17068a260ebf32652f51b5 --- /dev/null +++ b/drivers/infiniband/hw/xsc/mem.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include "xsc_ib.h" + +/* @umem: umem object to scan + * @addr: ib virtual address requested by the user + * @count: number of PAGE_SIZE pages covered by umem + * @shift: page shift for the compound pages found in the region + * @ncont: number of compund pages + * @order: log2 of the number of compound pages + */ +void __xsc_ib_cont_pages(struct ib_umem *umem, u64 addr, + unsigned long max_page_shift, + int *count, int *shift, + int *ncont, int *order) +{ + unsigned long tmp; + unsigned long m; + u64 base = ~0, p = 0; + u64 len, pfn; + int i = 0; + struct scatterlist *sg; + int entry; + // TODO: need peer mem support + unsigned long page_shift = PAGE_SHIFT; + + addr = addr >> page_shift; + tmp = (unsigned long)addr; + m = find_first_bit(&tmp, BITS_PER_LONG); + if (max_page_shift) + m = min_t(unsigned long, max_page_shift - page_shift, m); + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { + len = sg_dma_len(sg) >> page_shift; + pfn = sg_dma_address(sg) >> page_shift; + if (base + p != pfn) { + /* If either the offset or the new + * base are unaligned update m + */ + tmp = (unsigned long)(pfn | p); + if (!IS_ALIGNED(tmp, 1 << m)) + m = find_first_bit(&tmp, BITS_PER_LONG); + + base = pfn; + p = 0; + } + + p += len; + i += len; + } + + if (i) { + m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m); + + if (order) + *order = ilog2(roundup_pow_of_two(i) >> m); + + *ncont = DIV_ROUND_UP(i, (1 << m)); + } else { + m = 0; + + if (order) + *order = 0; + + *ncont = 0; + } + *shift = page_shift + m; + *count = i; +} + +void xsc_ib_cont_pages(struct ib_umem *umem, u64 addr, + int *count, int *shift, + int *ncont, int *order) +{ + // no limit for page_shift + __xsc_ib_cont_pages(umem, addr, 0, count, shift, ncont, order); +} + +void __xsc_ib_populate_pas(struct xsc_ib_dev *dev, struct ib_umem *umem, + int page_shift, size_t offset, size_t num_pages, + __be64 *pas, int access_flags, bool need_to_devide) +{ + unsigned long umem_page_shift = PAGE_SHIFT; + int shift = page_shift - umem_page_shift; + int mask = (1 << shift) - 1; + int k, idx; + u64 cur = 0; + u64 base; + int len; + struct scatterlist *sg; + int entry; + int i = 0; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { + len = sg_dma_len(sg) >> umem_page_shift; + if (need_to_devide) + len = sg_dma_len(sg) >> PAGE_SHIFT_4K; + else + len = sg_dma_len(sg) >> umem_page_shift; + base = sg_dma_address(sg); + + /* Skip elements below offset */ + if (i + len < offset << shift) { + i += len; + continue; + } + + /* Skip pages below offset */ + if (i < offset << shift) { + k = (offset << shift) - i; + i = offset << shift; + } else { + k = 0; + } + + for (; k < len; k++) { + if (!(i & mask)) { + if (need_to_devide) + cur = base + (k << PAGE_SHIFT_4K); + else + cur = base + (k << umem_page_shift); + cur |= access_flags; + idx = (i >> shift) - offset; + + pas[idx] = cpu_to_be64(cur); + xsc_ib_dbg(dev, "pas[%d] 0x%llx\n", + i >> shift, be64_to_cpu(pas[idx])); + } + i++; + + /* Stop after num_pages reached */ + if (i >> shift >= offset + num_pages) + return; + } + } +} + +void xsc_ib_populate_pas(struct xsc_ib_dev *dev, struct ib_umem *umem, + int page_shift, __be64 *pas, int npages, bool need_to_devide) +{ + return __xsc_ib_populate_pas(dev, umem, page_shift, 0, + npages, pas, 0, need_to_devide); +} + +int xsc_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset) +{ + u64 page_size; + u64 page_mask; + u64 off_size; + u64 off_mask; + u64 buf_off; + + page_size = 1 << page_shift; + page_mask = page_size - 1; + buf_off = addr & page_mask; + off_size = page_size >> 6; + off_mask = off_size - 1; + + if (buf_off & off_mask) + return -EINVAL; + + *offset = buf_off >> ilog2(off_size); + return 0; +} diff --git a/drivers/infiniband/hw/xsc/mr.c b/drivers/infiniband/hw/xsc/mr.c new file mode 100644 index 0000000000000000000000000000000000000000..e3804cc6fe3f7291e7a79b3681c837f376aa4f0a --- /dev/null +++ b/drivers/infiniband/hw/xsc/mr.c @@ -0,0 +1,837 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "ib_umem_ex.h" +#include "xsc_ib.h" + +static void xsc_invalidate_umem(void *invalidation_cookie, + struct ib_umem_ex *umem, unsigned long addr, size_t size); + +enum { + DEF_CACHE_SIZE = 10, +}; + +static __be64 *mr_align(__be64 *ptr, int align) +{ + unsigned long mask = align - 1; + + return (__be64 *)(((unsigned long)ptr + mask) & ~mask); +} + +static int order2idx(struct xsc_ib_dev *dev, int order) +{ + struct xsc_mr_cache *cache = &dev->cache; + + if (order < cache->ent[0].order) + return 0; + else + return order - cache->ent[0].order; +} + +static int add_keys(struct xsc_ib_dev *dev, int c, int num) +{ + struct device *ddev = dev->ib_dev.dma_device; + struct xsc_mr_cache *cache = &dev->cache; + struct xsc_cache_ent *ent = &cache->ent[c]; + struct xsc_register_mr_mbox_in *in; + struct xsc_ib_mr *mr; + int npages = 1 << ent->order; + int size = sizeof(u64) * npages; + int err = 0; + int i; + + in = kzalloc(sizeof(*in), GFP_KERNEL); + if (!in) + return -ENOMEM; + + for (i = 0; i < num; i++) { + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + err = -ENOMEM; + goto out; + } + mr->order = ent->order; + mr->pas = kmalloc(size + 0x3f, GFP_KERNEL); + if (!mr->pas) { + kfree(mr); + err = -ENOMEM; + goto out; + } + mr->dma = dma_map_single(ddev, mr_align(mr->pas, 0x40), size, + DMA_TO_DEVICE); + if (dma_mapping_error(ddev, mr->dma)) { + kfree(mr->pas); + kfree(mr); + err = -ENOMEM; + goto out; + } + + in->req.acc = XSC_ACCESS_MODE_MTT; + in->req.page_mode = 0; + + err = xsc_core_create_mkey(dev->xdev, &mr->mmr); + if (err) { + xsc_ib_warn(dev, "create mkey failed %d\n", err); + dma_unmap_single(ddev, mr->dma, size, DMA_TO_DEVICE); + kfree(mr->pas); + kfree(mr); + goto out; + } + in->req.mkey = cpu_to_be32(mr->mmr.key); + err = xsc_core_register_mr(dev->xdev, &mr->mmr, in, + sizeof(*in)); + if (err) { + xsc_ib_warn(dev, "register mr failed %d\n", err); + xsc_core_destroy_mkey(dev->xdev, &mr->mmr); + dma_unmap_single(ddev, mr->dma, size, DMA_TO_DEVICE); + kfree(mr->pas); + kfree(mr); + goto out; + } + cache->last_add = jiffies; + + spin_lock(&ent->lock); + list_add_tail(&mr->list, &ent->head); + ent->cur++; + ent->size++; + spin_unlock(&ent->lock); + } + +out: + kfree(in); + return err; +} + +static void remove_keys(struct xsc_ib_dev *dev, int c, int num) +{ + struct device *ddev = dev->ib_dev.dma_device; + struct xsc_mr_cache *cache = &dev->cache; + struct xsc_cache_ent *ent = &cache->ent[c]; + struct xsc_ib_mr *mr; + int size; + int err; + int i; + + for (i = 0; i < num; i++) { + spin_lock(&ent->lock); + if (list_empty(&ent->head)) { + spin_unlock(&ent->lock); + return; + } + mr = list_first_entry(&ent->head, struct xsc_ib_mr, list); + list_del(&mr->list); + ent->cur--; + ent->size--; + spin_unlock(&ent->lock); + err = xsc_core_destroy_mkey(dev->xdev, &mr->mmr); + if (err) { + xsc_ib_warn(dev, "failed destroy mkey\n"); + } else { + size = ALIGN(sizeof(u64) * (1 << mr->order), 0x40); + dma_unmap_single(ddev, mr->dma, size, DMA_TO_DEVICE); + kfree(mr->pas); + kfree(mr); + } + } +} + +static ssize_t size_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xsc_cache_ent *ent = filp->private_data; + struct xsc_ib_dev *dev = ent->dev; + char lbuf[20]; + u32 var; + int err; + int c; + + if (copy_from_user(lbuf, buf, sizeof(lbuf))) + return -EPERM; + + c = order2idx(dev, ent->order); + lbuf[sizeof(lbuf) - 1] = 0; + + if (kstrtou32(lbuf, 10, &var) != 1) + return -EINVAL; + + if (var < ent->limit) + return -EINVAL; + + if (var > ent->size) { + err = add_keys(dev, c, var - ent->size); + if (err) + return err; + } else if (var < ent->size) { + remove_keys(dev, c, ent->size - var); + } + + return count; +} + +static ssize_t size_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct xsc_cache_ent *ent = filp->private_data; + char lbuf[20]; + int err; + + if (*pos) + return 0; + + err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size); + if (err < 0) + return err; + + if (copy_to_user(buf, lbuf, err)) + return -EPERM; + + *pos += err; + + return err; +} + +static const struct file_operations size_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = size_write, + .read = size_read, +}; + +static ssize_t limit_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xsc_cache_ent *ent = filp->private_data; + struct xsc_ib_dev *dev = ent->dev; + char lbuf[20]; + u32 var; + int err; + int c; + + if (copy_from_user(lbuf, buf, sizeof(lbuf))) + return -EPERM; + + c = order2idx(dev, ent->order); + lbuf[sizeof(lbuf) - 1] = 0; + + if (kstrtou32(lbuf, 10, &var) != 1) + return -EINVAL; + + if (var > ent->size) + return -EINVAL; + + ent->limit = var; + + if (ent->cur < ent->limit) { + err = add_keys(dev, c, 2 * ent->limit - ent->cur); + if (err) + return err; + } + + return count; +} + +static ssize_t limit_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct xsc_cache_ent *ent = filp->private_data; + char lbuf[20]; + int err; + + if (*pos) + return 0; + + err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); + if (err < 0) + return err; + + if (copy_to_user(buf, lbuf, err)) + return -EPERM; + + *pos += err; + + return err; +} + +static const struct file_operations limit_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = limit_write, + .read = limit_read, +}; + +static int someone_adding(struct xsc_mr_cache *cache) +{ + int i; + + for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { + if (cache->ent[i].cur < cache->ent[i].limit) + return 1; + } + + return 0; +} + +static void __cache_work_func(struct xsc_cache_ent *ent) +{ + struct xsc_ib_dev *dev = ent->dev; + struct xsc_mr_cache *cache = &dev->cache; + int i = order2idx(dev, ent->order); + + if (cache->stopped) + return; + + ent = &dev->cache.ent[i]; + if (ent->cur < 2 * ent->limit) { + add_keys(dev, i, 1); + if (ent->cur < 2 * ent->limit) + queue_work(cache->wq, &ent->work); + } else if (ent->cur > 2 * ent->limit) { + if (!someone_adding(cache) && + time_after(jiffies, cache->last_add + 60 * HZ)) { + remove_keys(dev, i, 1); + if (ent->cur > ent->limit) + queue_work(cache->wq, &ent->work); + } else { + queue_delayed_work(cache->wq, &ent->dwork, 60 * HZ); + } + } +} + +static void delayed_cache_work_func(struct work_struct *work) +{ + struct xsc_cache_ent *ent; + + ent = container_of(work, struct xsc_cache_ent, dwork.work); + __cache_work_func(ent); +} + +static void cache_work_func(struct work_struct *work) +{ + struct xsc_cache_ent *ent; + + ent = container_of(work, struct xsc_cache_ent, work); + __cache_work_func(ent); +} + +static int xsc_mr_cache_debugfs_init(struct xsc_ib_dev *dev) +{ + struct xsc_mr_cache *cache = &dev->cache; + struct xsc_cache_ent *ent; + int i; + + if (!xsc_debugfs_root) + return 0; + + cache->root = debugfs_create_dir("mr_cache", dev->xdev->dev_res->dbg_root); + if (!cache->root) + return -ENOMEM; + + for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { + ent = &cache->ent[i]; + sprintf(ent->name, "%d", ent->order); + ent->dir = debugfs_create_dir(ent->name, cache->root); + if (!ent->dir) + return -ENOMEM; + + ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent, + &size_fops); + if (!ent->fsize) + return -ENOMEM; + + ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent, + &limit_fops); + if (!ent->flimit) + return -ENOMEM; + + debugfs_create_u32("cur", 0400, ent->dir, &ent->cur); + debugfs_create_u32("miss", 0600, ent->dir, &ent->miss); + } + + return 0; +} + +int xsc_mr_cache_init(struct xsc_ib_dev *dev) +{ + struct xsc_mr_cache *cache = &dev->cache; + struct xsc_cache_ent *ent; + int limit; + int size; + int err; + int i; + + cache->wq = create_singlethread_workqueue("mkey_cache"); + if (!cache->wq) { + xsc_ib_warn(dev, "failed to create work queue\n"); + return -ENOMEM; + } + + for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { + INIT_LIST_HEAD(&cache->ent[i].head); + spin_lock_init(&cache->ent[i].lock); + + ent = &cache->ent[i]; + INIT_LIST_HEAD(&ent->head); + spin_lock_init(&ent->lock); + ent->order = i + 2; + ent->dev = dev; + + if (dev->xdev->profile->mask & XSC_PROF_MASK_MR_CACHE) { + size = dev->xdev->profile->mr_cache[i].size; + limit = dev->xdev->profile->mr_cache[i].limit; + } else { + size = DEF_CACHE_SIZE; + limit = 0; + } + INIT_WORK(&ent->work, cache_work_func); + INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); + ent->limit = limit; + queue_work(cache->wq, &ent->work); + } + + err = xsc_mr_cache_debugfs_init(dev); + if (err) + xsc_ib_warn(dev, "cache debugfs failure\n"); + + return 0; +} + +struct ib_mr *xsc_ib_get_dma_mr(struct ib_pd *pd, int acc) +{ + struct xsc_ib_dev *dev = to_mdev(pd->device); + struct xsc_core_device *xdev = dev->xdev; + struct xsc_register_mr_mbox_in *in; + struct xsc_register_mr_request *req; + struct xsc_ib_mr *mr; + int err; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + pr_err("[%s:%d]", __func__, __LINE__); + return &mr->ibmr; + + in = kzalloc(sizeof(*in), GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err_free; + } + + req = &in->req; + req->acc = convert_access(acc) | XSC_ACCESS_MODE_PA; + req->pdn = cpu_to_be32(to_mpd(pd)->pdn | XSC_MKEY_LEN64); + req->va_base = 0; + + err = xsc_core_create_mkey(xdev, &mr->mmr); + if (err) + goto err_in; + req->mkey = cpu_to_be32(mr->mmr.key); + err = xsc_core_register_mr(xdev, &mr->mmr, in, sizeof(*in)); + if (err) + goto err_reg_mr; + kfree(in); + mr->ibmr.lkey = mr->mmr.key; + mr->ibmr.rkey = mr->mmr.key; + mr->umem = NULL; + + return &mr->ibmr; +err_reg_mr: + xsc_core_destroy_mkey(xdev, &mr->mmr); +err_in: + kfree(in); + +err_free: + kfree(mr); + + return ERR_PTR(err); +} + +void xsc_fill_pas(struct ib_umem *umem, int page_shift, __be64 *pas) +{ + struct scatterlist *sg; + int entry; + u64 base; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { + base = sg_dma_address(sg); + break; + } + + pas[0] = base & (~((1 << page_shift) - 1)); + pas[0] = cpu_to_be64(pas[0]); +} + +static struct xsc_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr, + u64 length, struct ib_umem *umem, + int npages, int page_shift, + int access_flags) +{ + struct xsc_ib_dev *dev = to_mdev(pd->device); + struct xsc_register_mr_mbox_in *in; + struct xsc_ib_mr *mr; + int inlen; + int err; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + inlen = sizeof(*in) + sizeof(*in->req.pas) * npages; + in = xsc_vzalloc(inlen); + if (!in) { + err = -ENOMEM; + goto err_1; + } + err = xsc_core_create_mkey(dev->xdev, &mr->mmr); + if (err) { + xsc_ib_warn(dev, "create mkey failed\n"); + goto err_2; + } + + if (npages != 1) + xsc_ib_populate_pas(dev, umem, page_shift, in->req.pas, npages, false); + else + xsc_fill_pas(umem, page_shift, in->req.pas); + + in->req.acc = convert_access(access_flags); + in->req.pa_num = cpu_to_be32(npages); + in->req.pdn = cpu_to_be32(to_mpd(pd)->pdn); + in->req.va_base = cpu_to_be64(virt_addr); + in->req.map_en = XSC_MPT_MAP_EN; + in->req.len = cpu_to_be32((u32)length); + in->req.page_mode = (page_shift == XSC_PAGE_SHIFT_4K ? XSC_PAGE_MODE_4K : + (page_shift == XSC_PAGE_SHIFT_64K ? XSC_PAGE_MODE_64K : + (page_shift == XSC_PAGE_SHIFT_2M ? XSC_PAGE_MODE_2M : XSC_PAGE_MODE_1G))); + in->req.mkey = cpu_to_be32(mr->mmr.key); + err = xsc_core_register_mr(dev->xdev, &mr->mmr, in, inlen); + if (err) { + xsc_ib_warn(dev, "register mr failed, err = %d\n", err); + goto err_reg_mr; + } + mr->umem = umem; + xsc_vfree(in); + + xsc_ib_dbg(dev, "mkey = 0x%x\n", mr->mmr.key); + + return mr; +err_reg_mr: + xsc_core_destroy_mkey(dev->xdev, &mr->mmr); +err_2: + xsc_vfree(in); + +err_1: + kfree(mr); + + return ERR_PTR(err); +} + +struct ib_mr *xsc_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int access_flags, + struct ib_udata *udata) +{ + struct xsc_ib_dev *dev = to_mdev(pd->device); + struct xsc_ib_mr *mr = NULL; + struct ib_umem_ex *umem_ex; + struct ib_umem *umem; + int page_shift; + int page_shift_adjust; + int npages; + int ncont; + int order; + int err; + int using_peer_mem = 0; + struct ib_peer_memory_client *ib_peer_mem = NULL; + struct xsc_ib_peer_id *xsc_ib_peer_id = NULL; + + xsc_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx\n", + start, virt_addr, length); + + umem = ib_umem_get(&dev->ib_dev, start, length, access_flags); + if (IS_ERR(umem)) { + // check client peer memory + u8 peer_exists = 0; + + umem_ex = ib_client_umem_get(pd->uobject->context, + start, length, access_flags, 0, &peer_exists); + if (!peer_exists) { + xsc_ib_dbg(dev, "umem get failed\n"); + return (void *)umem; + } + ib_peer_mem = umem_ex->ib_peer_mem; + xsc_ib_peer_id = kzalloc(sizeof(*xsc_ib_peer_id), GFP_KERNEL); + if (!xsc_ib_peer_id) { + err = -ENOMEM; + goto error; + } + init_completion(&xsc_ib_peer_id->comp); + err = ib_client_umem_activate_invalidation_notifier( + umem_ex, xsc_invalidate_umem, xsc_ib_peer_id); + if (err) + goto error; + using_peer_mem = 1; + + } else { + umem_ex = ib_umem_ex(umem); + if (IS_ERR(umem_ex)) { + err = -ENOMEM; + goto error; + } + } + umem = &umem_ex->umem; + + xsc_ib_cont_pages(umem, start, &npages, &page_shift, &ncont, &order); + if (!npages) { + xsc_ib_warn(dev, "avoid zero region\n"); + err = -EINVAL; + goto error; + } + + xsc_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n", + npages, ncont, order, page_shift); + + if (ncont == 1) { + page_shift_adjust = page_shift > XSC_PAGE_SHIFT_2M ? XSC_PAGE_SHIFT_1G : + page_shift > XSC_PAGE_SHIFT_64K ? XSC_PAGE_SHIFT_2M : + page_shift > XSC_PAGE_SHIFT_4K ? XSC_PAGE_SHIFT_64K : XSC_PAGE_SHIFT_4K; + } else { + page_shift_adjust = page_shift >= XSC_PAGE_SHIFT_1G ? XSC_PAGE_SHIFT_1G : + page_shift >= XSC_PAGE_SHIFT_2M ? XSC_PAGE_SHIFT_2M : + page_shift >= XSC_PAGE_SHIFT_64K ? XSC_PAGE_SHIFT_64K : XSC_PAGE_SHIFT_4K; + ncont = ncont << (page_shift - page_shift_adjust); + } + + if (using_peer_mem == 1) { + ncont = npages; + page_shift_adjust = PAGE_SHIFT; + } + xsc_ib_dbg(dev, "xsc pageshit=%d, npages=%d\n", page_shift_adjust, ncont); + mr = reg_create(pd, virt_addr, length, umem, ncont, page_shift_adjust, access_flags); + if (IS_ERR(mr)) { + err = PTR_ERR(mr); + goto error; + } + + xsc_ib_dbg(dev, "mkey 0x%x\n", mr->mmr.key); + + mr->umem = umem; + mr->npages = npages; + spin_lock(&dev->mr_lock); + dev->xdev->dev_res->reg_pages += npages; + spin_unlock(&dev->mr_lock); + mr->ibmr.lkey = mr->mmr.key; + mr->ibmr.rkey = mr->mmr.key; + mr->ibmr.length = length; + atomic_set(&mr->invalidated, 0); + if (ib_peer_mem) { + init_completion(&mr->invalidation_comp); + xsc_ib_peer_id->mr = mr; + mr->peer_id = xsc_ib_peer_id; + complete(&xsc_ib_peer_id->comp); + } + + return &mr->ibmr; + +error: + if (xsc_ib_peer_id) { + complete(&xsc_ib_peer_id->comp); + kfree(xsc_ib_peer_id); + xsc_ib_peer_id = NULL; + } + + ib_umem_ex_release(umem_ex); + return ERR_PTR(err); +} + +xsc_ib_dereg_mr_def() +{ + struct xsc_ib_dev *dev = to_mdev(ibmr->device); + struct xsc_ib_mr *mr = to_mmr(ibmr); + struct ib_umem *umem = mr->umem; + struct ib_umem_ex *umem_ex = (struct ib_umem_ex *)umem; + int npages = mr->npages; + int err; + + xsc_ib_dbg(dev, "dereg mkey = 0x%x\n", mr->mmr.key); + + if (atomic_inc_return(&mr->invalidated) > 1) { + /* In case there is inflight invalidation call pending for its termination */ + wait_for_completion(&mr->invalidation_comp); + kfree(mr); + return 0; + } + //printk("%s %d\n", __func__, __LINE__); + if (mr->npages) { + err = xsc_core_dereg_mr(dev->xdev, &mr->mmr); + if (err) { + xsc_ib_warn(dev, "failed to dereg mr 0x%x (%d)\n", + mr->mmr.key, err); + return err; + } + } + //printk("%s %d\n", __func__, __LINE__); + err = xsc_core_destroy_mkey(dev->xdev, &mr->mmr); + if (err) { + xsc_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n", + mr->mmr.key, err); + return err; + } + + if (umem_ex) { + ib_umem_ex_release(umem_ex); + spin_lock(&dev->mr_lock); + dev->xdev->dev_res->reg_pages -= npages; + spin_unlock(&dev->mr_lock); + } + + kfree(mr); + + return 0; +} + +static void xsc_invalidate_umem(void *invalidation_cookie, + struct ib_umem_ex *umem, unsigned long addr, size_t size) +{ + struct xsc_ib_mr *mr; + struct xsc_ib_dev *dev; + struct xsc_ib_peer_id *peer_id = (struct xsc_ib_peer_id *)invalidation_cookie; + + wait_for_completion(&peer_id->comp); + if (!peer_id->mr) + return; + + mr = peer_id->mr; + /* This function is called under client peer lock so its resources are race protected */ + if (atomic_inc_return(&mr->invalidated) > 1) { + umem->invalidation_ctx->inflight_invalidation = 1; + return; + } + + umem->invalidation_ctx->peer_callback = 1; + dev = to_mdev(mr->ibmr.device); + xsc_core_destroy_mkey(dev->xdev, &mr->mmr); + xsc_core_dereg_mr(dev->xdev, &mr->mmr); + complete(&mr->invalidation_comp); +} + +xsc_ib_alloc_mr_def() +{ + struct xsc_ib_dev *dev = to_mdev(pd->device); + struct xsc_ib_mr *mr; + int err; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return NULL; + mr->npages = 0; + mr->mmr.pd = to_mpd(pd)->pdn; + mr->pas = kcalloc(max_num_sg, sizeof(__be64), GFP_KERNEL); + if (!mr->pas) + goto err_alloc; + err = xsc_core_create_mkey(dev->xdev, &mr->mmr); + if (err) + goto err_create_mkey; + mr->ibmr.lkey = mr->mmr.key; + mr->ibmr.rkey = mr->mmr.key; + mr->ibmr.device = &dev->ib_dev; + + return &mr->ibmr; +err_create_mkey: + kfree(mr->pas); +err_alloc: + kfree(mr); + return NULL; +} + +static int xsc_set_page(struct ib_mr *ibmr, u64 pa) +{ + struct xsc_ib_mr *mmr = to_mmr(ibmr); + + mmr->pas[mmr->npages] = pa; + mmr->npages++; + return 0; +} + +int xsc_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + struct xsc_ib_mr *mmr = to_mmr(ibmr); + + mmr->npages = 0; + return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, xsc_set_page); +} + +int xsc_wr_reg_mr(struct xsc_ib_dev *dev, const struct ib_send_wr *wr) +{ + const struct ib_reg_wr *reg_wr = container_of(wr, struct ib_reg_wr, wr); + struct ib_mr *ibmr = reg_wr->mr; + struct xsc_ib_mr *mmr = to_mmr(ibmr); + struct xsc_register_mr_mbox_in *in; + int inlen; + int i; + int err; + __be64 *pas; + + inlen = sizeof(*in) + sizeof(__be64) * mmr->npages; + in = kzalloc(inlen, GFP_ATOMIC); + if (!in) + return -ENOMEM; + + in->req.pdn = cpu_to_be32(mmr->mmr.pd); + in->req.pa_num = cpu_to_be32(mmr->npages); + in->req.len = cpu_to_be32(ibmr->length); + in->req.mkey = cpu_to_be32(ibmr->rkey); + in->req.acc = convert_access(reg_wr->access); + in->req.page_mode = 0; + in->req.map_en = XSC_MPT_MAP_EN; + in->req.va_base = cpu_to_be64(ibmr->iova); + pas = in->req.pas; + for (i = 0; i < mmr->npages; i++) + pas[i] = cpu_to_be64(mmr->pas[i]); + err = xsc_core_register_mr(dev->xdev, &mmr->mmr, in, sizeof(*in)); + + kfree(in); + return err; +} + +int xsc_wr_invalidate_mr(struct xsc_ib_dev *dev, const struct ib_send_wr *wr) +{ + struct xsc_core_mr mr; + int err = 0; + + if (!wr) + return -1; + mr.key = wr->ex.invalidate_rkey; + //printk("%s %d key:0x%x\n", __func__, __LINE__, mr.key); + err = xsc_core_dereg_mr(dev->xdev, &mr); + return err; +} + +void xsc_reg_local_dma_mr(struct xsc_core_device *dev) +{ + struct xsc_register_mr_mbox_in in; + int err = 0; + + in.req.pdn = 0; + in.req.pa_num = 0; + in.req.len = 0; + in.req.mkey = cpu_to_be32(0xFF); + in.req.acc = XSC_PERM_LOCAL_WRITE | XSC_PERM_LOCAL_READ; + in.req.page_mode = 0; + in.req.map_en = !(XSC_MPT_MAP_EN); + in.req.va_base = 0; + + err = xsc_core_register_mr(dev, NULL, &in, sizeof(in)); + if (err) + xsc_core_err(dev, "\n"); +} + diff --git a/drivers/infiniband/hw/xsc/peer_mem.c b/drivers/infiniband/hw/xsc/peer_mem.c new file mode 100644 index 0000000000000000000000000000000000000000..2389eb92df59dad5595e47b3264f535319b27dc6 --- /dev/null +++ b/drivers/infiniband/hw/xsc/peer_mem.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include "ib_umem_ex.h" + +static DEFINE_MUTEX(peer_memory_mutex); +static LIST_HEAD(peer_memory_list); +static struct kobject *peers_kobj; + +static void complete_peer(struct kref *kref); +static struct ib_peer_memory_client *get_peer_by_kobj(void *kobj); +static ssize_t version_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct ib_peer_memory_client *ib_peer_client = get_peer_by_kobj(kobj); + + if (ib_peer_client) { + sprintf(buf, "%s\n", ib_peer_client->peer_mem->version); + kref_put(&ib_peer_client->ref, complete_peer); + return strlen(buf); + } + /* not found - nothing is return */ + return 0; +} + +static ssize_t num_alloc_mrs_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct ib_peer_memory_client *ib_peer_client = get_peer_by_kobj(kobj); + + if (ib_peer_client) { + sprintf(buf, "%llu\n", (u64)atomic64_read(&ib_peer_client->stats.num_alloc_mrs)); + kref_put(&ib_peer_client->ref, complete_peer); + return strlen(buf); + } + /* not found - nothing is return */ + return 0; +} + +static ssize_t num_dealloc_mrs_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct ib_peer_memory_client *ib_peer_client = get_peer_by_kobj(kobj); + + if (ib_peer_client) { + sprintf(buf, "%llu\n", (u64)atomic64_read(&ib_peer_client->stats.num_dealloc_mrs)); + kref_put(&ib_peer_client->ref, complete_peer); + return strlen(buf); + } + /* not found - nothing is return */ + return 0; +} + +static ssize_t num_reg_pages_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct ib_peer_memory_client *ib_peer_client = get_peer_by_kobj(kobj); + + if (ib_peer_client) { + sprintf(buf, "%llu\n", (u64)atomic64_read(&ib_peer_client->stats.num_reg_pages)); + kref_put(&ib_peer_client->ref, complete_peer); + return strlen(buf); + } + /* not found - nothing is return */ + return 0; +} + +static ssize_t num_dereg_pages_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct ib_peer_memory_client *ib_peer_client = get_peer_by_kobj(kobj); + + if (ib_peer_client) { + sprintf(buf, "%llu\n", (u64)atomic64_read(&ib_peer_client->stats.num_dereg_pages)); + kref_put(&ib_peer_client->ref, complete_peer); + return strlen(buf); + } + /* not found - nothing is return */ + return 0; +} + +static ssize_t num_reg_bytes_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct ib_peer_memory_client *ib_peer_client = get_peer_by_kobj(kobj); + + if (ib_peer_client) { + sprintf(buf, "%llu\n", (u64)atomic64_read(&ib_peer_client->stats.num_reg_bytes)); + kref_put(&ib_peer_client->ref, complete_peer); + return strlen(buf); + } + /* not found - nothing is return */ + return 0; +} + +static ssize_t num_dereg_bytes_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct ib_peer_memory_client *ib_peer_client = get_peer_by_kobj(kobj); + + if (ib_peer_client) { + sprintf(buf, "%llu\n", (u64)atomic64_read(&ib_peer_client->stats.num_dereg_bytes)); + kref_put(&ib_peer_client->ref, complete_peer); + return strlen(buf); + } + /* not found - nothing is return */ + return 0; +} + +static ssize_t num_free_callbacks_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct ib_peer_memory_client *ib_peer_client = get_peer_by_kobj(kobj); + + if (ib_peer_client) { + sprintf(buf, "%lu\n", ib_peer_client->stats.num_free_callbacks); + kref_put(&ib_peer_client->ref, complete_peer); + return strlen(buf); + } + /* not found - nothing is return */ + return 0; +} + +static struct kobj_attribute version_attr = __ATTR_RO(version); +static struct kobj_attribute num_alloc_mrs = __ATTR_RO(num_alloc_mrs); +static struct kobj_attribute num_dealloc_mrs = __ATTR_RO(num_dealloc_mrs); +static struct kobj_attribute num_reg_pages = __ATTR_RO(num_reg_pages); +static struct kobj_attribute num_dereg_pages = __ATTR_RO(num_dereg_pages); +static struct kobj_attribute num_reg_bytes = __ATTR_RO(num_reg_bytes); +static struct kobj_attribute num_dereg_bytes = __ATTR_RO(num_dereg_bytes); +static struct kobj_attribute num_free_callbacks = __ATTR_RO(num_free_callbacks); + +static struct attribute *peer_mem_attrs[] = { + &version_attr.attr, + &num_alloc_mrs.attr, + &num_dealloc_mrs.attr, + &num_reg_pages.attr, + &num_dereg_pages.attr, + &num_reg_bytes.attr, + &num_dereg_bytes.attr, + &num_free_callbacks.attr, + NULL, +}; + +static void destroy_peer_sysfs(struct ib_peer_memory_client *ib_peer_client) +{ + kobject_put(ib_peer_client->kobj); + if (list_empty(&peer_memory_list)) + kobject_put(peers_kobj); +} + +static int create_peer_sysfs(struct ib_peer_memory_client *ib_peer_client) +{ + int ret; + + if (list_empty(&peer_memory_list)) { + /* creating under /sys/kernel/mm */ + peers_kobj = kobject_create_and_add("memory_peers", mm_kobj); + if (!peers_kobj) + return -ENOMEM; + } + + ib_peer_client->peer_mem_attr_group.attrs = peer_mem_attrs; + /* Dir alreday was created explicitly to get its kernel object for further usage */ + ib_peer_client->peer_mem_attr_group.name = NULL; + ib_peer_client->kobj = kobject_create_and_add(ib_peer_client->peer_mem->name, + peers_kobj); + + if (!ib_peer_client->kobj) { + ret = -EINVAL; + goto free; + } + + /* Create the files associated with this kobject */ + ret = sysfs_create_group(ib_peer_client->kobj, + &ib_peer_client->peer_mem_attr_group); + if (ret) + goto peer_free; + + return 0; + +peer_free: + kobject_put(ib_peer_client->kobj); + +free: + if (list_empty(&peer_memory_list)) + kobject_put(peers_kobj); + + return ret; +} + +static struct ib_peer_memory_client *get_peer_by_kobj(void *kobj) +{ + struct ib_peer_memory_client *ib_peer_client; + + mutex_lock(&peer_memory_mutex); + list_for_each_entry(ib_peer_client, &peer_memory_list, core_peer_list) { + if (ib_peer_client->kobj == kobj) { + kref_get(&ib_peer_client->ref); + goto found; + } + } + + ib_peer_client = NULL; +found: + mutex_unlock(&peer_memory_mutex); + return ib_peer_client; +} + +/* Caller should be holding the peer client lock, ib_peer_client->lock */ +static struct core_ticket *ib_peer_search_context(struct ib_peer_memory_client *ib_peer_client, + u64 key) +{ + struct core_ticket *core_ticket; + + list_for_each_entry(core_ticket, &ib_peer_client->core_ticket_list, + ticket_list) { + if (core_ticket->key == key) + return core_ticket; + } + + return NULL; +} + +static int ib_invalidate_peer_memory(void *reg_handle, u64 core_context) +{ + struct ib_peer_memory_client *ib_peer_client = reg_handle; + struct invalidation_ctx *invalidation_ctx; + struct core_ticket *core_ticket; + int need_unlock = 1; + + mutex_lock(&ib_peer_client->lock); + ib_peer_client->stats.num_free_callbacks += 1; + core_ticket = ib_peer_search_context(ib_peer_client, core_context); + if (!core_ticket) + goto out; + + invalidation_ctx = (struct invalidation_ctx *)core_ticket->context; + /* If context is not ready yet, mark it to be invalidated */ + if (!invalidation_ctx->func) { + invalidation_ctx->peer_invalidated = 1; + goto out; + } + invalidation_ctx->func(invalidation_ctx->cookie, + invalidation_ctx->umem_ex, 0, 0); + if (invalidation_ctx->inflight_invalidation) { + /* init the completion to wait on before letting other thread to run */ + init_completion(&invalidation_ctx->comp); + mutex_unlock(&ib_peer_client->lock); + need_unlock = 0; + wait_for_completion(&invalidation_ctx->comp); + } + + kfree(invalidation_ctx); +out: + if (need_unlock) + mutex_unlock(&ib_peer_client->lock); + + return 0; +} + +static int ib_peer_insert_context(struct ib_peer_memory_client *ib_peer_client, + void *context, + u64 *context_ticket) +{ + struct core_ticket *core_ticket = kzalloc(sizeof(*core_ticket), GFP_KERNEL); + + if (!core_ticket) + return -ENOMEM; + + mutex_lock(&ib_peer_client->lock); + core_ticket->key = ib_peer_client->last_ticket++; + core_ticket->context = context; + list_add_tail(&core_ticket->ticket_list, + &ib_peer_client->core_ticket_list); + *context_ticket = core_ticket->key; + mutex_unlock(&ib_peer_client->lock); + + return 0; +} + +/* Caller should be holding the peer client lock, specifically, + * the caller should hold ib_peer_client->lock + */ +static int ib_peer_remove_context(struct ib_peer_memory_client *ib_peer_client, + u64 key) +{ + struct core_ticket *core_ticket; + + list_for_each_entry(core_ticket, &ib_peer_client->core_ticket_list, + ticket_list) { + if (core_ticket->key == key) { + list_del(&core_ticket->ticket_list); + kfree(core_ticket); + return 0; + } + } + + return 1; +} + +/* ib_peer_create_invalidation_ctx - creates invalidation context for a given umem + * @ib_peer_mem: peer client to be used + * @umem: umem struct belongs to that context + * @invalidation_ctx: output context + */ +int ib_peer_create_invalidation_ctx(struct ib_peer_memory_client *ib_peer_mem, + struct ib_umem_ex *umem_ex, struct invalidation_ctx **invalidation_ctx) +{ + int ret; + struct invalidation_ctx *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ret = ib_peer_insert_context(ib_peer_mem, ctx, + &ctx->context_ticket); + if (ret) { + kfree(ctx); + return ret; + } + + ctx->umem_ex = umem_ex; + umem_ex->invalidation_ctx = ctx; + *invalidation_ctx = ctx; + + return 0; +} + +/** + * ** ib_peer_destroy_invalidation_ctx - destroy a given invalidation context + * ** @ib_peer_mem: peer client to be used + * ** @invalidation_ctx: context to be invalidated + * **/ +void ib_peer_destroy_invalidation_ctx(struct ib_peer_memory_client *ib_peer_mem, + struct invalidation_ctx *invalidation_ctx) +{ + int peer_callback; + int inflight_invalidation; + + /* If we are under peer callback lock was already taken.*/ + if (!invalidation_ctx->peer_callback) + mutex_lock(&ib_peer_mem->lock); + ib_peer_remove_context(ib_peer_mem, invalidation_ctx->context_ticket); + /* make sure to check inflight flag after took the lock and remove from tree. + * in addition, from that point using local variables for peer_callback and + * inflight_invalidation as after the complete invalidation_ctx can't be accessed + * any more as it may be freed by the callback. + */ + peer_callback = invalidation_ctx->peer_callback; + inflight_invalidation = invalidation_ctx->inflight_invalidation; + if (inflight_invalidation) + complete(&invalidation_ctx->comp); + + /* On peer callback lock is handled externally */ + if (!peer_callback) + mutex_unlock(&ib_peer_mem->lock); + + /* in case under callback context or callback is pending + * let it free the invalidation context + */ + if (!peer_callback && !inflight_invalidation) + kfree(invalidation_ctx); +} +static int ib_memory_peer_check_mandatory(const struct peer_memory_client + *peer_client) +{ +#define PEER_MEM_MANDATORY_FUNC(x) { offsetof(struct peer_memory_client, x), #x } + static const struct { + size_t offset; + char *name; + } mandatory_table[] = { + PEER_MEM_MANDATORY_FUNC(acquire), + PEER_MEM_MANDATORY_FUNC(get_pages), + PEER_MEM_MANDATORY_FUNC(put_pages), + PEER_MEM_MANDATORY_FUNC(get_page_size), + PEER_MEM_MANDATORY_FUNC(dma_map), + PEER_MEM_MANDATORY_FUNC(dma_unmap) + }; + int i; + + for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { + if (!*(void **)((void *)peer_client + mandatory_table[i].offset)) { + pr_err("Peer memory %s is missing mandatory function %s\n", + peer_client->name, mandatory_table[i].name); + return -EINVAL; + } + } + + return 0; +} + +static void complete_peer(struct kref *kref) +{ + struct ib_peer_memory_client *ib_peer_client = + container_of(kref, struct ib_peer_memory_client, ref); + + complete(&ib_peer_client->unload_comp); +} + +void *ib_register_peer_memory_client(const struct peer_memory_client *peer_client, + invalidate_peer_memory *invalidate_callback) +{ + struct ib_peer_memory_client *ib_peer_client; + + if (ib_memory_peer_check_mandatory(peer_client)) + return NULL; + + ib_peer_client = kzalloc(sizeof(*ib_peer_client), GFP_KERNEL); + if (!ib_peer_client) + return NULL; + + INIT_LIST_HEAD(&ib_peer_client->core_ticket_list); + mutex_init(&ib_peer_client->lock); + init_completion(&ib_peer_client->unload_comp); + kref_init(&ib_peer_client->ref); + ib_peer_client->peer_mem = peer_client; + + /* Once peer supplied a non NULL callback it's an indication that + * invalidation support is required for any memory owning. + */ + if (invalidate_callback) { + *invalidate_callback = ib_invalidate_peer_memory; + ib_peer_client->invalidation_required = 1; + } + ib_peer_client->last_ticket = 1; + + mutex_lock(&peer_memory_mutex); + if (create_peer_sysfs(ib_peer_client)) { + kfree(ib_peer_client); + ib_peer_client = NULL; + goto end; + } + list_add_tail(&ib_peer_client->core_peer_list, &peer_memory_list); +end: + + mutex_unlock(&peer_memory_mutex); + return ib_peer_client; +} +EXPORT_SYMBOL(ib_register_peer_memory_client); + +void ib_unregister_peer_memory_client(void *reg_handle) +{ + struct ib_peer_memory_client *ib_peer_client = reg_handle; + + mutex_lock(&peer_memory_mutex); + list_del(&ib_peer_client->core_peer_list); + destroy_peer_sysfs(ib_peer_client); + mutex_unlock(&peer_memory_mutex); + + kref_put(&ib_peer_client->ref, complete_peer); + wait_for_completion(&ib_peer_client->unload_comp); + kfree(ib_peer_client); +} +EXPORT_SYMBOL(ib_unregister_peer_memory_client); + +struct ib_peer_memory_client *ib_get_peer_client(struct ib_ucontext *context, unsigned long addr, + size_t size, unsigned long peer_mem_flags, + void **peer_client_context) +{ + struct ib_peer_memory_client *ib_peer_client = NULL; + + int ret = 0; + + mutex_lock(&peer_memory_mutex); + list_for_each_entry(ib_peer_client, &peer_memory_list, core_peer_list) { + /* In case peer requires invalidation it can't own + * memory which doesn't support it + */ + if ((ib_peer_client->invalidation_required && + (!(peer_mem_flags & IB_PEER_MEM_INVAL_SUPP)))) + continue; + ret = ib_peer_client->peer_mem->acquire(addr, size, NULL, NULL, + peer_client_context); + if (ret > 0) + goto found; + } + + ib_peer_client = NULL; + +found: + if (ib_peer_client) + kref_get(&ib_peer_client->ref); + + mutex_unlock(&peer_memory_mutex); + + return ib_peer_client; +} +EXPORT_SYMBOL(ib_get_peer_client); + +void ib_put_peer_client(struct ib_peer_memory_client *ib_peer_client, + void *peer_client_context) +{ + if (ib_peer_client->peer_mem->release) + ib_peer_client->peer_mem->release(peer_client_context); + + kref_put(&ib_peer_client->ref, complete_peer); +} +EXPORT_SYMBOL(ib_put_peer_client); + +int ib_get_peer_private_data(struct ib_ucontext *context, u64 peer_id, + char *peer_name) +{ + pr_warn("predefine peer mem is not supported by now"); + return -1; +} +EXPORT_SYMBOL(ib_get_peer_private_data); + +void ib_put_peer_private_data(struct ib_ucontext *context) +{ + pr_warn("predefine peer mem is not supported by now"); +} +EXPORT_SYMBOL(ib_put_peer_private_data); diff --git a/drivers/infiniband/hw/xsc/peer_mem.h b/drivers/infiniband/hw/xsc/peer_mem.h new file mode 100644 index 0000000000000000000000000000000000000000..7e3f803ac246315558efab858a2fca505ccb26b4 --- /dev/null +++ b/drivers/infiniband/hw/xsc/peer_mem.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#if !defined(PEER_MEM_H) +#define PEER_MEM_H + +#include +#include +#include +#include +#include +#include + +#define IB_PEER_MEMORY_NAME_MAX 64 +#define IB_PEER_MEMORY_VER_MAX 16 +#define PEER_MEM_U64_CORE_CONTEXT + +/** + * struct peer_memory_client - registration information for peer client. + * @name: peer client name + * @version: peer client version + * @acquire: callback function to be used by IB core to detect whether a + * virtual address in under the responsibility of a specific peer client. + * @get_pages: callback function to be used by IB core asking the peer client to pin + * the physical pages of the given address range and returns that information. + * It equivalents to the kernel API of get_user_pages(), but targets peer memory. + * @dma_map: callback function to be used by IB core asking the peer client to fill + * the dma address mapping for a given address range. + * @dma_unmap: callback function to be used by IB core asking the peer client to take + * relevant actions to unmap the memory. + * @put_pages: callback function to be used by IB core asking the peer client to remove the + * pinning from the given memory. + * It's the peer-direct equivalent of the kernel API put_page. + * @get_page_size: callback function to be used by IB core to query the peer client for + * the page size for the given allocation. + * @release: callback function to be used by IB core asking peer client to release all + * resources associated with previous acquire call. The call will be performed + * only for contexts that have been successfully acquired (i.e. acquire returned a + * non-zero value). + * Additionally, IB core guarentees that there will be no pages pinned through this + * context when the callback is called. + * + * The subsections in this description contain detailed description + * of the callback arguments and expected return values for the + * callbacks defined in this struct. + * + * acquire: + * + * Callback function to be used by IB core to detect + * whether a virtual address in under the responsibility + * of a specific peer client. + * + * addr [IN] - virtual address to be checked whether belongs to peer. + * + * size [IN] - size of memory area starting at addr. + * + * peer_mem_private_data [IN] - The contents of ib_ucontext-> peer_mem_private_data. + * This parameter allows usage of the peer-direct + * API in implementations where it is impossible + * to detect if the memory belongs to the device + * based upon the virtual address alone. In such + * cases, the peer device can create a special + * ib_ucontext, which will be associated with the + * relevant peer memory. + * + * peer_mem_name [IN] - The contents of ib_ucontext-> peer_mem_name. + * Used to identify the peer memory client that + * initialized the ib_ucontext. + * This parameter is normally used along with + * peer_mem_private_data. + * client_context [OUT] - peer opaque data which holds a peer context for + * the acquired address range, will be provided + * back to the peer memory in subsequent + * calls for that given memory. + * + * If peer takes responsibility on the given address range further calls for memory + * management will be directed to the callbacks of this peer client. + * + * Return - 1 in case peer client takes responsibility on that range otherwise 0. + * Any peer internal error should resulted in a zero answer, in case address + * range really belongs to the peer, no owner will be found and application + * will get an error + * from IB Core as expected. + * + * get_pages: + * + * Callback function to be used by IB core asking the + * peer client to pin the physical pages of the given + * address range and returns that information. It + * equivalents to the kernel API of get_user_pages(), but + * targets peer memory. + * + * addr [IN] - start virtual address of that given allocation. + * + * size [IN] - size of memory area starting at addr. + * + * write [IN] - indicates whether the pages will be written to by the caller. + * Same meaning as of kernel API get_user_pages, can be + * ignored if not relevant. + * + * force [IN] - indicates whether to force write access even if user + * mapping is read only. Same meaning as of kernel API + * get_user_pages, can be ignored if not relevant. + * + * sg_head [IN/OUT] - pointer to head of struct sg_table. + * The peer client should allocate a table big + * enough to store all of the required entries. This + * function should fill the table with physical addresses + * and sizes of the memory segments composing this + * memory mapping. + * The table allocation can be done using sg_alloc_table. + * Filling in the physical memory addresses and size can + * be done using sg_set_page. + * + * client_context [IN] - peer context for the given allocation, as received from + * the acquire call. + * + * core_context [IN] - IB core context. If the peer client wishes to + * invalidate any of the pages pinned through this API, + * it must provide this context as an argument to the + * invalidate callback. + * + * Return - 0 success, otherwise errno error code. + * + * dma_map: + * + * Callback function to be used by IB core asking the peer client to fill + * the dma address mapping for a given address range. + * + * sg_head [IN/OUT] - pointer to head of struct sg_table. The peer memory + * should fill the dma_address & dma_length for + * each scatter gather entry in the table. + * + * client_context [IN] - peer context for the allocation mapped. + * + * dma_device [IN] - the RDMA capable device which requires access to the + * peer memory. + * + * dmasync [IN] - flush in-flight DMA when the memory region is written. + * Same meaning as with host memory mapping, can be ignored if + * not relevant. + * + * nmap [OUT] - number of mapped/set entries. + * + * Return - 0 success, otherwise errno error code. + * + * dma_unmap: + * + * Callback function to be used by IB core asking the peer client to take + * relevant actions to unmap the memory. + * + * sg_head [IN] - pointer to head of struct sg_table. The peer memory + * should fill the dma_address & dma_length for + * each scatter gather entry in the table. + * + * client_context [IN] - peer context for the allocation mapped. + * + * dma_device [IN] - the RDMA capable device which requires access to the + * peer memory. + * + * Return - 0 success, otherwise errno error code. + * + * put_pages: + * + * Callback function to be used by IB core asking the peer client to remove the + * pinning from the given memory. + * It's the peer-direct equivalent of the kernel API put_page. + * + * sg_head [IN] - pointer to head of struct sg_table. + * + * client_context [IN] - peer context for that given allocation. + * + * get_page_size: + * + * Callback function to be used by IB core to query the + * peer client for the page size for the given + * allocation. + * + * sg_head [IN] - pointer to head of struct sg_table. + * + * client_context [IN] - peer context for that given allocation. + * + * Return - Page size in bytes + * + * release: + * + * Callback function to be used by IB core asking peer + * client to release all resources associated with + * previous acquire call. The call will be performed only + * for contexts that have been successfully acquired + * (i.e. acquire returned a non-zero value). + * Additionally, IB core guarentees that there will be no + * pages pinned through this context when the callback is + * called. + * + * client_context [IN] - peer context for the given allocation. + * + **/ +struct peer_memory_client { + char name[IB_PEER_MEMORY_NAME_MAX]; + char version[IB_PEER_MEMORY_VER_MAX]; + int (*acquire)(unsigned long addr, size_t size, void *peer_mem_private_data, + char *peer_mem_name, void **client_context); + int (*get_pages)(unsigned long addr, + size_t size, int write, int force, + struct sg_table *sg_head, + void *client_context, u64 core_context); + int (*dma_map)(struct sg_table *sg_head, void *client_context, + struct device *dma_device, int dmasync, int *nmap); + int (*dma_unmap)(struct sg_table *sg_head, void *client_context, + struct device *dma_device); + void (*put_pages)(struct sg_table *sg_head, void *client_context); + unsigned long (*get_page_size)(void *client_context); + void (*release)(void *client_context); + void* (*get_context_private_data)(u64 peer_id); + void (*put_context_private_data)(void *context); +}; + +typedef int (*invalidate_peer_memory)(void *reg_handle, u64 core_context); + +void *ib_register_peer_memory_client(const struct peer_memory_client *peer_client, + invalidate_peer_memory *invalidate_callback); +void ib_unregister_peer_memory_client(void *reg_handle); + +#endif diff --git a/drivers/infiniband/hw/xsc/private_dev.c b/drivers/infiniband/hw/xsc/private_dev.c new file mode 100644 index 0000000000000000000000000000000000000000..ecb3728d71f86d076f136e5573ca4595a175a163 --- /dev/null +++ b/drivers/infiniband/hw/xsc/private_dev.c @@ -0,0 +1,1301 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "global.h" + +#define FEATURE_ONCHIP_FT_MASK (1<<4) +#define FEATURE_DMA_RW_TBL_MASK (1<<8) +#define FEATURE_PCT_EXP_MASK (1<<9) + +static int xsc_priv_dev_open(struct inode *inode, struct file *file) +{ + struct xsc_priv_device *priv_dev + = container_of(inode->i_cdev, struct xsc_priv_device, cdev); + + file->private_data = priv_dev; + + pr_err("[%s:%d] %s succ\n", + __func__, __LINE__, priv_dev->device_name); + return 0; +} + +static long xsc_ioctl_mem_free(struct xsc_priv_device *priv_dev, struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr) +{ + struct xsc_ioctl_mem_info *minfo; + struct xsc_ioctl_data_tl *tl; + struct xsc_ioctl_mbox_in *in; + struct xsc_mem_entry *mem_ent; + char tname[TASK_COMM_LEN]; + int in_size; + int err = 0; + u8 lfound = 0; + + in_size = sizeof(struct xsc_ioctl_mbox_in) + hdr->attr.length; + in = kvzalloc(in_size, GFP_KERNEL); + if (!in) + return -ENOMEM; + + in->len = hdr->attr.length; + err = copy_from_user(in->data, user_hdr->attr.data, hdr->attr.length); + if (err) { + kvfree(in); + return -EFAULT; + } + + if (in->len > sizeof(struct xsc_ioctl_data_tl)) { + tl = (struct xsc_ioctl_data_tl *)(in->data); + if (tl->length != sizeof(struct xsc_ioctl_mem_info)) { + kvfree(in); + return -EFAULT; + } + minfo = (struct xsc_ioctl_mem_info *)(tl + 1); + if (minfo->vir_addr && minfo->phy_addr) { + memset(tname, 0, sizeof(tname)); + get_task_comm(tname, current); + + spin_lock_irq(&priv_dev->mem_lock); + list_for_each_entry(mem_ent, &priv_dev->mem_list, list) { + if ((!strcmp(mem_ent->task_name, tname)) && + (mem_ent->mem_info.mem_num == minfo->mem_num) && + (mem_ent->mem_info.size == minfo->size)) { + if ((mem_ent->mem_info.phy_addr == minfo->phy_addr) && + (mem_ent->mem_info.vir_addr == minfo->vir_addr)) { + lfound = 1; + list_del(&mem_ent->list); + } else + err = -ENOMEM; + break; + } + } + spin_unlock_irq(&priv_dev->mem_lock); + + if (lfound) { + dma_free_coherent(&(xdev->pdev->dev), + minfo->size, + (void *)minfo->vir_addr, + minfo->phy_addr); + } + } else { + kvfree(in); + return -EFAULT; + } + } + + hdr->attr.error = err; + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + err = -EFAULT; + if (copy_to_user((void *)user_hdr->attr.data, in->data, in->len)) + err = -EFAULT; + + kvfree(in); + return err; +} + +static long xsc_ioctl_mem_alloc(struct xsc_priv_device *priv_dev, struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr) +{ + struct xsc_ioctl_mem_info *minfo; + struct xsc_ioctl_data_tl *tl; + struct xsc_ioctl_mbox_in *in; + struct xsc_mem_entry *mem_ent; + char tname[TASK_COMM_LEN]; + u64 vaddr = 0; + u64 paddr = 0; + int in_size; + int err = 0; + u8 lfound = 0; + u8 needfree = 0; + + in_size = sizeof(struct xsc_ioctl_mbox_in) + hdr->attr.length; + in = kvzalloc(in_size, GFP_KERNEL); + if (!in) + return -ENOMEM; + + in->len = hdr->attr.length; + err = copy_from_user(in->data, user_hdr->attr.data, hdr->attr.length); + if (err) { + kvfree(in); + return -EFAULT; + } + + if (in->len > sizeof(struct xsc_ioctl_data_tl)) { + tl = (struct xsc_ioctl_data_tl *)(in->data); + if (tl->length != sizeof(struct xsc_ioctl_mem_info)) { + kvfree(in); + return -EFAULT; + } + minfo = (struct xsc_ioctl_mem_info *)(tl + 1); + memset(tname, 0, sizeof(tname)); + get_task_comm(tname, current); + + spin_lock_irq(&priv_dev->mem_lock); + list_for_each_entry(mem_ent, &priv_dev->mem_list, list) { + if ((!strcmp(mem_ent->task_name, tname)) && + (mem_ent->mem_info.mem_num == minfo->mem_num)) { + if (mem_ent->mem_info.size == minfo->size) { + minfo->phy_addr = mem_ent->mem_info.phy_addr; + minfo->vir_addr = mem_ent->mem_info.vir_addr; + lfound = 1; + } else { + needfree = 1; + list_del(&mem_ent->list); + } + break; + } + } + spin_unlock_irq(&priv_dev->mem_lock); + + if (needfree) { + dma_free_coherent(&(xdev->pdev->dev), + mem_ent->mem_info.size, + (void *)mem_ent->mem_info.vir_addr, + mem_ent->mem_info.phy_addr); + } + + if (!lfound) { + vaddr = (u64)dma_alloc_coherent(&(xdev->pdev->dev), + minfo->size, + (dma_addr_t *)&paddr, + GFP_KERNEL); + if (vaddr) { + memset((void *)vaddr, 0, minfo->size); + minfo->phy_addr = paddr; + minfo->vir_addr = vaddr; + mem_ent = kzalloc(sizeof(struct xsc_mem_entry), GFP_KERNEL); + if (!mem_ent) { + kvfree(in); + return -ENOMEM; + } + strcpy(mem_ent->task_name, tname); + mem_ent->mem_info.mem_num = minfo->mem_num; + mem_ent->mem_info.size = minfo->size; + mem_ent->mem_info.phy_addr = paddr; + mem_ent->mem_info.vir_addr = vaddr; + spin_lock_irq(&priv_dev->mem_lock); + list_add(&mem_ent->list, &priv_dev->mem_list); + spin_unlock_irq(&priv_dev->mem_lock); + } else { + kvfree(in); + return -ENOMEM; + } + } + } + + hdr->attr.error = err; + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + err = -EFAULT; + if (copy_to_user((void *)user_hdr->attr.data, in->data, in->len)) + err = -EFAULT; + + kvfree(in); + return err; +} + +static long xsc_priv_dev_ioctl_mem(struct file *filp, unsigned long arg) +{ + struct xsc_priv_device *priv_dev = filp->private_data; + struct xsc_core_device *xdev; + struct xsc_ioctl_hdr __user *user_hdr = + (struct xsc_ioctl_hdr __user *)arg; + struct xsc_ioctl_hdr hdr; + int err; + + /* get xdev */ + xdev = container_of(priv_dev, struct xsc_core_device, priv_device); + + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + if (err) + return -EFAULT; + + /* check valid */ + if (hdr.check_filed != XSC_IOCTL_CHECK_FILED) + return -EINVAL; + + /* check ioctl cmd */ + switch (hdr.attr.opcode) { + case XSC_IOCTL_MEM_ALLOC: + return xsc_ioctl_mem_alloc(priv_dev, xdev, user_hdr, &hdr); + case XSC_IOCTL_MEM_FREE: + return xsc_ioctl_mem_free(priv_dev, xdev, user_hdr, &hdr); + default: + return -EINVAL; + } +} + +static int xsc_priv_modify_qp(struct xsc_core_device *xdev, void *in, void *out) +{ + int ret = 0, i = 0; + struct xsc_ioctl_qp_range *resp; + struct xsc_ioctl_data_tl *tl; + int insize; + struct xsc_modify_qp_mbox_in *mailin; + struct xsc_modify_qp_mbox_out mailout; + u32 qpn; + + tl = (struct xsc_ioctl_data_tl *)out; + resp = (struct xsc_ioctl_qp_range *)(tl + 1); + xsc_core_dbg(xdev, "xsc_ioctl_qp_range: qpn:%d, num:%d, opcode:%d\n", + resp->qpn, resp->num, resp->opcode); + if (resp->num == 0) { + xsc_core_dbg(xdev, "xsc_ioctl_qp_range: resp->num ==0\n"); + return 0; + } + qpn = resp->qpn; + insize = sizeof(struct xsc_modify_qp_mbox_in); + mailin = kvzalloc(insize, GFP_KERNEL); + if (!mailin) { + xsc_core_dbg(xdev, "xsc_ioctl_qp_range: enomem\n"); + return -ENOMEM; + } + if (resp->opcode == XSC_CMD_OP_RTR2RTS_QP) { + for (i = 0; i < resp->num; i++) { + mailin->hdr.opcode = cpu_to_be16(XSC_CMD_OP_RTR2RTS_QP); + mailin->qpn = cpu_to_be32(qpn + 0); + ret = xsc_cmd_exec(xdev, mailin, insize, &mailout, sizeof(mailout)); + xsc_core_dbg(xdev, "modify qp state qpn:%d\n", qpn + i); + } + } + + kvfree(mailin); + + return ret; +} + +static int xsc_priv_dev_ioctl_get_phy(struct xsc_core_device *xdev, + void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_data_tl *tl = (struct xsc_ioctl_data_tl *)out; + struct xsc_ioctl_get_phy_info_res *resp; + struct xsc_ioctl_get_vf_info_res *vf_res; + struct xsc_vf_info vf_info; + struct xsc_lag *ldev = xsc_lag_dev_get(xdev); + u16 lag_id = U16_MAX; + + if (ldev && __xsc_lag_is_active(ldev)) + lag_id = ldev->lag_id; + + switch (tl->opmod) { + case XSC_IOCTL_OP_GET_LOCAL: + resp = (struct xsc_ioctl_get_phy_info_res *)(tl + 1); + + resp->phy_port = xdev->pcie_port; + resp->func_id = xdev->glb_func_id; + resp->logic_in_port = xdev->logic_port; + resp->mac_phy_port = xdev->mac_port; + resp->mac_logic_in_port = xdev->mac_logic_port; + resp->lag_id = lag_id; + resp->raw_qp_id_base = xdev->caps.raweth_qp_id_base; + resp->raw_rss_qp_id_base = xdev->caps.raweth_rss_qp_id_base; + resp->lag_port_start = XSC_LAG_PORT_START; + resp->send_seg_num = xdev->caps.send_ds_num; + resp->recv_seg_num = xdev->caps.recv_ds_num; + resp->raw_tpe_qp_num = xdev->caps.raw_tpe_qp_num; + resp->chip_version = xdev->chip_ver_l; + resp->on_chip_tbl_vld = + (xdev->feature_flag & FEATURE_ONCHIP_FT_MASK) ? 1 : 0; + resp->dma_rw_tbl_vld = + (xdev->feature_flag & FEATURE_DMA_RW_TBL_MASK) ? 1 : 0; + resp->pct_compress_vld = + (xdev->feature_flag & FEATURE_PCT_EXP_MASK) ? 1 : 0; + + xsc_core_dbg(xdev, "%d,%d,%d,%d,%d,%d\n", resp->phy_port, + resp->func_id, resp->logic_in_port, + resp->mac_phy_port, resp->mac_logic_in_port, + resp->lag_id); + resp->funcid_encode[0] = XSC_PCIE0_VF0_FUNC_ID; + resp->funcid_encode[1] = XSC_PCIE0_VF_FUNC_ID_END; + resp->funcid_encode[2] = XSC_PCIE0_PF0_FUNC_ID; + resp->funcid_encode[3] = XSC_PCIE0_PF_FUNC_ID_END; + resp->funcid_encode[4] = XSC_PCIE1_VF0_FUNC_ID; + resp->funcid_encode[5] = XSC_PCIE1_VF_FUNC_ID_END; + resp->funcid_encode[6] = XSC_PCIE1_PF0_FUNC_ID; + resp->funcid_encode[7] = XSC_PCIE1_PF_FUNC_ID_END; + break; + + case XSC_IOCTL_OP_GET_VF_INFO: + vf_res = (struct xsc_ioctl_get_vf_info_res *)(tl + 1); + memcpy(&vf_info, vf_res, sizeof(struct xsc_vf_info)); + + xsc_pci_get_vf_info(xdev, &vf_info); + + vf_res->func_id = vf_info.func_id; + vf_res->logic_port = vf_info.logic_port; + break; + + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static int xsc_priv_dev_ioctl_get_global_pcp(struct xsc_core_device *xdev, void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_global_pcp *resp = (struct xsc_ioctl_global_pcp *)out; + + if (!XSC_IS_PF(xdev->glb_func_id)) { + ret = -EOPNOTSUPP; + return ret; + } + + resp->pcp = get_global_force_pcp(); + return 0; +} + +static int xsc_priv_dev_ioctl_get_global_dscp(struct xsc_core_device *xdev, void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_global_dscp *resp = (struct xsc_ioctl_global_dscp *)out; + + if (!XSC_IS_PF(xdev->glb_func_id)) { + ret = -EOPNOTSUPP; + return ret; + } + + resp->dscp = get_global_force_dscp(); + return 0; +} + +static int xsc_priv_dev_ioctl_set_global_pcp(struct xsc_core_device *xdev, void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_global_pcp *req = (struct xsc_ioctl_global_pcp *)out; + + if (!XSC_IS_PF(xdev->glb_func_id)) { + ret = -EOPNOTSUPP; + return ret; + } + + ret = set_global_force_pcp(req->pcp); + return ret; +} + +static int xsc_priv_dev_ioctl_set_global_dscp(struct xsc_core_device *xdev, void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_global_dscp *req = (struct xsc_ioctl_global_dscp *)out; + + if (!XSC_IS_PF(xdev->glb_func_id)) { + ret = -EOPNOTSUPP; + return ret; + } + + ret = set_global_force_dscp(req->dscp); + return ret; +} + +int xsc_priv_dev_exec_ioctl(struct xsc_core_device *xdev, void *in, int in_size, void *out, + int out_size) +{ + int opcode, ret = 0; + struct xsc_ioctl_attr *hdr; + + hdr = (struct xsc_ioctl_attr *)in; + opcode = hdr->opcode; + switch (opcode) { + case XSC_IOCTL_GET_PHY_INFO: + ret = xsc_priv_dev_ioctl_get_phy(xdev, in, out); + break; + case XSC_IOCTL_GET_GLOBAL_PCP: + xsc_core_dbg(xdev, "getting global pcp\n"); + ret = xsc_priv_dev_ioctl_get_global_pcp(xdev, in, out); + break; + case XSC_IOCTL_GET_GLOBAL_DSCP: + ret = xsc_priv_dev_ioctl_get_global_dscp(xdev, in, out); + break; + case XSC_IOCTL_SET_QP_STATUS: + xsc_core_dbg(xdev, "case XSC_IOCTL_SET_QP_STATUS:\n"); + ret = xsc_priv_modify_qp(xdev, in, out); + break; + case XSC_IOCTL_SET_GLOBAL_PCP: + xsc_core_dbg(xdev, "setting global pcp\n"); + ret = xsc_priv_dev_ioctl_set_global_pcp(xdev, in, out); + break; + case XSC_IOCTL_SET_GLOBAL_DSCP: + xsc_core_dbg(xdev, "setting global dscp\n"); + ret = xsc_priv_dev_ioctl_set_global_dscp(xdev, in, out); + break; + default: + ret = -EINVAL; + break; + } + + xsc_core_dbg(xdev, "%s failed ret=%u\n", __func__, ret); + + return ret; +} + +static long xsc_priv_dev_ioctl_getinfo(struct file *filp, unsigned long arg) +{ + struct xsc_priv_device *priv_dev = filp->private_data; + struct xsc_core_device *xdev; + struct xsc_ioctl_hdr __user *user_hdr = + (struct xsc_ioctl_hdr __user *)arg; + struct xsc_ioctl_hdr hdr; + struct xsc_ioctl_hdr *in; + int in_size; + int err; + + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + if (err) + return -EFAULT; + if (hdr.check_filed != XSC_IOCTL_CHECK_FILED) + return -EINVAL; + switch (hdr.attr.opcode) { + case XSC_IOCTL_GET_PHY_INFO: + case XSC_IOCTL_GET_GLOBAL_PCP: + case XSC_IOCTL_GET_GLOBAL_DSCP: + case XSC_IOCTL_SET_QP_STATUS: + case XSC_IOCTL_SET_GLOBAL_PCP: + case XSC_IOCTL_SET_GLOBAL_DSCP: + case XSC_IOCTL_GET_CONTEXT: + break; + default: + return -EINVAL; + } + in_size = sizeof(struct xsc_ioctl_hdr) + hdr.attr.length; + in = kvzalloc(in_size, GFP_KERNEL); + if (!in) + return -EFAULT; + in->attr.opcode = hdr.attr.opcode; + in->attr.length = hdr.attr.length; + err = copy_from_user(in->attr.data, user_hdr->attr.data, hdr.attr.length); + if (err) { + kvfree(in); + return -EFAULT; + } + xdev = container_of(priv_dev, struct xsc_core_device, priv_device); + err = xsc_priv_dev_exec_ioctl(xdev, &in->attr, (in_size-sizeof(u32)), in->attr.data, + hdr.attr.length); + in->attr.error = err; + if (copy_to_user((void *)arg, in, in_size)) + err = -EFAULT; + kvfree(in); + return err; +} + +static int xsc_ioctl_flow_cmdq(struct xsc_priv_device *priv_dev, struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr) +{ + struct xsc_ioctl_mbox_in *in; + struct xsc_ioctl_mbox_out *out; + int in_size; + int out_size; + int err; + + in_size = sizeof(struct xsc_ioctl_mbox_in) + hdr->attr.length; + in = kvzalloc(in_size, GFP_KERNEL); + if (!in) + return -EFAULT; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + in->len = __cpu_to_be16(hdr->attr.length); + err = copy_from_user(in->data, user_hdr->attr.data, hdr->attr.length); + if (err) { + kvfree(in); + return -EFAULT; + } + + out_size = sizeof(struct xsc_ioctl_mbox_out) + hdr->attr.length; + out = kvzalloc(out_size, GFP_KERNEL); + if (!out) { + kvfree(in); + return -ENOMEM; + } + memcpy(out->data, in->data, hdr->attr.length); + out->len = in->len; + err = xsc_cmd_exec(xdev, in, in_size, out, out_size); + + hdr->attr.error = __be32_to_cpu(out->error); + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + err = -EFAULT; + if (copy_to_user((void *)user_hdr->attr.data, out->data, hdr->attr.length)) + err = -EFAULT; + + kvfree(in); + kvfree(out); + return err; +} + +static int xsc_ioctl_qos(struct xsc_priv_device *priv_dev, struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr, u16 expect_req_size, + u16 expect_resp_size, void (*encode)(void *, u32), void (*decode)(void *)) +{ + struct xsc_qos_mbox_in *in; + struct xsc_qos_mbox_out *out; + u16 user_size; + int err; + + user_size = expect_req_size > expect_resp_size ? expect_req_size : expect_resp_size; + if (hdr->attr.length != user_size) + return -EINVAL; + + in = kvzalloc(sizeof(struct xsc_qos_mbox_in) + expect_req_size, GFP_KERNEL); + if (!in) + goto err_in; + out = kvzalloc(sizeof(struct xsc_qos_mbox_out) + expect_resp_size, GFP_KERNEL); + if (!out) + goto err_out; + + err = copy_from_user(&in->data, user_hdr->attr.data, expect_req_size); + if (err) + goto err; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + in->req_prfx.mac_port = xdev->mac_port; + + if (encode) + encode((void *)in->data, xdev->mac_port); + + err = xsc_cmd_exec( + xdev, in, sizeof(*in) + expect_req_size, out, sizeof(*out) + expect_resp_size); + + hdr->attr.error = out->hdr.status; + if (decode) + decode((void *)out->data); + + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + goto err; + if (copy_to_user((void *)user_hdr->attr.data, &out->data, expect_resp_size)) + goto err; + + kvfree(in); + kvfree(out); + return 0; + +err: + kvfree(out); +err_out: + kvfree(in); +err_in: + return -EFAULT; +} + +static int xsc_ioctl_cc(struct xsc_priv_device *priv_dev, struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr, u16 expect_req_size, + u16 expect_resp_size, void (*encode)(void *, u32), void (*decode)(void *)) +{ + struct xsc_cc_mbox_in *in; + struct xsc_cc_mbox_out *out; + u16 user_size; + int err; + + user_size = expect_req_size > expect_resp_size ? expect_req_size : expect_resp_size; + if (hdr->attr.length != user_size) + return -EINVAL; + + in = kvzalloc(sizeof(struct xsc_cc_mbox_in) + expect_req_size, GFP_KERNEL); + if (!in) + goto err_in; + out = kvzalloc(sizeof(struct xsc_cc_mbox_out) + expect_resp_size, GFP_KERNEL); + if (!out) + goto err_out; + + err = copy_from_user(&in->data, user_hdr->attr.data, expect_req_size); + if (err) + goto err; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + if (encode) + encode((void *)in->data, xdev->mac_port); + + err = xsc_cmd_exec( + xdev, in, sizeof(*in) + expect_req_size, out, sizeof(*out) + expect_resp_size); + + hdr->attr.error = out->hdr.status; + if (decode) + decode((void *)out->data); + + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + goto err; + if (copy_to_user((void *)user_hdr->attr.data, &out->data, expect_resp_size)) + goto err; + + kvfree(in); + kvfree(out); + return 0; + +err: + kvfree(out); +err_out: + kvfree(in); +err_in: + return -EFAULT; +} + +static int xsc_ioctl_hwconfig(struct xsc_priv_device *priv_dev, struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr, u16 expect_req_size, + u16 expect_resp_size, void (*encode)(void *, u32), void (*decode)(void *)) +{ + struct xsc_hwc_mbox_in *in; + struct xsc_hwc_mbox_out *out; + u16 user_size; + int err; + + user_size = expect_req_size > expect_resp_size ? expect_req_size : expect_resp_size; + if (hdr->attr.length != user_size) + return -EINVAL; + + in = kvzalloc(sizeof(struct xsc_hwc_mbox_in) + expect_req_size, GFP_KERNEL); + if (!in) + goto err_in; + out = kvzalloc(sizeof(struct xsc_hwc_mbox_out) + expect_resp_size, GFP_KERNEL); + if (!out) + goto err_out; + + err = copy_from_user(&in->data, user_hdr->attr.data, expect_req_size); + if (err) + goto err; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + if (encode) + encode((void *)in->data, xdev->mac_port); + + err = xsc_cmd_exec( + xdev, in, sizeof(*in) + expect_req_size, out, sizeof(*out) + expect_resp_size); + + hdr->attr.error = out->hdr.status; + if (decode) + decode((void *)out->data); + + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + goto err; + if (copy_to_user((void *)user_hdr->attr.data, &out->data, expect_resp_size)) + goto err; + + kvfree(in); + kvfree(out); + return 0; + +err: + kvfree(out); +err_out: + kvfree(in); +err_in: + return -EFAULT; +} + +static int xsc_ioctl_modify_raw_qp(struct xsc_priv_device *priv_dev, struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr) +{ + struct xsc_modify_raw_qp_mbox_in *in; + struct xsc_modify_raw_qp_mbox_out *out; + int err; + + if (hdr->attr.length != sizeof(struct xsc_modify_raw_qp_request)) + return -EINVAL; + + in = kvzalloc(sizeof(struct xsc_modify_raw_qp_mbox_in), GFP_KERNEL); + if (!in) + goto err_in; + out = kvzalloc(sizeof(struct xsc_modify_raw_qp_mbox_out), GFP_KERNEL); + if (!out) + goto err_out; + + err = copy_from_user(&in->req, user_hdr->attr.data, + sizeof(struct xsc_modify_raw_qp_request)); + if (err) + goto err; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + in->pcie_no = xsc_get_pcie_no(); + + err = xsc_cmd_exec(xdev, in, sizeof(struct xsc_modify_raw_qp_mbox_in), + out, sizeof(struct xsc_modify_raw_qp_mbox_out)); + + hdr->attr.error = __be32_to_cpu(out->hdr.status); + + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + goto err; + + kvfree(in); + kvfree(out); + return 0; + +err: + kvfree(out); +err_out: + kvfree(in); +err_in: + return -EFAULT; +} + +static void encode_rlimit_set(void *data, u32 mac_port) +{ + struct xsc_rate_limit_set *req = (struct xsc_rate_limit_set *) data; + + req->rate_cir = __cpu_to_be32(req->rate_cir); + req->limit_id = __cpu_to_be32(req->limit_id); +} + +static void decode_rlimit_get(void *data) +{ + struct xsc_rate_limit_get *resp = (struct xsc_rate_limit_get *) data; + int i; + + for (i = 0; i <= QOS_PRIO_MAX; i++) + resp->rate_cir[i] = __be32_to_cpu(resp->rate_cir[i]); + + resp->max_limit_id = __be32_to_cpu(resp->max_limit_id); +} + +static void encode_cc_cmd_enable_rp(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_enable_rp *cc_cmd = (struct xsc_cc_cmd_enable_rp *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->enable = __cpu_to_be32(cc_cmd->enable); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_enable_np(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_enable_np *cc_cmd = (struct xsc_cc_cmd_enable_np *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->enable = __cpu_to_be32(cc_cmd->enable); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_init_alpha(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_init_alpha *cc_cmd = (struct xsc_cc_cmd_init_alpha *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->alpha = __cpu_to_be32(cc_cmd->alpha); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_g(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_g *cc_cmd = (struct xsc_cc_cmd_g *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->g = __cpu_to_be32(cc_cmd->g); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_ai(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_ai *cc_cmd = (struct xsc_cc_cmd_ai *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->ai = __cpu_to_be32(cc_cmd->ai); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_hai(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_hai *cc_cmd = (struct xsc_cc_cmd_hai *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->hai = __cpu_to_be32(cc_cmd->hai); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_th(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_th *cc_cmd = (struct xsc_cc_cmd_th *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->threshold = __cpu_to_be32(cc_cmd->threshold); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_bc(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_bc *cc_cmd = (struct xsc_cc_cmd_bc *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->bytecount = __cpu_to_be32(cc_cmd->bytecount); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_cnp_opcode(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_opcode *cc_cmd = (struct xsc_cc_cmd_cnp_opcode *) data; + + cc_cmd->opcode = __cpu_to_be32(cc_cmd->opcode); +} + +static void encode_cc_cmd_cnp_bth_b(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_bth_b *cc_cmd = (struct xsc_cc_cmd_cnp_bth_b *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->bth_b = __cpu_to_be32(cc_cmd->bth_b); +} + +static void encode_cc_cmd_cnp_bth_f(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_bth_f *cc_cmd = (struct xsc_cc_cmd_cnp_bth_f *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->bth_f = __cpu_to_be32(cc_cmd->bth_f); +} + +static void encode_cc_cmd_cnp_ecn(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_ecn *cc_cmd = (struct xsc_cc_cmd_cnp_ecn *) data; + + cc_cmd->ecn = __cpu_to_be32(cc_cmd->ecn); +} + +static void encode_cc_cmd_data_ecn(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_data_ecn *cc_cmd = (struct xsc_cc_cmd_data_ecn *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->ecn = __cpu_to_be32(cc_cmd->ecn); +} + +static void encode_cc_cmd_cnp_tx_interval(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_tx_interval *cc_cmd = (struct xsc_cc_cmd_cnp_tx_interval *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->interval = __cpu_to_be32(cc_cmd->interval); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_evt_rsttime(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_evt_rsttime *cc_cmd = + (struct xsc_cc_cmd_evt_rsttime *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->period = __cpu_to_be32(cc_cmd->period); +} + +static void encode_cc_cmd_cnp_dscp(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_dscp *cc_cmd = (struct xsc_cc_cmd_cnp_dscp *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->dscp = __cpu_to_be32(cc_cmd->dscp); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_cnp_pcp(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_pcp *cc_cmd = (struct xsc_cc_cmd_cnp_pcp *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->pcp = __cpu_to_be32(cc_cmd->pcp); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_evt_period_alpha(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_evt_period_alpha *cc_cmd = (struct xsc_cc_cmd_evt_period_alpha *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->period = __cpu_to_be32(cc_cmd->period); +} + +static void encode_cc_get_cfg(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_get_cfg *cc_cmd = (struct xsc_cc_cmd_get_cfg *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void decode_cc_get_cfg(void *data) +{ + struct xsc_cc_cmd_get_cfg *cc_cmd = (struct xsc_cc_cmd_get_cfg *) data; + + cc_cmd->cmd = __be16_to_cpu(cc_cmd->cmd); + cc_cmd->len = __be16_to_cpu(cc_cmd->len); + cc_cmd->enable_rp = __be32_to_cpu(cc_cmd->enable_rp); + cc_cmd->enable_np = __be32_to_cpu(cc_cmd->enable_np); + cc_cmd->init_alpha = __be32_to_cpu(cc_cmd->init_alpha); + cc_cmd->g = __be32_to_cpu(cc_cmd->g); + cc_cmd->ai = __be32_to_cpu(cc_cmd->ai); + cc_cmd->hai = __be32_to_cpu(cc_cmd->hai); + cc_cmd->threshold = __be32_to_cpu(cc_cmd->threshold); + cc_cmd->bytecount = __be32_to_cpu(cc_cmd->bytecount); + cc_cmd->opcode = __be32_to_cpu(cc_cmd->opcode); + cc_cmd->bth_b = __be32_to_cpu(cc_cmd->bth_b); + cc_cmd->bth_f = __be32_to_cpu(cc_cmd->bth_f); + cc_cmd->cnp_ecn = __be32_to_cpu(cc_cmd->cnp_ecn); + cc_cmd->data_ecn = __be32_to_cpu(cc_cmd->data_ecn); + cc_cmd->cnp_tx_interval = __be32_to_cpu(cc_cmd->cnp_tx_interval); + cc_cmd->evt_period_rsttime = __be32_to_cpu(cc_cmd->evt_period_rsttime); + cc_cmd->cnp_dscp = __be32_to_cpu(cc_cmd->cnp_dscp); + cc_cmd->cnp_pcp = __be32_to_cpu(cc_cmd->cnp_pcp); + cc_cmd->evt_period_alpha = __be32_to_cpu(cc_cmd->evt_period_alpha); + cc_cmd->section = __be32_to_cpu(cc_cmd->section); +} + +static void encode_cc_get_stat(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_get_stat *cc_cmd = (struct xsc_cc_cmd_get_stat *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void decode_cc_get_stat(void *data) +{ + struct xsc_cc_cmd_stat *cc_cmd = (struct xsc_cc_cmd_stat *) data; + + cc_cmd->cnp_handled = __be32_to_cpu(cc_cmd->cnp_handled); + cc_cmd->alpha_recovery = __be32_to_cpu(cc_cmd->alpha_recovery); + cc_cmd->reset_timeout = __be32_to_cpu(cc_cmd->reset_timeout); + cc_cmd->reset_bytecount = __be32_to_cpu(cc_cmd->reset_bytecount); +} + +static long xsc_priv_dev_ioctl_cmdq(struct file *filp, unsigned long arg) +{ + struct xsc_priv_device *priv_dev = filp->private_data; + struct xsc_core_device *xdev; + struct xsc_ioctl_hdr __user *user_hdr = + (struct xsc_ioctl_hdr __user *)arg; + struct xsc_ioctl_hdr hdr; + int err; + void *in; + void *out; + + /* get xdev */ + xdev = container_of(priv_dev, struct xsc_core_device, priv_device); + + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + if (err) + return -EFAULT; + + /* check valid */ + if (hdr.check_filed != XSC_IOCTL_CHECK_FILED) + return -EINVAL; + + /* check ioctl cmd */ + switch (hdr.attr.opcode) { + case XSC_CMD_OP_IOCTL_FLOW: + return xsc_ioctl_flow_cmdq(priv_dev, xdev, user_hdr, &hdr); + case XSC_CMD_OP_CREATE_QP: + break; + case XSC_CMD_OP_DESTROY_QP: + break; + case XSC_CMD_OP_CREATE_CQ: + break; + case XSC_CMD_OP_DESTROY_CQ: + break; + case XSC_CMD_OP_CREATE_MULTI_QP: + break; + case XSC_CMD_OP_ALLOC_MULTI_VIRTQ_CQ: + break; + case XSC_CMD_OP_RELEASE_MULTI_VIRTQ_CQ: + break; + case XSC_CMD_OP_ALLOC_MULTI_VIRTQ: + break; + case XSC_CMD_OP_RELEASE_MULTI_VIRTQ: + break; + case XSC_CMD_OP_IOCTL_SET_DSCP_PMT: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_dscp_pmt_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_DSCP_PMT: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, 0, sizeof(struct xsc_dscp_pmt_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_TRUST_MODE: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_trust_mode_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_TRUST_MODE: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, 0, sizeof(struct xsc_trust_mode_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_PCP_PMT: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_pcp_pmt_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_PCP_PMT: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, 0, sizeof(struct xsc_pcp_pmt_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_DEFAULT_PRI: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_default_pri_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_DEFAULT_PRI: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, 0, sizeof(struct xsc_default_pri_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_PFC: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_pfc_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_PFC: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, 0, sizeof(struct xsc_pfc_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_RATE_LIMIT: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_rate_limit_set), 0, + encode_rlimit_set, NULL); + case XSC_CMD_OP_IOCTL_GET_RATE_LIMIT: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_rate_limit_get), + sizeof(struct xsc_rate_limit_get), NULL, decode_rlimit_get); + case XSC_CMD_OP_IOCTL_SET_SP: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_sp_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_SP: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, 0, sizeof(struct xsc_sp_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_WEIGHT: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_weight_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_WEIGHT: + return xsc_ioctl_qos( + priv_dev, xdev, user_hdr, &hdr, 0, sizeof(struct xsc_weight_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_ENABLE_RP: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_enable_rp), + 0, encode_cc_cmd_enable_rp, NULL); + case XSC_CMD_OP_IOCTL_SET_ENABLE_NP: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_enable_np), + 0, encode_cc_cmd_enable_np, NULL); + case XSC_CMD_OP_IOCTL_SET_INIT_ALPHA: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_init_alpha), + 0, encode_cc_cmd_init_alpha, NULL); + case XSC_CMD_OP_IOCTL_SET_G: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_g), + 0, encode_cc_cmd_g, NULL); + case XSC_CMD_OP_IOCTL_SET_AI: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_ai), + 0, encode_cc_cmd_ai, NULL); + case XSC_CMD_OP_IOCTL_SET_HAI: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_hai), + 0, encode_cc_cmd_hai, NULL); + case XSC_CMD_OP_IOCTL_SET_TH: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_th), + 0, encode_cc_cmd_th, NULL); + case XSC_CMD_OP_IOCTL_SET_BC_TH: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_bc), + 0, encode_cc_cmd_bc, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_OPCODE: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_opcode), + 0, encode_cc_cmd_cnp_opcode, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_BTH_B: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_bth_b), + 0, encode_cc_cmd_cnp_bth_b, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_BTH_F: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_bth_f), + 0, encode_cc_cmd_cnp_bth_f, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_ECN: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_ecn), + 0, encode_cc_cmd_cnp_ecn, NULL); + case XSC_CMD_OP_IOCTL_SET_DATA_ECN: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_data_ecn), + 0, encode_cc_cmd_data_ecn, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_TX_INTERVAL: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_tx_interval), + 0, encode_cc_cmd_cnp_tx_interval, NULL); + case XSC_CMD_OP_IOCTL_SET_EVT_PERIOD_RSTTIME: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, + sizeof(struct xsc_cc_cmd_evt_rsttime), 0, encode_cc_cmd_evt_rsttime, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_DSCP: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_dscp), + 0, encode_cc_cmd_cnp_dscp, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_PCP: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_pcp), + 0, encode_cc_cmd_cnp_pcp, NULL); + case XSC_CMD_OP_IOCTL_SET_EVT_PERIOD_ALPHA: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_evt_period_alpha), + 0, encode_cc_cmd_evt_period_alpha, NULL); + case XSC_CMD_OP_IOCTL_GET_CC_CFG: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_get_cfg), + sizeof(struct xsc_cc_cmd_get_cfg), encode_cc_get_cfg, decode_cc_get_cfg); + case XSC_CMD_OP_IOCTL_GET_CC_STAT: + return xsc_ioctl_cc( + priv_dev, xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_get_stat), + sizeof(struct xsc_cc_cmd_stat), encode_cc_get_stat, decode_cc_get_stat); + case XSC_CMD_OP_IOCTL_SET_HWC: + return xsc_ioctl_hwconfig(priv_dev, xdev, user_hdr, &hdr, + sizeof(struct hwc_set_t), 0, NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_HWC: + return xsc_ioctl_hwconfig(priv_dev, xdev, user_hdr, &hdr, + sizeof(struct hwc_get_t), sizeof(struct hwc_get_t), NULL, NULL); + case XSC_CMD_OP_MODIFY_RAW_QP: + return xsc_ioctl_modify_raw_qp(priv_dev, xdev, user_hdr, &hdr); + default: + return -EINVAL; + } + + in = kvzalloc(hdr.attr.length, GFP_KERNEL); + if (!in) + return -ENOMEM; + out = kvzalloc(hdr.attr.length, GFP_KERNEL); + if (!out) { + kfree(in); + return -ENOMEM; + } + + err = copy_from_user(in, user_hdr->attr.data, hdr.attr.length); + if (err) { + err = -EFAULT; + goto err_exit; + } + + xsc_cmd_exec(xdev, in, hdr.attr.length, out, hdr.attr.length); + + if (copy_to_user((void *)user_hdr, &hdr, sizeof(hdr))) + err = -EFAULT; + if (copy_to_user((void *)user_hdr->attr.data, out, hdr.attr.length)) + err = -EFAULT; +err_exit: + kfree(in); + kfree(out); + return err; +} + +static long xsc_priv_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + int err; + + switch (cmd) { + case XSC_IOCTL_CMDQ: + err = xsc_priv_dev_ioctl_cmdq(filp, arg); + break; + case XSC_IOCTL_DRV_GET: + case XSC_IOCTL_DRV_SET: + // TODO refactor to split driver get and set + err = xsc_priv_dev_ioctl_getinfo(filp, arg); + break; + case XSC_IOCTL_MEM: + err = xsc_priv_dev_ioctl_mem(filp, arg); + break; + default: + err = -EFAULT; + break; + + } + return err; +} + +static const struct file_operations dev_fops = { + .owner = THIS_MODULE, + .open = xsc_priv_dev_open, + .unlocked_ioctl = xsc_priv_dev_ioctl, + .compat_ioctl = xsc_priv_dev_ioctl, +}; + +int xsc_priv_dev_init(struct ib_device *ib_dev, struct xsc_core_device *dev) +{ + int ret; + struct xsc_priv_device *priv_dev = &dev->priv_device; + + sprintf(priv_dev->device_name, "%s", ib_dev->name); + + xsc_core_dbg(dev, "device_name %s\n", priv_dev->device_name); + + ret = alloc_chrdev_region(&priv_dev->devno, 0, 1, priv_dev->device_name); + if (ret) { + xsc_core_err(dev, "%s cant't get major %d\n", + priv_dev->device_name, MAJOR(priv_dev->devno)); + return ret; + } + + cdev_init(&priv_dev->cdev, &dev_fops); + priv_dev->cdev.owner = THIS_MODULE; + + ret = cdev_add(&priv_dev->cdev, priv_dev->devno, 1); + if (ret) { + xsc_core_err(dev, "%s cdev_add error ret:%d major:%d\n", + priv_dev->device_name, ret, MAJOR(priv_dev->devno)); + return ret; + } + + priv_dev->priv_class = class_create(THIS_MODULE, priv_dev->device_name); + device_create(priv_dev->priv_class, NULL, priv_dev->devno, + NULL, "%s", priv_dev->device_name); + + INIT_LIST_HEAD(&priv_dev->mem_list); + spin_lock_init(&priv_dev->mem_lock); + + xsc_core_dbg(dev, "init success\n"); + + return 0; +} + +void xsc_priv_dev_fini(struct ib_device *ib_dev, struct xsc_core_device *dev) +{ + struct xsc_priv_device *priv_dev; + struct cdev *char_dev; + + if (!dev || !ib_dev) { + pr_err("[%s:%d] device is null pointer\n", __func__, __LINE__); + return; + } + + priv_dev = &dev->priv_device; + if (!priv_dev || !priv_dev->priv_class) + return; + char_dev = &priv_dev->cdev; + if (!char_dev) + return; + + device_destroy(priv_dev->priv_class, priv_dev->devno); + cdev_del(&priv_dev->cdev); + unregister_chrdev_region(priv_dev->devno, 1); + class_destroy(priv_dev->priv_class); + + xsc_core_dbg(dev, "fini success\n"); +} diff --git a/drivers/infiniband/hw/xsc/qp.c b/drivers/infiniband/hw/xsc/qp.c new file mode 100644 index 0000000000000000000000000000000000000000..a744d70722f9bdffce30f170383823c7aa6c07b2 --- /dev/null +++ b/drivers/infiniband/hw/xsc/qp.c @@ -0,0 +1,1834 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include "xsc_ib.h" +#include "global.h" +#include "user.h" +#include +#include +#include +#include +#include + +/* not supported currently */ +static int wq_signature; + +#define MAD_QUEUE_DEPTH 128 + +enum { + XSC_IB_CACHE_LINE_SIZE = 64, +}; + +#define LAG_PORT_NUM_MASK 0xffff0000 +#define UDP_SPORT_MASK 0x0000ffff + +static const u32 xsc_ib_opcode[] = { + [IB_WR_SEND] = XSC_MSG_OPCODE_SEND, + [IB_WR_SEND_WITH_IMM] = XSC_MSG_OPCODE_SEND, + [IB_WR_RDMA_WRITE] = XSC_MSG_OPCODE_RDMA_WRITE, + [IB_WR_RDMA_WRITE_WITH_IMM] = XSC_MSG_OPCODE_RDMA_WRITE, + [IB_WR_RDMA_READ] = XSC_MSG_OPCODE_RDMA_READ, + [IB_WR_LOCAL_INV] = XSC_MSG_OPCODE_SEND, + [IB_WR_REG_MR] = XSC_MSG_OPCODE_SEND, +}; + +static int is_qp0(enum ib_qp_type qp_type) +{ + return qp_type == IB_QPT_SMI; +} + +static int is_qp1(enum ib_qp_type qp_type) +{ + return qp_type == IB_QPT_GSI; +} + +static int is_sqp(enum ib_qp_type qp_type) +{ + return is_qp0(qp_type) || is_qp1(qp_type); +} + +static void *get_wqe(struct xsc_ib_qp *qp, int offset) +{ + return xsc_buf_offset(&qp->buf, offset); +} + +static void *get_recv_wqe(struct xsc_ib_qp *qp, int n) +{ + return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); +} + +static void *get_seg_wqe(void *first, int n) +{ + return first + (n << XSC_BASE_WQE_SHIFT); +} + +void *xsc_get_send_wqe(struct xsc_ib_qp *qp, int n) +{ + return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift)); +} + +static int iboe_tos_to_sl(struct net_device *ndev, int tos) +{ + int prio; + struct net_device *dev; + + prio = rt_tos2priority(tos); + dev = is_vlan_dev(ndev) ? vlan_dev_real_dev(ndev) : ndev; + if (dev->num_tc) + return netdev_get_prio_tc_map(dev, prio); + +#if IS_ENABLED(CONFIG_VLAN_8021Q) + if (is_vlan_dev(ndev)) + return (vlan_dev_get_egress_qos_mask(ndev, prio) & + VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; +#endif + return 0; +} + +static inline void set_remote_addr_seg(struct xsc_wqe_data_seg *remote_seg, + u32 msg_len, u64 remote_addr, u32 rkey) +{ + remote_seg->in_line = 0; + WR_LE_32(remote_seg->seg_len, msg_len); + WR_LE_32(remote_seg->mkey, rkey); + WR_LE_64(remote_seg->va, remote_addr); +} + +static void set_local_data_seg(struct xsc_wqe_data_seg *data_seg, struct ib_sge *sg) +{ + data_seg->in_line = 0; + WR_LE_32(data_seg->seg_len, sg->length); + WR_LE_32(data_seg->mkey, sg->lkey); + WR_LE_64(data_seg->va, sg->addr); +} + +static int set_data_inl_seg(struct xsc_ib_qp *qp, const struct ib_send_wr *wr, void *ctrl) +{ + struct xsc_wqe_data_seg *data_seg; + unsigned int seg_index; + void *addr; + int len; + int i; + + for (i = 0, seg_index = 1; i < wr->num_sge; ++i, ++seg_index) { + if (likely(wr->sg_list[i].length)) { + addr = (void *)wr->sg_list[i].addr; + len = wr->sg_list[i].length; + + if (unlikely(len > qp->max_inline_data)) + return -ENOMEM; + + data_seg = get_seg_wqe(ctrl, seg_index); + data_seg->in_line = 1; + data_seg->len = len; + memcpy(data_seg->in_line_data, addr, len); + } + } + + return 0; +} + +static __be32 send_ieth(const struct ib_send_wr *wr) +{ + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_RDMA_WRITE_WITH_IMM: + return wr->ex.imm_data; + default: + return 0; + } +} + +static void xsc_ib_qp_event(struct xsc_core_qp *qp, int type) +{ + struct ib_qp *ibqp = &to_xibqp(qp)->ibqp; + struct ib_event event; + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + switch (type) { +// case XSC_EVENT_TYPE_COMM_EST: +// event.event = IB_EVENT_COMM_EST; +// break; + case XSC_EVENT_TYPE_WQ_CATAS_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + case XSC_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + event.event = IB_EVENT_QP_REQ_ERR; + break; + case XSC_EVENT_TYPE_WQ_ACCESS_ERROR: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + default: + pr_warn("xsc_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn); + return; + } + + ibqp->event_handler(&event, ibqp->qp_context); + } +} + +static int set_rq_size(struct xsc_ib_dev *dev, struct ib_qp_cap *cap, + int has_rq, struct xsc_ib_qp *qp, struct xsc_ib_create_qp *ucmd) +{ + /* Sanity check RQ size before proceeding */ + if (cap->max_recv_wr > dev->xdev->caps.max_wqes) + return -EINVAL; + + if (!has_rq) { + qp->rq.max_gs = 0; + qp->rq.wqe_cnt = 0; + qp->rq.wqe_shift = 0; + } else { + if (ucmd) { + qp->rq.wqe_cnt = ucmd->rq_wqe_count; + qp->rq.wqe_shift = ucmd->rq_wqe_shift; + qp->rq.max_gs = 1; + qp->rq.max_post = qp->rq.wqe_cnt; + } else { + qp->rq.wqe_cnt = roundup_pow_of_two(cap->max_recv_wr); + qp->rq.wqe_shift = dev->xdev->caps.recv_wqe_shift; + qp->rq.max_gs = dev->xdev->caps.recv_ds_num; + qp->rq.max_post = qp->rq.wqe_cnt; + } + } + + return 0; +} + +static int calc_sq_size(struct xsc_ib_dev *dev, struct ib_qp_init_attr *attr, + struct xsc_ib_qp *qp) +{ + int wqe_size; + int wq_size; + + if (!attr->cap.max_send_wr) + return -1; + + wqe_size = 1 << dev->xdev->caps.send_wqe_shift; + qp->max_inline_data = (dev->xdev->caps.send_ds_num - 2) * sizeof(struct xsc_wqe_data_seg); + attr->cap.max_inline_data = qp->max_inline_data; + + qp->sq.wqe_cnt = roundup_pow_of_two(attr->cap.max_send_wr); + qp->sq.ds_cnt = qp->sq.wqe_cnt << (dev->xdev->caps.send_wqe_shift - XSC_BASE_WQE_SHIFT); + wq_size = qp->sq.wqe_cnt * wqe_size; + qp->sq.wqe_shift = ilog2(wqe_size); + qp->sq.max_gs = attr->cap.max_send_wr; + qp->sq.max_post = qp->sq.wqe_cnt; + + return wq_size; +} + +static int qp_has_rq(struct ib_qp_init_attr *attr) +{ + if (attr->qp_type == IB_QPT_XRC_INI || + attr->qp_type == IB_QPT_XRC_TGT || attr->srq || + !attr->cap.max_recv_wr) + return 0; + + return 1; +} + +static enum xsc_qp_state to_xsc_state(enum ib_qp_state state) +{ + switch (state) { + case IB_QPS_RESET: return XSC_QP_STATE_RST; + case IB_QPS_INIT: return XSC_QP_STATE_INIT; + case IB_QPS_RTR: return XSC_QP_STATE_RTR; + case IB_QPS_RTS: return XSC_QP_STATE_RTS; + case IB_QPS_SQD: return XSC_QP_STATE_SQD; + case IB_QPS_SQE: return XSC_QP_STATE_SQER; + case IB_QPS_ERR: return XSC_QP_STATE_ERR; + default: return -1; + } +} + +static int create_user_qp(struct xsc_ib_dev *dev, struct ib_pd *pd, + struct xsc_ib_qp *qp, struct ib_udata *udata, + struct xsc_create_qp_mbox_in **in, + struct xsc_ib_create_qp_resp *resp, int *inlen) +{ + struct xsc_ib_ucontext *context; + struct xsc_ib_create_qp ucmd; + int page_shift; + int npages; + u32 offset; + int ncont; + int err; + int hw_npages; + + err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)); + xsc_ib_dbg(dev, + "buf_addr:0x%lx db_addr:0x%lx sq cnt:%u, rq cnt:%u, rq shift:%u\n", + (uintptr_t)ucmd.buf_addr, (uintptr_t)ucmd.db_addr, + ucmd.sq_wqe_count, ucmd.rq_wqe_count, ucmd.rq_wqe_shift); + if (err) { + xsc_ib_dbg(dev, "copy failed\n"); + return err; + } + + context = to_xucontext(pd->uobject->context); + + qp->sq.ds_cnt = ucmd.sq_wqe_count; + qp->sq.wqe_cnt = ucmd.sq_wqe_count; + qp->sq.wqe_shift = XSC_BASE_WQE_SHIFT; + qp->rq.ds_cnt = ucmd.rq_wqe_count; + qp->rq.wqe_cnt = ucmd.rq_wqe_count; + qp->rq.wqe_shift = XSC_BASE_WQE_SHIFT; + + qp->buf_size = (qp->sq.wqe_cnt << qp->sq.wqe_shift) + (qp->rq.wqe_cnt << qp->rq.wqe_shift); + + qp->umem = ib_umem_get(&dev->ib_dev, ucmd.buf_addr, qp->buf_size, 0); + if (IS_ERR(qp->umem)) { + xsc_ib_dbg(dev, "umem_get failed\n"); + err = PTR_ERR(qp->umem); + goto err_uuar; + } + + xsc_ib_cont_pages(qp->umem, ucmd.buf_addr, &npages, &page_shift, + &ncont, NULL); + if (ncont != npages) { + // TODO: peer memory support failed + page_shift = PAGE_SHIFT; + ncont = npages; + } + + hw_npages = DIV_ROUND_UP(qp->buf_size, PAGE_SIZE_4K); + err = xsc_ib_get_buf_offset(ucmd.buf_addr, page_shift, &offset); + if (err) { + xsc_ib_warn(dev, "bad offset\n"); + goto err_umem; + } + xsc_ib_dbg(dev, "npage:%d, page_shift:%d, ncont:%d, offset:%d, hw_npages %d\n", + npages, page_shift, ncont, offset, hw_npages); + + *inlen = sizeof(**in) + sizeof(*((*in)->req.pas)) * hw_npages; + *in = xsc_vzalloc(*inlen); + if (!*in) { + err = -ENOMEM; + goto err_umem; + } + xsc_ib_populate_pas(dev, qp->umem, page_shift, (*in)->req.pas, hw_npages, true); + (*in)->req.pa_num = cpu_to_be16(hw_npages); + + err = ib_copy_to_udata(udata, resp, sizeof(*resp)); + if (err) { + xsc_ib_dbg(dev, "copy failed\n"); + goto err_umem; + } + qp->create_type = XSC_QP_USER; + + return 0; + +err_umem: + ib_umem_release(qp->umem); + +err_uuar: + return err; +} + +static void destroy_qp_user(struct ib_pd *pd, struct xsc_ib_qp *qp) +{ + struct xsc_ib_ucontext *context; + + context = to_xucontext(pd->uobject->context); + ib_umem_release(qp->umem); +} + +#define MAX_QP1_SQ_HDR_SIZE_V2 512 +#define MAX_QP1_SQ_HDR_SIZE 86 + /* Ethernet header = 14 */ + /* ib_grh = 40 (provided by MAD) */ + /* ib_bth + ib_deth = 20 */ + /* MAD = 256 (provided by MAD) */ + /* iCRC = 4 */ +#define MAX_QP1_RQ_HDR_SIZE_V2 512 + +static int create_kernel_qp(struct xsc_ib_dev *dev, + struct ib_qp_init_attr *init_attr, + struct xsc_ib_qp *qp, + struct xsc_create_qp_mbox_in **in, int *inlen) +{ + int err; + int sq_size; + int hw_npages; + + sq_size = calc_sq_size(dev, init_attr, qp); + if (sq_size < 0) { + err = -ENOMEM; + xsc_ib_err(dev, "err %d\n", err); + return err; + } + + qp->rq.ds_cnt = qp->rq.wqe_cnt << (qp->rq.wqe_shift - XSC_BASE_WQE_SHIFT); + qp->rq.offset = 0; + qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; + qp->buf_size = qp->sq.offset + sq_size; + qp->send_psn = 0; + + err = xsc_buf_alloc(dev->xdev, qp->buf_size, PAGE_SIZE, &qp->buf); + if (err) { + xsc_ib_err(dev, "err %d\n", err); + return err; + } + + qp->sq.qend = qp->buf.direct.buf + qp->sq.offset + sq_size; + hw_npages = DIV_ROUND_UP(qp->buf_size, PAGE_SIZE_4K); + *inlen = sizeof(**in) + sizeof(*(*in)->req.pas) * hw_npages; + *in = xsc_vzalloc(*inlen); + if (!*in) { + err = -ENOMEM; + goto err_buf; + } + + xsc_fill_page_array(&qp->buf, (*in)->req.pas, hw_npages); + (*in)->req.pa_num = cpu_to_be16(hw_npages); + + qp->sq.wrid = kmalloc_array(qp->sq.wqe_cnt, sizeof(*qp->sq.wrid), GFP_KERNEL); + qp->sq.wr_data = kmalloc_array(qp->sq.wqe_cnt, sizeof(*qp->sq.wr_data), GFP_KERNEL); + qp->rq.wrid = kmalloc_array(qp->rq.wqe_cnt, sizeof(*qp->rq.wrid), GFP_KERNEL); + qp->sq.w_list = kmalloc_array(qp->sq.wqe_cnt, sizeof(*qp->sq.w_list), GFP_KERNEL); + qp->sq.wqe_head = kmalloc_array(qp->sq.wqe_cnt, sizeof(*qp->sq.wqe_head), GFP_KERNEL); + + if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid || + !qp->sq.w_list || !qp->sq.wqe_head) { + err = -ENOMEM; + goto err_wrid; + } + qp->create_type = XSC_QP_KERNEL; + + if (init_attr->qp_type == IB_QPT_GSI) { + qp->sq.mad_index = 0; + qp->sq.mad_queue_depth = MAD_QUEUE_DEPTH; + qp->sq.hdr_size = MAX_QP1_SQ_HDR_SIZE_V2 * MAD_QUEUE_DEPTH; + qp->sq.hdr_buf = ib_dma_alloc_coherent(&dev->ib_dev, + qp->sq.hdr_size, + &qp->sq.hdr_dma, GFP_KERNEL); + if (!qp->sq.hdr_buf) { + err = -ENOMEM; + xsc_ib_err(dev, "Failed to create sq_hdr_buf"); + goto err_wrid; + } + } + + return 0; + +err_wrid: + kfree(qp->sq.wqe_head); + kfree(qp->sq.w_list); + kfree(qp->sq.wrid); + kfree(qp->sq.wr_data); + kfree(qp->rq.wrid); + +err_buf: + xsc_buf_free(dev->xdev, &qp->buf); + return err; +} + +static void destroy_qp_kernel(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp) +{ + if (qp->sq.hdr_buf) + ib_dma_free_coherent(&dev->ib_dev, qp->sq.hdr_size, + qp->sq.hdr_buf, qp->sq.hdr_dma); + kfree(qp->sq.wqe_head); + kfree(qp->sq.w_list); + kfree(qp->sq.wrid); + kfree(qp->sq.wr_data); + kfree(qp->rq.wrid); + xsc_buf_free(dev->xdev, &qp->buf); +} + +static u8 ib_to_xsc_qp_type(enum ib_qp_type qp_type, __u32 flags) +{ + if (qp_type == IB_QPT_RC) { + return XSC_QUEUE_TYPE_RDMA_RC; + } else if ((qp_type == IB_QPT_GSI) || (qp_type == IB_QPT_SMI)) { + return XSC_QUEUE_TYPE_RDMA_MAD; + } else if (qp_type == IB_QPT_RAW_PACKET) { + if (flags & XSC_QP_FLAG_RAWPACKET_TSO) + return XSC_QUEUE_TYPE_RAW_TSO; + else + return XSC_QUEUE_TYPE_RAW; + } else { + return XSC_QUEUE_TYPE_INVALID; + } +} + +static int create_qp_common(struct xsc_ib_dev *dev, struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, struct xsc_ib_qp *qp) +{ + struct xsc_ib_resources *devr = &dev->devr; + struct xsc_ib_create_qp_resp resp; + struct xsc_create_qp_mbox_in *in = NULL; + struct xsc_ib_create_qp ucmd; + int inlen = sizeof(*in); + int err; + + mutex_init(&qp->mutex); + spin_lock_init(&qp->sq.lock); + spin_lock_init(&qp->rq.lock); + spin_lock_init(&qp->lock); + + if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) + qp->sq_signal_bits = XSC_WQE_CTRL_CQ_UPDATE; + + if (pd && pd->uobject) { + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { + xsc_ib_dbg(dev, "copy failed\n"); + return -EFAULT; + } + + qp->wq_sig = !!(ucmd.flags & XSC_QP_FLAG_SIGNATURE); + qp->scat_cqe = !!(ucmd.flags & XSC_QP_FLAG_SCATTER_CQE); + } else { + qp->wq_sig = !!wq_signature; + } + + xsc_ib_dbg(dev, "ucmd.flags=0x%x\n", ucmd.flags); + + qp->has_rq = qp_has_rq(init_attr); + + err = set_rq_size(dev, &init_attr->cap, qp->has_rq, + qp, (pd && pd->uobject) ? &ucmd : NULL); + if (err) { + xsc_ib_dbg(dev, "err %d\n", err); + return err; + } + + if (pd) { + if (pd->uobject) { + err = create_user_qp(dev, pd, qp, udata, &in, &resp, &inlen); + if (err) + xsc_ib_dbg(dev, "err %d\n", err); + } else { + err = create_kernel_qp(dev, init_attr, qp, &in, &inlen); + if (err) + xsc_ib_dbg(dev, "err %d\n", err); + else + qp->pa_lkey = to_mpd(pd)->pa_lkey; + } + + if (err) + return err; + } else { + in = xsc_vzalloc(sizeof(*in)); + if (!in) + return -ENOMEM; + + qp->create_type = XSC_QP_EMPTY; + } + + xsc_ib_dbg(dev, "[%s:%d]:qp_type=%d\n", __func__, __LINE__, init_attr->qp_type); + + if (is_sqp(init_attr->qp_type)) + qp->port = init_attr->port_num; + + in->req.qp_type = init_attr->qp_type; + if (is_qp1(init_attr->qp_type)) + in->req.input_qpn = cpu_to_be16(1); + + if (init_attr->qp_type != XSC_IB_QPT_REG_UMR) + in->req.pdn = cpu_to_be32(to_mpd(pd ? pd : devr->p0)->pdn); + + if (qp->rq.ds_cnt) + in->req.log_rq_sz = ilog2(qp->rq.ds_cnt); + + if (qp->sq.ds_cnt) + in->req.log_sq_sz = ilog2(qp->sq.ds_cnt); + else + in->req.log_sq_sz = ilog2(0x80); + + pr_debug("[%s:%d]\n", __func__, __LINE__); + + if (init_attr->send_cq) { + qp->send_cq = init_attr->send_cq; + in->req.cqn_send = to_xcq(init_attr->send_cq)->xcq.cqn; + in->req.cqn_send = cpu_to_be16(in->req.cqn_send); +#ifndef MSIX_SUPPORT + init_attr->send_cq->comp_handler(init_attr->send_cq, + init_attr->send_cq->cq_context); +#endif + } + + if (init_attr->recv_cq) { + qp->recv_cq = init_attr->recv_cq; + in->req.cqn_recv = to_xcq(init_attr->recv_cq)->xcq.cqn; + in->req.cqn_recv = cpu_to_be16(in->req.cqn_recv); + } + + in->req.qp_type = ib_to_xsc_qp_type(init_attr->qp_type, ucmd.flags); + + xsc_ib_dbg(dev, "[%s:%d]:req.qp_type=%d\n", __func__, __LINE__, in->req.qp_type); + + if (in->req.qp_type == XSC_QUEUE_TYPE_INVALID) + goto err_create; + in->req.glb_funcid = cpu_to_be16(dev->xdev->glb_func_id); + + qp->xqp.qp_type_internal = in->req.qp_type; + + err = xsc_core_create_qp(dev->xdev, &qp->xqp, in, inlen); + if (err) { + xsc_ib_dbg(dev, "create qp failed\n"); + goto err_create; + } + + xsc_vfree(in); + qp->doorbell_qpn = qp->xqp.qpn; + + qp->xqp.event = xsc_ib_qp_event; + qp->xqp.qp_type = init_attr->qp_type; + + return 0; + +err_create: + if (qp->create_type == XSC_QP_USER) + destroy_qp_user(pd, qp); + else if (qp->create_type == XSC_QP_KERNEL) + destroy_qp_kernel(dev, qp); + + xsc_vfree(in); + return err; +} + +static void xsc_ib_lock_cqs(struct xsc_ib_cq *send_cq, struct xsc_ib_cq *recv_cq) + __acquires(&send_cq->lock) __acquires(&recv_cq->lock) +{ + if (send_cq) { + if (recv_cq) { + if (send_cq->xcq.cqn < recv_cq->xcq.cqn) { + spin_lock_irq(&send_cq->lock); + spin_lock_nested(&recv_cq->lock, + SINGLE_DEPTH_NESTING); + } else if (send_cq->xcq.cqn == recv_cq->xcq.cqn) { + spin_lock_irq(&send_cq->lock); + __acquire(&recv_cq->lock); + } else { + spin_lock_irq(&recv_cq->lock); + spin_lock_nested(&send_cq->lock, + SINGLE_DEPTH_NESTING); + } + } else { + spin_lock_irq(&send_cq->lock); + } + } else if (recv_cq) { + spin_lock_irq(&recv_cq->lock); + } +} + +static void xsc_ib_unlock_cqs(struct xsc_ib_cq *send_cq, struct xsc_ib_cq *recv_cq) + __releases(&send_cq->lock) __releases(&recv_cq->lock) +{ + if (send_cq) { + if (recv_cq) { + if (send_cq->xcq.cqn < recv_cq->xcq.cqn) { + spin_unlock(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else if (send_cq->xcq.cqn == recv_cq->xcq.cqn) { + __release(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else { + spin_unlock(&send_cq->lock); + spin_unlock_irq(&recv_cq->lock); + } + } else { + spin_unlock_irq(&send_cq->lock); + } + } else if (recv_cq) { + spin_unlock_irq(&recv_cq->lock); + } +} + +static struct xsc_ib_pd *get_pd(struct xsc_ib_qp *qp) +{ + return to_mpd(qp->ibqp.pd); +} + +static void get_cqs(struct xsc_ib_qp *qp, + struct xsc_ib_cq **send_cq, struct xsc_ib_cq **recv_cq) +{ + switch (qp->ibqp.qp_type) { + case IB_QPT_XRC_TGT: + *send_cq = NULL; + *recv_cq = NULL; + break; + case XSC_IB_QPT_REG_UMR: + case IB_QPT_XRC_INI: + *send_cq = to_xcq(qp->ibqp.send_cq); + *recv_cq = NULL; + break; + + case IB_QPT_SMI: + case IB_QPT_GSI: + case IB_QPT_RC: + case IB_QPT_UC: + case IB_QPT_UD: + case IB_QPT_RAW_IPV6: + case IB_QPT_RAW_ETHERTYPE: + *send_cq = to_xcq(qp->ibqp.send_cq); + *recv_cq = to_xcq(qp->ibqp.recv_cq); + break; + + case IB_QPT_RAW_PACKET: + case IB_QPT_MAX: + default: + *send_cq = NULL; + *recv_cq = NULL; + break; + } +} + +static void destroy_qp_common(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp) +{ + struct xsc_ib_cq *send_cq, *recv_cq; + struct xsc_modify_qp_mbox_in *in; + int err; + + in = kzalloc(sizeof(*in), GFP_KERNEL); + if (!in) + return; + + if ((qp->xqp.qp_type_internal == XSC_QUEUE_TYPE_RAW) || + (qp->state != IB_QPS_RESET)) + if (xsc_core_qp_modify(dev->xdev, to_xsc_state(qp->state), + XSC_QP_STATE_RST, in, sizeof(*in), &qp->xqp)) + xsc_ib_warn(dev, "modify QP %06x to RESET failed\n", qp->xqp.qpn); + + get_cqs(qp, &send_cq, &recv_cq); + + if (qp->create_type == XSC_QP_KERNEL) { + xsc_ib_lock_cqs(send_cq, recv_cq); + __xsc_ib_cq_clean(recv_cq, qp->xqp.qpn); + if (send_cq != recv_cq) + __xsc_ib_cq_clean(send_cq, qp->xqp.qpn); + xsc_ib_unlock_cqs(send_cq, recv_cq); + } + + err = xsc_core_destroy_qp(dev->xdev, &qp->xqp); + if (err) + xsc_ib_warn(dev, "failed to destroy QP 0x%x\n", qp->xqp.qpn); + kfree(in); + + if (qp->create_type == XSC_QP_KERNEL) + destroy_qp_kernel(dev, qp); + else if (qp->create_type == XSC_QP_USER) + destroy_qp_user(&get_pd(qp)->ibpd, qp); +} + +static const char *ib_qp_type_str(enum ib_qp_type type) +{ + switch (type) { + case IB_QPT_SMI: + return "IB_QPT_SMI"; + case IB_QPT_GSI: + return "IB_QPT_GSI"; + case IB_QPT_RC: + return "IB_QPT_RC"; + case IB_QPT_UC: + return "IB_QPT_UC"; + case IB_QPT_UD: + return "IB_QPT_UD"; + case IB_QPT_RAW_IPV6: + return "IB_QPT_RAW_IPV6"; + case IB_QPT_RAW_ETHERTYPE: + return "IB_QPT_RAW_ETHERTYPE"; + case IB_QPT_XRC_INI: + return "IB_QPT_XRC_INI"; + case IB_QPT_XRC_TGT: + return "IB_QPT_XRC_TGT"; + case IB_QPT_RAW_PACKET: + return "IB_QPT_RAW_PACKET"; + case XSC_IB_QPT_REG_UMR: + return "XSC_IB_QPT_REG_UMR"; + case IB_QPT_MAX: + default: + return "Invalid QP type"; + } +} + +struct ib_qp *xsc_ib_create_qp(struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct xsc_ib_dev *dev; + struct xsc_ib_qp *qp; + int err; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + return ERR_PTR(-ENOMEM); + if (pd) { + dev = to_mdev(pd->device); + } else { + /* being cautious here */ + if (init_attr->qp_type != IB_QPT_XRC_TGT && + init_attr->qp_type != XSC_IB_QPT_REG_UMR) { + pr_warn("%s: no PD for transport %s\n", __func__, + ib_qp_type_str(init_attr->qp_type)); + return ERR_PTR(-EINVAL); + } + dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device); + } + + switch (init_attr->qp_type) { + case IB_QPT_RC: + case IB_QPT_SMI: + case IB_QPT_GSI: + case IB_QPT_RAW_PACKET: + err = create_qp_common(dev, pd, init_attr, udata, qp); + if (err) { + xsc_ib_dbg(dev, "create_qp_common failed\n"); + kfree(qp); + return ERR_PTR(err); + } + + if (is_qp0(init_attr->qp_type)) { + qp->ibqp.qp_num = 0; + } else if (is_qp1(init_attr->qp_type)) { + qp->ibqp.qp_num = 1; + dev->xdev->gsi_qpn = qp->xqp.qpn; + } else { + qp->ibqp.qp_num = qp->xqp.qpn; + } + + xsc_ib_dbg(dev, "ib qpnum 0x%x, qpn 0x%x, rcqn 0x%x, scqn 0x%x\n", + qp->ibqp.qp_num, qp->xqp.qpn, + to_xcq(init_attr->recv_cq)->xcq.cqn, + to_xcq(init_attr->send_cq)->xcq.cqn); + + break; + + case IB_QPT_RAW_IPV6: + case IB_QPT_RAW_ETHERTYPE: + case IB_QPT_MAX: + default: + xsc_ib_dbg(dev, "unsupported qp type %d\n", + init_attr->qp_type); + /* Don't support raw QPs */ + return ERR_PTR(-EINVAL); + } + + return &qp->ibqp; +} + +xsc_ib_destroy_qp_def() +{ + struct xsc_ib_dev *dev = to_mdev(qp->device); + struct xsc_ib_qp *xqp = to_xqp(qp); + + destroy_qp_common(dev, xqp); + + kfree(xqp); + return 0; +} + +static inline u16 xsc_calc_udp_sport(u32 lqpn, u32 rqpn) +{ + unsigned char *p; + u8 ports[2]; + u16 sport; + u64 tqpn; + + tqpn = ((u64)(lqpn & 0xffffff)) * ((u64)(rqpn & 0xffffff)); + p = (unsigned char *)&tqpn; + ports[0] = p[0] ^ p[2] ^ p[4]; + ports[1] = p[1] ^ p[3] ^ p[5]; + sport = *((u16 *)ports) | 0xC000; + + return sport; +} + +static inline void xsc_path_set_udp_sport(struct xsc_qp_path *path, + const struct rdma_ah_attr *ah, + u32 lqpn, u32 rqpn) +{ + if ((ah->grh.flow_label & UDP_SPORT_MASK) != 0) + path->sport = cpu_to_be16(xsc_flow_label_to_udp_sport(ah->grh.flow_label)); + else + path->sport = cpu_to_be16(xsc_calc_udp_sport(lqpn, rqpn)); +} + +static int xsc_set_path(struct xsc_ib_dev *dev, const struct rdma_ah_attr *ah, + struct xsc_qp_path *path, u8 port, int attr_mask, + u32 path_flags, const struct ib_qp_attr *attr, struct xsc_ib_qp *qp) +{ + struct ib_global_route *grh = rdma_ah_retrieve_grh((struct rdma_ah_attr *)ah); + const struct ib_gid_attr *sgid_attr = grh->sgid_attr; + union ib_gid *sgid = &((struct ib_gid_attr *)sgid_attr)->gid; + union ib_gid *dgid = &grh->dgid; + union { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } sgid_addr, dgid_addr; + int global_pcp, global_dscp; + + if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) { + if (!(rdma_ah_get_ah_flags(ah) & IB_AH_GRH)) + return -EINVAL; + + if (qp->ibqp.qp_type == IB_QPT_RC || + qp->ibqp.qp_type == IB_QPT_UC || + qp->ibqp.qp_type == IB_QPT_XRC_INI || + qp->ibqp.qp_type == IB_QPT_XRC_TGT) + xsc_path_set_udp_sport(path, ah, qp->ibqp.qp_num, attr->dest_qp_num); + +// if (ah->ddgrh.sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) +// path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f; + + global_dscp = get_global_force_dscp(); + if (global_dscp == GLOBAL_UNSET_FORCE_VALUE) + path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f; + else + path->ecn_dscp = global_dscp; + path->hop_limit = grh->hop_limit; + + rdma_gid2ip(&sgid_addr._sockaddr, sgid); + rdma_gid2ip(&dgid_addr._sockaddr, dgid); + + if (sgid_addr._sockaddr.sa_family == AF_INET && + dgid_addr._sockaddr.sa_family == AF_INET) { + memcpy(path->sip, &sgid_addr._sockaddr_in.sin_addr.s_addr, + sizeof(struct in_addr)); + memcpy(path->dip, &dgid_addr._sockaddr_in.sin_addr.s_addr, + sizeof(struct in_addr)); + path->af_type = AF_INET; + } else if (sgid_addr._sockaddr.sa_family == AF_INET6 && + dgid_addr._sockaddr.sa_family == AF_INET6) { + memcpy(path->sip, &sgid_addr._sockaddr_in6.sin6_addr.s6_addr, + sizeof(path->sip)); + memcpy(path->dip, &dgid_addr._sockaddr_in6.sin6_addr.s6_addr, + sizeof(path->dip)); + path->af_type = AF_INET6; + } else { + return -EINVAL; + } + + ether_addr_copy(path->smac, sgid_attr->ndev->dev_addr); + memcpy(path->dmac, ah->roce.dmac, sizeof(ah->roce.dmac)); + + if (is_vlan_dev(sgid_attr->ndev)) { + path->vlan_valid = 1; + path->vlan_id = cpu_to_be16(vlan_dev_vlan_id(sgid_attr->ndev)); + global_pcp = get_global_force_pcp(); + if (global_pcp == GLOBAL_UNSET_FORCE_VALUE) + path->dci_cfi_prio_sl = (ah->sl & 0x7); + else + path->dci_cfi_prio_sl = global_pcp; + } else { + path->vlan_valid = 0; + } + } + xsc_core_dbg(dev->xdev, "path dscp %d pcp %d\n", path->ecn_dscp, path->dci_cfi_prio_sl); + return 0; +} + +static int __xsc_ib_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, enum ib_qp_state new_state) +{ + struct xsc_ib_dev *dev = to_mdev(ibqp->device); + struct xsc_ib_qp *qp = to_xqp(ibqp); + struct xsc_ib_cq *send_cq, *recv_cq; + struct xsc_qp_context *context; + struct xsc_modify_qp_mbox_in *in; + struct xsc_qp_path path; + int sqd_event; + int err; + struct xsc_lag *ldev = xsc_lag_dev_get(dev->xdev); + u8 lag_port_num = ARRAY_SIZE(ldev->pf); + + in = kzalloc(sizeof(*in), GFP_KERNEL); + if (!in) + return -ENOMEM; + + context = &qp->ctx; + + if (attr_mask & IB_QP_PATH_MTU) { + if (attr->path_mtu != IB_MTU_1024 && + attr->path_mtu != IB_MTU_4096) { + xsc_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu); + } + + context->mtu_mode = (attr->path_mtu <= IB_MTU_1024) ? 0 : 1; + } + + if (attr_mask & IB_QP_DEST_QPN) + context->remote_qpn = cpu_to_be32(attr->dest_qp_num); + + if (attr_mask & IB_QP_AV) { + err = xsc_set_path(dev, &attr->ah_attr, &path, + attr_mask & IB_QP_PORT ? attr->port_num : qp->port, + attr_mask, 0, attr, qp); + if (err) + goto out; + + context->src_udp_port = path.sport; + context->dscp = path.ecn_dscp; + context->hop_limit = path.hop_limit; + context->ip_type = (path.af_type == AF_INET ? 0 : 1); + context->ip_type = cpu_to_be16(context->ip_type); + memcpy(context->dip, path.dip, sizeof(context->dip)); + memcpy(context->sip, path.sip, sizeof(context->sip)); + memcpy(context->dmac, path.dmac, sizeof(path.dmac)); + memcpy(context->smac, path.smac, sizeof(path.smac)); + + context->vlan_valid = path.vlan_valid; + context->dci_cfi_prio_sl = path.dci_cfi_prio_sl; + context->vlan_id = path.vlan_id; + + if (ldev && __xsc_lag_is_roce(ldev)) { + context->lag_id = cpu_to_be16(ldev->lag_id); + context->lag_sel_en = 1; + if ((attr->ah_attr.grh.flow_label & LAG_PORT_NUM_MASK) != 0) + context->lag_sel = ((attr->ah_attr.grh.flow_label + & LAG_PORT_NUM_MASK) >> 16) + % lag_port_num; + else + context->lag_sel = qp->xqp.qpn % XSC_MAX_PORTS; + } + } + + if (attr_mask & IB_QP_RNR_RETRY) + context->rnr_retry = attr->rnr_retry; + + if (attr_mask & IB_QP_RETRY_CNT) + context->retry_cnt = attr->retry_cnt; + + if (attr_mask & IB_QP_SQ_PSN) + context->next_send_psn = cpu_to_be32(attr->sq_psn); + + if (attr_mask & IB_QP_RQ_PSN) + context->next_recv_psn = cpu_to_be32(attr->rq_psn); + + if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD && + attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) + sqd_event = 1; + else + sqd_event = 0; + + memcpy(&in->ctx, context, sizeof(*context)); + err = xsc_core_qp_modify(dev->xdev, to_xsc_state(cur_state), + to_xsc_state(new_state), in, sqd_event, + &qp->xqp); + if (err) + goto out; + + qp->state = new_state; + + if (attr_mask & IB_QP_ACCESS_FLAGS) + qp->atomic_rd_en = attr->qp_access_flags; + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + qp->resp_depth = attr->max_dest_rd_atomic; + if (attr_mask & IB_QP_PORT) + qp->port = attr->port_num; + if (attr_mask & IB_QP_ALT_PATH) + qp->alt_port = attr->alt_port_num; + + /* If we moved a kernel QP to RESET, clean up all old CQ + * entries and reinitialize the QP. + */ + if (new_state == IB_QPS_RESET && !ibqp->uobject) { + get_cqs(qp, &send_cq, &recv_cq); + xsc_ib_cq_clean(recv_cq, qp->xqp.qpn); + if (send_cq != recv_cq) + xsc_ib_cq_clean(send_cq, qp->xqp.qpn); + + qp->rq.head = 0; + qp->rq.tail = 0; + qp->sq.head = 0; + qp->sq.tail = 0; + qp->sq.cur_post = 0; + qp->sq.last_poll = 0; + } + +out: + kfree(in); + return err; +} + +int xsc_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct xsc_ib_dev *dev = to_mdev(ibqp->device); + struct xsc_ib_qp *qp = to_xqp(ibqp); + enum ib_qp_state cur_state, new_state; + int err = -EINVAL; +#ifdef XSC_CHIP_RDMA_UNSUPPORTED + xsc_ib_dbg(dev, "rdma unsupported,%s no action.\n", __func__); + return 0; +#endif + + mutex_lock(&qp->mutex); + + cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; + new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; + + xsc_ib_dbg(dev, "cur_state:%u, new_state:%u attr_mask:0x%x\n", + cur_state, new_state, attr_mask); + + //if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask)) + // goto out; + + if ((attr_mask & IB_QP_PORT) && + (attr->port_num == 0 || attr->port_num > dev->xdev->caps.num_ports)) { + xsc_ib_dbg(dev, "erro port num\n"); + goto out; + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && + attr->max_rd_atomic > dev->xdev->caps.max_ra_res_qp) { + xsc_ib_err(dev, "rd atomic:%u exeeded", attr->max_rd_atomic); + goto out; + } + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && + attr->max_dest_rd_atomic > dev->xdev->caps.max_ra_req_qp) { + xsc_ib_err(dev, "dest rd atomic:%u exeeded", attr->max_dest_rd_atomic); + goto out; + } + + if (cur_state == new_state && cur_state == IB_QPS_RESET) { + err = 0; + goto out; + } + + err = __xsc_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); +out: + mutex_unlock(&qp->mutex); + return err; +} + +static int xsc_wq_overflow(struct xsc_ib_wq *wq, int nreq, struct xsc_ib_cq *cq) +{ + unsigned int cur; + + cur = wq->head - wq->tail; + if (likely(cur + nreq < wq->max_post)) + return 0; + + spin_lock(&cq->lock); + cur = wq->head - wq->tail; + spin_unlock(&cq->lock); + + return cur + nreq >= wq->max_post; +} +#ifdef XSC_DEBUG +static void dump_wqe(struct xsc_ib_qp *qp, int idx) +{ + struct xsc_send_wqe_ctrl_seg *seg; + struct xsc_ib_dev *dev = to_mdev(qp->ibqp.device); + u32 *p = NULL; + int i; + + seg = xsc_get_send_wqe(qp, idx); + xsc_ib_dbg(dev, "current wqe:%p index:%d\n", seg, idx); + for (i = 0; i < 4; i++) { + p = (u32 *)get_seg_wqe(seg, i); + xsc_ib_dbg(dev, "%08x %08x %08x %08x\n", p[0], p[1], p[2], p[3]); + } +} +#endif + +static inline void xsc_post_send_db(struct xsc_ib_qp *qp, + struct xsc_core_device *xdev, int nreq) +{ + u16 next_pid; + union xsc_db_data db; + + if (unlikely(!nreq)) + return; + + qp->sq.head += nreq; + + next_pid = qp->sq.head << (qp->sq.wqe_shift - XSC_BASE_WQE_SHIFT); + db.sq_next_pid = next_pid; + db.sqn = qp->doorbell_qpn; + + /* Make sure that descriptors are written before + * updating doorbell record and ringing the doorbell + */ + wmb(); + writel(db.raw_data, REG_ADDR(xdev, xdev->regs.tx_db)); +} + +static inline u32 xsc_crc32(struct xsc_ib_dev *dev, u32 crc, u8 *buf, size_t len) +{ + u32 i; + + for (i = 0; i < len; i++) + crc = dev->crc_32_table[(crc ^ buf[i]) & 0xff] ^ (crc >> 8); + + return crc; +} + +#define BTH_QPN_MASK (0x00ffffff) +#define BTH_PSN_MASK (0x00ffffff) + +/* Compute a partial ICRC for all the IB transport headers. */ +u32 xsc_icrc_hdr(struct xsc_ib_dev *dev, void *pkt, u32 size, u32 *icrc) +{ + struct iphdr *ip4h = NULL; + struct ipv6hdr *ip6h = NULL; + struct udphdr *udph; + struct ib_unpacked_eth *eth; + struct rxe_bth *bth; + struct ib_unpacked_deth *deth; + struct ib_unpacked_vlan *vlan; + int crc; + int crc_field_len; + __be16 l3_type; + u8 *l3_start; + + int hdr_size; + + /* pseudo header buffer size is calculate using ipv6 header size since + * it is bigger than ipv4 + */ + u8 pshdr[sizeof(struct udphdr) + + sizeof(struct ipv6hdr) + + sizeof(*bth) + sizeof(*deth)]; + + eth = pkt; + + if (eth->type == htons(ETH_P_8021Q)) { + vlan = (struct ib_unpacked_vlan *)(eth + 1); + l3_type = vlan->type; + l3_start = (u8 *)(vlan + 1); + size -= 4; + } else { + l3_type = eth->type; + l3_start = (u8 *)(eth + 1); + } + + hdr_size = sizeof(struct udphdr) + + (l3_type == htons(ETH_P_IP) ? + sizeof(struct iphdr) : sizeof(struct ipv6hdr)); + + crc_field_len = hdr_size + sizeof(*bth) + sizeof(*deth); + + if (crc_field_len != size) { + xsc_ib_err(dev, "Unmatched hdr: expect %d actual %d\n", + crc_field_len, size); + return -EINVAL; + } + + ip4h = (struct iphdr *)(l3_start); + ip6h = (struct ipv6hdr *)(l3_start); + udph = (struct udphdr *)(ip4h + 1); + bth = (struct rxe_bth *)(udph + 1); + + memcpy(pshdr, l3_start, crc_field_len); + + /* This seed is the result of computing a CRC with a seed of + * 0xfffffff and 8 bytes of 0xff representing a masked LRH. + */ + crc = 0xdebb20e3; + + if (l3_type == htons(ETH_P_IP)) { /* IPv4 */ + memcpy(pshdr, ip4h, hdr_size); + ip4h = (struct iphdr *)pshdr; + udph = (struct udphdr *)(ip4h + 1); + + ip4h->ttl = 0xff; + ip4h->check = CSUM_MANGLED_0; + ip4h->tos = 0xff; + } else { /* IPv6 */ + memcpy(pshdr, ip6h, hdr_size); + ip6h = (struct ipv6hdr *)pshdr; + udph = (struct udphdr *)(ip6h + 1); + + memset(ip6h->flow_lbl, 0xff, sizeof(ip6h->flow_lbl)); + ip6h->priority = 0xf; + ip6h->hop_limit = 0xff; + } + udph->check = CSUM_MANGLED_0; + + bth = (struct rxe_bth *)(udph + 1); + /* exclude bth.resv8a */ + bth->qpn |= cpu_to_be32(~BTH_QPN_MASK); + + *icrc = xsc_crc32(dev, crc, pshdr, crc_field_len); + + return 0; +} + +/* Routine for sending QP1 packets for RoCE V1 an V2 + */ + // TO BE DONE: sq hdr buf should be create dynamically for mult entry +int build_qp1_send_v2(struct xsc_ib_dev *dev, + struct xsc_ib_qp *qp, + const struct ib_send_wr *wr, + struct ib_sge *sge, + int payload_size, u32 *crc) +{ + struct xsc_ib_ah *ah = container_of(ud_wr(wr)->ah, struct xsc_ib_ah, + ibah); + const struct ib_gid_attr *sgid_attr = ah->ibah.sgid_attr; + u16 ether_type; + union ib_gid dgid; + bool is_eth = false; + bool is_vlan = false; + bool is_grh = false; + bool is_udp = false; + u8 ip_version = 0; + u16 vlan_id = 0xFFFF; + int rc = 0; + int cm_pcp = 0; + void *hdr_buf; + + if (!qp->sq.hdr_buf) { + xsc_ib_err(dev, "QP1 buffer is empty!"); + return -ENOMEM; + } + hdr_buf = (u8 *)qp->sq.hdr_buf + MAX_QP1_SQ_HDR_SIZE_V2 * qp->sq.mad_index; + + memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); + + if (is_vlan_dev(sgid_attr->ndev)) + vlan_id = vlan_dev_vlan_id(sgid_attr->ndev); + + memcpy(&dgid.raw, &ah->av.rgid, 16);// TO DO SOMETHING + is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; + if (is_udp) { + if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { + ip_version = 4; + ether_type = ETH_P_IP; + } else { + ip_version = 6; + ether_type = ETH_P_IPV6; + } + is_grh = false; + } else { + ether_type = ETH_P_IBOE; + is_grh = true; + } + + is_eth = true; + is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; + + ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, + ip_version, is_udp, 0, &qp->qp1_hdr); + + /* ETH */ + ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->av.rmac); + ether_addr_copy(qp->qp1_hdr.eth.smac_h, dev->netdev->dev_addr); + + /* For vlan, check the sgid for vlan existence */ + if (!is_vlan) { + qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); + } else { + if (dev->cm_pcp != DSCP_PCP_UNSET) + cm_pcp = dev->cm_pcp << 13; + else + cm_pcp = (iboe_tos_to_sl(sgid_attr->ndev, ah->av.tclass) << 13); + qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); + qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id|cm_pcp); + } + +// if (is_grh || (ip_version == 6)) { +// memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw, +// sizeof(sgid_attr->gid)); +// memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, +// sizeof(sgid_attr->gid)); +// qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; +// } + + if (ip_version == 4) { + if (dev->cm_dscp != DSCP_PCP_UNSET) + qp->qp1_hdr.ip4.tos = dev->cm_dscp << 2; + else + qp->qp1_hdr.ip4.tos = 0; + qp->qp1_hdr.ip4.id = 0; + qp->qp1_hdr.ip4.frag_off = htons(IP_DF); + qp->qp1_hdr.ip4.ttl = ah->av.hop_limit; + + memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4); + memcpy(&qp->qp1_hdr.ip4.daddr, ah->av.rgid + 12, 4); + qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); + } + + if (is_udp) { + qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); + qp->qp1_hdr.udp.sport = htons(ah->av.udp_sport); + qp->qp1_hdr.udp.csum = 0; + xsc_ib_dbg(dev, "CM packet used udp_sport=%d\n", ah->av.udp_sport); + } + + /* BTH */ + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + qp->qp1_hdr.immediate_present = 1; + } else { + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + } + if (wr->send_flags & IB_SEND_SOLICITED) + qp->qp1_hdr.bth.solicited_event = 1; + /* pad_count */ + qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; + + /* P_key for QP1 is for all members */ + qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); + qp->qp1_hdr.bth.destination_qpn = IB_QP1; + qp->qp1_hdr.bth.ack_req = 0; + qp->send_psn++; + qp->send_psn &= BTH_PSN_MASK; + qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); + /* DETH */ + /* Use the priviledged Q_Key for QP1 */ + qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); + qp->qp1_hdr.deth.source_qpn = IB_QP1; + + /* Pack the QP1 to the transmit buffer */ + sge->addr = (dma_addr_t)(qp->sq.hdr_dma + MAX_QP1_SQ_HDR_SIZE_V2 * qp->sq.mad_index); + sge->lkey = 0xFFFFFFFF; + sge->length = MAX_QP1_SQ_HDR_SIZE; + + ib_ud_header_pack(&qp->qp1_hdr, hdr_buf); + + /* Max Header buf size for IPV6 RoCE V2 is 86, + * which is same as the QP1 SQ header buffer. + * Header buf size for IPV4 RoCE V2 can be 66. + * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). + * Subtract 20 bytes from QP1 SQ header buf size + */ + if (is_udp && ip_version == 4) + sge->length -= 20; + + /* Max Header buf size for RoCE V1 is 78. + * ETH(14) + VLAN(4) + GRH(40) + BTH(20). + * Subtract 8 bytes from QP1 SQ header buf size + */ + if (!is_udp) + sge->length -= 8; + + /* Subtract 4 bytes for non vlan packets */ + if (!is_vlan) + sge->length -= 4; + + rc = xsc_icrc_hdr(dev, hdr_buf, sge->length - sizeof(struct ib_unpacked_eth), crc); + if (rc) { + xsc_ib_err(dev, "CRC error: hdr size %ld\n", + sge->length - sizeof(struct ib_unpacked_eth)); + } + return rc; +} + +int xsc_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct xsc_ib_qp *qp = to_xqp(ibqp); + struct xsc_ib_dev *dev = to_mdev(ibqp->device); + void *seg; + struct xsc_send_wqe_ctrl_seg *ctrl; + struct xsc_wqe_data_seg *data_seg; + u32 crc; + + int nreq; + int err = 0; + int i; + unsigned int idx; + unsigned long irqflag = 0; + struct ib_sge sg; + u8 *cur_p = NULL; + u8 *mad_send_base = NULL; + struct ib_wc wc; + unsigned long qp_irqflag = 0; +#ifdef XSC_CHIP_RDMA_UNSUPPORTED + xsc_ib_dbg(dev, "rdma unsupported,%s no action.\n", __func__); + return 0; +#endif + + if (wr->opcode == IB_WR_LOCAL_INV) { + spin_lock_irqsave(&qp->lock, qp_irqflag); + wc.status = IB_WC_SUCCESS; + wc.wr_cqe = wr->wr_cqe; + if (xsc_wr_invalidate_mr(dev, wr)) + wc.status = IB_WC_GENERAL_ERR; + + spin_unlock_irqrestore(&qp->lock, qp_irqflag); + if (wr->wr_cqe && wr->wr_cqe->done) + wr->wr_cqe->done(qp->send_cq, &wc); + return 0; + } + + if (wr->opcode == IB_WR_REG_MR) { + spin_lock_irqsave(&qp->lock, qp_irqflag); + wc.status = IB_WC_SUCCESS; + if (xsc_wr_reg_mr(dev, wr)) + wc.status = IB_WC_GENERAL_ERR; + if (wr->wr_cqe && wr->wr_cqe->done) + wr->wr_cqe->done(qp->send_cq, &wc); + spin_unlock_irqrestore(&qp->lock, qp_irqflag); + } + + spin_lock_irqsave(&qp->sq.lock, irqflag); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + unsigned int seg_index = 1; + unsigned int msg_len = 0; + struct ib_sge *sgl = &wr->sg_list[0]; + int sg_n = wr->num_sge; + + if (unlikely(wr->opcode < 0 + || wr->opcode >= ARRAY_SIZE(xsc_ib_opcode))) { + xsc_ib_err(dev, "bad opcode %d\n", wr->opcode); + err = EINVAL; + *bad_wr = wr; + goto out; + } + + if (unlikely(xsc_wq_overflow(&qp->sq, nreq, + to_xcq(qp->ibqp.send_cq)))) { + xsc_ib_err(dev, "send work queue overflow\n"); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > qp->sq.max_gs)) { + xsc_ib_err(dev, "max gs exceeded %d (max = %d)\n", + wr->num_sge, qp->sq.max_gs); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely((wr->opcode == IB_WR_RDMA_READ) && (wr->num_sge > 1))) { + xsc_ib_err(dev, "rdma read, max gs exceeded %d (max = 1)\n", + wr->num_sge); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); + ctrl = seg = xsc_get_send_wqe(qp, idx); + ctrl->wqe_id = cpu_to_le16(qp->sq.cur_post << + (qp->sq.wqe_shift - XSC_BASE_WQE_SHIFT)); + ctrl->ds_data_num = 0; + ctrl->se = wr->send_flags & IB_SEND_SOLICITED ? 1 : 0; + ctrl->ce = wr->send_flags & IB_SEND_SIGNALED ? 1 : 0; + for (i = 0; i < wr->num_sge; ++i) { + if (likely(wr->sg_list[i].length)) + msg_len += wr->sg_list[i].length; + } + ctrl->msg_len = msg_len; + ctrl->with_immdt = 0; + + switch (ibqp->qp_type) { + case IB_QPT_RC: + ctrl->ds_data_num = wr->num_sge; + switch (wr->opcode) { + case IB_WR_SEND: + break; + case IB_WR_SEND_WITH_IMM: + ctrl->with_immdt = 1; + ctrl->opcode_data = send_ieth(wr); + break; + case IB_WR_RDMA_WRITE_WITH_IMM: + ctrl->with_immdt = 1; + ctrl->opcode_data = send_ieth(wr); + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + ctrl->with_immdt = 0; + ctrl->ds_data_num++; + data_seg = get_seg_wqe(ctrl, seg_index); + set_remote_addr_seg( + data_seg, + msg_len, + rdma_wr(wr)->remote_addr, + rdma_wr(wr)->rkey); + seg_index++; + break; + case IB_WR_REG_MR: + break; + default: + xsc_ib_err(dev, "debug: opcode:%u NOT supported\n", wr->opcode); + err = EPERM; + *bad_wr = wr; + goto out; + } + ctrl->msg_opcode = xsc_ib_opcode[wr->opcode]; + break; + case IB_QPT_UD: + case IB_QPT_GSI: + xsc_ib_err(dev, "debug: %d UD/GSI hit\n", wr->opcode); + ctrl->msg_opcode = XSC_MSG_OPCODE_MAD; + ctrl->ds_data_num++; + data_seg = get_seg_wqe(ctrl, seg_index); + mad_send_base = (u8 *)qp->sq.hdr_buf + + MAX_QP1_SQ_HDR_SIZE_V2 * qp->sq.mad_index; + + build_qp1_send_v2(dev, qp, wr, &sg, msg_len, &crc); + + cur_p = mad_send_base + sg.length; + for (i = 0; i < wr->num_sge; ++i) { + if (likely(wr->sg_list[i].length)) + memcpy(cur_p, phys_to_virt( + dma_to_phys(dev->ib_dev.dma_device, + wr->sg_list[i].addr)), wr->sg_list[i].length); + cur_p += wr->sg_list[i].length; + } + crc = xsc_crc32(dev, crc, mad_send_base + sg.length, ctrl->msg_len); + ctrl->msg_len += sg.length; + seg_index++; + + *(u32 *)&mad_send_base[ctrl->msg_len] = ~crc; + ctrl->msg_len += sizeof(crc); + sg.length = ctrl->msg_len; + set_local_data_seg(data_seg, &sg); + xsc_ib_dbg(dev, "msg_len:%d\n", ctrl->msg_len); + qp->sq.mad_index = (qp->sq.mad_index + 1) % MAD_QUEUE_DEPTH; + + sg_n = 0; + break; + default: + xsc_ib_err(dev, "qp type:%u NOT supported\n", ibqp->qp_type); + err = EPERM; + *bad_wr = wr; + goto out; + } + + if (wr->opcode == IB_WR_REG_MR) { + nreq--; + continue; + } + + if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { + err = set_data_inl_seg(qp, wr, ctrl); + if (unlikely(err)) { + *bad_wr = wr; + xsc_ib_err(dev, "inline layout failed, err %d\n", err); + goto out; + } + } else { + for (i = 0; i < sg_n; ++i, ++seg_index) { + if (likely(sgl[i].length)) { + data_seg = get_seg_wqe(ctrl, seg_index); + set_local_data_seg(data_seg, &sgl[i]); + } + } + } + qp->sq.wrid[idx] = wr->wr_id; + qp->sq.wqe_head[idx] = qp->sq.head + nreq; + qp->sq.cur_post += 1; +#ifdef XSC_DEBUG + dump_wqe(qp, idx); +#endif + } +out: + xsc_ib_dbg(dev, "nreq:%d\n", nreq); + xsc_post_send_db(qp, dev->xdev, nreq); + spin_unlock_irqrestore(&qp->sq.lock, irqflag); + + return err; +} + +int xsc_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct xsc_ib_qp *qp = to_xqp(ibqp); + struct xsc_ib_dev *dev = to_mdev(ibqp->device); + struct xsc_core_device *xdev = dev->xdev; + struct xsc_wqe_data_seg *recv_head; + struct xsc_wqe_data_seg *data_seg; + unsigned long flags; + int err = 0; + u16 next_pid = 0; + union xsc_db_data db; + int nreq; + u16 idx; + int i; +#ifdef XSC_CHIP_RDMA_UNSUPPORTED + xsc_ib_dbg(dev, "rdma unsupported,%s no action.\n", __func__); + return 0; +#endif + + spin_lock_irqsave(&qp->rq.lock, flags); + + idx = qp->rq.head & (qp->rq.wqe_cnt - 1); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (unlikely(xsc_wq_overflow(&qp->rq, nreq, + to_xcq(qp->ibqp.recv_cq)))) { + xsc_ib_err(dev, "recv work queue overflow\n"); + err = ENOMEM; + *bad_wr = wr; + goto out; + } + + if (unlikely(wr->num_sge > qp->rq.max_gs)) { + xsc_ib_err(dev, "max gs exceeded %d (max = %d)\n", + wr->num_sge, qp->rq.max_gs); + err = EINVAL; + *bad_wr = wr; + goto out; + } + + recv_head = get_recv_wqe(qp, idx); + + for (i = 0; i < wr->num_sge; ++i) { + if (unlikely(!wr->sg_list[i].length)) + continue; + data_seg = get_seg_wqe(recv_head, i); + data_seg->in_line = 0; + WR_LE_64(data_seg->va, wr->sg_list[i].addr); + WR_LE_32(data_seg->mkey, wr->sg_list[i].lkey); + if (is_qp1(qp->xqp.qp_type)) + WR_LE_32(data_seg->seg_len, xdev->caps.rx_pkt_len_max); + else + WR_LE_32(data_seg->seg_len, wr->sg_list[i].length); + } + + qp->rq.wrid[idx] = wr->wr_id; + + idx = (idx + 1) & (qp->rq.wqe_cnt - 1); + } + +out: + if (likely(nreq)) { + qp->rq.head += nreq; + next_pid = qp->rq.head << (qp->rq.wqe_shift - XSC_BASE_WQE_SHIFT); + db.rq_next_pid = next_pid; + db.rqn = qp->doorbell_qpn; + + /* Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + writel(db.raw_data, REG_ADDR(xdev, xdev->regs.rx_db)); + } + + spin_unlock_irqrestore(&qp->rq.lock, flags); + + return err; +} + +static inline enum ib_qp_state to_ib_qp_state(enum xsc_qp_state xsc_state) +{ + switch (xsc_state) { + case XSC_QP_STATE_RST: return IB_QPS_RESET; + case XSC_QP_STATE_INIT: return IB_QPS_INIT; + case XSC_QP_STATE_RTR: return IB_QPS_RTR; + case XSC_QP_STATE_RTS: return IB_QPS_RTS; + case XSC_QP_STATE_SQ_DRAINING: + case XSC_QP_STATE_SQD: return IB_QPS_SQD; + case XSC_QP_STATE_SQER: return IB_QPS_SQE; + case XSC_QP_STATE_ERR: return IB_QPS_ERR; + default: return -1; + } +} + +static inline enum ib_mig_state to_ib_mig_state(int xsc_mig_state) +{ + switch (xsc_mig_state) { + case XSC_QP_PM_ARMED: return IB_MIG_ARMED; + case XSC_QP_PM_REARM: return IB_MIG_REARM; + case XSC_QP_PM_MIGRATED: return IB_MIG_MIGRATED; + default: return -1; + } +} + +int xsc_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct xsc_ib_dev *dev = to_mdev(ibqp->device); + struct xsc_ib_qp *qp = to_xqp(ibqp); + struct xsc_query_qp_mbox_out *outb; + struct xsc_qp_context *context; + int xsc_state; + int err = 0; +#ifdef XSC_CHIP_RDMA_UNSUPPORTED + xsc_ib_dbg(dev, "rdma unsupported,%s no action.\n", __func__); + return 0; +#endif + mutex_lock(&qp->mutex); + outb = kzalloc(sizeof(*outb), GFP_KERNEL); + if (!outb) { + err = -ENOMEM; + goto out; + } + context = &outb->ctx; + err = xsc_core_qp_query(dev->xdev, &qp->xqp, outb, sizeof(*outb)); + if (err) + goto out_free; + + xsc_state = context->state; + + qp->state = to_ib_qp_state(xsc_state); + qp_attr->qp_state = qp->state; + qp_attr->path_mtu = context->mtu_mode ? IB_MTU_4096 : IB_MTU_1024; +// qp_attr->path_mig_state = +// to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3); +// qp_attr->qkey = be32_to_cpu(context->qkey); + qp_attr->rq_psn = be32_to_cpu(context->next_recv_psn) & 0xffffff; + qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff; + qp_attr->dest_qp_num = be32_to_cpu(context->remote_qpn) & 0xffffff; +// qp_attr->qp_access_flags = +// to_ib_qp_access_flags(be32_to_cpu(context->params2)); + +// if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { +// to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path); +// to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path); +// qp_attr->alt_pkey_index = context->alt_path.pkey_index & 0x7f; +// qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; +// } + +// qp_attr->pkey_index = context->pri_path.pkey_index & 0x7f; +// qp_attr->port_num = context->pri_path.port; + + /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ + qp_attr->sq_draining = xsc_state == XSC_QP_STATE_SQ_DRAINING; +// +// qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7); +// +// qp_attr->max_dest_rd_atomic = +// 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7); +// qp_attr->min_rnr_timer = +// (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f; +// qp_attr->timeout = context->pri_path.ackto_lt >> 3; + qp_attr->retry_cnt = context->retry_cnt; + qp_attr->rnr_retry = context->rnr_retry; +// qp_attr->alt_timeout = context->alt_path.ackto_lt >> 3; + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; + qp_attr->cap.max_recv_sge = qp->rq.max_gs; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = qp->sq.max_gs; + } else { + qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = qp->sq.max_gs; + } + + /* We don't support inline sends for kernel QPs (yet), and we + * don't know what userspace's value should be. + */ + qp_attr->cap.max_inline_data = 0; + + qp_init_attr->cap = qp_attr->cap; + + qp_init_attr->create_flags = 0; + if (qp->flags & XSC_IB_QP_BLOCK_MULTICAST_LOOPBACK) + qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK; + + qp_init_attr->sq_sig_type = qp->sq_signal_bits & XSC_WQE_CTRL_CQ_UPDATE ? + IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; + +out_free: + kfree(outb); + +out: + mutex_unlock(&qp->mutex); + return err; +} + diff --git a/drivers/infiniband/hw/xsc/rtt.c b/drivers/infiniband/hw/xsc/rtt.c new file mode 100644 index 0000000000000000000000000000000000000000..3c6e461e7249acabffa07bd289c127c6e7ef12c9 --- /dev/null +++ b/drivers/infiniband/hw/xsc/rtt.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +struct xsc_rtt_interface { + struct xsc_core_device *xdev; + struct kobject kobj; +}; + +struct xsc_rtt_attributes { + struct attribute attr; + ssize_t (*show)(struct xsc_rtt_interface *interface, struct xsc_rtt_attributes *attr, + char *buf); + ssize_t (*store)(struct xsc_rtt_interface *interface, struct xsc_rtt_attributes *attr, + const char *buf, size_t count); +}; + +static ssize_t enable_show(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + char *buf) +{ + int err; + struct xsc_inbox_hdr in; + struct xsc_rtt_en_mbox_out out; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.opcode = __cpu_to_be16(XSC_CMD_OP_GET_RTT_EN); + err = xsc_cmd_exec(g->xdev, (void *)&in, sizeof(struct xsc_inbox_hdr), + (void *)&out, sizeof(struct xsc_rtt_en_mbox_out)); + if (err || out.hdr.status) { + xsc_core_err(g->xdev, "Failed to get rtt en, err(%u), status(%u)\n", + err, out.hdr.status); + return -EINVAL; + } + + return sprintf(buf, "%u\n", out.en); +} + +static ssize_t enable_store(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + const char *buf, size_t count) +{ + int err; + u16 rtt_enable; + struct xsc_rtt_en_mbox_in in; + struct xsc_rtt_en_mbox_out out; + + err = kstrtou16(buf, 0, &rtt_enable); + if (err != 0) + return -EINVAL; + + if (rtt_enable > 1) { + xsc_core_err(g->xdev, "Failed to set rtt en, rtt_enable(%hu) out of range[0,1]\n", + rtt_enable); + return -EINVAL; + } + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.hdr.opcode = __cpu_to_be16(XSC_CMD_OP_SET_RTT_EN); + in.en = rtt_enable; + + err = xsc_cmd_exec(g->xdev, (void *)&in, sizeof(struct xsc_rtt_en_mbox_in), + (void *)&out, sizeof(struct xsc_rtt_en_mbox_out)); + if (err || out.hdr.status) { + xsc_core_err(g->xdev, "Failed to set rtt en, err(%u), status(%u)\n", + err, out.hdr.status); + return -EINVAL; + } + + return count; +} + +static ssize_t qpn_show(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + char *buf) +{ + int err, i; + u32 count = 0; + struct xsc_inbox_hdr in; + struct xsc_get_rtt_qpn_mbox_out out; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.opcode = __cpu_to_be16(XSC_CMD_OP_GET_RTT_QPN); + err = xsc_cmd_exec(g->xdev, (void *)&in, sizeof(struct xsc_inbox_hdr), + (void *)&out, sizeof(struct xsc_get_rtt_qpn_mbox_out)); + if (err || out.hdr.status) { + xsc_core_err(g->xdev, "Failed to get rtt qpn, err(%u), status(%u)\n", + err, out.hdr.status); + return -EINVAL; + } + + for (i = 0; i < (XSC_RTT_CFG_QPN_MAX - 1); i++) + count += sprintf(&buf[count], "%hu,", __be16_to_cpu(out.qpn[i])); + + count += sprintf(&buf[count], "%hu\n", __be16_to_cpu(out.qpn[i])); + + return count; +} + +#define RTT_CFG_QPN_FORMAT "%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu," \ + "%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu,%hu" + +static ssize_t qpn_store(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + const char *buf, size_t count) +{ + int err, i; + struct xsc_rtt_qpn_mbox_in in; + struct xsc_rtt_qpn_mbox_out out; + u16 *ptr = in.qpn; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + err = sscanf(buf, RTT_CFG_QPN_FORMAT, &ptr[0], &ptr[1], &ptr[2], &ptr[3], &ptr[4], + &ptr[5], &ptr[6], &ptr[7], &ptr[8], &ptr[9], &ptr[10], &ptr[11], &ptr[12], + &ptr[13], &ptr[14], &ptr[15], &ptr[16], &ptr[17], &ptr[18], &ptr[19], + &ptr[20], &ptr[21], &ptr[22], &ptr[23], &ptr[24], &ptr[25], &ptr[26], + &ptr[27], &ptr[28], &ptr[29], &ptr[30], &ptr[31]); + if (err != XSC_RTT_CFG_QPN_MAX) + return -EINVAL; + + in.hdr.opcode = __cpu_to_be16(XSC_CMD_OP_SET_RTT_QPN); + + for (i = 0 ; i < XSC_RTT_CFG_QPN_MAX; i++) + in.qpn[i] = __cpu_to_be16(ptr[i]); + + err = xsc_cmd_exec(g->xdev, (void *)&in, sizeof(struct xsc_rtt_qpn_mbox_in), + (void *)&out, sizeof(struct xsc_rtt_qpn_mbox_out)); + if (err || out.hdr.status) { + xsc_core_err(g->xdev, "Failed to set rtt qpn, err(%u), status(%u)\n", + err, out.hdr.status); + return -EINVAL; + } + + return count; +} + +static ssize_t period_show(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + char *buf) +{ + int err; + struct xsc_inbox_hdr in; + struct xsc_rtt_period_mbox_out out; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.opcode = __cpu_to_be16(XSC_CMD_OP_GET_RTT_PERIOD); + err = xsc_cmd_exec(g->xdev, (void *)&in, sizeof(struct xsc_inbox_hdr), + (void *)&out, sizeof(struct xsc_rtt_period_mbox_out)); + if (err || out.hdr.status) { + xsc_core_err(g->xdev, "Failed to get rtt period, err(%u), status(%u)\n", + err, out.hdr.status); + return -EINVAL; + } + + return sprintf(buf, "%u\n", __be32_to_cpu(out.period)); +} + +#define RTT_CFG_PERIOD_MAX 10000 //ms, 10s +static ssize_t period_store(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + const char *buf, size_t count) +{ + int err; + u32 rtt_period; + struct xsc_rtt_period_mbox_in in; + struct xsc_rtt_period_mbox_out out; + + err = kstrtouint(buf, 0, &rtt_period); + if (err != 0) + return -EINVAL; + + if (rtt_period > RTT_CFG_PERIOD_MAX) + return -EINVAL; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.hdr.opcode = __cpu_to_be16(XSC_CMD_OP_SET_RTT_PERIOD); + in.period = __cpu_to_be32(rtt_period); + + err = xsc_cmd_exec(g->xdev, (void *)&in, sizeof(struct xsc_rtt_period_mbox_in), + (void *)&out, sizeof(struct xsc_rtt_period_mbox_out)); + if (err || out.hdr.status) { + xsc_core_err(g->xdev, "Failed to set rtt period, err(%u), status(%u)\n", + err, out.hdr.status); + return -EINVAL; + } + + return count; +} + +static ssize_t result_show(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + char *buf) +{ + int i, err; + u32 count = 0; + struct xsc_inbox_hdr in; + struct xsc_rtt_result_mbox_out out; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.opcode = __cpu_to_be16(XSC_CMD_OP_GET_RTT_RESULT); + + err = xsc_cmd_exec(g->xdev, (void *)&in, sizeof(struct xsc_inbox_hdr), + (void *)&out, sizeof(struct xsc_rtt_result_mbox_out)); + if (err || out.hdr.status) { + xsc_core_err(g->xdev, "Failed to get rtt result, err(%u), status(%u)\n", + err, out.hdr.status); + return -EINVAL; + } + + for (i = 0; i < (XSC_RTT_CFG_QPN_MAX - 1); i++) + count += sprintf(&buf[count], "%lld,", __be64_to_cpu(out.result[i])); + + count += sprintf(&buf[count], "%lld\n", __be64_to_cpu(out.result[i])); + + return count; +} + +static ssize_t result_store(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + const char *buf, size_t count) +{ + return -EOPNOTSUPP; +} + +static ssize_t stats_show(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + char *buf) +{ + int err; + u32 count = 0; + struct xsc_inbox_hdr in; + struct xsc_rtt_stats_mbox_out out; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.opcode = __cpu_to_be16(XSC_CMD_OP_GET_RTT_STATS); + + err = xsc_cmd_exec(g->xdev, (void *)&in, sizeof(struct xsc_inbox_hdr), + (void *)&out, sizeof(struct xsc_rtt_stats_mbox_out)); + if (err || out.hdr.status) { + xsc_core_err(g->xdev, "Failed to get rtt stats, err(%u), status(%u)\n", + err, out.hdr.status); + return -EINVAL; + } + + count += sprintf(&buf[count], "rtt_succ_snd_req_cnt %llu\n", + __be64_to_cpu(out.stats.rtt_succ_snd_req_cnt)); + count += sprintf(&buf[count], "rtt_succ_snd_rsp_cnt %llu\n", + __be64_to_cpu(out.stats.rtt_succ_snd_rsp_cnt)); + count += sprintf(&buf[count], "rtt_fail_snd_req_cnt %llu\n", + __be64_to_cpu(out.stats.rtt_fail_snd_req_cnt)); + count += sprintf(&buf[count], "rtt_fail_snd_rsp_cnt %llu\n", + __be64_to_cpu(out.stats.rtt_fail_snd_rsp_cnt)); + count += sprintf(&buf[count], "rtt_rcv_req_cnt %llu\n", + __be64_to_cpu(out.stats.rtt_rcv_req_cnt)); + count += sprintf(&buf[count], "rtt_rcv_rsp_cnt %llu\n", + __be64_to_cpu(out.stats.rtt_rcv_rsp_cnt)); + count += sprintf(&buf[count], "rtt_rcv_unk_cnt %llu\n", + __be64_to_cpu(out.stats.rtt_rcv_unk_cnt)); + count += sprintf(&buf[count], "rtt_grp_invalid_cnt %llu\n", + __be64_to_cpu(out.stats.rtt_grp_invalid_cnt)); + + return count; +} + +static ssize_t stats_store(struct xsc_rtt_interface *g, struct xsc_rtt_attributes *a, + const char *buf, size_t count) +{ + return -EOPNOTSUPP; +} + +#define RTT_ATTR(_name) struct xsc_rtt_attributes xsc_rtt_attr_##_name = \ + __ATTR(rtt_probe_##_name, 0644, _name##_show, _name##_store) + +RTT_ATTR(enable); +RTT_ATTR(qpn); +RTT_ATTR(period); +RTT_ATTR(result); +RTT_ATTR(stats); + +static ssize_t rtt_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct xsc_rtt_attributes *ga = + container_of(attr, struct xsc_rtt_attributes, attr); + struct xsc_rtt_interface *g = container_of(kobj, struct xsc_rtt_interface, kobj); + + if (!ga->show) + return -EIO; + + return ga->show(g, ga, buf); +} + +static ssize_t rtt_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t size) +{ + struct xsc_rtt_attributes *ga = + container_of(attr, struct xsc_rtt_attributes, attr); + struct xsc_rtt_interface *g = container_of(kobj, struct xsc_rtt_interface, kobj); + + if (!ga->store) + return -EIO; + + return ga->store(g, ga, buf, size); +} + +static const struct sysfs_ops rtt_sysfs_ops = { + .show = rtt_attr_show, + .store = rtt_attr_store, +}; + +static struct attribute *rtt_attrs[] = { + &xsc_rtt_attr_enable.attr, + &xsc_rtt_attr_qpn.attr, + &xsc_rtt_attr_period.attr, + &xsc_rtt_attr_result.attr, + &xsc_rtt_attr_stats.attr, + NULL +}; + +static struct kobj_type rtt_ktype = { + .sysfs_ops = &rtt_sysfs_ops, + .default_attrs = rtt_attrs +}; + +int xsc_rtt_sysfs_init(struct ib_device *ib_dev, struct xsc_core_device *xdev) +{ + struct xsc_rtt_interface *tmp; + int err; + + if (!xdev || !xsc_core_is_pf(xdev) || xdev->pf_id != 0) + return -EACCES; + + tmp = kzalloc(sizeof(struct xsc_rtt_interface), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + err = kobject_init_and_add(&tmp->kobj, &rtt_ktype, + &(ib_dev->dev.kobj), "rtt"); + if (err) + goto rtt_attr_err; + + xdev->rtt_priv = tmp; + tmp->xdev = xdev; + return 0; + +rtt_attr_err: + kobject_put(&tmp->kobj); + kfree(tmp); + return err; +} + +void xsc_rtt_sysfs_fini(struct xsc_core_device *xdev) +{ + int err; + struct xsc_rtt_en_mbox_in in; + struct xsc_rtt_en_mbox_out out; + struct xsc_rtt_period_mbox_in period_in; + struct xsc_rtt_period_mbox_out period_out; + struct xsc_rtt_interface *rtt; + + if (!xdev || !xdev->rtt_priv) + return; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.hdr.opcode = __cpu_to_be16(XSC_CMD_OP_SET_RTT_EN); + in.en = 0; + + err = xsc_cmd_exec(xdev, (void *)&in, sizeof(struct xsc_rtt_en_mbox_in), + (void *)&out, sizeof(struct xsc_rtt_en_mbox_out)); + if (err || out.hdr.status) + xsc_core_err(xdev, "Failed to set rtt disable, err(%u), status(%u)\n", + err, out.hdr.status); + + memset(&period_in, 0, sizeof(period_in)); + memset(&period_out, 0, sizeof(period_out)); + + period_in.hdr.opcode = __cpu_to_be16(XSC_CMD_OP_SET_RTT_PERIOD); + period_in.period = __cpu_to_be32(RTT_CFG_PERIOD_MAX); + + err = xsc_cmd_exec(xdev, (void *)&period_in, sizeof(struct xsc_rtt_period_mbox_in), + (void *)&period_out, sizeof(struct xsc_rtt_period_mbox_out)); + if (err || period_out.hdr.status) + xsc_core_err(xdev, "Failed to set rtt period default, err(%u), status(%u)\n", + err, out.hdr.status); + + rtt = xdev->rtt_priv; + kobject_put(&rtt->kobj); + kfree(rtt); + xdev->rtt_priv = NULL; +} + diff --git a/drivers/infiniband/hw/xsc/user.h b/drivers/infiniband/hw/xsc/user.h new file mode 100644 index 0000000000000000000000000000000000000000..cccc5de39d1e26d2bbf987895fbacfbc5d6b4f84 --- /dev/null +++ b/drivers/infiniband/hw/xsc/user.h @@ -0,0 +1,255 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_IB_USER_H +#define XSC_IB_USER_H + +#include +#include /* For ETH_ALEN. */ +#include + +enum xsc_ib_devx_methods { + XSC_IB_METHOD_DEVX_OTHER = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_METHOD_DEVX_QUERY_UAR, + XSC_IB_METHOD_DEVX_QUERY_EQN, +}; + +enum xsc_ib_devx_other_attrs { + XSC_IB_ATTR_DEVX_OTHER_CMD_IN = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_ATTR_DEVX_OTHER_CMD_OUT, +}; + +enum xsc_ib_objects { + XSC_IB_OBJECT_DEVX = (1U << UVERBS_ID_NS_SHIFT), + XSC_IB_OBJECT_DEVX_OBJ, + XSC_IB_OBJECT_DEVX_UMEM, + XSC_IB_OBJECT_FLOW_MATCHER, +}; + +/* Increment this value if any changes that break userspace ABI + * compatibility are made. + */ +#define XSC_IB_UVERBS_ABI_VERSION 1 + +/* Make sure that all structs defined in this file remain laid out so + * that they pack the same way on 32-bit and 64-bit architectures (to + * avoid incompatibility between 32-bit userspace and 64-bit kernels). + * In particular do not use pointer types -- pass pointers in __u64 + * instead. + */ + +enum { + XSC_QP_FLAG_SIGNATURE = 1 << 0, + XSC_QP_FLAG_SCATTER_CQE = 1 << 1, + XSC_QP_FLAG_TUNNEL_OFFLOADS = 1 << 2, + XSC_QP_FLAG_BFREG_INDEX = 1 << 3, + XSC_QP_FLAG_TYPE_DCT = 1 << 4, + XSC_QP_FLAG_TYPE_DCI = 1 << 5, + XSC_QP_FLAG_TIR_ALLOW_SELF_LB_UC = 1 << 6, + XSC_QP_FLAG_TIR_ALLOW_SELF_LB_MC = 1 << 7, + XSC_QP_FLAG_ALLOW_SCATTER_CQE = 1 << 8, + XSC_QP_FLAG_RAWPACKET_TSO = 1 << 9, +}; + +struct xsc_ib_alloc_ucontext_req { + __u32 rsvd0; + __u32 rsvd1; +}; + +enum xsc_user_cmds_supp_uhw { + XSC_USER_CMDS_SUPP_UHW_QUERY_DEVICE = 1 << 0, + XSC_USER_CMDS_SUPP_UHW_CREATE_AH = 1 << 1, +}; + +struct xsc_ib_alloc_ucontext_resp { + __u32 qp_tab_size; + __u32 cache_line_size; + __u16 max_sq_desc_sz; + __u16 max_rq_desc_sz; + __u32 max_send_wqebb; + __u32 max_recv_wr; + __u16 num_ports; + __u16 reserved; + __u64 qpm_tx_db; + __u64 qpm_rx_db; + __u64 cqm_next_cid_reg; + __u64 cqm_armdb; + __u32 send_ds_num; + __u32 recv_ds_num; + __u32 cmds_supp_uhw; +}; + +struct xsc_ib_create_qp { + __u64 buf_addr; + __u64 db_addr; + __u32 sq_wqe_count; + __u32 rq_wqe_count; + __u32 rq_wqe_shift; + __u32 flags; +}; + +struct xsc_ib_create_qp_resp { + __u32 uuar_index; + __u32 reserved; +}; + +struct xsc_ib_create_cq { + __u64 buf_addr; + __u64 db_addr; + __u32 cqe_size; +}; + +struct xsc_ib_create_cq_resp { + __u32 cqn; + __u32 reserved; +}; + +struct xsc_ib_create_ah_resp { + __u32 response_length; + __u8 dmac[ETH_ALEN]; + __u8 reserved[6]; +}; + +struct xsc_ib_alloc_pd_resp { + __u32 pdn; +}; + +struct xsc_ib_tso_caps { + __u32 max_tso; /* Maximum tso payload size in bytes */ + + /* Corresponding bit will be set if qp type from + * 'enum ib_qp_type' is supported, e.g. + * supported_qpts |= 1 << IB_QPT_UD + */ + __u32 supported_qpts; +}; + +/* RX Hash function flags */ +enum xsc_rx_hash_function_flags { + XSC_RX_HASH_FUNC_TOEPLITZ = 1 << 0, +}; + +/* RX Hash flags, these flags allows to set which incoming packet's field should + * participates in RX Hash. Each flag represent certain packet's field, + * when the flag is set the field that is represented by the flag will + * participate in RX Hash calculation. + * Note: *IPV4 and *IPV6 flags can't be enabled together on the same QP + * and *TCP and *UDP flags can't be enabled together on the same QP. + */ +enum xsc_rx_hash_fields { + XSC_RX_HASH_SRC_IPV4 = 1 << 0, + XSC_RX_HASH_DST_IPV4 = 1 << 1, + XSC_RX_HASH_SRC_IPV6 = 1 << 2, + XSC_RX_HASH_DST_IPV6 = 1 << 3, + XSC_RX_HASH_SRC_PORT_TCP = 1 << 4, + XSC_RX_HASH_DST_PORT_TCP = 1 << 5, + XSC_RX_HASH_SRC_PORT_UDP = 1 << 6, + XSC_RX_HASH_DST_PORT_UDP = 1 << 7, + XSC_RX_HASH_IPSEC_SPI = 1 << 8, + /* Save bits for future fields */ + XSC_RX_HASH_INNER = (1UL << 31), +}; + +struct xsc_ib_rss_caps { + __aligned_u64 rx_hash_fields_mask; /* enum xsc_rx_hash_fields */ + __u8 rx_hash_function; /* enum xsc_rx_hash_function_flags */ + __u8 reserved[7]; +}; + +enum xsc_ib_cqe_comp_res_format { + XSC_IB_CQE_RES_FORMAT_HASH = 1 << 0, + XSC_IB_CQE_RES_FORMAT_CSUM = 1 << 1, + XSC_IB_CQE_RES_FORMAT_CSUM_STRIDX = 1 << 2, +}; + +struct xsc_ib_cqe_comp_caps { + __u32 max_num; + __u32 supported_format; /* enum xsc_ib_cqe_comp_res_format */ +}; + +enum xsc_ib_packet_pacing_cap_flags { + XSC_IB_PP_SUPPORT_BURST = 1 << 0, +}; + +struct xsc_packet_pacing_caps { + __u32 qp_rate_limit_min; + __u32 qp_rate_limit_max; /* In kpbs */ + + /* Corresponding bit will be set if qp type from + * 'enum ib_qp_type' is supported, e.g. + * supported_qpts |= 1 << IB_QPT_RAW_PACKET + */ + __u32 supported_qpts; + __u8 cap_flags; /* enum xsc_ib_packet_pacing_cap_flags */ + __u8 reserved[3]; +}; + +enum xsc_ib_mpw_caps { + MPW_RESERVED = 1 << 0, + XSC_IB_ALLOW_MPW = 1 << 1, + XSC_IB_SUPPORT_EMPW = 1 << 2, +}; + +enum xsc_ib_sw_parsing_offloads { + XSC_IB_SW_PARSING = 1 << 0, + XSC_IB_SW_PARSING_CSUM = 1 << 1, + XSC_IB_SW_PARSING_LSO = 1 << 2, +}; + +struct xsc_ib_sw_parsing_caps { + __u32 sw_parsing_offloads; /* enum xsc_ib_sw_parsing_offloads */ + + /* Corresponding bit will be set if qp type from + * 'enum ib_qp_type' is supported, e.g. + * supported_qpts |= 1 << IB_QPT_RAW_PACKET + */ + __u32 supported_qpts; +}; + +struct xsc_ib_striding_rq_caps { + __u32 min_single_stride_log_num_of_bytes; + __u32 max_single_stride_log_num_of_bytes; + __u32 min_single_wqe_log_num_of_strides; + __u32 max_single_wqe_log_num_of_strides; + + /* Corresponding bit will be set if qp type from + * 'enum ib_qp_type' is supported, e.g. + * supported_qpts |= 1 << IB_QPT_RAW_PACKET + */ + __u32 supported_qpts; + __u32 reserved; +}; + +enum xsc_ib_query_dev_resp_flags { + /* Support 128B CQE compression */ + XSC_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP = 1 << 0, + XSC_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD = 1 << 1, +}; + +enum xsc_ib_tunnel_offloads { + XSC_IB_TUNNELED_OFFLOADS_VXLAN = 1 << 0, + XSC_IB_TUNNELED_OFFLOADS_GRE = 1 << 1, + XSC_IB_TUNNELED_OFFLOADS_GENEVE = 1 << 2, + XSC_IB_TUNNELED_OFFLOADS_MPLS_GRE = 1 << 3, + XSC_IB_TUNNELED_OFFLOADS_MPLS_UDP = 1 << 4, +}; + +struct xsc_ib_query_device_resp { + __u32 comp_mask; + __u32 response_length; + struct xsc_ib_tso_caps tso_caps; + struct xsc_ib_rss_caps rss_caps; + struct xsc_ib_cqe_comp_caps cqe_comp_caps; + struct xsc_packet_pacing_caps packet_pacing_caps; + __u32 xsc_ib_support_multi_pkt_send_wqes; + __u32 flags; /* Use enum xsc_ib_query_dev_resp_flags */ + struct xsc_ib_sw_parsing_caps sw_parsing_caps; + struct xsc_ib_striding_rq_caps striding_rq_caps; + __u32 tunnel_offloads_caps; /* enum xsc_ib_tunnel_offloads */ + __u32 reserved; +}; + +#endif /* XSC_IB_USER_H */ diff --git a/drivers/infiniband/hw/xsc/xsc_ib.h b/drivers/infiniband/hw/xsc/xsc_ib.h new file mode 100644 index 0000000000000000000000000000000000000000..121f2ad682e864b65f9d422d251efd8f85b1d14e --- /dev/null +++ b/drivers/infiniband/hw/xsc/xsc_ib.h @@ -0,0 +1,486 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_IB_H +#define XSC_IB_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xsc_ib_compat.h" + +#define DISABLE_XSC_IB_DBG + +#ifdef DISABLE_XSC_IB_DBG +#define xsc_ib_dbg(dev, format, arg...) \ +pr_debug("%s:%d:(pid %d): " format, (dev)->ib_dev.name, \ + __LINE__, current->pid, ##arg) + +#define xsc_ib_err(dev, format, arg...) \ +pr_err("%s:%d:(pid %d): " format, (dev)->ib_dev.name, \ + __LINE__, current->pid, ##arg) + +#define xsc_ib_warn(dev, format, arg...) \ +pr_warn("%s:%d:(pid %d): " format, (dev)->ib_dev.name, \ + __LINE__, current->pid, ##arg) +#else +#define xsc_ib_dbg(dev, format, arg...) \ +pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \ + __LINE__, current->pid, ##arg) + +#define xsc_ib_err(dev, format, arg...) \ +pr_err("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \ + __LINE__, current->pid, ##arg) + +#define xsc_ib_warn(dev, format, arg...) \ +pr_warn("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \ + __LINE__, current->pid, ##arg) +#endif + +struct xsc_ib_ucontext { + struct ib_ucontext ibucontext; + struct list_head db_page_list; + + /* protect doorbell record alloc/free + */ + struct mutex db_page_mutex; +}; + +#define field_avail(type, fld, sz) (offsetof(type, fld) + \ + sizeof(((type *)0)->fld) <= (sz)) + +static inline struct xsc_ib_ucontext *to_xucontext(struct ib_ucontext *ibucontext) +{ + return container_of(ibucontext, struct xsc_ib_ucontext, ibucontext); +} + +struct xsc_ib_pd { + struct ib_pd ibpd; + u32 pdn; + u32 pa_lkey; +}; + +/* Use macros here so that don't have to duplicate + * enum ib_send_flags and enum ib_qp_type for low-level driver + */ + +#define XSC_IB_QPT_REG_UMR IB_QPT_RESERVED1 + +enum { + XSC_PAGE_SHIFT_4K = 12, + XSC_PAGE_SHIFT_64K = 16, + XSC_PAGE_SHIFT_2M = 21, + XSC_PAGE_SHIFT_1G = 30, +}; + +enum { + XSC_PAGE_MODE_4K = 0, + XSC_PAGE_MODE_64K = 1, + XSC_PAGE_MODE_2M = 2, + XSC_PAGE_MODE_1G = 3, +}; + +struct wr_list { + u16 opcode; + u16 next; +}; + +struct xsc_ib_wq { + u64 *wrid; + u32 *wr_data; + struct wr_list *w_list; + unsigned int *wqe_head; + u16 unsig_count; + + /* serialize post to the work queue + */ + spinlock_t lock; + int wqe_cnt; + int ds_cnt; + int max_post; + int max_gs; + int offset; + int wqe_shift; + unsigned int head; + unsigned int tail; + u16 cur_post; + u16 last_poll; + void *qend; + void *hdr_buf; + u32 hdr_size; + dma_addr_t hdr_dma; + int mad_queue_depth; + int mad_index; +}; + +enum { + XSC_QP_USER, + XSC_QP_KERNEL, + XSC_QP_EMPTY +}; + +struct xsc_ib_qp { + struct ib_qp ibqp; + struct xsc_core_qp xqp; + struct xsc_buf buf; + + struct xsc_db db; + struct xsc_ib_wq rq; + + u32 doorbell_qpn; + u8 sq_signal_bits; + u8 fm_cache; + int sq_max_wqes_per_wr; + int sq_spare_wqes; + struct xsc_ib_wq sq; + + struct ib_umem *umem; + int buf_size; + + /* serialize qp state modifications + */ + struct mutex mutex; + u16 xrcdn; + u32 flags; + u8 port; + u8 alt_port; + u8 atomic_rd_en; + u8 resp_depth; + u8 state; + int xsc_type; + int wq_sig; + int scat_cqe; + int max_inline_data; + int has_rq; + + int create_type; + u32 pa_lkey; + /* For QP1 */ + struct ib_ud_header qp1_hdr; + u32 send_psn; + struct xsc_qp_context ctx; + struct ib_cq *send_cq; + struct ib_cq *recv_cq; + spinlock_t lock; +}; + +struct xsc_ib_cq_buf { + struct xsc_buf buf; + struct ib_umem *umem; + int cqe_size; +}; + +enum xsc_ib_qp_flags { + XSC_IB_QP_BLOCK_MULTICAST_LOOPBACK = 1 << 0, + XSC_IB_QP_SIGNATURE_HANDLING = 1 << 1, +}; + +struct xsc_shared_mr_info { + int mr_id; + struct ib_umem *umem; +}; + +struct xsc_ib_cq { + struct ib_cq ibcq; + struct xsc_core_cq xcq; + struct xsc_ib_cq_buf buf; + struct xsc_db db; + + /* serialize access to the CQ + */ + spinlock_t lock; + + /* protect resize cq + */ + struct mutex resize_mutex; + struct xsc_ib_cq_resize *resize_buf; + struct ib_umem *resize_umem; + int cqe_size; +}; + +struct xsc_ib_xrcd { + struct ib_xrcd ibxrcd; + u32 xrcdn; +}; + +struct xsc_ib_peer_id; + +struct xsc_ib_mr { + struct ib_mr ibmr; + struct xsc_core_mr mmr; + struct ib_umem *umem; + struct xsc_shared_mr_info *smr_info; + struct list_head list; + int order; + __be64 *pas; + dma_addr_t dma; + int npages; + struct completion done; + enum ib_wc_status status; + struct xsc_ib_peer_id *peer_id; + atomic_t invalidated; + struct completion invalidation_comp; +}; + +struct xsc_ib_peer_id { + struct completion comp; + struct xsc_ib_mr *mr; +}; + +struct xsc_cache_ent { + struct list_head head; + /* sync access to the cahce entry + */ + spinlock_t lock; + + struct dentry *dir; + char name[4]; + u32 order; + u32 size; + u32 cur; + u32 miss; + u32 limit; + + struct dentry *fsize; + struct dentry *fcur; + struct dentry *fmiss; + struct dentry *flimit; + + struct xsc_ib_dev *dev; + struct work_struct work; + struct delayed_work dwork; +}; + +struct xsc_mr_cache { + struct workqueue_struct *wq; + struct xsc_cache_ent ent[MAX_MR_CACHE_ENTRIES]; + int stopped; + struct dentry *root; + unsigned long last_add; +}; + +struct xsc_gid { + u8 data[16]; +}; + +struct xsc_sgid_tbl { + struct xsc_gid *tbl; + u32 max; + u32 count; +}; + +struct xsc_ib_res { + struct xsc_sgid_tbl sgid_tbl; +}; + +struct xsc_ib_resources { + struct ib_cq *c0; + struct ib_xrcd *x0; + struct ib_xrcd *x1; + struct ib_pd *p0; + struct ib_srq *s0; +}; + +struct xsc_ib_dev { + struct ib_device ib_dev; + struct uverbs_object_tree_def *driver_trees[6]; + struct net_device *netdev; + struct xsc_core_device *xdev; + XSC_DECLARE_DOORBELL_LOCK(uar_lock); + struct list_head eqs_list; + int num_ports; + int num_comp_vectors; + /* serialize update of capability mask + */ + struct mutex cap_mask_mutex; + bool ib_active; + /* sync used page count stats + */ + spinlock_t mr_lock; + struct xsc_ib_res ib_res; + struct xsc_ib_resources devr; + struct xsc_mr_cache cache; + u32 crc_32_table[256]; + int cm_pcp; + int cm_dscp; +}; + +union xsc_ib_fw_ver { + u64 data; + struct { + u16 chip_ver_h; + u16 hotfix_num; + u16 chip_ver_l; + u16 feature_flag; + } s; +}; + +static inline struct xsc_ib_cq *to_xibcq(struct xsc_core_cq *xcq) +{ + return container_of(xcq, struct xsc_ib_cq, xcq); +} + +static inline struct xsc_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd) +{ + return container_of(ibxrcd, struct xsc_ib_xrcd, ibxrcd); +} + +static inline struct xsc_ib_dev *to_mdev(struct ib_device *ibdev) +{ + return container_of(ibdev, struct xsc_ib_dev, ib_dev); +} + +static inline struct xsc_ib_cq *to_xcq(struct ib_cq *ibcq) +{ + return container_of(ibcq, struct xsc_ib_cq, ibcq); +} + +static inline struct xsc_ib_qp *to_xibqp(struct xsc_core_qp *xqp) +{ + return container_of(xqp, struct xsc_ib_qp, xqp); +} + +static inline struct xsc_ib_pd *to_mpd(struct ib_pd *ibpd) +{ + return container_of(ibpd, struct xsc_ib_pd, ibpd); +} + +static inline struct xsc_ib_qp *to_xqp(struct ib_qp *ibqp) +{ + return container_of(ibqp, struct xsc_ib_qp, ibqp); +} + +static inline struct xsc_ib_mr *to_mmr(struct ib_mr *ibmr) +{ + return container_of(ibmr, struct xsc_ib_mr, ibmr); +} + +struct xsc_ib_ah { + struct ib_ah ibah; + struct xsc_av av; +}; + +static inline struct xsc_ib_ah *to_mah(struct ib_ah *ibah) +{ + return container_of(ibah, struct xsc_ib_ah, ibah); +} + +static inline struct xsc_ib_dev *xdev2ibdev(struct xsc_core_device *xdev) +{ + return container_of(xdev, struct xsc_ib_dev, xdev); +} + +int xsc_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props); + +struct ib_qp *xsc_ib_create_qp(struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); + +void __xsc_ib_cq_clean(struct xsc_ib_cq *cq, u32 qpn); +void xsc_ib_cq_clean(struct xsc_ib_cq *cq, u32 qpn); +int xsc_MAD_IFC(struct xsc_ib_dev *dev, int ignore_mkey, int ignore_bkey, + int port, struct ib_wc *in_wc, struct ib_grh *in_grh, + void *in_mad, void *response_mad); + +int xsc_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); +int xsc_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata); +int xsc_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr); +int xsc_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); +int xsc_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); +void *xsc_get_send_wqe(struct xsc_ib_qp *qp, int n); +int xsc_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); +int xsc_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); +struct ib_mr *xsc_ib_get_dma_mr(struct ib_pd *pd, int acc); +struct ib_mr *xsc_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int access_flags, + struct ib_udata *udata); +int xsc_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + struct ib_wc *in_wc, struct ib_grh *in_grh, + struct ib_mad *in_mad, struct ib_mad *out_mad); +int xsc_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset); +void xsc_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift, + int *ncont, int *order); +void xsc_ib_populate_pas(struct xsc_ib_dev *dev, struct ib_umem *umem, + int page_shift, __be64 *pas, int npages, bool need_to_devide); +int xsc_mr_cache_init(struct xsc_ib_dev *dev); +const struct uverbs_object_tree_def *xsc_ib_get_devx_tree(void); + +int xsc_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset); +int xsc_wr_reg_mr(struct xsc_ib_dev *dev, const struct ib_send_wr *wr); +int xsc_wr_invalidate_mr(struct xsc_ib_dev *dev, const struct ib_send_wr *wr); + +static inline void init_query_mad(struct ib_smp *mad) +{ + mad->base_version = 1; + mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; + mad->class_version = 1; + mad->method = IB_MGMT_METHOD_GET; +} + +static inline u8 convert_access(int acc) +{ + return (acc & IB_ACCESS_REMOTE_ATOMIC ? XSC_PERM_ATOMIC : 0) | + (acc & IB_ACCESS_REMOTE_WRITE ? XSC_PERM_REMOTE_WRITE : 0) | + (acc & IB_ACCESS_REMOTE_READ ? XSC_PERM_REMOTE_READ : 0) | + (acc & IB_ACCESS_LOCAL_WRITE ? XSC_PERM_LOCAL_WRITE : 0) | + XSC_PERM_LOCAL_READ; +} + +static inline enum ib_mtu xsc_net_to_ib_mtu(unsigned int mtu) +{ + mtu = mtu - (IB_GRH_BYTES + IB_UDP_BYTES + IB_BTH_BYTES + + IB_EXT_XRC_BYTES + IB_EXT_ATOMICETH_BYTES + + IB_ICRC_BYTES); + + if (mtu >= ib_mtu_enum_to_int(IB_MTU_4096)) + return IB_MTU_4096; + else if (mtu >= ib_mtu_enum_to_int(IB_MTU_1024)) + return IB_MTU_1024; + else + return 0; +} + +/** + * UDP source port selection must adhere IANA port allocation ranges. Thus + * we will be using IANA recommendation for Ephemeral port range of: + * 49152-65535, or in hex: 0xC000-0xFFFF. + */ +#define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000) +#define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF) +#define IB_GRH_FLOWLABEL_MASK (0x000FFFFF) + +/** + * rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based + * on the flow_label + * + * This function will convert the 20 bit flow_label input to a valid RoCE v2 + * UDP src port 14 bit value. All RoCE V2 drivers should use this same + * convention. + */ +static inline u16 xsc_flow_label_to_udp_sport(u32 fl) +{ + u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000; + + fl_low ^= fl_high >> 14; + return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN); +} + +#endif /* XSC_IB_H */ diff --git a/drivers/infiniband/hw/xsc/xsc_ib_compat.h b/drivers/infiniband/hw/xsc/xsc_ib_compat.h new file mode 100644 index 0000000000000000000000000000000000000000..9c67be098f700eb16be0c9e2750c453206087e31 --- /dev/null +++ b/drivers/infiniband/hw/xsc/xsc_ib_compat.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_IB_COMPAT_H +#define XSC_IB_COMPAT_H + +/* adaptive to different ib_core versions + */ + +struct xsc_ib_ucontext; + +int xsc_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *ah_attr, + struct ib_udata *udata); +int xsc_ib_destroy_ah(struct ib_ah *ibah, u32 destroy_flags); +#define xsc_ib_create_ah_def() int xsc_ib_create_ah(\ + struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata) +#define xsc_ib_destroy_ah_def() int xsc_ib_destroy_ah(struct ib_ah *ah, u32 flags) + +int xsc_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); +int xsc_ib_create_cq(struct ib_cq *ibcq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata); +int xsc_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); + +// from main.c static functions +int xsc_ib_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata); +void xsc_ib_dealloc_ucontext(struct ib_ucontext *ibcontext); +int xsc_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); + +int xsc_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); +int xsc_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +struct ib_mr *xsc_ib_alloc_mr( + struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg); +#define xsc_ib_alloc_mr_def() struct ib_mr *xsc_ib_alloc_mr(\ + struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg) + +#define xsc_ib_destroy_qp_def() int xsc_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) +#define xsc_ib_create_cq_def() int xsc_ib_create_cq(struct ib_cq *ibcq,\ + const struct ib_cq_init_attr *attr, struct ib_udata *udata) +#define xsc_ib_destroy_cq_def() int xsc_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +#define xsc_ib_dereg_mr_def() int xsc_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) +#define xsc_ib_alloc_ucontext_def() int xsc_ib_alloc_ucontext(\ + struct ib_ucontext *uctx, struct ib_udata *udata) +#define xsc_ib_dealloc_ucontext_def() void xsc_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) +#define xsc_ib_alloc_pd_def() int xsc_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) + +#define xsc_ib_dealloc_pd_def() int xsc_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) + +#define RET_VALUE(x) (x) + +#endif diff --git a/drivers/infiniband/hw/xsc/xsc_rdma_ctrl.c b/drivers/infiniband/hw/xsc/xsc_rdma_ctrl.c new file mode 100644 index 0000000000000000000000000000000000000000..7c49a038554c3005e39dcfaa10b2b30f3b00f8f1 --- /dev/null +++ b/drivers/infiniband/hw/xsc/xsc_rdma_ctrl.c @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "global.h" +#include "xsc_ib.h" + +#define XSC_RDMA_CTRL_NAME "rdma_ctrl" + +static void encode_cc_cmd_enable_rp(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_enable_rp *cc_cmd = (struct xsc_cc_cmd_enable_rp *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->enable = __cpu_to_be32(cc_cmd->enable); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_enable_np(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_enable_np *cc_cmd = (struct xsc_cc_cmd_enable_np *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->enable = __cpu_to_be32(cc_cmd->enable); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_init_alpha(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_init_alpha *cc_cmd = (struct xsc_cc_cmd_init_alpha *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->alpha = __cpu_to_be32(cc_cmd->alpha); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_g(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_g *cc_cmd = (struct xsc_cc_cmd_g *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->g = __cpu_to_be32(cc_cmd->g); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_ai(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_ai *cc_cmd = (struct xsc_cc_cmd_ai *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->ai = __cpu_to_be32(cc_cmd->ai); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_hai(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_hai *cc_cmd = (struct xsc_cc_cmd_hai *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->hai = __cpu_to_be32(cc_cmd->hai); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_th(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_th *cc_cmd = (struct xsc_cc_cmd_th *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->threshold = __cpu_to_be32(cc_cmd->threshold); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_bc(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_bc *cc_cmd = (struct xsc_cc_cmd_bc *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->bytecount = __cpu_to_be32(cc_cmd->bytecount); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_cnp_opcode(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_opcode *cc_cmd = (struct xsc_cc_cmd_cnp_opcode *) data; + + cc_cmd->opcode = __cpu_to_be32(cc_cmd->opcode); +} + +static void encode_cc_cmd_cnp_bth_b(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_bth_b *cc_cmd = (struct xsc_cc_cmd_cnp_bth_b *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->bth_b = __cpu_to_be32(cc_cmd->bth_b); +} + +static void encode_cc_cmd_cnp_bth_f(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_bth_f *cc_cmd = (struct xsc_cc_cmd_cnp_bth_f *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->bth_f = __cpu_to_be32(cc_cmd->bth_f); +} + +static void encode_cc_cmd_cnp_ecn(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_ecn *cc_cmd = (struct xsc_cc_cmd_cnp_ecn *) data; + + cc_cmd->ecn = __cpu_to_be32(cc_cmd->ecn); +} + +static void encode_cc_cmd_data_ecn(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_data_ecn *cc_cmd = (struct xsc_cc_cmd_data_ecn *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->ecn = __cpu_to_be32(cc_cmd->ecn); +} + +static void encode_cc_cmd_cnp_tx_interval(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_tx_interval *cc_cmd = (struct xsc_cc_cmd_cnp_tx_interval *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->interval = __cpu_to_be32(cc_cmd->interval); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_evt_rsttime(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_evt_rsttime *cc_cmd = + (struct xsc_cc_cmd_evt_rsttime *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->period = __cpu_to_be32(cc_cmd->period); +} + +static void encode_cc_cmd_cnp_dscp(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_dscp *cc_cmd = (struct xsc_cc_cmd_cnp_dscp *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->dscp = __cpu_to_be32(cc_cmd->dscp); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_cnp_pcp(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_cnp_pcp *cc_cmd = (struct xsc_cc_cmd_cnp_pcp *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->pcp = __cpu_to_be32(cc_cmd->pcp); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_cmd_evt_period_alpha(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_evt_period_alpha *cc_cmd = (struct xsc_cc_cmd_evt_period_alpha *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->period = __cpu_to_be32(cc_cmd->period); +} + +static void encode_cc_cmd_clamp_tgt_rate(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_clamp_tgt_rate *cc_cmd = (struct xsc_cc_cmd_clamp_tgt_rate *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->clamp_tgt_rate = __cpu_to_be32(cc_cmd->clamp_tgt_rate); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void encode_cc_get_cfg(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_get_cfg *cc_cmd = (struct xsc_cc_cmd_get_cfg *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void decode_cc_get_cfg(void *data) +{ + struct xsc_cc_cmd_get_cfg *cc_cmd = (struct xsc_cc_cmd_get_cfg *) data; + + cc_cmd->cmd = __be16_to_cpu(cc_cmd->cmd); + cc_cmd->len = __be16_to_cpu(cc_cmd->len); + cc_cmd->enable_rp = __be32_to_cpu(cc_cmd->enable_rp); + cc_cmd->enable_np = __be32_to_cpu(cc_cmd->enable_np); + cc_cmd->init_alpha = __be32_to_cpu(cc_cmd->init_alpha); + cc_cmd->g = __be32_to_cpu(cc_cmd->g); + cc_cmd->ai = __be32_to_cpu(cc_cmd->ai); + cc_cmd->hai = __be32_to_cpu(cc_cmd->hai); + cc_cmd->threshold = __be32_to_cpu(cc_cmd->threshold); + cc_cmd->bytecount = __be32_to_cpu(cc_cmd->bytecount); + cc_cmd->opcode = __be32_to_cpu(cc_cmd->opcode); + cc_cmd->bth_b = __be32_to_cpu(cc_cmd->bth_b); + cc_cmd->bth_f = __be32_to_cpu(cc_cmd->bth_f); + cc_cmd->cnp_ecn = __be32_to_cpu(cc_cmd->cnp_ecn); + cc_cmd->data_ecn = __be32_to_cpu(cc_cmd->data_ecn); + cc_cmd->cnp_tx_interval = __be32_to_cpu(cc_cmd->cnp_tx_interval); + cc_cmd->evt_period_rsttime = __be32_to_cpu(cc_cmd->evt_period_rsttime); + cc_cmd->cnp_dscp = __be32_to_cpu(cc_cmd->cnp_dscp); + cc_cmd->cnp_pcp = __be32_to_cpu(cc_cmd->cnp_pcp); + cc_cmd->evt_period_alpha = __be32_to_cpu(cc_cmd->evt_period_alpha); + cc_cmd->clamp_tgt_rate = __be32_to_cpu(cc_cmd->clamp_tgt_rate); + cc_cmd->section = __be32_to_cpu(cc_cmd->section); +} + +static void encode_cc_get_stat(void *data, u32 mac_port) +{ + struct xsc_cc_cmd_get_stat *cc_cmd = (struct xsc_cc_cmd_get_stat *) data; + + cc_cmd->cmd = __cpu_to_be16(cc_cmd->cmd); + cc_cmd->len = __cpu_to_be16(cc_cmd->len); + cc_cmd->section = __cpu_to_be32(mac_port); +} + +static void decode_cc_get_stat(void *data) +{ + struct xsc_cc_cmd_stat *cc_cmd = (struct xsc_cc_cmd_stat *) data; + + cc_cmd->cnp_handled = __be32_to_cpu(cc_cmd->cnp_handled); + cc_cmd->alpha_recovery = __be32_to_cpu(cc_cmd->alpha_recovery); + cc_cmd->reset_timeout = __be32_to_cpu(cc_cmd->reset_timeout); + cc_cmd->reset_bytecount = __be32_to_cpu(cc_cmd->reset_bytecount); +} + +static int xsc_priv_dev_ioctl_get_global_pcp(struct xsc_core_device *xdev, void *in, void *out) +{ + struct xsc_ioctl_global_pcp *resp = (struct xsc_ioctl_global_pcp *)out; + + resp->pcp = get_global_force_pcp(); + return 0; +} + +static int xsc_priv_dev_ioctl_get_global_dscp(struct xsc_core_device *xdev, void *in, void *out) +{ + struct xsc_ioctl_global_dscp *resp = (struct xsc_ioctl_global_dscp *)out; + + resp->dscp = get_global_force_dscp(); + return 0; +} + +static int xsc_priv_dev_ioctl_set_global_pcp(struct xsc_core_device *xdev, void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_global_pcp *req = (struct xsc_ioctl_global_pcp *)out; + + ret = set_global_force_pcp(req->pcp); + return ret; +} + +static int xsc_priv_dev_ioctl_set_global_dscp(struct xsc_core_device *xdev, void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_global_dscp *req = (struct xsc_ioctl_global_dscp *)out; + + ret = set_global_force_dscp(req->dscp); + return ret; +} + +static int xsc_priv_dev_ioctl_get_cma_pcp(struct xsc_core_device *xdev, void *in, void *out) +{ + struct xsc_ib_dev *ib_dev = xdev->xsc_ib_dev; + struct xsc_ioctl_cma_pcp *resp = (struct xsc_ioctl_cma_pcp *)out; + + if (!XSC_IS_PF(xdev->glb_func_id)) + return -EOPNOTSUPP; + + resp->pcp = ib_dev->cm_pcp; + return 0; +} + +static int xsc_priv_dev_ioctl_get_cma_dscp(struct xsc_core_device *xdev, void *in, void *out) +{ + struct xsc_ib_dev *ib_dev = xdev->xsc_ib_dev; + struct xsc_ioctl_cma_dscp *resp = (struct xsc_ioctl_cma_dscp *)out; + + if (!XSC_IS_PF(xdev->glb_func_id)) + return -EOPNOTSUPP; + + resp->dscp = ib_dev->cm_dscp; + return 0; +} + +static int xsc_priv_dev_ioctl_set_cma_pcp(struct xsc_core_device *xdev, void *in, void *out) +{ + struct xsc_ib_dev *ib_dev = xdev->xsc_ib_dev; + struct xsc_ioctl_cma_pcp *req = (struct xsc_ioctl_cma_pcp *)out; + + if (!XSC_IS_PF(xdev->glb_func_id)) + return -EOPNOTSUPP; + + if (req->pcp < 0 || req->pcp > QOS_PCP_MAX) + return -EINVAL; + + ib_dev->cm_pcp = req->pcp; + return 0; +} + +static int xsc_priv_dev_ioctl_set_cma_dscp(struct xsc_core_device *xdev, void *in, void *out) +{ + struct xsc_ib_dev *ib_dev = xdev->xsc_ib_dev; + struct xsc_ioctl_cma_dscp *req = (struct xsc_ioctl_cma_dscp *)out; + + if (!XSC_IS_PF(xdev->glb_func_id)) + return -EOPNOTSUPP; + + if (req->dscp < 0 || req->dscp > QOS_DSCP_MAX) + return -EINVAL; + + ib_dev->cm_dscp = req->dscp; + return 0; +} + +static int _rdma_ctrl_ioctl_cc(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr, u16 expect_req_size, + u16 expect_resp_size, void (*encode)(void *, u32), void (*decode)(void *)) +{ + struct xsc_cc_mbox_in *in; + struct xsc_cc_mbox_out *out; + u16 user_size; + int err; + + user_size = expect_req_size > expect_resp_size ? expect_req_size : expect_resp_size; + if (hdr->attr.length != user_size) + return -EINVAL; + + in = kvzalloc(sizeof(struct xsc_cc_mbox_in) + expect_req_size, GFP_KERNEL); + if (!in) + goto err_in; + out = kvzalloc(sizeof(struct xsc_cc_mbox_out) + expect_resp_size, GFP_KERNEL); + if (!out) + goto err_out; + + err = copy_from_user(&in->data, user_hdr->attr.data, expect_req_size); + if (err) + goto err; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + if (encode) + encode((void *)in->data, xdev->mac_port); + + err = xsc_cmd_exec( + xdev, in, sizeof(*in) + expect_req_size, out, sizeof(*out) + expect_resp_size); + + hdr->attr.error = __be32_to_cpu(out->hdr.status); + if (decode) + decode((void *)out->data); + + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + goto err; + if (copy_to_user((void *)user_hdr->attr.data, &out->data, expect_resp_size)) + goto err; + + kvfree(in); + kvfree(out); + return 0; + +err: + kvfree(out); +err_out: + kvfree(in); +err_in: + return -EFAULT; +} + +int _rdma_ctrl_exec_ioctl(struct xsc_core_device *xdev, void *in, int in_size, void *out, + int out_size) +{ + int opcode, ret = 0; + struct xsc_ioctl_attr *hdr; + + hdr = (struct xsc_ioctl_attr *)in; + opcode = hdr->opcode; + switch (opcode) { + case XSC_IOCTL_GET_GLOBAL_PCP: + ret = xsc_priv_dev_ioctl_get_global_pcp(xdev, in, out); + break; + case XSC_IOCTL_GET_GLOBAL_DSCP: + ret = xsc_priv_dev_ioctl_get_global_dscp(xdev, in, out); + break; + case XSC_IOCTL_GET_CMA_PCP: + ret = xsc_priv_dev_ioctl_get_cma_pcp(xdev, in, out); + break; + case XSC_IOCTL_GET_CMA_DSCP: + ret = xsc_priv_dev_ioctl_get_cma_dscp(xdev, in, out); + break; + case XSC_IOCTL_SET_GLOBAL_PCP: + xsc_core_dbg(xdev, "setting global pcp\n"); + ret = xsc_priv_dev_ioctl_set_global_pcp(xdev, in, out); + break; + case XSC_IOCTL_SET_GLOBAL_DSCP: + xsc_core_dbg(xdev, "setting global dscp\n"); + ret = xsc_priv_dev_ioctl_set_global_dscp(xdev, in, out); + break; + case XSC_IOCTL_SET_CMA_PCP: + ret = xsc_priv_dev_ioctl_set_cma_pcp(xdev, in, out); + break; + case XSC_IOCTL_SET_CMA_DSCP: + ret = xsc_priv_dev_ioctl_set_cma_dscp(xdev, in, out); + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static long _rdma_ctrl_ioctl_getinfo(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr) +{ + struct xsc_ioctl_hdr hdr; + struct xsc_ioctl_hdr *in; + int in_size; + int err; + + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + if (err) + return -EFAULT; + if (hdr.check_filed != XSC_IOCTL_CHECK_FILED) + return -EINVAL; + switch (hdr.attr.opcode) { + case XSC_IOCTL_GET_GLOBAL_PCP: + case XSC_IOCTL_GET_GLOBAL_DSCP: + case XSC_IOCTL_SET_GLOBAL_PCP: + case XSC_IOCTL_SET_GLOBAL_DSCP: + case XSC_IOCTL_GET_CMA_PCP: + case XSC_IOCTL_GET_CMA_DSCP: + case XSC_IOCTL_SET_CMA_PCP: + case XSC_IOCTL_SET_CMA_DSCP: + break; + default: + return -EINVAL; + } + in_size = sizeof(struct xsc_ioctl_hdr) + hdr.attr.length; + in = kvzalloc(in_size, GFP_KERNEL); + if (!in) + return -EFAULT; + in->attr.opcode = hdr.attr.opcode; + in->attr.length = hdr.attr.length; + err = copy_from_user(in->attr.data, user_hdr->attr.data, hdr.attr.length); + if (err) { + kvfree(in); + return -EFAULT; + } + + err = _rdma_ctrl_exec_ioctl(xdev, &in->attr, (in_size-sizeof(u32)), in->attr.data, + hdr.attr.length); + in->attr.error = err; + if (copy_to_user(user_hdr, in, in_size)) + err = -EFAULT; + kvfree(in); + return err; +} + +static long _rdma_ctrl_ioctl_cmdq(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr) +{ + struct xsc_ioctl_hdr hdr; + int err; + void *in; + void *out; + + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + if (err) + return -EFAULT; + + /* check valid */ + if (hdr.check_filed != XSC_IOCTL_CHECK_FILED) + return -EINVAL; + + /* check ioctl cmd */ + switch (hdr.attr.opcode) { + case XSC_CMD_OP_IOCTL_SET_ENABLE_RP: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_enable_rp), + 0, encode_cc_cmd_enable_rp, NULL); + case XSC_CMD_OP_IOCTL_SET_ENABLE_NP: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_enable_np), + 0, encode_cc_cmd_enable_np, NULL); + case XSC_CMD_OP_IOCTL_SET_INIT_ALPHA: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_init_alpha), + 0, encode_cc_cmd_init_alpha, NULL); + case XSC_CMD_OP_IOCTL_SET_G: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_g), + 0, encode_cc_cmd_g, NULL); + case XSC_CMD_OP_IOCTL_SET_AI: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_ai), + 0, encode_cc_cmd_ai, NULL); + case XSC_CMD_OP_IOCTL_SET_HAI: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_hai), + 0, encode_cc_cmd_hai, NULL); + case XSC_CMD_OP_IOCTL_SET_TH: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_th), + 0, encode_cc_cmd_th, NULL); + case XSC_CMD_OP_IOCTL_SET_BC_TH: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_bc), + 0, encode_cc_cmd_bc, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_OPCODE: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_opcode), + 0, encode_cc_cmd_cnp_opcode, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_BTH_B: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_bth_b), + 0, encode_cc_cmd_cnp_bth_b, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_BTH_F: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_bth_f), + 0, encode_cc_cmd_cnp_bth_f, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_ECN: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_ecn), + 0, encode_cc_cmd_cnp_ecn, NULL); + case XSC_CMD_OP_IOCTL_SET_DATA_ECN: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_data_ecn), + 0, encode_cc_cmd_data_ecn, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_TX_INTERVAL: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_tx_interval), + 0, encode_cc_cmd_cnp_tx_interval, NULL); + case XSC_CMD_OP_IOCTL_SET_EVT_PERIOD_RSTTIME: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, + sizeof(struct xsc_cc_cmd_evt_rsttime), 0, encode_cc_cmd_evt_rsttime, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_DSCP: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_dscp), + 0, encode_cc_cmd_cnp_dscp, NULL); + case XSC_CMD_OP_IOCTL_SET_CNP_PCP: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_cnp_pcp), + 0, encode_cc_cmd_cnp_pcp, NULL); + case XSC_CMD_OP_IOCTL_SET_EVT_PERIOD_ALPHA: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_evt_period_alpha), + 0, encode_cc_cmd_evt_period_alpha, NULL); + case XSC_CMD_OP_IOCTL_SET_CLAMP_TGT_RATE: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_clamp_tgt_rate), + 0, encode_cc_cmd_clamp_tgt_rate, NULL); + case XSC_CMD_OP_IOCTL_GET_CC_CFG: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_get_cfg), + sizeof(struct xsc_cc_cmd_get_cfg), encode_cc_get_cfg, decode_cc_get_cfg); + case XSC_CMD_OP_IOCTL_GET_CC_STAT: + return _rdma_ctrl_ioctl_cc( + xdev, user_hdr, &hdr, sizeof(struct xsc_cc_cmd_get_stat), + sizeof(struct xsc_cc_cmd_stat), encode_cc_get_stat, decode_cc_get_stat); + default: + return -EINVAL; + } + + in = kvzalloc(hdr.attr.length, GFP_KERNEL); + if (!in) + return -ENOMEM; + out = kvzalloc(hdr.attr.length, GFP_KERNEL); + if (!out) { + kfree(in); + return -ENOMEM; + } + + err = copy_from_user(in, user_hdr->attr.data, hdr.attr.length); + if (err) { + err = -EFAULT; + goto err_exit; + } + + xsc_cmd_exec(xdev, in, hdr.attr.length, out, hdr.attr.length); + + if (copy_to_user((void *)user_hdr, &hdr, sizeof(hdr))) + err = -EFAULT; + if (copy_to_user((void *)user_hdr->attr.data, out, hdr.attr.length)) + err = -EFAULT; +err_exit: + kfree(in); + kfree(out); + return err; +} + +static void _rdma_ctrl_reg_cb(struct xsc_core_device *xdev, unsigned int cmd, + struct xsc_ioctl_hdr __user *user_hdr, void *data) +{ + int err; + + switch (cmd) { + case XSC_IOCTL_CMDQ: + err = _rdma_ctrl_ioctl_cmdq(xdev, user_hdr); + break; + case XSC_IOCTL_DRV_GET: + case XSC_IOCTL_DRV_SET: + // TODO refactor to split driver get and set + err = _rdma_ctrl_ioctl_getinfo(xdev, user_hdr); + break; + default: + err = -EFAULT; + break; + } +} + +static void _rdma_ctrl_reg_fini(void) +{ + xsc_port_ctrl_cb_dereg(XSC_RDMA_CTRL_NAME); +} + +static int _rdma_ctrl_reg_init(void) +{ + int ret; + + ret = xsc_port_ctrl_cb_reg(XSC_RDMA_CTRL_NAME, _rdma_ctrl_reg_cb, NULL); + if (ret != 0) + pr_err("failed to register port control node for %s\n", + XSC_RDMA_CTRL_NAME); + + return ret; +} + +void xsc_rdma_ctrl_fini(void) +{ + _rdma_ctrl_reg_fini(); +} + +int xsc_rdma_ctrl_init(void) +{ + return _rdma_ctrl_reg_init(); +} + diff --git a/drivers/infiniband/hw/xsc/xsc_rdma_ctrl.h b/drivers/infiniband/hw/xsc/xsc_rdma_ctrl.h new file mode 100644 index 0000000000000000000000000000000000000000..5049377101f9a34c3132ad4b913cbd0b3fa8ec75 --- /dev/null +++ b/drivers/infiniband/hw/xsc/xsc_rdma_ctrl.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_RDMA_CTRL_H +#define XSC_RDMA_CTRL_H + +void xsc_rdma_ctrl_fini(void); +int xsc_rdma_ctrl_init(void); + +#endif diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig index 48c9601adf2e43e48ee2327730a97210e900ad07..d7fe74daf197dc400edee38fa97303fa7e803e36 100644 --- a/drivers/net/ethernet/Kconfig +++ b/drivers/net/ethernet/Kconfig @@ -185,5 +185,6 @@ source "drivers/net/ethernet/wiznet/Kconfig" source "drivers/net/ethernet/xilinx/Kconfig" source "drivers/net/ethernet/xircom/Kconfig" source "drivers/net/ethernet/nebula-matrix/Kconfig" +source "drivers/net/ethernet/yunsilicon/Kconfig" endif # ETHERNET diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile index d637fd528a145f413fb5fca9b4be3fd5e7920098..25dc570ec43a4b93cf2462880d428d5f1f25798d 100644 --- a/drivers/net/ethernet/Makefile +++ b/drivers/net/ethernet/Makefile @@ -98,3 +98,4 @@ obj-$(CONFIG_NET_VENDOR_SYNOPSYS) += synopsys/ obj-$(CONFIG_NET_VENDOR_PENSANDO) += pensando/ obj-$(CONFIG_NET_VENDOR_NETSWIFT) += netswift/ obj-$(CONFIG_NET_VENDOR_NEBULA_MATRIX) += nebula-matrix/ +obj-$(CONFIG_NET_VENDOR_YUNSILICON) += yunsilicon/ diff --git a/drivers/net/ethernet/yunsilicon/Kconfig b/drivers/net/ethernet/yunsilicon/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..a387a8ddeba4b2c5274d131aa6080fa7ba586a66 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. +# All rights reserved. +# Yunsilicon driver configuration +# + +config NET_VENDOR_YUNSILICON + bool "Yunsilicon devices" + default y + depends on PCI || NET + depends on ARM64 || X86_64 + help + If you have a network (Ethernet or RDMA) device belonging to this + class, say Y. + + Note that the answer to this question doesn't directly affect the + kernel: saying N will just cause the configurator to skip all + the questions about Yunsilicon devices. If you say Y, you will be + asked for your specific card in the following questions. + +if NET_VENDOR_YUNSILICON + +source "drivers/net/ethernet/yunsilicon/xsc/net/Kconfig" +source "drivers/net/ethernet/yunsilicon/xsc/pci/Kconfig" + +endif # NET_VENDOR_YUNSILICON diff --git a/drivers/net/ethernet/yunsilicon/Makefile b/drivers/net/ethernet/yunsilicon/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0c603d2bf207e4e589ce7ed68261aff7321879d6 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. +# All rights reserved. +# Makefile for the Yunsilicon device drivers. +# + +obj-$(CONFIG_YUNSILICON_XSC_ETH) += xsc/net/ +obj-$(CONFIG_YUNSILICON_XSC_PCI) += xsc/pci/ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/andes/chip_scale_defines.h b/drivers/net/ethernet/yunsilicon/xsc/common/andes/chip_scale_defines.h new file mode 100644 index 0000000000000000000000000000000000000000..76885db35120f13034e376e0472eb319bc523295 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/andes/chip_scale_defines.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _CHIP_SCALE_DEFINES_H_ +#define _CHIP_SCALE_DEFINES_H_ + +#define MAIN_CLK_FREQ (100 * 1000 * 1000) +#define NIF_PORT_NUM 2 +#define PCIE_PORT_NUM 1 +#define FUNC_ID_NUM 1026 +#define MSIX_VEC_NUM 4096 +#define PIO_TLPQ_NUM 2 + +#define PCIE0_PF_NUM 2 +#define PCIE0_PF0_VF_NUM 512 +#define PCIE0_PF1_VF_NUM 512 + +#define PCIE1_PF_NUM 0 +#define QP_NUM_MAX 32768 +#define RAW_QP_NUM_MAX 8192 +#define TSO_QP_NUM_MAX 1024 +#define CQ_NUM_MAX 32768 +#define SQ_SIZE_MAX 1024 +#define RQ_SIZE_MAX 1024 +#define CQ_SIZE_MAX 32768 +#define MPT_SIZE 32768 +#define MTT_SIZE 65536 +#define GRP_NUM_MAX 1024 +#define CLUSTER_NUM_MAX 1 + +#define PP_PCT_DEPTH 512 +#define PP_PCT_KEY_WIDTH 352 +#define PP_PCT_AD_WIDTH 42 + +#define PP_WCT_DEPTH 64 +#define PP_WCT_SHORT_KEY_WIDTH 240 +#define PP_WCT_LONG_KEY_WIDTH 480 +#define PP_WCT_AD_WIDTH 19 + +#define PP_IACL_DEPTH 16 +#define PP_IACL_KEY_WIDTH 463 +#define PP_IACL_AD_WIDTH 38 + +#define PP_TUNNEL_ENCAP_TBL_DEPTH 10240 +#define PP_TUNNEL_ENCAP_TBL_WIDTH 45 + +#define PP_MIRROR_TBL_DEPTH 256 +#define PP_MIRROR_TBL_WIDTH 106 + +#define PP_IPAT_DEPTH 2048 +#define PP_IPAT_WIDTH 119 + +#define PP_EPAT_DEPTH 2048 +#define PP_EPAT_WIDTH 160 + +#define PP_ONCHIP_FT_DEPTH 16384 +#define PP_ONCHIP_FT_WIDTH 439 + +#define PP_ONCHIP_FAT_DEPTH (16384 + 128) +#define PP_ONCHIP_FAT_WIDTH 326 + +#define PP_ONCHIP_CT_DEPTH (16384 + 128) +#define PP_ONCHIP_CT_WIDTH 156 + +#define PP_ONCHIP_VER_TBL_DEPTH 2048 +#define PP_ONCHIP_VER_TBL_WIDTH 13 + +#define PP_BOMT_DEPTH 1040 +#define PP_BOMT_WIDTH 12 + +#define PP_PST_DEPTH 2048 +#define PP_PST_WIDTH 1 + +#define PRI_NUM 8 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/andes/chip_version.h b/drivers/net/ethernet/yunsilicon/xsc/common/andes/chip_version.h new file mode 100644 index 0000000000000000000000000000000000000000..8505c6a687964eea7985b8723a293cd204566a65 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/andes/chip_version.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _CHIP_VERSION_H_ +#define _CHIP_VERSION_H_ + +#define CHIP_VERSION_H 0x100 +#define CHIP_VERSION_M 0x47a9bdfd +#define CHIP_VERSION_L 0xb13 +#define CHIP_HOTFIX_NUM 0x30 +#define CHIP_FEATURE_FLAG 0b1110010101 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/andes/clsf_dma_csr_defines.h b/drivers/net/ethernet/yunsilicon/xsc/common/andes/clsf_dma_csr_defines.h new file mode 100644 index 0000000000000000000000000000000000000000..90f0dc17a9381ea3bf80424a0dfb6e5842a78d24 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/andes/clsf_dma_csr_defines.h @@ -0,0 +1,751 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _CLSF_DMA_CSR_DEFINES_H_ +#define _CLSF_DMA_CSR_DEFINES_H_ + +#define CLSF_DMA_SOFT_RESET_REG_ADDR 0xa6010000 +#define CLSF_DMA_SOFT_RESET_REG_WIDTH 32 +#define CLSF_DMA_SOFT_RESET_REG_LENGTH 32 +#define CLSF_DMA_SOFT_RESET_REG_SOFT_RESET_MASK 0x1 +#define CLSF_DMA_SOFT_RESET_REG_SOFT_RESET_SHIFT 0 +#define CLSF_DMA_SOFT_RESET_REG_SOFT_RESET_WIDTH 1 +#define CLSF_DMA_SOFT_RESET_REG_SOFT_RESET_MAX_VAL 0x1 +#define CLSF_DMA_SOFT_RESET_REG_SOFT_RESET_MIN_VAL 0x0 + +#define CLSF_DMA_SCRATCH_PAD_REG_ADDR 0xa601000c +#define CLSF_DMA_SCRATCH_PAD_REG_WIDTH 32 +#define CLSF_DMA_SCRATCH_PAD_REG_LENGTH 32 +#define CLSF_DMA_SCRATCH_PAD_REG_SCRATCH_PAD_MASK 0xffffffff +#define CLSF_DMA_SCRATCH_PAD_REG_SCRATCH_PAD_SHIFT 0 +#define CLSF_DMA_SCRATCH_PAD_REG_SCRATCH_PAD_WIDTH 32 +#define CLSF_DMA_SCRATCH_PAD_REG_SCRATCH_PAD_MAX_VAL 0xffffffff +#define CLSF_DMA_SCRATCH_PAD_REG_SCRATCH_PAD_MIN_VAL 0x0 + +#define CLSF_DMA_DMA_RD_TABLE_ID_REG_ADDR 0xa6010010 +#define CLSF_DMA_DMA_RD_TABLE_ID_REG_WIDTH 32 +#define CLSF_DMA_DMA_RD_TABLE_ID_REG_LENGTH 32 +#define CLSF_DMA_DMA_RD_TABLE_ID_REG_DMA_RD_TBL_ID_MASK 0x7f +#define CLSF_DMA_DMA_RD_TABLE_ID_REG_DMA_RD_TBL_ID_SHIFT 0 +#define CLSF_DMA_DMA_RD_TABLE_ID_REG_DMA_RD_TBL_ID_WIDTH 7 +#define CLSF_DMA_DMA_RD_TABLE_ID_REG_DMA_RD_TBL_ID_MAX_VAL 0x7f +#define CLSF_DMA_DMA_RD_TABLE_ID_REG_DMA_RD_TBL_ID_MIN_VAL 0x0 + +#define CLSF_DMA_DMA_RD_ADDR_REG_ADDR 0xa6010014 +#define CLSF_DMA_DMA_RD_ADDR_REG_WIDTH 32 +#define CLSF_DMA_DMA_RD_ADDR_REG_LENGTH 32 +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_START_ADDR_MASK 0xffff +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_START_ADDR_SHIFT 0 +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_START_ADDR_WIDTH 16 +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_START_ADDR_MAX_VAL 0xffff +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_START_ADDR_MIN_VAL 0x0 +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_BURST_NUM_MASK 0xffff0000 +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_BURST_NUM_SHIFT 16 +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_BURST_NUM_WIDTH 16 +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_BURST_NUM_MAX_VAL 0xffff +#define CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_BURST_NUM_MIN_VAL 0x0 + +#define CLSF_DMA_INDRW_RD_START_REG_ADDR 0xa6010018 +#define CLSF_DMA_INDRW_RD_START_REG_WIDTH 32 +#define CLSF_DMA_INDRW_RD_START_REG_LENGTH 32 +#define CLSF_DMA_INDRW_RD_START_REG_INDRW_START_MASK 0x1 +#define CLSF_DMA_INDRW_RD_START_REG_INDRW_START_SHIFT 0 +#define CLSF_DMA_INDRW_RD_START_REG_INDRW_START_WIDTH 1 +#define CLSF_DMA_INDRW_RD_START_REG_INDRW_START_MAX_VAL 0x1 +#define CLSF_DMA_INDRW_RD_START_REG_INDRW_START_MIN_VAL 0x0 + +#define CLSF_DMA_DMA_UL_TIMEOUT_REG_ADDR 0xa6010030 +#define CLSF_DMA_DMA_UL_TIMEOUT_REG_WIDTH 32 +#define CLSF_DMA_DMA_UL_TIMEOUT_REG_LENGTH 32 +#define CLSF_DMA_DMA_UL_TIMEOUT_REG_DMA_UL_TIMEOUT_MASK 0x1 +#define CLSF_DMA_DMA_UL_TIMEOUT_REG_DMA_UL_TIMEOUT_SHIFT 0 +#define CLSF_DMA_DMA_UL_TIMEOUT_REG_DMA_UL_TIMEOUT_WIDTH 1 + +#define CLSF_DMA_DMA_UL_TIMEOUT_INT_MASK_REG_ADDR 0xa6010034 +#define CLSF_DMA_DMA_UL_TIMEOUT_INT_MASK_REG_WIDTH 32 +#define CLSF_DMA_DMA_UL_TIMEOUT_INT_MASK_REG_LENGTH 32 +#define CLSF_DMA_DMA_UL_TIMEOUT_INT_MASK_REG_DMA_UL_TIMEOUT_INT_MASK_MASK 0x1 +#define CLSF_DMA_DMA_UL_TIMEOUT_INT_MASK_REG_DMA_UL_TIMEOUT_INT_MASK_SHIFT 0 +#define CLSF_DMA_DMA_UL_TIMEOUT_INT_MASK_REG_DMA_UL_TIMEOUT_INT_MASK_WIDTH 1 +#define CLSF_DMA_DMA_UL_TIMEOUT_INT_MASK_REG_DMA_UL_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_TIMEOUT_INT_MASK_REG_DMA_UL_TIMEOUT_INT_MASK_MIN_VAL 0x0 + +#define CLSF_DMA_DMA_UL_BUSY_REG_ADDR 0xa6010038 +#define CLSF_DMA_DMA_UL_BUSY_REG_WIDTH 32 +#define CLSF_DMA_DMA_UL_BUSY_REG_LENGTH 32 +#define CLSF_DMA_DMA_UL_BUSY_REG_DMA_UL_BUSY_MASK 0x1 +#define CLSF_DMA_DMA_UL_BUSY_REG_DMA_UL_BUSY_SHIFT 0 +#define CLSF_DMA_DMA_UL_BUSY_REG_DMA_UL_BUSY_WIDTH 1 + +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ADDR 0xa6010040 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_WIDTH 512 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_LENGTH 512 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_1_MASK 0xffffffff +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_1_SHIFT 0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_1_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_2_MASK 0xffffffff00000000 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_2_SHIFT 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_2_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_3_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_3_SHIFT 64 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_3_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_4_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_4_SHIFT 96 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_4_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_5_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_5_SHIFT 128 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_5_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_6_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_6_SHIFT 160 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_6_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_7_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_7_SHIFT 192 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_7_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_8_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_8_SHIFT 224 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_8_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_9_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_9_SHIFT 256 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_9_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_10_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_10_SHIFT 288 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_10_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_11_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_11_SHIFT 320 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_11_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_12_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_12_SHIFT 352 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_12_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_13_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_13_SHIFT 384 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_13_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_14_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_14_SHIFT 416 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_14_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_15_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_15_SHIFT 448 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_15_WIDTH 32 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_16_MASK 0x0 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_16_SHIFT 480 +#define CLSF_DMA_EM_ERR_CODE_STS_REG_ERR_CODE_16_WIDTH 32 + +#define CLSF_DMA_DMA_DL_SUCCESS_REG_ADDR 0xa6010080 +#define CLSF_DMA_DMA_DL_SUCCESS_REG_WIDTH 128 +#define CLSF_DMA_DMA_DL_SUCCESS_REG_LENGTH 128 +#define CLSF_DMA_DMA_DL_SUCCESS_REG_SUCCESS_MASK 0xffffffffffffffff +#define CLSF_DMA_DMA_DL_SUCCESS_REG_SUCCESS_SHIFT 0 +#define CLSF_DMA_DMA_DL_SUCCESS_REG_SUCCESS_WIDTH 128 + +#define CLSF_DMA_DMA_DL_DONE_REG_ADDR 0xa6010090 +#define CLSF_DMA_DMA_DL_DONE_REG_WIDTH 32 +#define CLSF_DMA_DMA_DL_DONE_REG_LENGTH 32 +#define CLSF_DMA_DMA_DL_DONE_REG_DL_DONE_MASK 0x1 +#define CLSF_DMA_DMA_DL_DONE_REG_DL_DONE_SHIFT 0 +#define CLSF_DMA_DMA_DL_DONE_REG_DL_DONE_WIDTH 1 +#define CLSF_DMA_DMA_DL_DONE_REG_DL_STS_MASK 0x2 +#define CLSF_DMA_DMA_DL_DONE_REG_DL_STS_SHIFT 1 +#define CLSF_DMA_DMA_DL_DONE_REG_DL_STS_WIDTH 1 + +#define CLSF_DMA_ERR_CODE_CLR_REG_ADDR 0xa6010094 +#define CLSF_DMA_ERR_CODE_CLR_REG_WIDTH 32 +#define CLSF_DMA_ERR_CODE_CLR_REG_LENGTH 32 +#define CLSF_DMA_ERR_CODE_CLR_REG_ERR_CODE_CLR_MASK 0x1 +#define CLSF_DMA_ERR_CODE_CLR_REG_ERR_CODE_CLR_SHIFT 0 +#define CLSF_DMA_ERR_CODE_CLR_REG_ERR_CODE_CLR_WIDTH 1 +#define CLSF_DMA_ERR_CODE_CLR_REG_ERR_CODE_CLR_MAX_VAL 0x1 +#define CLSF_DMA_ERR_CODE_CLR_REG_ERR_CODE_CLR_MIN_VAL 0x0 + +#define CLSF_DMA_UL_CFG_ERR_INT_REG_ADDR 0xa6010098 +#define CLSF_DMA_UL_CFG_ERR_INT_REG_WIDTH 32 +#define CLSF_DMA_UL_CFG_ERR_INT_REG_LENGTH 32 +#define CLSF_DMA_UL_CFG_ERR_INT_REG_UL_CFG_ERR_MASK 0x1 +#define CLSF_DMA_UL_CFG_ERR_INT_REG_UL_CFG_ERR_SHIFT 0 +#define CLSF_DMA_UL_CFG_ERR_INT_REG_UL_CFG_ERR_WIDTH 1 + +#define CLSF_DMA_UL_CFG_ERR_INT_MASK_REG_ADDR 0xa601009c +#define CLSF_DMA_UL_CFG_ERR_INT_MASK_REG_WIDTH 32 +#define CLSF_DMA_UL_CFG_ERR_INT_MASK_REG_LENGTH 32 +#define CLSF_DMA_UL_CFG_ERR_INT_MASK_REG_UL_CFG_ERR_INT_MASK_MASK 0x1 +#define CLSF_DMA_UL_CFG_ERR_INT_MASK_REG_UL_CFG_ERR_INT_MASK_SHIFT 0 +#define CLSF_DMA_UL_CFG_ERR_INT_MASK_REG_UL_CFG_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_CFG_ERR_INT_MASK_REG_UL_CFG_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_CFG_ERR_INT_MASK_REG_UL_CFG_ERR_INT_MASK_MIN_VAL 0x0 + +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_ADDR 0xa60100a0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_WIDTH 32 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_LENGTH 32 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_EM_MONITOR_EOF_ERR_MASK 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_EM_MONITOR_EOF_ERR_SHIFT 0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_EM_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_EM_MONITOR_SOF_ERR_MASK 0x2 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_EM_MONITOR_SOF_ERR_SHIFT 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_EM_MONITOR_SOF_ERR_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_TCAM_MONITOR_EOF_ERR_MASK 0x4 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_TCAM_MONITOR_EOF_ERR_SHIFT 2 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_TCAM_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_TCAM_MONITOR_SOF_ERR_MASK 0x8 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_TCAM_MONITOR_SOF_ERR_SHIFT 3 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_TCAM_MONITOR_SOF_ERR_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_DIR_MONITOR_EOF_ERR_MASK 0x10 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_DIR_MONITOR_EOF_ERR_SHIFT 4 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_DIR_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_DIR_MONITOR_SOF_ERR_MASK 0x20 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_DIR_MONITOR_SOF_ERR_SHIFT 5 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_DIR_MONITOR_SOF_ERR_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_RWE_MONITOR_EOF_ERR_MASK 0x40 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_RWE_MONITOR_EOF_ERR_SHIFT 6 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_RWE_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_RWE_MONITOR_SOF_ERR_MASK 0x80 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_RWE_MONITOR_SOF_ERR_SHIFT 7 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_REG_DL_RWE_MONITOR_SOF_ERR_WIDTH 1 + +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_ADDR 0xa60100a4 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_WIDTH 32 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_LENGTH 32 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_EOF_ERR_INT_MASK_MASK 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_EOF_ERR_INT_MASK_SHIFT 0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_SOF_ERR_INT_MASK_MASK 0x2 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_SOF_ERR_INT_MASK_SHIFT 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_EM_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_EOF_ERR_INT_MASK_MASK 0x4 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_EOF_ERR_INT_MASK_SHIFT 2 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_SOF_ERR_INT_MASK_MASK 0x8 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_SOF_ERR_INT_MASK_SHIFT 3 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_TCAM_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_EOF_ERR_INT_MASK_MASK 0x10 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_EOF_ERR_INT_MASK_SHIFT 4 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_SOF_ERR_INT_MASK_MASK 0x20 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_SOF_ERR_INT_MASK_SHIFT 5 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_DIR_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_EOF_ERR_INT_MASK_MASK 0x40 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_EOF_ERR_INT_MASK_SHIFT 6 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_SOF_ERR_INT_MASK_MASK 0x80 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_SOF_ERR_INT_MASK_SHIFT 7 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_DL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_DL_RWE_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 + +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_ADDR 0xa60100a8 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_WIDTH 32 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_LENGTH 32 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_EM_MONITOR_EOF_ERR_MASK 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_EM_MONITOR_EOF_ERR_SHIFT 0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_EM_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_TCAM_MONITOR_EOF_ERR_MASK 0x2 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_TCAM_MONITOR_EOF_ERR_SHIFT 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_TCAM_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_DIR_MONITOR_EOF_ERR_MASK 0x4 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_DIR_MONITOR_EOF_ERR_SHIFT 2 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_DIR_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_RWE_MONITOR_EOF_ERR_MASK 0x8 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_RWE_MONITOR_EOF_ERR_SHIFT 3 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_RWE_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_STAT_MONITOR_EOF_ERR_MASK 0x10 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_STAT_MONITOR_EOF_ERR_SHIFT 4 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_STAT_MONITOR_EOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_EM_MONITOR_SOF_ERR_MASK 0x20 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_EM_MONITOR_SOF_ERR_SHIFT 5 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_EM_MONITOR_SOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_TCAM_MONITOR_SOF_ERR_MASK 0x40 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_TCAM_MONITOR_SOF_ERR_SHIFT 6 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_TCAM_MONITOR_SOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_DIR_MONITOR_SOF_ERR_MASK 0x80 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_DIR_MONITOR_SOF_ERR_SHIFT 7 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_DIR_MONITOR_SOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_RWE_MONITOR_SOF_ERR_MASK 0x100 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_RWE_MONITOR_SOF_ERR_SHIFT 8 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_RWE_MONITOR_SOF_ERR_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_STAT_MONITOR_SOF_ERR_MASK 0x200 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_STAT_MONITOR_SOF_ERR_SHIFT 9 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_REG_UL_STAT_MONITOR_SOF_ERR_WIDTH 1 + +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_ADDR 0xa60100ac +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_WIDTH 32 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_LENGTH 32 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_EOF_ERR_INT_MASK_MASK 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_EOF_ERR_INT_MASK_SHIFT 0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_EOF_ERR_INT_MASK_MASK 0x2 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_EOF_ERR_INT_MASK_SHIFT 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_EOF_ERR_INT_MASK_MASK 0x4 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_EOF_ERR_INT_MASK_SHIFT 2 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_EOF_ERR_INT_MASK_MASK 0x8 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_EOF_ERR_INT_MASK_SHIFT 3 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_EOF_ERR_INT_MASK_MASK 0x10 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_EOF_ERR_INT_MASK_SHIFT 4 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_EOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_EOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_EOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_SOF_ERR_INT_MASK_MASK 0x20 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_SOF_ERR_INT_MASK_SHIFT 5 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_EM_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_SOF_ERR_INT_MASK_MASK 0x40 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_SOF_ERR_INT_MASK_SHIFT 6 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_TCAM_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_SOF_ERR_INT_MASK_MASK 0x80 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_SOF_ERR_INT_MASK_SHIFT 7 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_DIR_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_SOF_ERR_INT_MASK_MASK 0x100 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_SOF_ERR_INT_MASK_SHIFT 8 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_RWE_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_SOF_ERR_INT_MASK_MASK 0x200 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_SOF_ERR_INT_MASK_SHIFT 9 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_SOF_ERR_INT_MASK_WIDTH 1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_SOF_ERR_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_UL_MONITOR_SOF_EOF_ERR_INT_MASK_REG_UL_STAT_MONITOR_SOF_ERR_INT_MASK_MIN_VAL 0x0 + +#define CLSF_DMA_FIFO_AFUL_TH_REG_ADDR 0xa60100b0 +#define CLSF_DMA_FIFO_AFUL_TH_REG_WIDTH 32 +#define CLSF_DMA_FIFO_AFUL_TH_REG_LENGTH 32 +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_DL_FIFO_AFUL_TH_MASK 0x3f +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_DL_FIFO_AFUL_TH_SHIFT 0 +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_DL_FIFO_AFUL_TH_WIDTH 6 +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_DL_FIFO_AFUL_TH_MAX_VAL 0x3f +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_DL_FIFO_AFUL_TH_MIN_VAL 0x0 +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_UL_FIFO_AFUL_TH_MASK 0xfc0 +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_UL_FIFO_AFUL_TH_SHIFT 6 +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_UL_FIFO_AFUL_TH_WIDTH 6 +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_UL_FIFO_AFUL_TH_MAX_VAL 0x3f +#define CLSF_DMA_FIFO_AFUL_TH_REG_DMA_UL_FIFO_AFUL_TH_MIN_VAL 0x0 + +#define CLSF_DMA_DMA_DL_WMH_REG_ADDR 0xa60100b8 +#define CLSF_DMA_DMA_DL_WMH_REG_WIDTH 32 +#define CLSF_DMA_DMA_DL_WMH_REG_LENGTH 32 +#define CLSF_DMA_DMA_DL_WMH_REG_DMA_DL_FIFO_USED_CNT_WMH_MASK 0x3f +#define CLSF_DMA_DMA_DL_WMH_REG_DMA_DL_FIFO_USED_CNT_WMH_SHIFT 0 +#define CLSF_DMA_DMA_DL_WMH_REG_DMA_DL_FIFO_USED_CNT_WMH_WIDTH 6 + +#define CLSF_DMA_DMA_UL_WMH_REG_ADDR 0xa60100bc +#define CLSF_DMA_DMA_UL_WMH_REG_WIDTH 32 +#define CLSF_DMA_DMA_UL_WMH_REG_LENGTH 32 +#define CLSF_DMA_DMA_UL_WMH_REG_DMA_UL_FIFO_USED_CNT_WMH_MASK 0x3f +#define CLSF_DMA_DMA_UL_WMH_REG_DMA_UL_FIFO_USED_CNT_WMH_SHIFT 0 +#define CLSF_DMA_DMA_UL_WMH_REG_DMA_UL_FIFO_USED_CNT_WMH_WIDTH 6 + +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_ADDR 0xa60100c0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_WIDTH 32 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_LENGTH 32 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC0_MASK 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC0_SHIFT 0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC0_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC0_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC0_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC1_MASK 0x2 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC1_SHIFT 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC1_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC1_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_EM_RSP_FORCE_FC1_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC0_MASK 0x4 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC0_SHIFT 2 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC0_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC0_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC0_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC1_MASK 0x8 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC1_SHIFT 3 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC1_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC1_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_TCAM_RSP_FORCE_FC1_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC0_MASK 0x10 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC0_SHIFT 4 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC0_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC0_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC0_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC1_MASK 0x20 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC1_SHIFT 5 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC1_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC1_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_DIR_RSP_FORCE_FC1_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC0_MASK 0x40 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC0_SHIFT 6 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC0_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC0_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC0_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC1_MASK 0x80 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC1_SHIFT 7 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC1_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC1_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_STAT_RSP_FORCE_FC1_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC0_MASK 0x100 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC0_SHIFT 8 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC0_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC0_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC0_MIN_VAL 0x0 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC1_MASK 0x200 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC1_SHIFT 9 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC1_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC1_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_DEBUG_CFG_REG_DMA_UL_RWE_RSP_FORCE_FC1_MIN_VAL 0x0 + +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_ADDR 0xa60100c4 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_WIDTH 32 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_LENGTH 32 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC0_MASK 0x1 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC0_SHIFT 0 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC0_WIDTH 1 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC0_MAX_VAL 0x1 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC0_MIN_VAL 0x0 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC1_MASK 0x2 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC1_SHIFT 1 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC1_WIDTH 1 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC1_MAX_VAL 0x1 +#define CLSF_DMA_DMA_DL_FC_DEBUG_CFG_REG_DMA_DL_FORCE_FC1_MIN_VAL 0x0 + +#define CLSF_DMA_DMA_DL_FC_CNT_IN_REVS_REG_ADDR 0xa60100c8 +#define CLSF_DMA_DMA_DL_FC_CNT_IN_REVS_REG_WIDTH 32 +#define CLSF_DMA_DMA_DL_FC_CNT_IN_REVS_REG_LENGTH 32 +#define CLSF_DMA_DMA_DL_FC_CNT_IN_REVS_REG_DMA_DL_FC_CNT_IN_REVS_MASK 0x1 +#define CLSF_DMA_DMA_DL_FC_CNT_IN_REVS_REG_DMA_DL_FC_CNT_IN_REVS_SHIFT 0 +#define CLSF_DMA_DMA_DL_FC_CNT_IN_REVS_REG_DMA_DL_FC_CNT_IN_REVS_WIDTH 1 +#define CLSF_DMA_DMA_DL_FC_CNT_IN_REVS_REG_DMA_DL_FC_CNT_IN_REVS_MAX_VAL 0x1 +#define CLSF_DMA_DMA_DL_FC_CNT_IN_REVS_REG_DMA_DL_FC_CNT_IN_REVS_MIN_VAL 0x0 + +#define CLSF_DMA_DMA_UL_RSP_FC_CNT_IN_REVS_REG_ADDR 0xa60100cc +#define CLSF_DMA_DMA_UL_RSP_FC_CNT_IN_REVS_REG_WIDTH 32 +#define CLSF_DMA_DMA_UL_RSP_FC_CNT_IN_REVS_REG_LENGTH 32 +#define CLSF_DMA_DMA_UL_RSP_FC_CNT_IN_REVS_REG_DMA_UL_RSP_FC_CNT_IN_REVS_MASK 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_CNT_IN_REVS_REG_DMA_UL_RSP_FC_CNT_IN_REVS_SHIFT 0 +#define CLSF_DMA_DMA_UL_RSP_FC_CNT_IN_REVS_REG_DMA_UL_RSP_FC_CNT_IN_REVS_WIDTH 1 +#define CLSF_DMA_DMA_UL_RSP_FC_CNT_IN_REVS_REG_DMA_UL_RSP_FC_CNT_IN_REVS_MAX_VAL 0x1 +#define CLSF_DMA_DMA_UL_RSP_FC_CNT_IN_REVS_REG_DMA_UL_RSP_FC_CNT_IN_REVS_MIN_VAL 0x0 + +#define CLSF_DMA_HIF2IPP_FC_CNT_IN_REVS_REG_ADDR 0xa60100d0 +#define CLSF_DMA_HIF2IPP_FC_CNT_IN_REVS_REG_WIDTH 32 +#define CLSF_DMA_HIF2IPP_FC_CNT_IN_REVS_REG_LENGTH 32 +#define CLSF_DMA_HIF2IPP_FC_CNT_IN_REVS_REG_HIF2IPP_UL_FC_CNT_IN_REVS_MASK 0x1 +#define CLSF_DMA_HIF2IPP_FC_CNT_IN_REVS_REG_HIF2IPP_UL_FC_CNT_IN_REVS_SHIFT 0 +#define CLSF_DMA_HIF2IPP_FC_CNT_IN_REVS_REG_HIF2IPP_UL_FC_CNT_IN_REVS_WIDTH 1 +#define CLSF_DMA_HIF2IPP_FC_CNT_IN_REVS_REG_HIF2IPP_UL_FC_CNT_IN_REVS_MAX_VAL 0x1 +#define CLSF_DMA_HIF2IPP_FC_CNT_IN_REVS_REG_HIF2IPP_UL_FC_CNT_IN_REVS_MIN_VAL 0x0 + +#define CLSF_DMA_OVFL_UNFL_INT_REG_ADDR 0xa60100d8 +#define CLSF_DMA_OVFL_UNFL_INT_REG_WIDTH 32 +#define CLSF_DMA_OVFL_UNFL_INT_REG_LENGTH 32 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_DL_FIFO_OVFL_MASK 0x1 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_DL_FIFO_OVFL_SHIFT 0 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_DL_FIFO_OVFL_WIDTH 1 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_DL_FIFO_UNFL_MASK 0x2 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_DL_FIFO_UNFL_SHIFT 1 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_DL_FIFO_UNFL_WIDTH 1 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_UL_FIFO_OVFL_MASK 0x4 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_UL_FIFO_OVFL_SHIFT 2 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_UL_FIFO_OVFL_WIDTH 1 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_UL_FIFO_UNFL_MASK 0x8 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_UL_FIFO_UNFL_SHIFT 3 +#define CLSF_DMA_OVFL_UNFL_INT_REG_DMA_UL_FIFO_UNFL_WIDTH 1 + +#define CLSF_DMA_PP_DMA_FC_CNT_REG_ADDR 0xa60100e0 +#define CLSF_DMA_PP_DMA_FC_CNT_REG_WIDTH 32 +#define CLSF_DMA_PP_DMA_FC_CNT_REG_LENGTH 32 +#define CLSF_DMA_PP_DMA_FC_CNT_REG_TX_IPP2HIF_DL_FC_CNT_MASK 0xffff +#define CLSF_DMA_PP_DMA_FC_CNT_REG_TX_IPP2HIF_DL_FC_CNT_SHIFT 0 +#define CLSF_DMA_PP_DMA_FC_CNT_REG_TX_IPP2HIF_DL_FC_CNT_WIDTH 16 +#define CLSF_DMA_PP_DMA_FC_CNT_REG_RX_HIF2IPP_UL_FC_CNT_MASK 0xffff0000 +#define CLSF_DMA_PP_DMA_FC_CNT_REG_RX_HIF2IPP_UL_FC_CNT_SHIFT 16 +#define CLSF_DMA_PP_DMA_FC_CNT_REG_RX_HIF2IPP_UL_FC_CNT_WIDTH 16 + +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_ADDR 0xa60100f0 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_WIDTH 128 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_LENGTH 96 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_EM_UL_RSP_FC_CNT_MASK 0xffff +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_EM_UL_RSP_FC_CNT_SHIFT 0 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_EM_UL_RSP_FC_CNT_WIDTH 16 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_TCAM_UL_RSP_FC_CNT_MASK 0xffff0000 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_TCAM_UL_RSP_FC_CNT_SHIFT 16 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_TCAM_UL_RSP_FC_CNT_WIDTH 16 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_DIR_UL_RSP_FC_CNT_MASK 0xffff00000000 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_DIR_UL_RSP_FC_CNT_SHIFT 32 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_DIR_UL_RSP_FC_CNT_WIDTH 16 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_STAT_UL_RSP_FC_CNT_MASK 0xffff000000000000 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_STAT_UL_RSP_FC_CNT_SHIFT 48 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_STAT_UL_RSP_FC_CNT_WIDTH 16 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_RWE_UL_RSP_FC_CNT_MASK 0x0 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_RWE_UL_RSP_FC_CNT_SHIFT 64 +#define CLSF_DMA_PP_DMA_UL_RSP_CNT_REG_TX_RWE_UL_RSP_FC_CNT_WIDTH 16 + +#define CLSF_DMA_RX_DL_HIF_CNT_REG_ADDR 0xa6010100 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_WIDTH 128 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_LENGTH 96 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_BURST_VLD_CNT_MASK 0xffff +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_BURST_VLD_CNT_SHIFT 0 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_BURST_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_SOF_CNT_MASK 0xffff0000 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_SOF_CNT_SHIFT 16 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_SOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_EOF_CNT_MASK 0xffff00000000 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_EOF_CNT_SHIFT 32 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_EOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_DATA_VLD_CNT_MASK 0xffff000000000000 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_DATA_VLD_CNT_SHIFT 48 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_HIF_DATA_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_LAST_FLAG_CNT_MASK 0x0 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_LAST_FLAG_CNT_SHIFT 64 +#define CLSF_DMA_RX_DL_HIF_CNT_REG_RX_LAST_FLAG_CNT_WIDTH 16 + +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_ADDR 0xa6010120 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_WIDTH 256 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_LENGTH 224 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_FALSE_RSP_VLD_CNT_MASK 0xffff +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_FALSE_RSP_VLD_CNT_SHIFT 0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_FALSE_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_SOF_CNT_MASK 0xffff0000 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_SOF_CNT_SHIFT 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_EOF_CNT_MASK 0xffff00000000 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_EOF_CNT_SHIFT 32 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_VLD_CNT_MASK 0xffff000000000000 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_VLD_CNT_SHIFT 48 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_EM_REQ_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_SOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_SOF_CNT_SHIFT 64 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_EOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_EOF_CNT_SHIFT 80 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_VLD_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_VLD_CNT_SHIFT 96 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_TCAM_REQ_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_SOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_SOF_CNT_SHIFT 112 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_EOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_EOF_CNT_SHIFT 128 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_VLD_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_VLD_CNT_SHIFT 144 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_DIR_REQ_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_SOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_SOF_CNT_SHIFT 160 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_EOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_EOF_CNT_SHIFT 176 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_VLD_CNT_MASK 0x0 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_VLD_CNT_SHIFT 192 +#define CLSF_DMA_TX_DL_CLSF_CNT_REG_TX_RWE_REQ_VLD_CNT_WIDTH 16 + +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_ADDR 0xa6010140 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_WIDTH 64 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_LENGTH 64 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_EM_DL_RSP_VLD_CNT_MASK 0xffff +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_EM_DL_RSP_VLD_CNT_SHIFT 0 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_EM_DL_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_TCAM_DL_RSP_VLD_CNT_MASK 0xffff0000 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_TCAM_DL_RSP_VLD_CNT_SHIFT 16 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_TCAM_DL_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_DIR_DL_RSP_VLD_CNT_MASK 0xffff00000000 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_DIR_DL_RSP_VLD_CNT_SHIFT 32 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_DIR_DL_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_RWE_DL_RSP_VLD_CNT_MASK 0xffff000000000000 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_RWE_DL_RSP_VLD_CNT_SHIFT 48 +#define CLSF_DMA_RX_DL_CLSF_RSP_CNT_REG_RX_RWE_DL_RSP_VLD_CNT_WIDTH 16 + +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_ADDR 0xa6010160 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_WIDTH 256 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_LENGTH 256 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_VLD_CNT_MASK 0xffff +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_VLD_CNT_SHIFT 0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_SOF_CNT_MASK 0xffff0000 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_SOF_CNT_SHIFT 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_EOF_CNT_MASK 0xffff00000000 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_EOF_CNT_SHIFT 32 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_EM_UL_REQ_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_VLD_CNT_MASK 0xffff000000000000 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_VLD_CNT_SHIFT 48 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_SOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_SOF_CNT_SHIFT 64 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_EOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_EOF_CNT_SHIFT 80 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_TCAM_UL_REQ_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_VLD_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_VLD_CNT_SHIFT 96 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_SOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_SOF_CNT_SHIFT 112 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_EOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_EOF_CNT_SHIFT 128 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_DIR_UL_REQ_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_VLD_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_VLD_CNT_SHIFT 144 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_SOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_SOF_CNT_SHIFT 160 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_EOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_EOF_CNT_SHIFT 176 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_STAT_UL_REQ_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_VLD_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_VLD_CNT_SHIFT 192 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_SOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_SOF_CNT_SHIFT 208 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_EOF_CNT_MASK 0x0 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_EOF_CNT_SHIFT 224 +#define CLSF_DMA_TX_UL_CLSF_CNT_REG_TX_RWE_UL_REQ_EOF_CNT_WIDTH 16 + +#define CLSF_DMA_TX_UL_HIF_CNT_REG_ADDR 0xa6010180 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_WIDTH 64 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_LENGTH 64 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_VLD_CNT_MASK 0xffff +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_VLD_CNT_SHIFT 0 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_VLD_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_SOF_CNT_MASK 0xffff0000 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_SOF_CNT_SHIFT 16 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_SOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_EOF_CNT_MASK 0xffff00000000 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_EOF_CNT_SHIFT 32 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_UL_EOF_CNT_WIDTH 16 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_DMA_RD_DONE_CNT_MASK 0xffff000000000000 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_DMA_RD_DONE_CNT_SHIFT 48 +#define CLSF_DMA_TX_UL_HIF_CNT_REG_TX_IPP2HIF_DMA_RD_DONE_CNT_WIDTH 16 + +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_ADDR 0xa60101a0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_WIDTH 256 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_LENGTH 256 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_VLD_CNT_MASK 0xffff +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_VLD_CNT_SHIFT 0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_SOF_CNT_MASK 0xffff0000 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_SOF_CNT_SHIFT 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_SOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_EOF_CNT_MASK 0xffff00000000 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_EOF_CNT_SHIFT 32 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_EM_UL_RSP_EOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_VLD_CNT_MASK 0xffff000000000000 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_VLD_CNT_SHIFT 48 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_SOF_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_SOF_CNT_SHIFT 64 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_SOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_EOF_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_EOF_CNT_SHIFT 80 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_TCAM_UL_RSP_EOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_VLD_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_VLD_CNT_SHIFT 96 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_SOF_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_SOF_CNT_SHIFT 112 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_SOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_EOF_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_EOF_CNT_SHIFT 128 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_DIR_UL_RSP_EOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_VLD_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_VLD_CNT_SHIFT 144 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_SOF_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_SOF_CNT_SHIFT 160 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_SOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_EOF_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_EOF_CNT_SHIFT 176 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_STAT_UL_RSP_EOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_VLD_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_VLD_CNT_SHIFT 192 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_VLD_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_SOF_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_SOF_CNT_SHIFT 208 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_SOF_CNT_WIDTH 16 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_EOF_CNT_MASK 0x0 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_EOF_CNT_SHIFT 224 +#define CLSF_DMA_RX_UL_CLSF_CNT_REG_RX_RWE_UL_RSP_EOF_CNT_WIDTH 16 + +#define CLSF_DMA_RX_CFG_RD_REQ_POS_CNT_REG_ADDR 0xa60101c0 +#define CLSF_DMA_RX_CFG_RD_REQ_POS_CNT_REG_WIDTH 32 +#define CLSF_DMA_RX_CFG_RD_REQ_POS_CNT_REG_LENGTH 32 +#define CLSF_DMA_RX_CFG_RD_REQ_POS_CNT_REG_CFG_RD_REQ_POS_CNT_MASK 0xffff +#define CLSF_DMA_RX_CFG_RD_REQ_POS_CNT_REG_CFG_RD_REQ_POS_CNT_SHIFT 0 +#define CLSF_DMA_RX_CFG_RD_REQ_POS_CNT_REG_CFG_RD_REQ_POS_CNT_WIDTH 16 + +#define CLSF_DMA_NEW_EOF_CNT_REG_ADDR 0xa60101c4 +#define CLSF_DMA_NEW_EOF_CNT_REG_WIDTH 32 +#define CLSF_DMA_NEW_EOF_CNT_REG_LENGTH 32 +#define CLSF_DMA_NEW_EOF_CNT_REG_NEW_EOF_CNT_MASK 0xffff +#define CLSF_DMA_NEW_EOF_CNT_REG_NEW_EOF_CNT_SHIFT 0 +#define CLSF_DMA_NEW_EOF_CNT_REG_NEW_EOF_CNT_WIDTH 16 + +#define CLSF_DMA_RD_LAST_FLAG_CNT_REG_ADDR 0xa60101c8 +#define CLSF_DMA_RD_LAST_FLAG_CNT_REG_WIDTH 32 +#define CLSF_DMA_RD_LAST_FLAG_CNT_REG_LENGTH 32 +#define CLSF_DMA_RD_LAST_FLAG_CNT_REG_RD_LAST_FLAG_CNT_MASK 0xffff +#define CLSF_DMA_RD_LAST_FLAG_CNT_REG_RD_LAST_FLAG_CNT_SHIFT 0 +#define CLSF_DMA_RD_LAST_FLAG_CNT_REG_RD_LAST_FLAG_CNT_WIDTH 16 + +#define CLSF_DMA_TIMEOUT_INT_REG_ADDR 0xa60101cc +#define CLSF_DMA_TIMEOUT_INT_REG_WIDTH 32 +#define CLSF_DMA_TIMEOUT_INT_REG_LENGTH 32 +#define CLSF_DMA_TIMEOUT_INT_REG_DMA_DL_SUCCESS_REG_TIMEOUT_MASK 0x1 +#define CLSF_DMA_TIMEOUT_INT_REG_DMA_DL_SUCCESS_REG_TIMEOUT_SHIFT 0 +#define CLSF_DMA_TIMEOUT_INT_REG_DMA_DL_SUCCESS_REG_TIMEOUT_WIDTH 1 +#define CLSF_DMA_TIMEOUT_INT_REG_DMA_DL_DONE_REG_TIMEOUT_MASK 0x2 +#define CLSF_DMA_TIMEOUT_INT_REG_DMA_DL_DONE_REG_TIMEOUT_SHIFT 1 +#define CLSF_DMA_TIMEOUT_INT_REG_DMA_DL_DONE_REG_TIMEOUT_WIDTH 1 + +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_ADDR 0xa60101d0 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_WIDTH 32 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_LENGTH 32 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_SUCCESS_REG_TIMEOUT_INT_MASK_MASK 0x1 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_SUCCESS_REG_TIMEOUT_INT_MASK_SHIFT 0 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_SUCCESS_REG_TIMEOUT_INT_MASK_WIDTH 1 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_SUCCESS_REG_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_SUCCESS_REG_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_DONE_REG_TIMEOUT_INT_MASK_MASK 0x2 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_DONE_REG_TIMEOUT_INT_MASK_SHIFT 1 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_DONE_REG_TIMEOUT_INT_MASK_WIDTH 1 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_DONE_REG_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define CLSF_DMA_TIMEOUT_INT_MASK_REG_DMA_DL_DONE_REG_TIMEOUT_INT_MASK_MIN_VAL 0x0 + +#define CLSF_DMA_TIMEOUT_CFG_REG_ADDR 0xa60101d4 +#define CLSF_DMA_TIMEOUT_CFG_REG_WIDTH 32 +#define CLSF_DMA_TIMEOUT_CFG_REG_LENGTH 32 +#define CLSF_DMA_TIMEOUT_CFG_REG_TIMEOUT_PARA_MASK 0xffff +#define CLSF_DMA_TIMEOUT_CFG_REG_TIMEOUT_PARA_SHIFT 0 +#define CLSF_DMA_TIMEOUT_CFG_REG_TIMEOUT_PARA_WIDTH 16 +#define CLSF_DMA_TIMEOUT_CFG_REG_TIMEOUT_PARA_MAX_VAL 0xffff +#define CLSF_DMA_TIMEOUT_CFG_REG_TIMEOUT_PARA_MIN_VAL 0x0 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_cmdqm_csr_defines.h b/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_cmdqm_csr_defines.h new file mode 100644 index 0000000000000000000000000000000000000000..b54f97b0c8bdb931f05cd06ff0ff3f1002ed999e --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_cmdqm_csr_defines.h @@ -0,0 +1,978 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _HIF_CMDQM_CSR_DEFINES_H_ +#define _HIF_CMDQM_CSR_DEFINES_H_ + +#define HIF_CMDQM_SOFT_RESET_REG_ADDR 0xa1020000 +#define HIF_CMDQM_SOFT_RESET_REG_WIDTH 32 +#define HIF_CMDQM_SOFT_RESET_REG_LENGTH 32 +#define HIF_CMDQM_SOFT_RESET_REG_SOFT_RESET_MASK 0x1 +#define HIF_CMDQM_SOFT_RESET_REG_SOFT_RESET_SHIFT 0 +#define HIF_CMDQM_SOFT_RESET_REG_SOFT_RESET_WIDTH 1 +#define HIF_CMDQM_SOFT_RESET_REG_SOFT_RESET_MAX_VAL 0x1 +#define HIF_CMDQM_SOFT_RESET_REG_SOFT_RESET_MIN_VAL 0x0 + +#define HIF_CMDQM_SCRATCH_PAD_REG_ADDR 0xa102000c +#define HIF_CMDQM_SCRATCH_PAD_REG_WIDTH 32 +#define HIF_CMDQM_SCRATCH_PAD_REG_LENGTH 32 +#define HIF_CMDQM_SCRATCH_PAD_REG_SCRATCH_PAD_MASK 0xffffffff +#define HIF_CMDQM_SCRATCH_PAD_REG_SCRATCH_PAD_SHIFT 0 +#define HIF_CMDQM_SCRATCH_PAD_REG_SCRATCH_PAD_WIDTH 32 +#define HIF_CMDQM_SCRATCH_PAD_REG_SCRATCH_PAD_MAX_VAL 0xffffffff +#define HIF_CMDQM_SCRATCH_PAD_REG_SCRATCH_PAD_MIN_VAL 0x0 + +#define HIF_CMDQM_CSR_ERR_FLAG_REG_ADDR 0xa1020010 +#define HIF_CMDQM_CSR_ERR_FLAG_REG_WIDTH 32 +#define HIF_CMDQM_CSR_ERR_FLAG_REG_LENGTH 32 +#define HIF_CMDQM_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_MASK 0x1 +#define HIF_CMDQM_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_SHIFT 0 +#define HIF_CMDQM_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_WIDTH 1 + +#define HIF_CMDQM_CSR_ERR_ADDR_REG_ADDR 0xa1020014 +#define HIF_CMDQM_CSR_ERR_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_CSR_ERR_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_MASK 0xffffffff +#define HIF_CMDQM_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_SHIFT 0 +#define HIF_CMDQM_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_WIDTH 32 + +#define HIF_CMDQM_CSR_ERR_LEN_REG_ADDR 0xa1020018 +#define HIF_CMDQM_CSR_ERR_LEN_REG_WIDTH 32 +#define HIF_CMDQM_CSR_ERR_LEN_REG_LENGTH 32 +#define HIF_CMDQM_CSR_ERR_LEN_REG_CSR_ERR_LEN_MASK 0x3ff +#define HIF_CMDQM_CSR_ERR_LEN_REG_CSR_ERR_LEN_SHIFT 0 +#define HIF_CMDQM_CSR_ERR_LEN_REG_CSR_ERR_LEN_WIDTH 10 + +#define HIF_CMDQM_CSR_ERR_TYPE_REG_ADDR 0xa102001c +#define HIF_CMDQM_CSR_ERR_TYPE_REG_WIDTH 32 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_LENGTH 32 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_MASK 0x1 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_SHIFT 0 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_WIDTH 1 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_MASK 0x6 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_SHIFT 1 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_WIDTH 2 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_MASK 0x8 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_SHIFT 3 +#define HIF_CMDQM_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_WIDTH 1 + +#define HIF_CMDQM_Q_ELEMENT_SZ_REG_ADDR 0xa1020020 +#define HIF_CMDQM_Q_ELEMENT_SZ_REG_WIDTH 32 +#define HIF_CMDQM_Q_ELEMENT_SZ_REG_LENGTH 32 +#define HIF_CMDQM_Q_ELEMENT_SZ_REG_Q_ELEMENT_SZ_MASK 0xf +#define HIF_CMDQM_Q_ELEMENT_SZ_REG_Q_ELEMENT_SZ_SHIFT 0 +#define HIF_CMDQM_Q_ELEMENT_SZ_REG_Q_ELEMENT_SZ_WIDTH 4 +#define HIF_CMDQM_Q_ELEMENT_SZ_REG_Q_ELEMENT_SZ_MAX_VAL 0xf +#define HIF_CMDQM_Q_ELEMENT_SZ_REG_Q_ELEMENT_SZ_MIN_VAL 0x0 + +#define HIF_CMDQM_PCIE1_GLB_FUNC_ID_BASE_REG_ADDR 0xa1020024 +#define HIF_CMDQM_PCIE1_GLB_FUNC_ID_BASE_REG_WIDTH 32 +#define HIF_CMDQM_PCIE1_GLB_FUNC_ID_BASE_REG_LENGTH 32 +#define HIF_CMDQM_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_MASK 0x7ff +#define HIF_CMDQM_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_SHIFT 0 +#define HIF_CMDQM_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_WIDTH 11 +#define HIF_CMDQM_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_MAX_VAL 0x7ff +#define HIF_CMDQM_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_MIN_VAL 0x0 + +#define HIF_CMDQM_HOST_Q_DEPTH_REG_ADDR 0xa1020028 +#define HIF_CMDQM_HOST_Q_DEPTH_REG_WIDTH 32 +#define HIF_CMDQM_HOST_Q_DEPTH_REG_LENGTH 32 +#define HIF_CMDQM_HOST_Q_DEPTH_REG_HOST_Q_DEPTH_MASK 0x7ff +#define HIF_CMDQM_HOST_Q_DEPTH_REG_HOST_Q_DEPTH_SHIFT 0 +#define HIF_CMDQM_HOST_Q_DEPTH_REG_HOST_Q_DEPTH_WIDTH 11 +#define HIF_CMDQM_HOST_Q_DEPTH_REG_HOST_Q_DEPTH_MAX_VAL 0x7ff +#define HIF_CMDQM_HOST_Q_DEPTH_REG_HOST_Q_DEPTH_MIN_VAL 0x0 + +#define HIF_CMDQM_HOST_Q_BURST_NUM_REG_ADDR 0xa102002c +#define HIF_CMDQM_HOST_Q_BURST_NUM_REG_WIDTH 32 +#define HIF_CMDQM_HOST_Q_BURST_NUM_REG_LENGTH 32 +#define HIF_CMDQM_HOST_Q_BURST_NUM_REG_HOST_Q_BURST_NUM_MASK 0x7 +#define HIF_CMDQM_HOST_Q_BURST_NUM_REG_HOST_Q_BURST_NUM_SHIFT 0 +#define HIF_CMDQM_HOST_Q_BURST_NUM_REG_HOST_Q_BURST_NUM_WIDTH 3 +#define HIF_CMDQM_HOST_Q_BURST_NUM_REG_HOST_Q_BURST_NUM_MAX_VAL 0x7 +#define HIF_CMDQM_HOST_Q_BURST_NUM_REG_HOST_Q_BURST_NUM_MIN_VAL 0x0 + +#define HIF_CMDQM_ARM_Q_DEPTH_REG_ADDR 0xa1020030 +#define HIF_CMDQM_ARM_Q_DEPTH_REG_WIDTH 32 +#define HIF_CMDQM_ARM_Q_DEPTH_REG_LENGTH 32 +#define HIF_CMDQM_ARM_Q_DEPTH_REG_ARM_Q_DEPTH_MASK 0x7ff +#define HIF_CMDQM_ARM_Q_DEPTH_REG_ARM_Q_DEPTH_SHIFT 0 +#define HIF_CMDQM_ARM_Q_DEPTH_REG_ARM_Q_DEPTH_WIDTH 11 +#define HIF_CMDQM_ARM_Q_DEPTH_REG_ARM_Q_DEPTH_MAX_VAL 0x7ff +#define HIF_CMDQM_ARM_Q_DEPTH_REG_ARM_Q_DEPTH_MIN_VAL 0x0 + +#define HIF_CMDQM_ARM_Q_BURST_NUM_REG_ADDR 0xa1020034 +#define HIF_CMDQM_ARM_Q_BURST_NUM_REG_WIDTH 32 +#define HIF_CMDQM_ARM_Q_BURST_NUM_REG_LENGTH 32 +#define HIF_CMDQM_ARM_Q_BURST_NUM_REG_ARM_Q_BURST_NUM_MASK 0x7 +#define HIF_CMDQM_ARM_Q_BURST_NUM_REG_ARM_Q_BURST_NUM_SHIFT 0 +#define HIF_CMDQM_ARM_Q_BURST_NUM_REG_ARM_Q_BURST_NUM_WIDTH 3 +#define HIF_CMDQM_ARM_Q_BURST_NUM_REG_ARM_Q_BURST_NUM_MAX_VAL 0x7 +#define HIF_CMDQM_ARM_Q_BURST_NUM_REG_ARM_Q_BURST_NUM_MIN_VAL 0x0 + +#define HIF_CMDQM_ARM_REQ_BASE_ADDR_REG_ADDR 0xa1020038 +#define HIF_CMDQM_ARM_REQ_BASE_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_ARM_REQ_BASE_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_ARM_REQ_BASE_ADDR_REG_ARM_REQ_BASE_ADDR_MASK 0xffffffff +#define HIF_CMDQM_ARM_REQ_BASE_ADDR_REG_ARM_REQ_BASE_ADDR_SHIFT 0 +#define HIF_CMDQM_ARM_REQ_BASE_ADDR_REG_ARM_REQ_BASE_ADDR_WIDTH 32 +#define HIF_CMDQM_ARM_REQ_BASE_ADDR_REG_ARM_REQ_BASE_ADDR_MAX_VAL 0xffffffff +#define HIF_CMDQM_ARM_REQ_BASE_ADDR_REG_ARM_REQ_BASE_ADDR_MIN_VAL 0x0 + +#define HIF_CMDQM_ARM_REQ_CID_REG_ADDR 0xa102003c +#define HIF_CMDQM_ARM_REQ_CID_REG_WIDTH 32 +#define HIF_CMDQM_ARM_REQ_CID_REG_LENGTH 32 +#define HIF_CMDQM_ARM_REQ_CID_REG_ARM_REQ_CID_MASK 0x7ff +#define HIF_CMDQM_ARM_REQ_CID_REG_ARM_REQ_CID_SHIFT 0 +#define HIF_CMDQM_ARM_REQ_CID_REG_ARM_REQ_CID_WIDTH 11 +#define HIF_CMDQM_ARM_REQ_CID_REG_ARM_REQ_CID_MAX_VAL 0x7ff +#define HIF_CMDQM_ARM_REQ_CID_REG_ARM_REQ_CID_MIN_VAL 0x0 + +#define HIF_CMDQM_HW_ARM_REQ_CID_REG_ADDR 0xa1020040 +#define HIF_CMDQM_HW_ARM_REQ_CID_REG_WIDTH 32 +#define HIF_CMDQM_HW_ARM_REQ_CID_REG_LENGTH 32 +#define HIF_CMDQM_HW_ARM_REQ_CID_REG_HW_ARM_REQ_CID_MASK 0x7ff +#define HIF_CMDQM_HW_ARM_REQ_CID_REG_HW_ARM_REQ_CID_SHIFT 0 +#define HIF_CMDQM_HW_ARM_REQ_CID_REG_HW_ARM_REQ_CID_WIDTH 11 + +#define HIF_CMDQM_ARM_REQ_PID_REG_ADDR 0xa1020044 +#define HIF_CMDQM_ARM_REQ_PID_REG_WIDTH 32 +#define HIF_CMDQM_ARM_REQ_PID_REG_LENGTH 32 +#define HIF_CMDQM_ARM_REQ_PID_REG_ARM_REQ_PID_MASK 0x7ff +#define HIF_CMDQM_ARM_REQ_PID_REG_ARM_REQ_PID_SHIFT 0 +#define HIF_CMDQM_ARM_REQ_PID_REG_ARM_REQ_PID_WIDTH 11 + +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_REG_ADDR 0xa1020048 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_REG_WIDTH 32 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_REG_LENGTH 32 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_REG_HW2ARM_REQ_MASK 0x1 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_REG_HW2ARM_REQ_SHIFT 0 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_REG_HW2ARM_REQ_WIDTH 1 + +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_INT_MASK_REG_ADDR 0xa102004c +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_INT_MASK_REG_WIDTH 32 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_INT_MASK_REG_LENGTH 32 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_INT_MASK_REG_HW2ARM_REQ_INT_MASK_MASK 0x1 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_INT_MASK_REG_HW2ARM_REQ_INT_MASK_SHIFT 0 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_INT_MASK_REG_HW2ARM_REQ_INT_MASK_WIDTH 1 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_INT_MASK_REG_HW2ARM_REQ_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_CMDQM2IRQ_CMDQ_REQ_INT_MASK_REG_HW2ARM_REQ_INT_MASK_MIN_VAL 0x0 + +#define HIF_CMDQM_ARM_RSP_BASE_ADDR_REG_ADDR 0xa1020050 +#define HIF_CMDQM_ARM_RSP_BASE_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_ARM_RSP_BASE_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_ARM_RSP_BASE_ADDR_REG_ARM_RSP_BASE_ADDR_MASK 0xffffffff +#define HIF_CMDQM_ARM_RSP_BASE_ADDR_REG_ARM_RSP_BASE_ADDR_SHIFT 0 +#define HIF_CMDQM_ARM_RSP_BASE_ADDR_REG_ARM_RSP_BASE_ADDR_WIDTH 32 +#define HIF_CMDQM_ARM_RSP_BASE_ADDR_REG_ARM_RSP_BASE_ADDR_MAX_VAL 0xffffffff +#define HIF_CMDQM_ARM_RSP_BASE_ADDR_REG_ARM_RSP_BASE_ADDR_MIN_VAL 0x0 + +#define HIF_CMDQM_ARM_RSP_CID_REG_ADDR 0xa1020054 +#define HIF_CMDQM_ARM_RSP_CID_REG_WIDTH 32 +#define HIF_CMDQM_ARM_RSP_CID_REG_LENGTH 32 +#define HIF_CMDQM_ARM_RSP_CID_REG_ARM_RSP_CID_MASK 0x7ff +#define HIF_CMDQM_ARM_RSP_CID_REG_ARM_RSP_CID_SHIFT 0 +#define HIF_CMDQM_ARM_RSP_CID_REG_ARM_RSP_CID_WIDTH 11 + +#define HIF_CMDQM_ARM_RSP_PID_REG_ADDR 0xa1020058 +#define HIF_CMDQM_ARM_RSP_PID_REG_WIDTH 32 +#define HIF_CMDQM_ARM_RSP_PID_REG_LENGTH 32 +#define HIF_CMDQM_ARM_RSP_PID_REG_ARM_RSP_PID_MASK 0x7ff +#define HIF_CMDQM_ARM_RSP_PID_REG_ARM_RSP_PID_SHIFT 0 +#define HIF_CMDQM_ARM_RSP_PID_REG_ARM_RSP_PID_WIDTH 11 +#define HIF_CMDQM_ARM_RSP_PID_REG_ARM_RSP_PID_MAX_VAL 0x7ff +#define HIF_CMDQM_ARM_RSP_PID_REG_ARM_RSP_PID_MIN_VAL 0x0 + +#define HIF_CMDQM_HW_ARM_RSP_PID_REG_ADDR 0xa102005c +#define HIF_CMDQM_HW_ARM_RSP_PID_REG_WIDTH 32 +#define HIF_CMDQM_HW_ARM_RSP_PID_REG_LENGTH 32 +#define HIF_CMDQM_HW_ARM_RSP_PID_REG_HW_ARM_RSP_PID_MASK 0x7ff +#define HIF_CMDQM_HW_ARM_RSP_PID_REG_HW_ARM_RSP_PID_SHIFT 0 +#define HIF_CMDQM_HW_ARM_RSP_PID_REG_HW_ARM_RSP_PID_WIDTH 11 + +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_ADDR 0xa1020060 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_WIDTH 32 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_LENGTH 32 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_SIZE 2 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_DMA_RX_PCIE_PORT_MASK 0x1 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_DMA_RX_PCIE_PORT_SHIFT 0 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_DMA_RX_PCIE_PORT_WIDTH 1 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_DMA_RX_PCIE_PORT_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_RX_PCIE_PORT_REG_DMA_RX_PCIE_PORT_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_ADDR 0xa1020068 +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_WIDTH 32 +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_LENGTH 32 +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_SIZE 2 +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_DMA_RX_FUNCTION_ID_MASK 0x7ff +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_DMA_RX_FUNCTION_ID_SHIFT 0 +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_DMA_RX_FUNCTION_ID_WIDTH 11 +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_DMA_RX_FUNCTION_ID_MAX_VAL 0x7ff +#define HIF_CMDQM_DMA_RX_FUNCTION_ID_REG_DMA_RX_FUNCTION_ID_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_ADDR 0xa1020070 +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_SIZE 2 +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_DMA_RX_SRC_H_ADDR_MASK 0xffffffff +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_DMA_RX_SRC_H_ADDR_SHIFT 0 +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_DMA_RX_SRC_H_ADDR_WIDTH 32 +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_DMA_RX_SRC_H_ADDR_MAX_VAL 0xffffffff +#define HIF_CMDQM_DMA_RX_SRC_H_ADDR_REG_DMA_RX_SRC_H_ADDR_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_ADDR 0xa1020078 +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_SIZE 2 +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_DMA_RX_SRC_L_ADDR_MASK 0xffffffff +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_DMA_RX_SRC_L_ADDR_SHIFT 0 +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_DMA_RX_SRC_L_ADDR_WIDTH 32 +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_DMA_RX_SRC_L_ADDR_MAX_VAL 0xffffffff +#define HIF_CMDQM_DMA_RX_SRC_L_ADDR_REG_DMA_RX_SRC_L_ADDR_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_ADDR 0xa1020080 +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_SIZE 2 +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_DMA_RX_DST_ADDR_MASK 0xffffffff +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_DMA_RX_DST_ADDR_SHIFT 0 +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_DMA_RX_DST_ADDR_WIDTH 32 +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_DMA_RX_DST_ADDR_MAX_VAL 0xffffffff +#define HIF_CMDQM_DMA_RX_DST_ADDR_REG_DMA_RX_DST_ADDR_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_ADDR 0xa1020088 +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_WIDTH 32 +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_LENGTH 32 +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_SIZE 2 +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_DMA_RX_DATA_LEN_MASK 0x1ffff +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_DMA_RX_DATA_LEN_SHIFT 0 +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_DMA_RX_DATA_LEN_WIDTH 17 +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_DMA_RX_DATA_LEN_MAX_VAL 0x1ffff +#define HIF_CMDQM_DMA_RX_DATA_LEN_REG_DMA_RX_DATA_LEN_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_RX_STRAT_REG_ADDR 0xa1020090 +#define HIF_CMDQM_DMA_RX_STRAT_REG_WIDTH 32 +#define HIF_CMDQM_DMA_RX_STRAT_REG_LENGTH 32 +#define HIF_CMDQM_DMA_RX_STRAT_REG_SIZE 2 +#define HIF_CMDQM_DMA_RX_STRAT_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_RX_STRAT_REG_DMA_RX_STRAT_MASK 0x1 +#define HIF_CMDQM_DMA_RX_STRAT_REG_DMA_RX_STRAT_SHIFT 0 +#define HIF_CMDQM_DMA_RX_STRAT_REG_DMA_RX_STRAT_WIDTH 1 +#define HIF_CMDQM_DMA_RX_STRAT_REG_DMA_RX_STRAT_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_RX_STRAT_REG_DMA_RX_STRAT_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_RX_STATE_REG_ADDR 0xa1020098 +#define HIF_CMDQM_DMA_RX_STATE_REG_WIDTH 32 +#define HIF_CMDQM_DMA_RX_STATE_REG_LENGTH 32 +#define HIF_CMDQM_DMA_RX_STATE_REG_SIZE 2 +#define HIF_CMDQM_DMA_RX_STATE_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_RX_STATE_REG_DMA_RX_STATE_MASK 0x1 +#define HIF_CMDQM_DMA_RX_STATE_REG_DMA_RX_STATE_SHIFT 0 +#define HIF_CMDQM_DMA_RX_STATE_REG_DMA_RX_STATE_WIDTH 1 + +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_REG_ADDR 0xa10200a0 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_REG_WIDTH 32 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_REG_LENGTH 32 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_REG_SIZE 2 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_REG_STRIDE 0x8 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_REG_DMA_RX_MASK 0x1 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_REG_DMA_RX_SHIFT 0 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_REG_DMA_RX_WIDTH 1 + +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_ADDR 0xa10200a4 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_WIDTH 32 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_LENGTH 32 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_SIZE 2 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_STRIDE 0x8 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_DMA_RX_INT_MASK_MASK 0x1 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_DMA_RX_INT_MASK_SHIFT 0 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_DMA_RX_INT_MASK_WIDTH 1 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_DMA_RX_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_CMDQM2IRQ_DMA_RX_INT_MASK_REG_DMA_RX_INT_MASK_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_ADDR 0xa10200b0 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_WIDTH 32 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_LENGTH 32 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_SIZE 2 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_DMA_TX_PCIE_PORT_MASK 0x1 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_DMA_TX_PCIE_PORT_SHIFT 0 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_DMA_TX_PCIE_PORT_WIDTH 1 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_DMA_TX_PCIE_PORT_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_TX_PCIE_PORT_REG_DMA_TX_PCIE_PORT_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_ADDR 0xa10200b8 +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_WIDTH 32 +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_LENGTH 32 +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_SIZE 2 +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_DMA_TX_FUNCTION_ID_MASK 0x7ff +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_DMA_TX_FUNCTION_ID_SHIFT 0 +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_DMA_TX_FUNCTION_ID_WIDTH 11 +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_DMA_TX_FUNCTION_ID_MAX_VAL 0x7ff +#define HIF_CMDQM_DMA_TX_FUNCTION_ID_REG_DMA_TX_FUNCTION_ID_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_ADDR 0xa10200c0 +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_SIZE 2 +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_DMA_TX_SRC_ADDR_MASK 0xffffffff +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_DMA_TX_SRC_ADDR_SHIFT 0 +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_DMA_TX_SRC_ADDR_WIDTH 32 +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_DMA_TX_SRC_ADDR_MAX_VAL 0xffffffff +#define HIF_CMDQM_DMA_TX_SRC_ADDR_REG_DMA_TX_SRC_ADDR_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_ADDR 0xa10200c8 +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_SIZE 2 +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_DMA_TX_DST_H_ADDR_MASK 0xffffffff +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_DMA_TX_DST_H_ADDR_SHIFT 0 +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_DMA_TX_DST_H_ADDR_WIDTH 32 +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_DMA_TX_DST_H_ADDR_MAX_VAL 0xffffffff +#define HIF_CMDQM_DMA_TX_DST_H_ADDR_REG_DMA_TX_DST_H_ADDR_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_ADDR 0xa10200d0 +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_SIZE 2 +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_DMA_TX_DST_L_ADDR_MASK 0xffffffff +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_DMA_TX_DST_L_ADDR_SHIFT 0 +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_DMA_TX_DST_L_ADDR_WIDTH 32 +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_DMA_TX_DST_L_ADDR_MAX_VAL 0xffffffff +#define HIF_CMDQM_DMA_TX_DST_L_ADDR_REG_DMA_TX_DST_L_ADDR_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_ADDR 0xa10200d8 +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_WIDTH 32 +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_LENGTH 32 +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_SIZE 2 +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_DMA_TX_DATA_LEN_MASK 0x1ffff +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_DMA_TX_DATA_LEN_SHIFT 0 +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_DMA_TX_DATA_LEN_WIDTH 17 +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_DMA_TX_DATA_LEN_MAX_VAL 0x1ffff +#define HIF_CMDQM_DMA_TX_DATA_LEN_REG_DMA_TX_DATA_LEN_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_TX_STRAT_REG_ADDR 0xa10200e0 +#define HIF_CMDQM_DMA_TX_STRAT_REG_WIDTH 32 +#define HIF_CMDQM_DMA_TX_STRAT_REG_LENGTH 32 +#define HIF_CMDQM_DMA_TX_STRAT_REG_SIZE 2 +#define HIF_CMDQM_DMA_TX_STRAT_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_TX_STRAT_REG_DMA_TX_STRAT_MASK 0x1 +#define HIF_CMDQM_DMA_TX_STRAT_REG_DMA_TX_STRAT_SHIFT 0 +#define HIF_CMDQM_DMA_TX_STRAT_REG_DMA_TX_STRAT_WIDTH 1 +#define HIF_CMDQM_DMA_TX_STRAT_REG_DMA_TX_STRAT_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_TX_STRAT_REG_DMA_TX_STRAT_MIN_VAL 0x0 + +#define HIF_CMDQM_DMA_TX_STATE_REG_ADDR 0xa10200e8 +#define HIF_CMDQM_DMA_TX_STATE_REG_WIDTH 32 +#define HIF_CMDQM_DMA_TX_STATE_REG_LENGTH 32 +#define HIF_CMDQM_DMA_TX_STATE_REG_SIZE 2 +#define HIF_CMDQM_DMA_TX_STATE_REG_STRIDE 0x4 +#define HIF_CMDQM_DMA_TX_STATE_REG_DMA_TX_STATE_MASK 0x1 +#define HIF_CMDQM_DMA_TX_STATE_REG_DMA_TX_STATE_SHIFT 0 +#define HIF_CMDQM_DMA_TX_STATE_REG_DMA_TX_STATE_WIDTH 1 + +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_REG_ADDR 0xa10200f0 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_REG_WIDTH 32 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_REG_LENGTH 32 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_REG_SIZE 2 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_REG_STRIDE 0x8 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_REG_DMA_TX_MASK 0x1 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_REG_DMA_TX_SHIFT 0 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_REG_DMA_TX_WIDTH 1 + +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_ADDR 0xa10200f4 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_WIDTH 32 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_LENGTH 32 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_SIZE 2 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_STRIDE 0x8 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_DMA_TX_INT_MASK_MASK 0x1 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_DMA_TX_INT_MASK_SHIFT 0 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_DMA_TX_INT_MASK_WIDTH 1 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_DMA_TX_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_CMDQM2IRQ_DMA_TX_INT_MASK_REG_DMA_TX_INT_MASK_MIN_VAL 0x0 + +#define HIF_CMDQM_FIFO_INT_REG_ADDR 0xa102010c +#define HIF_CMDQM_FIFO_INT_REG_WIDTH 32 +#define HIF_CMDQM_FIFO_INT_REG_LENGTH 32 +#define HIF_CMDQM_FIFO_INT_REG_REQ_DATA_FIFO_OVFL_MASK 0x1 +#define HIF_CMDQM_FIFO_INT_REG_REQ_DATA_FIFO_OVFL_SHIFT 0 +#define HIF_CMDQM_FIFO_INT_REG_REQ_DATA_FIFO_OVFL_WIDTH 1 +#define HIF_CMDQM_FIFO_INT_REG_REQ_DATA_FIFO_UNFL_MASK 0x2 +#define HIF_CMDQM_FIFO_INT_REG_REQ_DATA_FIFO_UNFL_SHIFT 1 +#define HIF_CMDQM_FIFO_INT_REG_REQ_DATA_FIFO_UNFL_WIDTH 1 +#define HIF_CMDQM_FIFO_INT_REG_RSP_DATA_FIFO_OVFL_MASK 0x4 +#define HIF_CMDQM_FIFO_INT_REG_RSP_DATA_FIFO_OVFL_SHIFT 2 +#define HIF_CMDQM_FIFO_INT_REG_RSP_DATA_FIFO_OVFL_WIDTH 1 +#define HIF_CMDQM_FIFO_INT_REG_RSP_DATA_FIFO_UNFL_MASK 0x8 +#define HIF_CMDQM_FIFO_INT_REG_RSP_DATA_FIFO_UNFL_SHIFT 3 +#define HIF_CMDQM_FIFO_INT_REG_RSP_DATA_FIFO_UNFL_WIDTH 1 +#define HIF_CMDQM_FIFO_INT_REG_HOST_RSP_INT_FIFO_OVFL_MASK 0x10 +#define HIF_CMDQM_FIFO_INT_REG_HOST_RSP_INT_FIFO_OVFL_SHIFT 4 +#define HIF_CMDQM_FIFO_INT_REG_HOST_RSP_INT_FIFO_OVFL_WIDTH 1 +#define HIF_CMDQM_FIFO_INT_REG_HOST_RSP_INT_FIFO_UNFL_MASK 0x20 +#define HIF_CMDQM_FIFO_INT_REG_HOST_RSP_INT_FIFO_UNFL_SHIFT 5 +#define HIF_CMDQM_FIFO_INT_REG_HOST_RSP_INT_FIFO_UNFL_WIDTH 1 +#define HIF_CMDQM_FIFO_INT_REG_HOST_ERR_INT_FIFO_OVFL_MASK 0x40 +#define HIF_CMDQM_FIFO_INT_REG_HOST_ERR_INT_FIFO_OVFL_SHIFT 6 +#define HIF_CMDQM_FIFO_INT_REG_HOST_ERR_INT_FIFO_OVFL_WIDTH 1 +#define HIF_CMDQM_FIFO_INT_REG_HOST_ERR_INT_FIFO_UNFL_MASK 0x80 +#define HIF_CMDQM_FIFO_INT_REG_HOST_ERR_INT_FIFO_UNFL_SHIFT 7 +#define HIF_CMDQM_FIFO_INT_REG_HOST_ERR_INT_FIFO_UNFL_WIDTH 1 + +#define HIF_CMDQM_CREDIT_INT_REG_ADDR 0xa102011c +#define HIF_CMDQM_CREDIT_INT_REG_WIDTH 32 +#define HIF_CMDQM_CREDIT_INT_REG_LENGTH 32 +#define HIF_CMDQM_CREDIT_INT_REG_CREDIT_OVFL_MASK 0x1 +#define HIF_CMDQM_CREDIT_INT_REG_CREDIT_OVFL_SHIFT 0 +#define HIF_CMDQM_CREDIT_INT_REG_CREDIT_OVFL_WIDTH 1 +#define HIF_CMDQM_CREDIT_INT_REG_CREDIT_UNFL_MASK 0x2 +#define HIF_CMDQM_CREDIT_INT_REG_CREDIT_UNFL_SHIFT 1 +#define HIF_CMDQM_CREDIT_INT_REG_CREDIT_UNFL_WIDTH 1 + +#define HIF_CMDQM_HOST_VF_RSP_INT_EN_REG_ADDR 0xa1020124 +#define HIF_CMDQM_HOST_VF_RSP_INT_EN_REG_WIDTH 32 +#define HIF_CMDQM_HOST_VF_RSP_INT_EN_REG_LENGTH 32 +#define HIF_CMDQM_HOST_VF_RSP_INT_EN_REG_HOST_VF_RSP_INT_EN_MASK 0x1 +#define HIF_CMDQM_HOST_VF_RSP_INT_EN_REG_HOST_VF_RSP_INT_EN_SHIFT 0 +#define HIF_CMDQM_HOST_VF_RSP_INT_EN_REG_HOST_VF_RSP_INT_EN_WIDTH 1 +#define HIF_CMDQM_HOST_VF_RSP_INT_EN_REG_HOST_VF_RSP_INT_EN_MAX_VAL 0x1 +#define HIF_CMDQM_HOST_VF_RSP_INT_EN_REG_HOST_VF_RSP_INT_EN_MIN_VAL 0x0 + +#define HIF_CMDQM_HOST_VF_ERR_INFO_0_REG_ADDR 0xa1020130 +#define HIF_CMDQM_HOST_VF_ERR_INFO_0_REG_WIDTH 32 +#define HIF_CMDQM_HOST_VF_ERR_INFO_0_REG_LENGTH 32 +#define HIF_CMDQM_HOST_VF_ERR_INFO_0_REG_SIZE 4 +#define HIF_CMDQM_HOST_VF_ERR_INFO_0_REG_STRIDE 0x4 +#define HIF_CMDQM_HOST_VF_ERR_INFO_0_REG_HOST_VF_ERR_INFO_VF_MASK 0x7ff +#define HIF_CMDQM_HOST_VF_ERR_INFO_0_REG_HOST_VF_ERR_INFO_VF_SHIFT 0 +#define HIF_CMDQM_HOST_VF_ERR_INFO_0_REG_HOST_VF_ERR_INFO_VF_WIDTH 11 + +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_ADDR 0xa1020140 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_WIDTH 32 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_LENGTH 32 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_SIZE 4 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_STRIDE 0x4 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_HOST_VF_ERR_INFO_PID_MASK 0x7ff +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_HOST_VF_ERR_INFO_PID_SHIFT 0 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_HOST_VF_ERR_INFO_PID_WIDTH 11 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_HOST_VF_ERR_INFO_CID_MASK 0x3ff800 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_HOST_VF_ERR_INFO_CID_SHIFT 11 +#define HIF_CMDQM_HOST_VF_ERR_INFO_1_REG_HOST_VF_ERR_INFO_CID_WIDTH 11 + +#define HIF_CMDQM_ARM_ERR_INT_REG_ADDR 0xa1020150 +#define HIF_CMDQM_ARM_ERR_INT_REG_WIDTH 32 +#define HIF_CMDQM_ARM_ERR_INT_REG_LENGTH 32 +#define HIF_CMDQM_ARM_ERR_INT_REG_ARM_ERR_INT_MASK 0x1 +#define HIF_CMDQM_ARM_ERR_INT_REG_ARM_ERR_INT_SHIFT 0 +#define HIF_CMDQM_ARM_ERR_INT_REG_ARM_ERR_INT_WIDTH 1 + +#define HIF_CMDQM_ARM_ERR_INT_MASK_REG_ADDR 0xa1020154 +#define HIF_CMDQM_ARM_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CMDQM_ARM_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CMDQM_ARM_ERR_INT_MASK_REG_ARM_ERR_INT_INT_MASK_MASK 0x1 +#define HIF_CMDQM_ARM_ERR_INT_MASK_REG_ARM_ERR_INT_INT_MASK_SHIFT 0 +#define HIF_CMDQM_ARM_ERR_INT_MASK_REG_ARM_ERR_INT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_ARM_ERR_INT_MASK_REG_ARM_ERR_INT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_ARM_ERR_INT_MASK_REG_ARM_ERR_INT_INT_MASK_MIN_VAL 0x0 + +#define HIF_CMDQM_ARM_ERR_STS_REG_ADDR 0xa1020158 +#define HIF_CMDQM_ARM_ERR_STS_REG_WIDTH 32 +#define HIF_CMDQM_ARM_ERR_STS_REG_LENGTH 32 +#define HIF_CMDQM_ARM_ERR_STS_REG_ARM_ERR_STS_MASK 0x3f +#define HIF_CMDQM_ARM_ERR_STS_REG_ARM_ERR_STS_SHIFT 0 +#define HIF_CMDQM_ARM_ERR_STS_REG_ARM_ERR_STS_WIDTH 6 + +#define HIF_CMDQM_ARM_ERR_INFO_REG_ADDR 0xa102015c +#define HIF_CMDQM_ARM_ERR_INFO_REG_WIDTH 32 +#define HIF_CMDQM_ARM_ERR_INFO_REG_LENGTH 32 +#define HIF_CMDQM_ARM_ERR_INFO_REG_ARM_ERR_INFO_PID_MASK 0x7ff +#define HIF_CMDQM_ARM_ERR_INFO_REG_ARM_ERR_INFO_PID_SHIFT 0 +#define HIF_CMDQM_ARM_ERR_INFO_REG_ARM_ERR_INFO_PID_WIDTH 11 +#define HIF_CMDQM_ARM_ERR_INFO_REG_ARM_ERR_INFO_CID_MASK 0x3ff800 +#define HIF_CMDQM_ARM_ERR_INFO_REG_ARM_ERR_INFO_CID_SHIFT 11 +#define HIF_CMDQM_ARM_ERR_INFO_REG_ARM_ERR_INFO_CID_WIDTH 11 + +#define HIF_CMDQM_ARM_ERR_INFO_RSP_BASE_ADDR_REG_ADDR 0xa1020160 +#define HIF_CMDQM_ARM_ERR_INFO_RSP_BASE_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_ARM_ERR_INFO_RSP_BASE_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_ARM_ERR_INFO_RSP_BASE_ADDR_REG_ARM_ERR_INFO_RSP_BASE_ADDR_MASK 0xffffffff +#define HIF_CMDQM_ARM_ERR_INFO_RSP_BASE_ADDR_REG_ARM_ERR_INFO_RSP_BASE_ADDR_SHIFT 0 +#define HIF_CMDQM_ARM_ERR_INFO_RSP_BASE_ADDR_REG_ARM_ERR_INFO_RSP_BASE_ADDR_WIDTH 32 + +#define HIF_CMDQM_ARM_ERR_INFO_REQ_BASE_ADDR_REG_ADDR 0xa1020164 +#define HIF_CMDQM_ARM_ERR_INFO_REQ_BASE_ADDR_REG_WIDTH 32 +#define HIF_CMDQM_ARM_ERR_INFO_REQ_BASE_ADDR_REG_LENGTH 32 +#define HIF_CMDQM_ARM_ERR_INFO_REQ_BASE_ADDR_REG_ARM_ERR_INFO_REQ_BASE_ADDR_MASK 0xffffffff +#define HIF_CMDQM_ARM_ERR_INFO_REQ_BASE_ADDR_REG_ARM_ERR_INFO_REQ_BASE_ADDR_SHIFT 0 +#define HIF_CMDQM_ARM_ERR_INFO_REQ_BASE_ADDR_REG_ARM_ERR_INFO_REQ_BASE_ADDR_WIDTH 32 + +#define HIF_CMDQM_DMA_ERR_INT_REG_ADDR 0xa1020170 +#define HIF_CMDQM_DMA_ERR_INT_REG_WIDTH 32 +#define HIF_CMDQM_DMA_ERR_INT_REG_LENGTH 32 +#define HIF_CMDQM_DMA_ERR_INT_REG_SIZE 2 +#define HIF_CMDQM_DMA_ERR_INT_REG_STRIDE 0x8 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_RD_MASK 0x1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_RD_SHIFT 0 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_RD_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_WR_MASK 0x2 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_WR_SHIFT 1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_WR_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_RECFG_MASK 0x4 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_RECFG_SHIFT 2 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_RECFG_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_CFG_LEN_MASK 0x8 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_CFG_LEN_SHIFT 3 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_RX_CFG_LEN_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_RD_MASK 0x10 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_RD_SHIFT 4 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_RD_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_WR_MASK 0x20 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_WR_SHIFT 5 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_WR_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_RECFG_MASK 0x40 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_RECFG_SHIFT 6 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_RECFG_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_CFG_LEN_MASK 0x80 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_CFG_LEN_SHIFT 7 +#define HIF_CMDQM_DMA_ERR_INT_REG_DMA_ERR_TX_CFG_LEN_WIDTH 1 + +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_ADDR 0xa1020174 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_SIZE 2 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_STRIDE 0x8 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RD_INT_MASK_MASK 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RD_INT_MASK_SHIFT 0 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RD_INT_MASK_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RD_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RD_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_WR_INT_MASK_MASK 0x2 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_WR_INT_MASK_SHIFT 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_WR_INT_MASK_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_WR_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_WR_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RECFG_INT_MASK_MASK 0x4 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RECFG_INT_MASK_SHIFT 2 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RECFG_INT_MASK_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RECFG_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_RECFG_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_CFG_LEN_INT_MASK_MASK 0x8 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_CFG_LEN_INT_MASK_SHIFT 3 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_CFG_LEN_INT_MASK_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_CFG_LEN_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_RX_CFG_LEN_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RD_INT_MASK_MASK 0x10 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RD_INT_MASK_SHIFT 4 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RD_INT_MASK_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RD_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RD_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_WR_INT_MASK_MASK 0x20 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_WR_INT_MASK_SHIFT 5 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_WR_INT_MASK_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_WR_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_WR_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RECFG_INT_MASK_MASK 0x40 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RECFG_INT_MASK_SHIFT 6 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RECFG_INT_MASK_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RECFG_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_RECFG_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_CFG_LEN_INT_MASK_MASK 0x80 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_CFG_LEN_INT_MASK_SHIFT 7 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_CFG_LEN_INT_MASK_WIDTH 1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_CFG_LEN_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_DMA_ERR_INT_MASK_REG_DMA_ERR_TX_CFG_LEN_INT_MASK_MIN_VAL 0x0 + +#define HIF_CMDQM_RD_ADPT_DEBUG_CNT_REG_ADDR 0xa1020180 +#define HIF_CMDQM_RD_ADPT_DEBUG_CNT_REG_WIDTH 32 +#define HIF_CMDQM_RD_ADPT_DEBUG_CNT_REG_LENGTH 32 +#define HIF_CMDQM_RD_ADPT_DEBUG_CNT_REG_RD_ADPT_CNT_MASK 0xffff +#define HIF_CMDQM_RD_ADPT_DEBUG_CNT_REG_RD_ADPT_CNT_SHIFT 0 +#define HIF_CMDQM_RD_ADPT_DEBUG_CNT_REG_RD_ADPT_CNT_WIDTH 16 + +#define HIF_CMDQM_RD_ADPT_ACK_DEBUG_CNT_REG_ADDR 0xa1020184 +#define HIF_CMDQM_RD_ADPT_ACK_DEBUG_CNT_REG_WIDTH 32 +#define HIF_CMDQM_RD_ADPT_ACK_DEBUG_CNT_REG_LENGTH 32 +#define HIF_CMDQM_RD_ADPT_ACK_DEBUG_CNT_REG_RD_ADPT_ACK_CNT_MASK 0xffff +#define HIF_CMDQM_RD_ADPT_ACK_DEBUG_CNT_REG_RD_ADPT_ACK_CNT_SHIFT 0 +#define HIF_CMDQM_RD_ADPT_ACK_DEBUG_CNT_REG_RD_ADPT_ACK_CNT_WIDTH 16 + +#define HIF_CMDQM_WR_ADPT_DEBUG_CNT_REG_ADDR 0xa1020188 +#define HIF_CMDQM_WR_ADPT_DEBUG_CNT_REG_WIDTH 32 +#define HIF_CMDQM_WR_ADPT_DEBUG_CNT_REG_LENGTH 32 +#define HIF_CMDQM_WR_ADPT_DEBUG_CNT_REG_WR_ADPT_CNT_MASK 0xffff +#define HIF_CMDQM_WR_ADPT_DEBUG_CNT_REG_WR_ADPT_CNT_SHIFT 0 +#define HIF_CMDQM_WR_ADPT_DEBUG_CNT_REG_WR_ADPT_CNT_WIDTH 16 + +#define HIF_CMDQM_WR_ADPT_ACK_DEBUG_CNT_REG_ADDR 0xa102018c +#define HIF_CMDQM_WR_ADPT_ACK_DEBUG_CNT_REG_WIDTH 32 +#define HIF_CMDQM_WR_ADPT_ACK_DEBUG_CNT_REG_LENGTH 32 +#define HIF_CMDQM_WR_ADPT_ACK_DEBUG_CNT_REG_WR_ADPT_ACK_CNT_MASK 0xffff +#define HIF_CMDQM_WR_ADPT_ACK_DEBUG_CNT_REG_WR_ADPT_ACK_CNT_SHIFT 0 +#define HIF_CMDQM_WR_ADPT_ACK_DEBUG_CNT_REG_WR_ADPT_ACK_CNT_WIDTH 16 + +#define HIF_CMDQM_RD_AXI_DEBUG_CNT_REG_ADDR 0xa1020190 +#define HIF_CMDQM_RD_AXI_DEBUG_CNT_REG_WIDTH 32 +#define HIF_CMDQM_RD_AXI_DEBUG_CNT_REG_LENGTH 32 +#define HIF_CMDQM_RD_AXI_DEBUG_CNT_REG_RD_AXI_CNT_MASK 0xffff +#define HIF_CMDQM_RD_AXI_DEBUG_CNT_REG_RD_AXI_CNT_SHIFT 0 +#define HIF_CMDQM_RD_AXI_DEBUG_CNT_REG_RD_AXI_CNT_WIDTH 16 + +#define HIF_CMDQM_RD_AXI_ACK_DEBUG_CNT_REG_ADDR 0xa1020194 +#define HIF_CMDQM_RD_AXI_ACK_DEBUG_CNT_REG_WIDTH 32 +#define HIF_CMDQM_RD_AXI_ACK_DEBUG_CNT_REG_LENGTH 32 +#define HIF_CMDQM_RD_AXI_ACK_DEBUG_CNT_REG_RD_AXI_ACK_CNT_MASK 0xffff +#define HIF_CMDQM_RD_AXI_ACK_DEBUG_CNT_REG_RD_AXI_ACK_CNT_SHIFT 0 +#define HIF_CMDQM_RD_AXI_ACK_DEBUG_CNT_REG_RD_AXI_ACK_CNT_WIDTH 16 + +#define HIF_CMDQM_WR_AXI_DEBUG_CNT_REG_ADDR 0xa1020198 +#define HIF_CMDQM_WR_AXI_DEBUG_CNT_REG_WIDTH 32 +#define HIF_CMDQM_WR_AXI_DEBUG_CNT_REG_LENGTH 32 +#define HIF_CMDQM_WR_AXI_DEBUG_CNT_REG_WR_AXI_CNT_MASK 0xffff +#define HIF_CMDQM_WR_AXI_DEBUG_CNT_REG_WR_AXI_CNT_SHIFT 0 +#define HIF_CMDQM_WR_AXI_DEBUG_CNT_REG_WR_AXI_CNT_WIDTH 16 + +#define HIF_CMDQM_WR_AXI_ACK_DEBUG_CNT_REG_ADDR 0xa102019c +#define HIF_CMDQM_WR_AXI_ACK_DEBUG_CNT_REG_WIDTH 32 +#define HIF_CMDQM_WR_AXI_ACK_DEBUG_CNT_REG_LENGTH 32 +#define HIF_CMDQM_WR_AXI_ACK_DEBUG_CNT_REG_WR_AXI_ACK_CNT_MASK 0xffff +#define HIF_CMDQM_WR_AXI_ACK_DEBUG_CNT_REG_WR_AXI_ACK_CNT_SHIFT 0 +#define HIF_CMDQM_WR_AXI_ACK_DEBUG_CNT_REG_WR_AXI_ACK_CNT_WIDTH 16 + +#define HIF_CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_ADDR 0xa1022000 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_HOST_REQ_BUF_BASE_H_ADDR_MASK 0xffffffff +#define HIF_CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_HOST_REQ_BUF_BASE_H_ADDR_SHIFT 0 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_HOST_REQ_BUF_BASE_H_ADDR_WIDTH 32 + +#define HIF_CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_ADDR 0xa1024000 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_HOST_REQ_BUF_BASE_L_ADDR_MASK 0xffffffff +#define HIF_CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_HOST_REQ_BUF_BASE_L_ADDR_SHIFT 0 +#define HIF_CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_HOST_REQ_BUF_BASE_L_ADDR_WIDTH 32 + +#define HIF_CMDQM_HOST_REQ_PID_MEM_ADDR 0xa1026000 +#define HIF_CMDQM_HOST_REQ_PID_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_REQ_PID_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_REQ_PID_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_REQ_PID_MEM_HOST_REQ_PID_MASK 0x7ff +#define HIF_CMDQM_HOST_REQ_PID_MEM_HOST_REQ_PID_SHIFT 0 +#define HIF_CMDQM_HOST_REQ_PID_MEM_HOST_REQ_PID_WIDTH 11 + +#define HIF_CMDQM_HOST_REQ_CID_MEM_ADDR 0xa1028000 +#define HIF_CMDQM_HOST_REQ_CID_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_REQ_CID_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_REQ_CID_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_REQ_CID_MEM_HOST_REQ_CID_MASK 0x7ff +#define HIF_CMDQM_HOST_REQ_CID_MEM_HOST_REQ_CID_SHIFT 0 +#define HIF_CMDQM_HOST_REQ_CID_MEM_HOST_REQ_CID_WIDTH 11 + +#define HIF_CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_ADDR 0xa102a000 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_HOST_RSP_BUF_BASE_H_ADDR_MASK 0xffffffff +#define HIF_CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_HOST_RSP_BUF_BASE_H_ADDR_SHIFT 0 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_HOST_RSP_BUF_BASE_H_ADDR_WIDTH 32 + +#define HIF_CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_ADDR 0xa102c000 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_HOST_RSP_BUF_BASE_L_ADDR_MASK 0xffffffff +#define HIF_CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_HOST_RSP_BUF_BASE_L_ADDR_SHIFT 0 +#define HIF_CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_HOST_RSP_BUF_BASE_L_ADDR_WIDTH 32 + +#define HIF_CMDQM_HOST_RSP_PID_MEM_ADDR 0xa102e000 +#define HIF_CMDQM_HOST_RSP_PID_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_RSP_PID_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_RSP_PID_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_RSP_PID_MEM_HOST_RSP_PID_MASK 0x7ff +#define HIF_CMDQM_HOST_RSP_PID_MEM_HOST_RSP_PID_SHIFT 0 +#define HIF_CMDQM_HOST_RSP_PID_MEM_HOST_RSP_PID_WIDTH 11 + +#define HIF_CMDQM_HOST_RSP_CID_MEM_ADDR 0xa1030000 +#define HIF_CMDQM_HOST_RSP_CID_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_RSP_CID_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_RSP_CID_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_RSP_CID_MEM_HOST_RSP_CID_MASK 0x7ff +#define HIF_CMDQM_HOST_RSP_CID_MEM_HOST_RSP_CID_SHIFT 0 +#define HIF_CMDQM_HOST_RSP_CID_MEM_HOST_RSP_CID_WIDTH 11 + +#define HIF_CMDQM_HOST_VF_ERR_STS_MEM_ADDR 0xa1032000 +#define HIF_CMDQM_HOST_VF_ERR_STS_MEM_WIDTH 32 +#define HIF_CMDQM_HOST_VF_ERR_STS_MEM_LENGTH 32 +#define HIF_CMDQM_HOST_VF_ERR_STS_MEM_DEPTH 1056 +#define HIF_CMDQM_HOST_VF_ERR_STS_MEM_HOST_VF_ERR_STS_MASK 0xf +#define HIF_CMDQM_HOST_VF_ERR_STS_MEM_HOST_VF_ERR_STS_SHIFT 0 +#define HIF_CMDQM_HOST_VF_ERR_STS_MEM_HOST_VF_ERR_STS_WIDTH 4 + +#define HIF_CMDQM_VECTOR_ID_MEM_ADDR 0xa1034000 +#define HIF_CMDQM_VECTOR_ID_MEM_WIDTH 32 +#define HIF_CMDQM_VECTOR_ID_MEM_LENGTH 32 +#define HIF_CMDQM_VECTOR_ID_MEM_DEPTH 1056 +#define HIF_CMDQM_VECTOR_ID_MEM_VECTOR_ID_MASK 0xfff +#define HIF_CMDQM_VECTOR_ID_MEM_VECTOR_ID_SHIFT 0 +#define HIF_CMDQM_VECTOR_ID_MEM_VECTOR_ID_WIDTH 12 + +#define HIF_CMDQM_MEM_ERROR_INT_ADDR 0xa1036000 +#define HIF_CMDQM_MEM_ERROR_INT_WIDTH 32 +#define HIF_CMDQM_MEM_ERROR_INT_LENGTH 32 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_H_ADDR_MEM_SB_ERR_MASK 0x1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_H_ADDR_MEM_SB_ERR_SHIFT 0 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_H_ADDR_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_H_ADDR_MEM_DB_ERR_MASK 0x2 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_H_ADDR_MEM_DB_ERR_SHIFT 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_H_ADDR_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_L_ADDR_MEM_SB_ERR_MASK 0x4 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_L_ADDR_MEM_SB_ERR_SHIFT 2 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_L_ADDR_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_L_ADDR_MEM_DB_ERR_MASK 0x8 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_L_ADDR_MEM_DB_ERR_SHIFT 3 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_BUF_BASE_L_ADDR_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_PID_MEM_SB_ERR_MASK 0x10 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_PID_MEM_SB_ERR_SHIFT 4 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_PID_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_PID_MEM_DB_ERR_MASK 0x20 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_PID_MEM_DB_ERR_SHIFT 5 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_PID_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_CID_MEM_SB_ERR_MASK 0x40 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_CID_MEM_SB_ERR_SHIFT 6 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_CID_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_CID_MEM_DB_ERR_MASK 0x80 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_CID_MEM_DB_ERR_SHIFT 7 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_REQ_CID_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_H_ADDR_MEM_SB_ERR_MASK 0x100 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_H_ADDR_MEM_SB_ERR_SHIFT 8 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_H_ADDR_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_H_ADDR_MEM_DB_ERR_MASK 0x200 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_H_ADDR_MEM_DB_ERR_SHIFT 9 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_H_ADDR_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_L_ADDR_MEM_SB_ERR_MASK 0x400 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_L_ADDR_MEM_SB_ERR_SHIFT 10 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_L_ADDR_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_L_ADDR_MEM_DB_ERR_MASK 0x800 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_L_ADDR_MEM_DB_ERR_SHIFT 11 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_BUF_BASE_L_ADDR_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_PID_MEM_SB_ERR_MASK 0x1000 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_PID_MEM_SB_ERR_SHIFT 12 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_PID_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_PID_MEM_DB_ERR_MASK 0x2000 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_PID_MEM_DB_ERR_SHIFT 13 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_PID_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_CID_MEM_SB_ERR_MASK 0x4000 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_CID_MEM_SB_ERR_SHIFT 14 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_CID_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_CID_MEM_DB_ERR_MASK 0x8000 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_CID_MEM_DB_ERR_SHIFT 15 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_RSP_CID_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_VF_ERR_STS_MEM_SB_ERR_MASK 0x10000 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_VF_ERR_STS_MEM_SB_ERR_SHIFT 16 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_VF_ERR_STS_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_VF_ERR_STS_MEM_DB_ERR_MASK 0x20000 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_VF_ERR_STS_MEM_DB_ERR_SHIFT 17 +#define HIF_CMDQM_MEM_ERROR_INT_HOST_VF_ERR_STS_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_VECTOR_ID_MEM_SB_ERR_MASK 0x40000 +#define HIF_CMDQM_MEM_ERROR_INT_VECTOR_ID_MEM_SB_ERR_SHIFT 18 +#define HIF_CMDQM_MEM_ERROR_INT_VECTOR_ID_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_VECTOR_ID_MEM_DB_ERR_MASK 0x80000 +#define HIF_CMDQM_MEM_ERROR_INT_VECTOR_ID_MEM_DB_ERR_SHIFT 19 +#define HIF_CMDQM_MEM_ERROR_INT_VECTOR_ID_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_REQ_DATA_FIFO_MEM_SB_ERR_MASK 0x100000 +#define HIF_CMDQM_MEM_ERROR_INT_REQ_DATA_FIFO_MEM_SB_ERR_SHIFT 20 +#define HIF_CMDQM_MEM_ERROR_INT_REQ_DATA_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_REQ_DATA_FIFO_MEM_DB_ERR_MASK 0x200000 +#define HIF_CMDQM_MEM_ERROR_INT_REQ_DATA_FIFO_MEM_DB_ERR_SHIFT 21 +#define HIF_CMDQM_MEM_ERROR_INT_REQ_DATA_FIFO_MEM_DB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_RSP_DATA_FIFO_MEM_SB_ERR_MASK 0x400000 +#define HIF_CMDQM_MEM_ERROR_INT_RSP_DATA_FIFO_MEM_SB_ERR_SHIFT 22 +#define HIF_CMDQM_MEM_ERROR_INT_RSP_DATA_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_CMDQM_MEM_ERROR_INT_RSP_DATA_FIFO_MEM_DB_ERR_MASK 0x800000 +#define HIF_CMDQM_MEM_ERROR_INT_RSP_DATA_FIFO_MEM_DB_ERR_SHIFT 23 +#define HIF_CMDQM_MEM_ERROR_INT_RSP_DATA_FIFO_MEM_DB_ERR_WIDTH 1 + +#define HIF_CMDQM_MEM_INIT_CTRL_ADDR 0xa1036008 +#define HIF_CMDQM_MEM_INIT_CTRL_WIDTH 32 +#define HIF_CMDQM_MEM_INIT_CTRL_LENGTH 32 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_H_ADDR_MEM_INIT_RST_N_MASK 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_H_ADDR_MEM_INIT_RST_N_SHIFT 0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_H_ADDR_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_H_ADDR_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_H_ADDR_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_L_ADDR_MEM_INIT_RST_N_MASK 0x2 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_L_ADDR_MEM_INIT_RST_N_SHIFT 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_L_ADDR_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_L_ADDR_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_BUF_BASE_L_ADDR_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_PID_MEM_INIT_RST_N_MASK 0x4 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_PID_MEM_INIT_RST_N_SHIFT 2 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_PID_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_PID_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_PID_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_CID_MEM_INIT_RST_N_MASK 0x8 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_CID_MEM_INIT_RST_N_SHIFT 3 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_CID_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_CID_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_REQ_CID_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_H_ADDR_MEM_INIT_RST_N_MASK 0x10 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_H_ADDR_MEM_INIT_RST_N_SHIFT 4 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_H_ADDR_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_H_ADDR_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_H_ADDR_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_L_ADDR_MEM_INIT_RST_N_MASK 0x20 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_L_ADDR_MEM_INIT_RST_N_SHIFT 5 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_L_ADDR_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_L_ADDR_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_BUF_BASE_L_ADDR_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_PID_MEM_INIT_RST_N_MASK 0x40 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_PID_MEM_INIT_RST_N_SHIFT 6 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_PID_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_PID_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_PID_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_CID_MEM_INIT_RST_N_MASK 0x80 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_CID_MEM_INIT_RST_N_SHIFT 7 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_CID_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_CID_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_RSP_CID_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_VF_ERR_STS_MEM_INIT_RST_N_MASK 0x100 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_VF_ERR_STS_MEM_INIT_RST_N_SHIFT 8 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_VF_ERR_STS_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_VF_ERR_STS_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_HOST_VF_ERR_STS_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_CMDQM_MEM_INIT_CTRL_VECTOR_ID_MEM_INIT_RST_N_MASK 0x200 +#define HIF_CMDQM_MEM_INIT_CTRL_VECTOR_ID_MEM_INIT_RST_N_SHIFT 9 +#define HIF_CMDQM_MEM_INIT_CTRL_VECTOR_ID_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CMDQM_MEM_INIT_CTRL_VECTOR_ID_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CMDQM_MEM_INIT_CTRL_VECTOR_ID_MEM_INIT_RST_N_MIN_VAL 0x0 + +#define HIF_CMDQM_TIMEOUT_INT_REG_ADDR 0xa1036010 +#define HIF_CMDQM_TIMEOUT_INT_REG_WIDTH 32 +#define HIF_CMDQM_TIMEOUT_INT_REG_LENGTH 32 +#define HIF_CMDQM_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_MASK 0x1 +#define HIF_CMDQM_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_SHIFT 0 +#define HIF_CMDQM_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_ARM_ERR_STS_REG_TIMEOUT_MASK 0x2 +#define HIF_CMDQM_TIMEOUT_INT_REG_ARM_ERR_STS_REG_TIMEOUT_SHIFT 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_ARM_ERR_STS_REG_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_BUF_BASE_H_ADDR_MEM_TIMEOUT_MASK 0x4 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_BUF_BASE_H_ADDR_MEM_TIMEOUT_SHIFT 2 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_BUF_BASE_H_ADDR_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_BUF_BASE_L_ADDR_MEM_TIMEOUT_MASK 0x8 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_BUF_BASE_L_ADDR_MEM_TIMEOUT_SHIFT 3 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_BUF_BASE_L_ADDR_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_PID_MEM_TIMEOUT_MASK 0x10 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_PID_MEM_TIMEOUT_SHIFT 4 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_PID_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_CID_MEM_TIMEOUT_MASK 0x20 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_CID_MEM_TIMEOUT_SHIFT 5 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_REQ_CID_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_BUF_BASE_H_ADDR_MEM_TIMEOUT_MASK 0x40 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_BUF_BASE_H_ADDR_MEM_TIMEOUT_SHIFT 6 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_BUF_BASE_H_ADDR_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_BUF_BASE_L_ADDR_MEM_TIMEOUT_MASK 0x80 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_BUF_BASE_L_ADDR_MEM_TIMEOUT_SHIFT 7 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_BUF_BASE_L_ADDR_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_PID_MEM_TIMEOUT_MASK 0x100 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_PID_MEM_TIMEOUT_SHIFT 8 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_PID_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_CID_MEM_TIMEOUT_MASK 0x200 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_CID_MEM_TIMEOUT_SHIFT 9 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_RSP_CID_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_VF_ERR_STS_MEM_TIMEOUT_MASK 0x400 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_VF_ERR_STS_MEM_TIMEOUT_SHIFT 10 +#define HIF_CMDQM_TIMEOUT_INT_REG_HOST_VF_ERR_STS_MEM_TIMEOUT_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_REG_VECTOR_ID_MEM_TIMEOUT_MASK 0x800 +#define HIF_CMDQM_TIMEOUT_INT_REG_VECTOR_ID_MEM_TIMEOUT_SHIFT 11 +#define HIF_CMDQM_TIMEOUT_INT_REG_VECTOR_ID_MEM_TIMEOUT_WIDTH 1 + +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_ADDR 0xa1036014 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_WIDTH 32 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_LENGTH 32 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MASK 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_SHIFT 0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_ARM_ERR_STS_REG_TIMEOUT_INT_MASK_MASK 0x2 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_ARM_ERR_STS_REG_TIMEOUT_INT_MASK_SHIFT 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_ARM_ERR_STS_REG_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_ARM_ERR_STS_REG_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_ARM_ERR_STS_REG_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_MASK 0x4 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_SHIFT 2 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_MASK 0x8 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_SHIFT 3 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_PID_MEM_TIMEOUT_INT_MASK_MASK 0x10 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_PID_MEM_TIMEOUT_INT_MASK_SHIFT 4 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_PID_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_PID_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_PID_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_CID_MEM_TIMEOUT_INT_MASK_MASK 0x20 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_CID_MEM_TIMEOUT_INT_MASK_SHIFT 5 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_CID_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_CID_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_REQ_CID_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_MASK 0x40 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_SHIFT 6 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_H_ADDR_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_MASK 0x80 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_SHIFT 7 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_BUF_BASE_L_ADDR_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_PID_MEM_TIMEOUT_INT_MASK_MASK 0x100 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_PID_MEM_TIMEOUT_INT_MASK_SHIFT 8 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_PID_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_PID_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_PID_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_CID_MEM_TIMEOUT_INT_MASK_MASK 0x200 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_CID_MEM_TIMEOUT_INT_MASK_SHIFT 9 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_CID_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_CID_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_RSP_CID_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_VF_ERR_STS_MEM_TIMEOUT_INT_MASK_MASK 0x400 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_VF_ERR_STS_MEM_TIMEOUT_INT_MASK_SHIFT 10 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_VF_ERR_STS_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_VF_ERR_STS_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_HOST_VF_ERR_STS_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_VECTOR_ID_MEM_TIMEOUT_INT_MASK_MASK 0x800 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_VECTOR_ID_MEM_TIMEOUT_INT_MASK_SHIFT 11 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_VECTOR_ID_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_VECTOR_ID_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CMDQM_TIMEOUT_INT_MASK_REG_VECTOR_ID_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 + +#define HIF_CMDQM_TIMEOUT_CFG_REG_ADDR 0xa1036018 +#define HIF_CMDQM_TIMEOUT_CFG_REG_WIDTH 32 +#define HIF_CMDQM_TIMEOUT_CFG_REG_LENGTH 32 +#define HIF_CMDQM_TIMEOUT_CFG_REG_TIMEOUT_PARA_MASK 0xffff +#define HIF_CMDQM_TIMEOUT_CFG_REG_TIMEOUT_PARA_SHIFT 0 +#define HIF_CMDQM_TIMEOUT_CFG_REG_TIMEOUT_PARA_WIDTH 16 +#define HIF_CMDQM_TIMEOUT_CFG_REG_TIMEOUT_PARA_MAX_VAL 0xffff +#define HIF_CMDQM_TIMEOUT_CFG_REG_TIMEOUT_PARA_MIN_VAL 0x0 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_cpm_csr_defines.h b/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_cpm_csr_defines.h new file mode 100644 index 0000000000000000000000000000000000000000..36a7399fb9ac191a5060a20289707698d36d0cf1 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_cpm_csr_defines.h @@ -0,0 +1,598 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _HIF_CPM_CSR_DEFINES_H_ +#define _HIF_CPM_CSR_DEFINES_H_ + +#define HIF_CPM_SOFT_RESET_REG_ADDR 0xa0000000 +#define HIF_CPM_SOFT_RESET_REG_WIDTH 32 +#define HIF_CPM_SOFT_RESET_REG_LENGTH 32 +#define HIF_CPM_SOFT_RESET_REG_SOFT_RESET_MASK 0x1 +#define HIF_CPM_SOFT_RESET_REG_SOFT_RESET_SHIFT 0 +#define HIF_CPM_SOFT_RESET_REG_SOFT_RESET_WIDTH 1 +#define HIF_CPM_SOFT_RESET_REG_SOFT_RESET_MAX_VAL 0x1 +#define HIF_CPM_SOFT_RESET_REG_SOFT_RESET_MIN_VAL 0x0 + +#define HIF_CPM_CHIP_VERSION_H_REG_ADDR 0xa0000010 +#define HIF_CPM_CHIP_VERSION_H_REG_WIDTH 32 +#define HIF_CPM_CHIP_VERSION_H_REG_LENGTH 32 +#define HIF_CPM_CHIP_VERSION_H_REG_CHIP_VERSION_H_MASK 0xffffffff +#define HIF_CPM_CHIP_VERSION_H_REG_CHIP_VERSION_H_SHIFT 0 +#define HIF_CPM_CHIP_VERSION_H_REG_CHIP_VERSION_H_WIDTH 32 + +#define HIF_CPM_CHIP_VERSION_M_REG_ADDR 0xa0000014 +#define HIF_CPM_CHIP_VERSION_M_REG_WIDTH 32 +#define HIF_CPM_CHIP_VERSION_M_REG_LENGTH 32 +#define HIF_CPM_CHIP_VERSION_M_REG_CHIP_VERSION_M_MASK 0xffffffff +#define HIF_CPM_CHIP_VERSION_M_REG_CHIP_VERSION_M_SHIFT 0 +#define HIF_CPM_CHIP_VERSION_M_REG_CHIP_VERSION_M_WIDTH 32 + +#define HIF_CPM_CHIP_VERSION_L_REG_ADDR 0xa0000018 +#define HIF_CPM_CHIP_VERSION_L_REG_WIDTH 32 +#define HIF_CPM_CHIP_VERSION_L_REG_LENGTH 32 +#define HIF_CPM_CHIP_VERSION_L_REG_CHIP_VERSION_L_MASK 0xffffffff +#define HIF_CPM_CHIP_VERSION_L_REG_CHIP_VERSION_L_SHIFT 0 +#define HIF_CPM_CHIP_VERSION_L_REG_CHIP_VERSION_L_WIDTH 32 + +#define HIF_CPM_SIMULATION_MODE_REG_ADDR 0xa000001c +#define HIF_CPM_SIMULATION_MODE_REG_WIDTH 32 +#define HIF_CPM_SIMULATION_MODE_REG_LENGTH 32 +#define HIF_CPM_SIMULATION_MODE_REG_SIMULATION_MODE_MASK 0x1 +#define HIF_CPM_SIMULATION_MODE_REG_SIMULATION_MODE_SHIFT 0 +#define HIF_CPM_SIMULATION_MODE_REG_SIMULATION_MODE_WIDTH 1 + +#define HIF_CPM_IDA_CMD_REG_ADDR 0xa0000020 +#define HIF_CPM_IDA_CMD_REG_WIDTH 32 +#define HIF_CPM_IDA_CMD_REG_LENGTH 32 +#define HIF_CPM_IDA_CMD_REG_IDA_IDX_MASK 0x1f +#define HIF_CPM_IDA_CMD_REG_IDA_IDX_SHIFT 0 +#define HIF_CPM_IDA_CMD_REG_IDA_IDX_WIDTH 5 +#define HIF_CPM_IDA_CMD_REG_IDA_IDX_MAX_VAL 0x1f +#define HIF_CPM_IDA_CMD_REG_IDA_IDX_MIN_VAL 0x0 +#define HIF_CPM_IDA_CMD_REG_IDA_LEN_MASK 0x1e0 +#define HIF_CPM_IDA_CMD_REG_IDA_LEN_SHIFT 5 +#define HIF_CPM_IDA_CMD_REG_IDA_LEN_WIDTH 4 +#define HIF_CPM_IDA_CMD_REG_IDA_LEN_MAX_VAL 0xf +#define HIF_CPM_IDA_CMD_REG_IDA_LEN_MIN_VAL 0x0 +#define HIF_CPM_IDA_CMD_REG_IDA_R0W1_MASK 0x200 +#define HIF_CPM_IDA_CMD_REG_IDA_R0W1_SHIFT 9 +#define HIF_CPM_IDA_CMD_REG_IDA_R0W1_WIDTH 1 +#define HIF_CPM_IDA_CMD_REG_IDA_R0W1_MAX_VAL 0x1 +#define HIF_CPM_IDA_CMD_REG_IDA_R0W1_MIN_VAL 0x0 + +#define HIF_CPM_IDA_ADDR_REG_ADDR 0xa0000080 +#define HIF_CPM_IDA_ADDR_REG_WIDTH 32 +#define HIF_CPM_IDA_ADDR_REG_LENGTH 32 +#define HIF_CPM_IDA_ADDR_REG_SIZE 32 +#define HIF_CPM_IDA_ADDR_REG_STRIDE 0x4 +#define HIF_CPM_IDA_ADDR_REG_IDA_ADDR_MASK 0xffffffff +#define HIF_CPM_IDA_ADDR_REG_IDA_ADDR_SHIFT 0 +#define HIF_CPM_IDA_ADDR_REG_IDA_ADDR_WIDTH 32 +#define HIF_CPM_IDA_ADDR_REG_IDA_ADDR_MAX_VAL 0xffffffff +#define HIF_CPM_IDA_ADDR_REG_IDA_ADDR_MIN_VAL 0x0 + +#define HIF_CPM_IDA_BUSY_REG_ADDR 0xa0000100 +#define HIF_CPM_IDA_BUSY_REG_WIDTH 32 +#define HIF_CPM_IDA_BUSY_REG_LENGTH 32 +#define HIF_CPM_IDA_BUSY_REG_IDA_BUSY_MASK 0xffffffff +#define HIF_CPM_IDA_BUSY_REG_IDA_BUSY_SHIFT 0 +#define HIF_CPM_IDA_BUSY_REG_IDA_BUSY_WIDTH 32 + +#define HIF_CPM_LOCK_GET_REG_ADDR 0xa0000104 +#define HIF_CPM_LOCK_GET_REG_WIDTH 32 +#define HIF_CPM_LOCK_GET_REG_LENGTH 32 +#define HIF_CPM_LOCK_GET_REG_LOCK_IDX_MASK 0x1f +#define HIF_CPM_LOCK_GET_REG_LOCK_IDX_SHIFT 0 +#define HIF_CPM_LOCK_GET_REG_LOCK_IDX_WIDTH 5 +#define HIF_CPM_LOCK_GET_REG_LOCK_VLD_MASK 0x20 +#define HIF_CPM_LOCK_GET_REG_LOCK_VLD_SHIFT 5 +#define HIF_CPM_LOCK_GET_REG_LOCK_VLD_WIDTH 1 + +#define HIF_CPM_LOCK_PUT_REG_ADDR 0xa0000108 +#define HIF_CPM_LOCK_PUT_REG_WIDTH 32 +#define HIF_CPM_LOCK_PUT_REG_LENGTH 32 +#define HIF_CPM_LOCK_PUT_REG_LOCK_IDX_MASK 0x1f +#define HIF_CPM_LOCK_PUT_REG_LOCK_IDX_SHIFT 0 +#define HIF_CPM_LOCK_PUT_REG_LOCK_IDX_WIDTH 5 +#define HIF_CPM_LOCK_PUT_REG_LOCK_IDX_MAX_VAL 0x1f +#define HIF_CPM_LOCK_PUT_REG_LOCK_IDX_MIN_VAL 0x0 + +#define HIF_CPM_LOCK_AVAIL_REG_ADDR 0xa000010c +#define HIF_CPM_LOCK_AVAIL_REG_WIDTH 32 +#define HIF_CPM_LOCK_AVAIL_REG_LENGTH 32 +#define HIF_CPM_LOCK_AVAIL_REG_LOCK_AVAIL_MASK 0xffffffff +#define HIF_CPM_LOCK_AVAIL_REG_LOCK_AVAIL_SHIFT 0 +#define HIF_CPM_LOCK_AVAIL_REG_LOCK_AVAIL_WIDTH 32 + +#define HIF_CPM_BUSY_SWITCH_REG_ADDR 0xa0000110 +#define HIF_CPM_BUSY_SWITCH_REG_WIDTH 32 +#define HIF_CPM_BUSY_SWITCH_REG_LENGTH 32 +#define HIF_CPM_BUSY_SWITCH_REG_BUSY_SWITCH_MASK 0x1 +#define HIF_CPM_BUSY_SWITCH_REG_BUSY_SWITCH_SHIFT 0 +#define HIF_CPM_BUSY_SWITCH_REG_BUSY_SWITCH_WIDTH 1 +#define HIF_CPM_BUSY_SWITCH_REG_BUSY_SWITCH_MAX_VAL 0x1 +#define HIF_CPM_BUSY_SWITCH_REG_BUSY_SWITCH_MIN_VAL 0x0 + +#define HIF_CPM_AXIS_IDA_WR_REQ_CNT_REG_ADDR 0xa0000114 +#define HIF_CPM_AXIS_IDA_WR_REQ_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_WR_REQ_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_WR_REQ_CNT_REG_AXIS_IDA_WR_REQ_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_IDA_WR_REQ_CNT_REG_AXIS_IDA_WR_REQ_CNT_SHIFT 0 +#define HIF_CPM_AXIS_IDA_WR_REQ_CNT_REG_AXIS_IDA_WR_REQ_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_DA_WR_REQ_CNT_REG_ADDR 0xa0000118 +#define HIF_CPM_AXIS_DA_WR_REQ_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_WR_REQ_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_WR_REQ_CNT_REG_AXIS_DA_WR_REQ_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_DA_WR_REQ_CNT_REG_AXIS_DA_WR_REQ_CNT_SHIFT 0 +#define HIF_CPM_AXIS_DA_WR_REQ_CNT_REG_AXIS_DA_WR_REQ_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_IDA_WR_RSP_CNT_REG_ADDR 0xa000011c +#define HIF_CPM_AXIS_IDA_WR_RSP_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_WR_RSP_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_WR_RSP_CNT_REG_AXIS_IDA_WR_RSP_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_IDA_WR_RSP_CNT_REG_AXIS_IDA_WR_RSP_CNT_SHIFT 0 +#define HIF_CPM_AXIS_IDA_WR_RSP_CNT_REG_AXIS_IDA_WR_RSP_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_DA_WR_RSP_CNT_REG_ADDR 0xa0000120 +#define HIF_CPM_AXIS_DA_WR_RSP_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_WR_RSP_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_WR_RSP_CNT_REG_AXIS_DA_WR_RSP_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_DA_WR_RSP_CNT_REG_AXIS_DA_WR_RSP_CNT_SHIFT 0 +#define HIF_CPM_AXIS_DA_WR_RSP_CNT_REG_AXIS_DA_WR_RSP_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_CNT_REG_ADDR 0xa0000124 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_CNT_REG_AXIS_IDA_WR_RSP_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_CNT_REG_AXIS_IDA_WR_RSP_ERR_CNT_SHIFT 0 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_CNT_REG_AXIS_IDA_WR_RSP_ERR_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_CNT_REG_ADDR 0xa0000128 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_CNT_REG_AXIS_DA_WR_RSP_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_CNT_REG_AXIS_DA_WR_RSP_ERR_CNT_SHIFT 0 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_CNT_REG_AXIS_DA_WR_RSP_ERR_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_IDA_WR_ERROR_ADDR_REG_ADDR 0xa000012c +#define HIF_CPM_AXIS_IDA_WR_ERROR_ADDR_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_WR_ERROR_ADDR_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_WR_ERROR_ADDR_REG_AXIS_IDA_WR_ERROR_ADDR_MASK 0xffffffff +#define HIF_CPM_AXIS_IDA_WR_ERROR_ADDR_REG_AXIS_IDA_WR_ERROR_ADDR_SHIFT 0 +#define HIF_CPM_AXIS_IDA_WR_ERROR_ADDR_REG_AXIS_IDA_WR_ERROR_ADDR_WIDTH 32 + +#define HIF_CPM_AXIS_DA_WR_ERROR_ADDR_REG_ADDR 0xa0000130 +#define HIF_CPM_AXIS_DA_WR_ERROR_ADDR_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_WR_ERROR_ADDR_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_WR_ERROR_ADDR_REG_AXIS_DA_WR_ERROR_ADDR_MASK 0xffffffff +#define HIF_CPM_AXIS_DA_WR_ERROR_ADDR_REG_AXIS_DA_WR_ERROR_ADDR_SHIFT 0 +#define HIF_CPM_AXIS_DA_WR_ERROR_ADDR_REG_AXIS_DA_WR_ERROR_ADDR_WIDTH 32 + +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_REG_ADDR 0xa0000134 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_REG_AXIS_IDA_WR_RSP_ERR_MASK 0x1 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_REG_AXIS_IDA_WR_RSP_ERR_SHIFT 0 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_REG_AXIS_IDA_WR_RSP_ERR_WIDTH 1 + +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_INT_MASK_REG_ADDR 0xa0000138 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_INT_MASK_REG_AXIS_IDA_WR_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_INT_MASK_REG_AXIS_IDA_WR_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_INT_MASK_REG_AXIS_IDA_WR_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_INT_MASK_REG_AXIS_IDA_WR_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_AXIS_IDA_WR_RSP_ERR_INT_MASK_REG_AXIS_IDA_WR_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_REG_ADDR 0xa000013c +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_REG_AXIS_DA_WR_RSP_ERR_MASK 0x1 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_REG_AXIS_DA_WR_RSP_ERR_SHIFT 0 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_REG_AXIS_DA_WR_RSP_ERR_WIDTH 1 + +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_INT_MASK_REG_ADDR 0xa0000140 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_INT_MASK_REG_AXIS_DA_WR_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_INT_MASK_REG_AXIS_DA_WR_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_INT_MASK_REG_AXIS_DA_WR_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_INT_MASK_REG_AXIS_DA_WR_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_AXIS_DA_WR_RSP_ERR_INT_MASK_REG_AXIS_DA_WR_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_AXIS_IDA_RD_REQ_CNT_REG_ADDR 0xa0000144 +#define HIF_CPM_AXIS_IDA_RD_REQ_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_RD_REQ_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_RD_REQ_CNT_REG_AXIS_IDA_RD_REQ_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_IDA_RD_REQ_CNT_REG_AXIS_IDA_RD_REQ_CNT_SHIFT 0 +#define HIF_CPM_AXIS_IDA_RD_REQ_CNT_REG_AXIS_IDA_RD_REQ_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_DA_RD_REQ_CNT_REG_ADDR 0xa0000148 +#define HIF_CPM_AXIS_DA_RD_REQ_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_RD_REQ_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_RD_REQ_CNT_REG_AXIS_DA_RD_REQ_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_DA_RD_REQ_CNT_REG_AXIS_DA_RD_REQ_CNT_SHIFT 0 +#define HIF_CPM_AXIS_DA_RD_REQ_CNT_REG_AXIS_DA_RD_REQ_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_IDA_RD_RSP_CNT_REG_ADDR 0xa000014c +#define HIF_CPM_AXIS_IDA_RD_RSP_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_RD_RSP_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_RD_RSP_CNT_REG_AXIS_IDA_RD_RSP_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_IDA_RD_RSP_CNT_REG_AXIS_IDA_RD_RSP_CNT_SHIFT 0 +#define HIF_CPM_AXIS_IDA_RD_RSP_CNT_REG_AXIS_IDA_RD_RSP_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_DA_RD_RSP_CNT_REG_ADDR 0xa0000150 +#define HIF_CPM_AXIS_DA_RD_RSP_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_RD_RSP_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_RD_RSP_CNT_REG_AXIS_DA_RD_RSP_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_DA_RD_RSP_CNT_REG_AXIS_DA_RD_RSP_CNT_SHIFT 0 +#define HIF_CPM_AXIS_DA_RD_RSP_CNT_REG_AXIS_DA_RD_RSP_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_CNT_REG_ADDR 0xa0000154 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_CNT_REG_AXIS_IDA_RD_RSP_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_CNT_REG_AXIS_IDA_RD_RSP_ERR_CNT_SHIFT 0 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_CNT_REG_AXIS_IDA_RD_RSP_ERR_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_CNT_REG_ADDR 0xa0000158 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_CNT_REG_AXIS_DA_RD_RSP_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_CNT_REG_AXIS_DA_RD_RSP_ERR_CNT_SHIFT 0 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_CNT_REG_AXIS_DA_RD_RSP_ERR_CNT_WIDTH 32 + +#define HIF_CPM_AXIS_IDA_RD_ERROR_ADDR_REG_ADDR 0xa000015c +#define HIF_CPM_AXIS_IDA_RD_ERROR_ADDR_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_RD_ERROR_ADDR_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_RD_ERROR_ADDR_REG_AXIS_IDA_RD_ERROR_ADDR_MASK 0xffffffff +#define HIF_CPM_AXIS_IDA_RD_ERROR_ADDR_REG_AXIS_IDA_RD_ERROR_ADDR_SHIFT 0 +#define HIF_CPM_AXIS_IDA_RD_ERROR_ADDR_REG_AXIS_IDA_RD_ERROR_ADDR_WIDTH 32 + +#define HIF_CPM_AXIS_DA_RD_ERROR_ADDR_REG_ADDR 0xa0000160 +#define HIF_CPM_AXIS_DA_RD_ERROR_ADDR_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_RD_ERROR_ADDR_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_RD_ERROR_ADDR_REG_AXIS_DA_RD_ERROR_ADDR_MASK 0xffffffff +#define HIF_CPM_AXIS_DA_RD_ERROR_ADDR_REG_AXIS_DA_RD_ERROR_ADDR_SHIFT 0 +#define HIF_CPM_AXIS_DA_RD_ERROR_ADDR_REG_AXIS_DA_RD_ERROR_ADDR_WIDTH 32 + +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_REG_ADDR 0xa0000164 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_REG_AXIS_IDA_RD_RSP_ERR_MASK 0x1 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_REG_AXIS_IDA_RD_RSP_ERR_SHIFT 0 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_REG_AXIS_IDA_RD_RSP_ERR_WIDTH 1 + +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_INT_MASK_REG_ADDR 0xa0000168 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_INT_MASK_REG_AXIS_IDA_RD_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_INT_MASK_REG_AXIS_IDA_RD_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_INT_MASK_REG_AXIS_IDA_RD_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_INT_MASK_REG_AXIS_IDA_RD_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_AXIS_IDA_RD_RSP_ERR_INT_MASK_REG_AXIS_IDA_RD_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_REG_ADDR 0xa000016c +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_REG_AXIS_DA_RD_RSP_ERR_MASK 0x1 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_REG_AXIS_DA_RD_RSP_ERR_SHIFT 0 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_REG_AXIS_DA_RD_RSP_ERR_WIDTH 1 + +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_INT_MASK_REG_ADDR 0xa0000170 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_INT_MASK_REG_AXIS_DA_RD_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_INT_MASK_REG_AXIS_DA_RD_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_INT_MASK_REG_AXIS_DA_RD_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_INT_MASK_REG_AXIS_DA_RD_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_AXIS_DA_RD_RSP_ERR_INT_MASK_REG_AXIS_DA_RD_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_AXIM_IDA_WR_REQ_CNT_REG_ADDR 0xa0000174 +#define HIF_CPM_AXIM_IDA_WR_REQ_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_WR_REQ_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_WR_REQ_CNT_REG_AXIM_IDA_WR_REQ_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_IDA_WR_REQ_CNT_REG_AXIM_IDA_WR_REQ_CNT_SHIFT 0 +#define HIF_CPM_AXIM_IDA_WR_REQ_CNT_REG_AXIM_IDA_WR_REQ_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_DA_WR_REQ_CNT_REG_ADDR 0xa0000178 +#define HIF_CPM_AXIM_DA_WR_REQ_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_WR_REQ_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_WR_REQ_CNT_REG_AXIM_DA_WR_REQ_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_DA_WR_REQ_CNT_REG_AXIM_DA_WR_REQ_CNT_SHIFT 0 +#define HIF_CPM_AXIM_DA_WR_REQ_CNT_REG_AXIM_DA_WR_REQ_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_IDA_WR_RSP_CNT_REG_ADDR 0xa000017c +#define HIF_CPM_AXIM_IDA_WR_RSP_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_WR_RSP_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_WR_RSP_CNT_REG_AXIM_IDA_WR_RSP_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_IDA_WR_RSP_CNT_REG_AXIM_IDA_WR_RSP_CNT_SHIFT 0 +#define HIF_CPM_AXIM_IDA_WR_RSP_CNT_REG_AXIM_IDA_WR_RSP_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_DA_WR_RSP_CNT_REG_ADDR 0xa0000180 +#define HIF_CPM_AXIM_DA_WR_RSP_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_WR_RSP_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_WR_RSP_CNT_REG_AXIM_DA_WR_RSP_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_DA_WR_RSP_CNT_REG_AXIM_DA_WR_RSP_CNT_SHIFT 0 +#define HIF_CPM_AXIM_DA_WR_RSP_CNT_REG_AXIM_DA_WR_RSP_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_CNT_REG_ADDR 0xa0000184 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_CNT_REG_AXIM_IDA_WR_RSP_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_CNT_REG_AXIM_IDA_WR_RSP_ERR_CNT_SHIFT 0 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_CNT_REG_AXIM_IDA_WR_RSP_ERR_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_CNT_REG_ADDR 0xa0000188 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_CNT_REG_AXIM_DA_WR_RSP_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_CNT_REG_AXIM_DA_WR_RSP_ERR_CNT_SHIFT 0 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_CNT_REG_AXIM_DA_WR_RSP_ERR_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_IDA_WR_ERROR_ADDR_REG_ADDR 0xa000018c +#define HIF_CPM_AXIM_IDA_WR_ERROR_ADDR_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_WR_ERROR_ADDR_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_WR_ERROR_ADDR_REG_AXIM_IDA_WR_ERROR_ADDR_MASK 0xffffffff +#define HIF_CPM_AXIM_IDA_WR_ERROR_ADDR_REG_AXIM_IDA_WR_ERROR_ADDR_SHIFT 0 +#define HIF_CPM_AXIM_IDA_WR_ERROR_ADDR_REG_AXIM_IDA_WR_ERROR_ADDR_WIDTH 32 + +#define HIF_CPM_AXIM_DA_WR_ERROR_ADDR_REG_ADDR 0xa0000190 +#define HIF_CPM_AXIM_DA_WR_ERROR_ADDR_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_WR_ERROR_ADDR_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_WR_ERROR_ADDR_REG_AXIM_DA_WR_ERROR_ADDR_MASK 0xffffffff +#define HIF_CPM_AXIM_DA_WR_ERROR_ADDR_REG_AXIM_DA_WR_ERROR_ADDR_SHIFT 0 +#define HIF_CPM_AXIM_DA_WR_ERROR_ADDR_REG_AXIM_DA_WR_ERROR_ADDR_WIDTH 32 + +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_REG_ADDR 0xa0000194 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_REG_AXIM_IDA_WR_RSP_ERR_MASK 0x1 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_REG_AXIM_IDA_WR_RSP_ERR_SHIFT 0 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_REG_AXIM_IDA_WR_RSP_ERR_WIDTH 1 + +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_INT_MASK_REG_ADDR 0xa0000198 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_INT_MASK_REG_AXIM_IDA_WR_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_INT_MASK_REG_AXIM_IDA_WR_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_INT_MASK_REG_AXIM_IDA_WR_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_INT_MASK_REG_AXIM_IDA_WR_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_AXIM_IDA_WR_RSP_ERR_INT_MASK_REG_AXIM_IDA_WR_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_REG_ADDR 0xa000019c +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_REG_AXIM_DA_WR_RSP_ERR_MASK 0x1 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_REG_AXIM_DA_WR_RSP_ERR_SHIFT 0 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_REG_AXIM_DA_WR_RSP_ERR_WIDTH 1 + +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_INT_MASK_REG_ADDR 0xa00001a0 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_INT_MASK_REG_AXIM_DA_WR_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_INT_MASK_REG_AXIM_DA_WR_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_INT_MASK_REG_AXIM_DA_WR_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_INT_MASK_REG_AXIM_DA_WR_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_AXIM_DA_WR_RSP_ERR_INT_MASK_REG_AXIM_DA_WR_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_AXIM_IDA_RD_REQ_CNT_REG_ADDR 0xa00001a4 +#define HIF_CPM_AXIM_IDA_RD_REQ_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_RD_REQ_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_RD_REQ_CNT_REG_AXIM_IDA_RD_REQ_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_IDA_RD_REQ_CNT_REG_AXIM_IDA_RD_REQ_CNT_SHIFT 0 +#define HIF_CPM_AXIM_IDA_RD_REQ_CNT_REG_AXIM_IDA_RD_REQ_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_DA_RD_REQ_CNT_REG_ADDR 0xa00001a8 +#define HIF_CPM_AXIM_DA_RD_REQ_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_RD_REQ_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_RD_REQ_CNT_REG_AXIM_DA_RD_REQ_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_DA_RD_REQ_CNT_REG_AXIM_DA_RD_REQ_CNT_SHIFT 0 +#define HIF_CPM_AXIM_DA_RD_REQ_CNT_REG_AXIM_DA_RD_REQ_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_IDA_RD_RSP_CNT_REG_ADDR 0xa00001ac +#define HIF_CPM_AXIM_IDA_RD_RSP_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_RD_RSP_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_RD_RSP_CNT_REG_AXIM_IDA_RD_RSP_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_IDA_RD_RSP_CNT_REG_AXIM_IDA_RD_RSP_CNT_SHIFT 0 +#define HIF_CPM_AXIM_IDA_RD_RSP_CNT_REG_AXIM_IDA_RD_RSP_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_DA_RD_RSP_CNT_REG_ADDR 0xa00001b0 +#define HIF_CPM_AXIM_DA_RD_RSP_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_RD_RSP_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_RD_RSP_CNT_REG_AXIM_DA_RD_RSP_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_DA_RD_RSP_CNT_REG_AXIM_DA_RD_RSP_CNT_SHIFT 0 +#define HIF_CPM_AXIM_DA_RD_RSP_CNT_REG_AXIM_DA_RD_RSP_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_CNT_REG_ADDR 0xa00001b4 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_CNT_REG_AXIM_IDA_RD_RSP_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_CNT_REG_AXIM_IDA_RD_RSP_ERR_CNT_SHIFT 0 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_CNT_REG_AXIM_IDA_RD_RSP_ERR_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_CNT_REG_ADDR 0xa00001b8 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_CNT_REG_AXIM_DA_RD_RSP_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_CNT_REG_AXIM_DA_RD_RSP_ERR_CNT_SHIFT 0 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_CNT_REG_AXIM_DA_RD_RSP_ERR_CNT_WIDTH 32 + +#define HIF_CPM_AXIM_IDA_RD_ERROR_ADDR_REG_ADDR 0xa00001bc +#define HIF_CPM_AXIM_IDA_RD_ERROR_ADDR_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_RD_ERROR_ADDR_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_RD_ERROR_ADDR_REG_AXIM_IDA_RD_ERROR_ADDR_MASK 0xffffffff +#define HIF_CPM_AXIM_IDA_RD_ERROR_ADDR_REG_AXIM_IDA_RD_ERROR_ADDR_SHIFT 0 +#define HIF_CPM_AXIM_IDA_RD_ERROR_ADDR_REG_AXIM_IDA_RD_ERROR_ADDR_WIDTH 32 + +#define HIF_CPM_AXIM_DA_RD_ERROR_ADDR_REG_ADDR 0xa00001c0 +#define HIF_CPM_AXIM_DA_RD_ERROR_ADDR_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_RD_ERROR_ADDR_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_RD_ERROR_ADDR_REG_AXIM_DA_RD_ERROR_ADDR_MASK 0xffffffff +#define HIF_CPM_AXIM_DA_RD_ERROR_ADDR_REG_AXIM_DA_RD_ERROR_ADDR_SHIFT 0 +#define HIF_CPM_AXIM_DA_RD_ERROR_ADDR_REG_AXIM_DA_RD_ERROR_ADDR_WIDTH 32 + +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_REG_ADDR 0xa00001c4 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_REG_AXIM_IDA_RD_RSP_ERR_MASK 0x1 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_REG_AXIM_IDA_RD_RSP_ERR_SHIFT 0 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_REG_AXIM_IDA_RD_RSP_ERR_WIDTH 1 + +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_INT_MASK_REG_ADDR 0xa00001c8 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_INT_MASK_REG_AXIM_IDA_RD_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_INT_MASK_REG_AXIM_IDA_RD_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_INT_MASK_REG_AXIM_IDA_RD_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_INT_MASK_REG_AXIM_IDA_RD_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_AXIM_IDA_RD_RSP_ERR_INT_MASK_REG_AXIM_IDA_RD_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_REG_ADDR 0xa00001cc +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_REG_AXIM_DA_RD_RSP_ERR_MASK 0x1 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_REG_AXIM_DA_RD_RSP_ERR_SHIFT 0 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_REG_AXIM_DA_RD_RSP_ERR_WIDTH 1 + +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_INT_MASK_REG_ADDR 0xa00001d0 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_INT_MASK_REG_AXIM_DA_RD_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_INT_MASK_REG_AXIM_DA_RD_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_INT_MASK_REG_AXIM_DA_RD_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_INT_MASK_REG_AXIM_DA_RD_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_AXIM_DA_RD_RSP_ERR_INT_MASK_REG_AXIM_DA_RD_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_STRB_ERR_REG_ADDR 0xa00001d4 +#define HIF_CPM_STRB_ERR_REG_WIDTH 32 +#define HIF_CPM_STRB_ERR_REG_LENGTH 32 +#define HIF_CPM_STRB_ERR_REG_STRB_ERR_MASK 0x1 +#define HIF_CPM_STRB_ERR_REG_STRB_ERR_SHIFT 0 +#define HIF_CPM_STRB_ERR_REG_STRB_ERR_WIDTH 1 + +#define HIF_CPM_STRB_ERR_INT_MASK_REG_ADDR 0xa00001d8 +#define HIF_CPM_STRB_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_STRB_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_STRB_ERR_INT_MASK_REG_STRB_ERR_INT_MASK_MASK 0x1 +#define HIF_CPM_STRB_ERR_INT_MASK_REG_STRB_ERR_INT_MASK_SHIFT 0 +#define HIF_CPM_STRB_ERR_INT_MASK_REG_STRB_ERR_INT_MASK_WIDTH 1 +#define HIF_CPM_STRB_ERR_INT_MASK_REG_STRB_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_STRB_ERR_INT_MASK_REG_STRB_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_IDA_OPCMD_BUSY_IDX_ERR_CNT_REG_ADDR 0xa00001dc +#define HIF_CPM_IDA_OPCMD_BUSY_IDX_ERR_CNT_REG_WIDTH 32 +#define HIF_CPM_IDA_OPCMD_BUSY_IDX_ERR_CNT_REG_LENGTH 32 +#define HIF_CPM_IDA_OPCMD_BUSY_IDX_ERR_CNT_REG_IDA_OPCMD_BUSY_IDX_ERR_CNT_MASK 0xffffffff +#define HIF_CPM_IDA_OPCMD_BUSY_IDX_ERR_CNT_REG_IDA_OPCMD_BUSY_IDX_ERR_CNT_SHIFT 0 +#define HIF_CPM_IDA_OPCMD_BUSY_IDX_ERR_CNT_REG_IDA_OPCMD_BUSY_IDX_ERR_CNT_WIDTH 32 + +#define HIF_CPM_CSR_MON_ADDR_REG_ADDR 0xa00001e0 +#define HIF_CPM_CSR_MON_ADDR_REG_WIDTH 32 +#define HIF_CPM_CSR_MON_ADDR_REG_LENGTH 32 +#define HIF_CPM_CSR_MON_ADDR_REG_CSR_MON_ADDR_MASK 0xffffffff +#define HIF_CPM_CSR_MON_ADDR_REG_CSR_MON_ADDR_SHIFT 0 +#define HIF_CPM_CSR_MON_ADDR_REG_CSR_MON_ADDR_WIDTH 32 +#define HIF_CPM_CSR_MON_ADDR_REG_CSR_MON_ADDR_MAX_VAL 0xffffffff +#define HIF_CPM_CSR_MON_ADDR_REG_CSR_MON_ADDR_MIN_VAL 0x0 + +#define HIF_CPM_CSR_MON_WR_CNT_REG_ADDR 0xa00001e4 +#define HIF_CPM_CSR_MON_WR_CNT_REG_WIDTH 32 +#define HIF_CPM_CSR_MON_WR_CNT_REG_LENGTH 32 +#define HIF_CPM_CSR_MON_WR_CNT_REG_CSR_MON_WR_CNT_MASK 0xff +#define HIF_CPM_CSR_MON_WR_CNT_REG_CSR_MON_WR_CNT_SHIFT 0 +#define HIF_CPM_CSR_MON_WR_CNT_REG_CSR_MON_WR_CNT_WIDTH 8 + +#define HIF_CPM_CSR_MON_WR_DATA_REG_ADDR 0xa00001e8 +#define HIF_CPM_CSR_MON_WR_DATA_REG_WIDTH 32 +#define HIF_CPM_CSR_MON_WR_DATA_REG_LENGTH 32 +#define HIF_CPM_CSR_MON_WR_DATA_REG_CSR_MON_WR_DATA_MASK 0xffffffff +#define HIF_CPM_CSR_MON_WR_DATA_REG_CSR_MON_WR_DATA_SHIFT 0 +#define HIF_CPM_CSR_MON_WR_DATA_REG_CSR_MON_WR_DATA_WIDTH 32 + +#define HIF_CPM_CSR_MON_RD_CNT_REG_ADDR 0xa00001ec +#define HIF_CPM_CSR_MON_RD_CNT_REG_WIDTH 32 +#define HIF_CPM_CSR_MON_RD_CNT_REG_LENGTH 32 +#define HIF_CPM_CSR_MON_RD_CNT_REG_CSR_MON_RD_CNT_MASK 0xff +#define HIF_CPM_CSR_MON_RD_CNT_REG_CSR_MON_RD_CNT_SHIFT 0 +#define HIF_CPM_CSR_MON_RD_CNT_REG_CSR_MON_RD_CNT_WIDTH 8 + +#define HIF_CPM_CHIP_HOTFIX_NUM_REG_ADDR 0xa00001f0 +#define HIF_CPM_CHIP_HOTFIX_NUM_REG_WIDTH 32 +#define HIF_CPM_CHIP_HOTFIX_NUM_REG_LENGTH 32 +#define HIF_CPM_CHIP_HOTFIX_NUM_REG_CHIP_HOTFIX_NUM_MASK 0xffffffff +#define HIF_CPM_CHIP_HOTFIX_NUM_REG_CHIP_HOTFIX_NUM_SHIFT 0 +#define HIF_CPM_CHIP_HOTFIX_NUM_REG_CHIP_HOTFIX_NUM_WIDTH 32 + +#define HIF_CPM_CHIP_FEATURE_FLAG_REG_ADDR 0xa00001f4 +#define HIF_CPM_CHIP_FEATURE_FLAG_REG_WIDTH 32 +#define HIF_CPM_CHIP_FEATURE_FLAG_REG_LENGTH 32 +#define HIF_CPM_CHIP_FEATURE_FLAG_REG_CHIP_FEATURE_FLAG_MASK 0xffffffff +#define HIF_CPM_CHIP_FEATURE_FLAG_REG_CHIP_FEATURE_FLAG_SHIFT 0 +#define HIF_CPM_CHIP_FEATURE_FLAG_REG_CHIP_FEATURE_FLAG_WIDTH 32 + +#define HIF_CPM_IDA_DATA_MEM_ADDR 0xa0000800 +#define HIF_CPM_IDA_DATA_MEM_WIDTH 32 +#define HIF_CPM_IDA_DATA_MEM_LENGTH 32 +#define HIF_CPM_IDA_DATA_MEM_DEPTH 512 +#define HIF_CPM_IDA_DATA_MEM_DATA_MASK 0xffffffff +#define HIF_CPM_IDA_DATA_MEM_DATA_SHIFT 0 +#define HIF_CPM_IDA_DATA_MEM_DATA_WIDTH 32 + +#define HIF_CPM_MEM_ERROR_INT_ADDR 0xa0001000 +#define HIF_CPM_MEM_ERROR_INT_WIDTH 32 +#define HIF_CPM_MEM_ERROR_INT_LENGTH 32 +#define HIF_CPM_MEM_ERROR_INT_IDA_DATA_MEM_SB_ERR_MASK 0x1 +#define HIF_CPM_MEM_ERROR_INT_IDA_DATA_MEM_SB_ERR_SHIFT 0 +#define HIF_CPM_MEM_ERROR_INT_IDA_DATA_MEM_SB_ERR_WIDTH 1 +#define HIF_CPM_MEM_ERROR_INT_IDA_DATA_MEM_DB_ERR_MASK 0x2 +#define HIF_CPM_MEM_ERROR_INT_IDA_DATA_MEM_DB_ERR_SHIFT 1 +#define HIF_CPM_MEM_ERROR_INT_IDA_DATA_MEM_DB_ERR_WIDTH 1 + +#define HIF_CPM_MEM_INIT_CTRL_ADDR 0xa0001008 +#define HIF_CPM_MEM_INIT_CTRL_WIDTH 32 +#define HIF_CPM_MEM_INIT_CTRL_LENGTH 32 +#define HIF_CPM_MEM_INIT_CTRL_IDA_DATA_MEM_INIT_RST_N_MASK 0x1 +#define HIF_CPM_MEM_INIT_CTRL_IDA_DATA_MEM_INIT_RST_N_SHIFT 0 +#define HIF_CPM_MEM_INIT_CTRL_IDA_DATA_MEM_INIT_RST_N_WIDTH 1 +#define HIF_CPM_MEM_INIT_CTRL_IDA_DATA_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_CPM_MEM_INIT_CTRL_IDA_DATA_MEM_INIT_RST_N_MIN_VAL 0x0 + +#define HIF_CPM_TIMEOUT_INT_REG_ADDR 0xa0001010 +#define HIF_CPM_TIMEOUT_INT_REG_WIDTH 32 +#define HIF_CPM_TIMEOUT_INT_REG_LENGTH 32 +#define HIF_CPM_TIMEOUT_INT_REG_LOCK_GET_REG_TIMEOUT_MASK 0x1 +#define HIF_CPM_TIMEOUT_INT_REG_LOCK_GET_REG_TIMEOUT_SHIFT 0 +#define HIF_CPM_TIMEOUT_INT_REG_LOCK_GET_REG_TIMEOUT_WIDTH 1 +#define HIF_CPM_TIMEOUT_INT_REG_IDA_DATA_MEM_TIMEOUT_MASK 0x2 +#define HIF_CPM_TIMEOUT_INT_REG_IDA_DATA_MEM_TIMEOUT_SHIFT 1 +#define HIF_CPM_TIMEOUT_INT_REG_IDA_DATA_MEM_TIMEOUT_WIDTH 1 + +#define HIF_CPM_TIMEOUT_INT_MASK_REG_ADDR 0xa0001014 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_WIDTH 32 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_LENGTH 32 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_LOCK_GET_REG_TIMEOUT_INT_MASK_MASK 0x1 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_LOCK_GET_REG_TIMEOUT_INT_MASK_SHIFT 0 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_LOCK_GET_REG_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_LOCK_GET_REG_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_LOCK_GET_REG_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_IDA_DATA_MEM_TIMEOUT_INT_MASK_MASK 0x2 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_IDA_DATA_MEM_TIMEOUT_INT_MASK_SHIFT 1 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_IDA_DATA_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_IDA_DATA_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_CPM_TIMEOUT_INT_MASK_REG_IDA_DATA_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 + +#define HIF_CPM_TIMEOUT_CFG_REG_ADDR 0xa0001018 +#define HIF_CPM_TIMEOUT_CFG_REG_WIDTH 32 +#define HIF_CPM_TIMEOUT_CFG_REG_LENGTH 32 +#define HIF_CPM_TIMEOUT_CFG_REG_TIMEOUT_PARA_MASK 0xffff +#define HIF_CPM_TIMEOUT_CFG_REG_TIMEOUT_PARA_SHIFT 0 +#define HIF_CPM_TIMEOUT_CFG_REG_TIMEOUT_PARA_WIDTH 16 +#define HIF_CPM_TIMEOUT_CFG_REG_TIMEOUT_PARA_MAX_VAL 0xffff +#define HIF_CPM_TIMEOUT_CFG_REG_TIMEOUT_PARA_MIN_VAL 0x0 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_irq_csr_defines.h b/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_irq_csr_defines.h new file mode 100644 index 0000000000000000000000000000000000000000..efdea561a28605c3645b9a9b6e98dcfcf272be90 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_irq_csr_defines.h @@ -0,0 +1,1778 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _HIF_IRQ_CSR_DEFINES_H_ +#define _HIF_IRQ_CSR_DEFINES_H_ + +#define HIF_IRQ_SOFT_RESET_REG_ADDR 0xa1100000 +#define HIF_IRQ_SOFT_RESET_REG_WIDTH 32 +#define HIF_IRQ_SOFT_RESET_REG_LENGTH 32 +#define HIF_IRQ_SOFT_RESET_REG_SOFT_RESET_MASK 0x1 +#define HIF_IRQ_SOFT_RESET_REG_SOFT_RESET_SHIFT 0 +#define HIF_IRQ_SOFT_RESET_REG_SOFT_RESET_WIDTH 1 +#define HIF_IRQ_SOFT_RESET_REG_SOFT_RESET_MAX_VAL 0x1 +#define HIF_IRQ_SOFT_RESET_REG_SOFT_RESET_MIN_VAL 0x0 + +#define HIF_IRQ_SCRATCH_PAD_REG_ADDR 0xa110000c +#define HIF_IRQ_SCRATCH_PAD_REG_WIDTH 32 +#define HIF_IRQ_SCRATCH_PAD_REG_LENGTH 32 +#define HIF_IRQ_SCRATCH_PAD_REG_SCRATCH_PAD_MASK 0xffffffff +#define HIF_IRQ_SCRATCH_PAD_REG_SCRATCH_PAD_SHIFT 0 +#define HIF_IRQ_SCRATCH_PAD_REG_SCRATCH_PAD_WIDTH 32 +#define HIF_IRQ_SCRATCH_PAD_REG_SCRATCH_PAD_MAX_VAL 0xffffffff +#define HIF_IRQ_SCRATCH_PAD_REG_SCRATCH_PAD_MIN_VAL 0x0 + +#define HIF_IRQ_CSR_ERR_FLAG_REG_ADDR 0xa1100010 +#define HIF_IRQ_CSR_ERR_FLAG_REG_WIDTH 32 +#define HIF_IRQ_CSR_ERR_FLAG_REG_LENGTH 32 +#define HIF_IRQ_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_MASK 0x1 +#define HIF_IRQ_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_SHIFT 0 +#define HIF_IRQ_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_WIDTH 1 + +#define HIF_IRQ_CSR_ERR_ADDR_REG_ADDR 0xa1100014 +#define HIF_IRQ_CSR_ERR_ADDR_REG_WIDTH 32 +#define HIF_IRQ_CSR_ERR_ADDR_REG_LENGTH 32 +#define HIF_IRQ_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_MASK 0xffffffff +#define HIF_IRQ_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_SHIFT 0 +#define HIF_IRQ_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_WIDTH 32 + +#define HIF_IRQ_CSR_ERR_LEN_REG_ADDR 0xa1100018 +#define HIF_IRQ_CSR_ERR_LEN_REG_WIDTH 32 +#define HIF_IRQ_CSR_ERR_LEN_REG_LENGTH 32 +#define HIF_IRQ_CSR_ERR_LEN_REG_CSR_ERR_LEN_MASK 0x3ff +#define HIF_IRQ_CSR_ERR_LEN_REG_CSR_ERR_LEN_SHIFT 0 +#define HIF_IRQ_CSR_ERR_LEN_REG_CSR_ERR_LEN_WIDTH 10 + +#define HIF_IRQ_CSR_ERR_TYPE_REG_ADDR 0xa110001c +#define HIF_IRQ_CSR_ERR_TYPE_REG_WIDTH 32 +#define HIF_IRQ_CSR_ERR_TYPE_REG_LENGTH 32 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_MASK 0x1 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_SHIFT 0 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_WIDTH 1 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_MASK 0x6 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_SHIFT 1 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_WIDTH 2 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_MASK 0x8 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_SHIFT 3 +#define HIF_IRQ_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_WIDTH 1 + +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_ADDR 0xa1100020 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_WIDTH 32 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_LENGTH 32 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_VECTOR_ID_MAX_VAL 0xfff +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_EN_MASK 0x1000 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_EN_SHIFT 12 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_EN_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_EN_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MSIX_REG_TOP_INT_SUM_MSIX_EN_MIN_VAL 0x0 + +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_ADDR 0xa1100024 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_WIDTH 32 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_LENGTH 32 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_VECTOR_ID_MAX_VAL 0xfff +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_EN_MASK 0x1000 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_EN_SHIFT 12 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_EN_WIDTH 1 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_EN_MAX_VAL 0x1 +#define HIF_IRQ_PCIE2HIF_PCIE0_MSG_INT_MSIX_REG_PCIE2HIF_PCIE0_MSG_INT_MSIX_EN_MIN_VAL 0x0 + +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_ADDR 0xa1100028 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_WIDTH 32 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_LENGTH 32 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_VECTOR_ID_MAX_VAL 0xfff +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_EN_MASK 0x1000 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_EN_SHIFT 12 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_EN_WIDTH 1 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_EN_MAX_VAL 0x1 +#define HIF_IRQ_PCIE2HIF_PCIE1_MSG_INT_MSIX_REG_PCIE2HIF_PCIE1_MSG_INT_MSIX_EN_MIN_VAL 0x0 + +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_ADDR 0xa1100030 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_WIDTH 32 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_LENGTH 32 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_SIZE 2 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_STRIDE 0x4 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_VECTOR_ID_MAX_VAL 0xfff +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_EN_MASK 0x1000 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_EN_SHIFT 12 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_EN_WIDTH 1 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_EN_MAX_VAL 0x1 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_MSIX_REG_PIO2IRQ_PIO_REQ_INT_MSIX_EN_MIN_VAL 0x0 + +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_ADDR 0xa1100038 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_WIDTH 32 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_LENGTH 32 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_VECTOR_ID_MAX_VAL 0xfff +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_EN_MASK 0x1000 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_EN_SHIFT 12 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_EN_WIDTH 1 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_EN_MAX_VAL 0x1 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_REG_CMDQM2IRQ_CMDQ_REQ_INT_MSIX_EN_MIN_VAL 0x0 + +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_ADDR 0xa1100040 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_WIDTH 32 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_LENGTH 32 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_SIZE 2 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_STRIDE 0x4 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_VECTOR_ID_MAX_VAL 0xfff +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_EN_MASK 0x1000 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_EN_SHIFT 12 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_EN_WIDTH 1 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_EN_MAX_VAL 0x1 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_MSIX_REG_CMDQM2IRQ_DMA_RX_INT_MSIX_EN_MIN_VAL 0x0 + +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_ADDR 0xa1100048 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_WIDTH 32 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_LENGTH 32 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_SIZE 2 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_STRIDE 0x4 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_VECTOR_ID_MAX_VAL 0xfff +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_EN_MASK 0x1000 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_EN_SHIFT 12 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_EN_WIDTH 1 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_EN_MAX_VAL 0x1 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_MSIX_REG_CMDQM2IRQ_DMA_TX_INT_MSIX_EN_MIN_VAL 0x0 + +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_ADDR 0xa1100050 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_WIDTH 32 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_LENGTH 32 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_VECTOR_ID_MAX_VAL \ + 0xfff +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_EN_MASK 0x1000 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_EN_SHIFT 12 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_EN_WIDTH 1 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_EN_MAX_VAL 0x1 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_TBL2IRQ_TBL_RD_DONE_INT_MSIX_EN_MIN_VAL 0x0 + +#define HIF_IRQ_BUBBLE_INSERT_TH_REG_ADDR 0xa1100054 +#define HIF_IRQ_BUBBLE_INSERT_TH_REG_WIDTH 32 +#define HIF_IRQ_BUBBLE_INSERT_TH_REG_LENGTH 32 +#define HIF_IRQ_BUBBLE_INSERT_TH_REG_BUBBLE_INSERT_TH_MASK 0xffff +#define HIF_IRQ_BUBBLE_INSERT_TH_REG_BUBBLE_INSERT_TH_SHIFT 0 +#define HIF_IRQ_BUBBLE_INSERT_TH_REG_BUBBLE_INSERT_TH_WIDTH 16 +#define HIF_IRQ_BUBBLE_INSERT_TH_REG_BUBBLE_INSERT_TH_MAX_VAL 0xffff +#define HIF_IRQ_BUBBLE_INSERT_TH_REG_BUBBLE_INSERT_TH_MIN_VAL 0x0 + +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_ADDR 0xa1100058 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_WIDTH 32 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_LENGTH 32 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_PCIE0_INT_SUM_STS_MASK 0x1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_PCIE0_INT_SUM_STS_SHIFT 0 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_PCIE0_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_PCIE1_INT_SUM_STS_MASK 0x2 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_PCIE1_INT_SUM_STS_SHIFT 1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_PCIE1_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_NIF_INT_SUM_STS_MASK 0x4 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_NIF_INT_SUM_STS_SHIFT 2 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_NIF_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_EPP_INT_SUM_STS_MASK 0x8 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_EPP_INT_SUM_STS_SHIFT 3 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_EPP_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_IPP_INT_SUM_STS_MASK 0x10 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_IPP_INT_SUM_STS_SHIFT 4 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_IPP_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_TPE_INT_SUM_STS_MASK 0x20 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_TPE_INT_SUM_STS_SHIFT 5 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_TPE_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_TM_INT_SUM_STS_MASK 0x40 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_TM_INT_SUM_STS_SHIFT 6 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_TM_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_HIF_INT_SUM_STS_MASK 0x80 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_HIF_INT_SUM_STS_SHIFT 7 +#define HIF_IRQ_TOP_INT_SUM_STATUS_REG_TOP_HIF_INT_SUM_STS_WIDTH 1 + +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_ADDR 0xa110005c +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_WIDTH 32 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_LENGTH 32 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE0_INT_SUM_MASK_MASK 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE0_INT_SUM_MASK_SHIFT 0 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE0_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE0_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE0_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE1_INT_SUM_MASK_MASK 0x2 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE1_INT_SUM_MASK_SHIFT 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE1_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE1_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_PCIE1_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_NIF_INT_SUM_MASK_MASK 0x4 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_NIF_INT_SUM_MASK_SHIFT 2 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_NIF_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_NIF_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_NIF_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_EPP_INT_SUM_MASK_MASK 0x8 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_EPP_INT_SUM_MASK_SHIFT 3 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_EPP_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_EPP_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_EPP_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_IPP_INT_SUM_MASK_MASK 0x10 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_IPP_INT_SUM_MASK_SHIFT 4 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_IPP_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_IPP_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_IPP_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TPE_INT_SUM_MASK_MASK 0x20 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TPE_INT_SUM_MASK_SHIFT 5 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TPE_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TPE_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TPE_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TM_INT_SUM_MASK_MASK 0x40 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TM_INT_SUM_MASK_SHIFT 6 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TM_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TM_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_TM_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_HIF_INT_SUM_MASK_MASK 0x80 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_HIF_INT_SUM_MASK_SHIFT 7 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_HIF_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_HIF_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TOP_INT_SUM_MASK_REG_TOP_HIF_INT_SUM_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_ADDR 0xa1100060 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_WIDTH 32 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_LENGTH 32 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_CPM_INT_SUM_STS_MASK 0x1 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_CPM_INT_SUM_STS_SHIFT 0 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_CPM_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_PIO_INT_SUM_STS_MASK 0x2 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_PIO_INT_SUM_STS_SHIFT 1 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_PIO_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_ADPT_INT_SUM_STS_MASK 0x4 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_ADPT_INT_SUM_STS_SHIFT 2 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_ADPT_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_TBL_INT_SUM_STS_MASK 0x8 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_TBL_INT_SUM_STS_SHIFT 3 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_TBL_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_CMDQM_INT_SUM_STS_MASK 0x10 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_CMDQM_INT_SUM_STS_SHIFT 4 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_CMDQM_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_IRQ_INT_SUM_STS_MASK 0x20 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_IRQ_INT_SUM_STS_SHIFT 5 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_IRQ_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_NOC_INT_SUM_STS_MASK 0x40 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_NOC_INT_SUM_STS_SHIFT 6 +#define HIF_IRQ_HIF_INT_SUM_STATUS_REG_HIF_NOC_INT_SUM_STS_WIDTH 1 + +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_ADDR 0xa1100064 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_WIDTH 32 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_LENGTH 32 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CPM_INT_SUM_MASK_MASK 0x1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CPM_INT_SUM_MASK_SHIFT 0 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CPM_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CPM_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CPM_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_PIO_INT_SUM_MASK_MASK 0x2 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_PIO_INT_SUM_MASK_SHIFT 1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_PIO_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_PIO_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_PIO_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_ADPT_INT_SUM_MASK_MASK 0x4 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_ADPT_INT_SUM_MASK_SHIFT 2 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_ADPT_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_ADPT_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_ADPT_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_TBL_INT_SUM_MASK_MASK 0x8 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_TBL_INT_SUM_MASK_SHIFT 3 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_TBL_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_TBL_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_TBL_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CMDQM_INT_SUM_MASK_MASK 0x10 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CMDQM_INT_SUM_MASK_SHIFT 4 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CMDQM_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CMDQM_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_CMDQM_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_IRQ_INT_SUM_MASK_MASK 0x20 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_IRQ_INT_SUM_MASK_SHIFT 5 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_IRQ_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_IRQ_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_IRQ_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_NOC_INT_SUM_MASK_MASK 0x40 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_NOC_INT_SUM_MASK_SHIFT 6 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_NOC_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_NOC_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_HIF_INT_SUM_MASK_REG_HIF_NOC_INT_SUM_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_ADDR 0xa1100068 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_WIDTH 32 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_LENGTH 32 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_MTR_INT_SUM_STS_MASK 0x1 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_MTR_INT_SUM_STS_SHIFT 0 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_MTR_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_QMU_INT_SUM_STS_MASK 0x2 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_QMU_INT_SUM_STS_SHIFT 1 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_QMU_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_FRG_INT_SUM_STS_MASK 0x4 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_FRG_INT_SUM_STS_SHIFT 2 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_FRG_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_PDB_INT_SUM_STS_MASK 0x8 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_PDB_INT_SUM_STS_SHIFT 3 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_PDB_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_PRA_INT_SUM_STS_MASK 0x10 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_PRA_INT_SUM_STS_SHIFT 4 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_PRA_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_SCA_INT_SUM_STS_MASK 0x20 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_SCA_INT_SUM_STS_SHIFT 5 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_SCA_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_OCA_INT_SUM_STS_MASK 0x40 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_OCA_INT_SUM_STS_SHIFT 6 +#define HIF_IRQ_TM_INT_SUM_STATUS_REG_TM_OCA_INT_SUM_STS_WIDTH 1 + +#define HIF_IRQ_TM_INT_SUM_MASK_REG_ADDR 0xa110006c +#define HIF_IRQ_TM_INT_SUM_MASK_REG_WIDTH 32 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_LENGTH 32 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_MTR_INT_SUM_MASK_MASK 0x1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_MTR_INT_SUM_MASK_SHIFT 0 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_MTR_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_MTR_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_MTR_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_QMU_INT_SUM_MASK_MASK 0x2 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_QMU_INT_SUM_MASK_SHIFT 1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_QMU_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_QMU_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_QMU_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_FRG_INT_SUM_MASK_MASK 0x4 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_FRG_INT_SUM_MASK_SHIFT 2 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_FRG_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_FRG_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_FRG_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PDB_INT_SUM_MASK_MASK 0x8 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PDB_INT_SUM_MASK_SHIFT 3 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PDB_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PDB_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PDB_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PRA_INT_SUM_MASK_MASK 0x10 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PRA_INT_SUM_MASK_SHIFT 4 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PRA_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PRA_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_PRA_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_SCA_INT_SUM_MASK_MASK 0x20 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_SCA_INT_SUM_MASK_SHIFT 5 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_SCA_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_SCA_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_SCA_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_OCA_INT_SUM_MASK_MASK 0x40 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_OCA_INT_SUM_MASK_SHIFT 6 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_OCA_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_OCA_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TM_INT_SUM_MASK_REG_TM_OCA_INT_SUM_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_ADDR 0xa1100070 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_WIDTH 32 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_LENGTH 32 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MMC_INT_SUM_STS_MASK 0x1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MMC_INT_SUM_STS_SHIFT 0 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MMC_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_QPM_INT_SUM_STS_MASK 0x2 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_QPM_INT_SUM_STS_SHIFT 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_QPM_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_SV_INT_SUM_STS_MASK 0x4 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_SV_INT_SUM_STS_SHIFT 2 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_SV_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MER_INT_SUM_STS_MASK 0x8 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MER_INT_SUM_STS_SHIFT 3 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MER_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_TOP_INT_SUM_STS_MASK 0x10 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_TOP_INT_SUM_STS_SHIFT 4 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_TOP_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_CRDT_INT_SUM_STS_MASK 0x20 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_CRDT_INT_SUM_STS_SHIFT 5 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_CRDT_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MET_INT_SUM_STS_MASK 0x40 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MET_INT_SUM_STS_SHIFT 6 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MET_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_SHAP_INT_SUM_STS_MASK 0x80 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_SHAP_INT_SUM_STS_SHIFT 7 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_SHAP_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MA_INT_SUM_STS_MASK 0x100 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MA_INT_SUM_STS_SHIFT 8 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_MA_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_CEM_INT_SUM_STS_MASK 0x200 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_CEM_INT_SUM_STS_SHIFT 9 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_CEM_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_PET_INT_SUM_STS_MASK 0x400 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_PET_INT_SUM_STS_SHIFT 10 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_PET_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_PG_INT_SUM_STS_MASK 0x800 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_PG_INT_SUM_STS_SHIFT 11 +#define HIF_IRQ_TPE_INT_SUM_STATUS_REG_TPE_PG_INT_SUM_STS_WIDTH 1 + +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_ADDR 0xa1100074 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_WIDTH 32 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_LENGTH 32 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MMC_INT_SUM_MASK_MASK 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MMC_INT_SUM_MASK_SHIFT 0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MMC_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MMC_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MMC_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_QPM_INT_SUM_MASK_MASK 0x2 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_QPM_INT_SUM_MASK_SHIFT 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_QPM_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_QPM_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_QPM_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SV_INT_SUM_MASK_MASK 0x4 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SV_INT_SUM_MASK_SHIFT 2 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SV_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SV_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SV_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MER_INT_SUM_MASK_MASK 0x8 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MER_INT_SUM_MASK_SHIFT 3 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MER_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MER_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MER_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_TOP_INT_SUM_MASK_MASK 0x10 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_TOP_INT_SUM_MASK_SHIFT 4 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_TOP_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_TOP_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_TOP_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CRDT_INT_SUM_MASK_MASK 0x20 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CRDT_INT_SUM_MASK_SHIFT 5 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CRDT_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CRDT_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CRDT_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MET_INT_SUM_MASK_MASK 0x40 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MET_INT_SUM_MASK_SHIFT 6 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MET_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MET_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MET_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SHAP_INT_SUM_MASK_MASK 0x80 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SHAP_INT_SUM_MASK_SHIFT 7 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SHAP_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SHAP_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_SHAP_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MA_INT_SUM_MASK_MASK 0x100 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MA_INT_SUM_MASK_SHIFT 8 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MA_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MA_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_MA_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CEM_INT_SUM_MASK_MASK 0x200 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CEM_INT_SUM_MASK_SHIFT 9 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CEM_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CEM_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_CEM_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PET_INT_SUM_MASK_MASK 0x400 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PET_INT_SUM_MASK_SHIFT 10 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PET_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PET_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PET_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PG_INT_SUM_MASK_MASK 0x800 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PG_INT_SUM_MASK_SHIFT 11 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PG_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PG_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TPE_INT_SUM_MASK_REG_TPE_PG_INT_SUM_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_ADDR 0xa1100078 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_WIDTH 32 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_LENGTH 32 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_EM_INT_SUM_STS_MASK 0x1 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_EM_INT_SUM_STS_SHIFT 0 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_EM_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_STAT_INT_SUM_STS_MASK 0x2 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_STAT_INT_SUM_STS_SHIFT 1 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_STAT_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_CLSF_DMA_INT_SUM_STS_MASK 0x4 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_CLSF_DMA_INT_SUM_STS_SHIFT 2 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_CLSF_DMA_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_DIR_INT_SUM_STS_MASK 0x8 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_DIR_INT_SUM_STS_SHIFT 3 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_DIR_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_TCAM_INT_SUM_STS_MASK 0x10 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_TCAM_INT_SUM_STS_SHIFT 4 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_TCAM_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_CLSF_CTRL_INT_SUM_STS_MASK 0x20 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_CLSF_CTRL_INT_SUM_STS_SHIFT 5 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_CLSF_CTRL_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_PRS_INT_SUM_STS_MASK 0x40 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_PRS_INT_SUM_STS_SHIFT 6 +#define HIF_IRQ_IPP_INT_SUM_STATUS_REG_IPP_PRS_INT_SUM_STS_WIDTH 1 + +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_ADDR 0xa110007c +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_WIDTH 32 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_LENGTH 32 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_EM_INT_SUM_MASK_MASK 0x1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_EM_INT_SUM_MASK_SHIFT 0 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_EM_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_EM_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_EM_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_STAT_INT_SUM_MASK_MASK 0x2 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_STAT_INT_SUM_MASK_SHIFT 1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_STAT_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_STAT_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_STAT_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_DMA_INT_SUM_MASK_MASK 0x4 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_DMA_INT_SUM_MASK_SHIFT 2 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_DMA_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_DMA_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_DMA_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_DIR_INT_SUM_MASK_MASK 0x8 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_DIR_INT_SUM_MASK_SHIFT 3 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_DIR_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_DIR_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_DIR_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_TCAM_INT_SUM_MASK_MASK 0x10 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_TCAM_INT_SUM_MASK_SHIFT 4 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_TCAM_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_TCAM_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_TCAM_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_CTRL_INT_SUM_MASK_MASK 0x20 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_CTRL_INT_SUM_MASK_SHIFT 5 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_CTRL_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_CTRL_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_CLSF_CTRL_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_PRS_INT_SUM_MASK_MASK 0x40 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_PRS_INT_SUM_MASK_SHIFT 6 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_PRS_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_PRS_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_IPP_INT_SUM_MASK_REG_IPP_PRS_INT_SUM_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_EPP_INT_SUM_STATUS_REG_ADDR 0xa1100080 +#define HIF_IRQ_EPP_INT_SUM_STATUS_REG_WIDTH 32 +#define HIF_IRQ_EPP_INT_SUM_STATUS_REG_LENGTH 32 +#define HIF_IRQ_EPP_INT_SUM_STATUS_REG_EPP_RWE_INT_SUM_STS_MASK 0x1 +#define HIF_IRQ_EPP_INT_SUM_STATUS_REG_EPP_RWE_INT_SUM_STS_SHIFT 0 +#define HIF_IRQ_EPP_INT_SUM_STATUS_REG_EPP_RWE_INT_SUM_STS_WIDTH 1 + +#define HIF_IRQ_EPP_INT_SUM_MASK_REG_ADDR 0xa1100084 +#define HIF_IRQ_EPP_INT_SUM_MASK_REG_WIDTH 32 +#define HIF_IRQ_EPP_INT_SUM_MASK_REG_LENGTH 32 +#define HIF_IRQ_EPP_INT_SUM_MASK_REG_EPP_RWE_INT_SUM_MASK_MASK 0x1 +#define HIF_IRQ_EPP_INT_SUM_MASK_REG_EPP_RWE_INT_SUM_MASK_SHIFT 0 +#define HIF_IRQ_EPP_INT_SUM_MASK_REG_EPP_RWE_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_EPP_INT_SUM_MASK_REG_EPP_RWE_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_EPP_INT_SUM_MASK_REG_EPP_RWE_INT_SUM_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_ADDR 0xa1100088 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_WIDTH 32 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_LENGTH 32 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NDP_INT_SUM_STS_MASK 0x1 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NDP_INT_SUM_STS_SHIFT 0 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NDP_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NEA_INT_SUM_STS_MASK 0x2 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NEA_INT_SUM_STS_SHIFT 1 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NEA_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NEM_INT_SUM_STS_MASK 0x4 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NEM_INT_SUM_STS_SHIFT 2 +#define HIF_IRQ_NIF_INT_SUM_STATUS_REG_NIF_NEM_INT_SUM_STS_WIDTH 1 + +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_ADDR 0xa110008c +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_WIDTH 32 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_LENGTH 32 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NDP_INT_SUM_MASK_MASK 0x1 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NDP_INT_SUM_MASK_SHIFT 0 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NDP_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NDP_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NDP_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEA_INT_SUM_MASK_MASK 0x2 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEA_INT_SUM_MASK_SHIFT 1 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEA_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEA_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEA_INT_SUM_MASK_MIN_VAL 0x0 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEM_INT_SUM_MASK_MASK 0x4 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEM_INT_SUM_MASK_SHIFT 2 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEM_INT_SUM_MASK_WIDTH 1 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEM_INT_SUM_MASK_MAX_VAL 0x1 +#define HIF_IRQ_NIF_INT_SUM_MASK_REG_NIF_NEM_INT_SUM_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_PCIE1_GLB_FUNC_ID_BASE_REG_ADDR 0xa1100090 +#define HIF_IRQ_PCIE1_GLB_FUNC_ID_BASE_REG_WIDTH 32 +#define HIF_IRQ_PCIE1_GLB_FUNC_ID_BASE_REG_LENGTH 32 +#define HIF_IRQ_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_MASK 0x7ff +#define HIF_IRQ_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_SHIFT 0 +#define HIF_IRQ_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_WIDTH 11 +#define HIF_IRQ_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_MAX_VAL 0x7ff +#define HIF_IRQ_PCIE1_GLB_FUNC_ID_BASE_REG_PCIE1_GLB_FUNC_ID_BASE_MIN_VAL 0x0 + +#define HIF_IRQ_INT_DB_REG_ADDR 0xa1100094 +#define HIF_IRQ_INT_DB_REG_WIDTH 32 +#define HIF_IRQ_INT_DB_REG_LENGTH 32 +#define HIF_IRQ_INT_DB_REG_INT_DB_VECTOR_ID_MASK 0xfff +#define HIF_IRQ_INT_DB_REG_INT_DB_VECTOR_ID_SHIFT 0 +#define HIF_IRQ_INT_DB_REG_INT_DB_VECTOR_ID_WIDTH 12 +#define HIF_IRQ_INT_DB_REG_INT_DB_VECTOR_ID_MAX_VAL 0xfff +#define HIF_IRQ_INT_DB_REG_INT_DB_VECTOR_ID_MIN_VAL 0x0 +#define HIF_IRQ_INT_DB_REG_INT_DB_OP_MASK 0x1f000 +#define HIF_IRQ_INT_DB_REG_INT_DB_OP_SHIFT 12 +#define HIF_IRQ_INT_DB_REG_INT_DB_OP_WIDTH 5 +#define HIF_IRQ_INT_DB_REG_INT_DB_OP_MAX_VAL 0x1f +#define HIF_IRQ_INT_DB_REG_INT_DB_OP_MIN_VAL 0x0 + +#define HIF_IRQ_BRS_SCAN_TH_REG_ADDR 0xa11000b4 +#define HIF_IRQ_BRS_SCAN_TH_REG_WIDTH 32 +#define HIF_IRQ_BRS_SCAN_TH_REG_LENGTH 32 +#define HIF_IRQ_BRS_SCAN_TH_REG_BRS_SCAN_TH_MASK 0xff +#define HIF_IRQ_BRS_SCAN_TH_REG_BRS_SCAN_TH_SHIFT 0 +#define HIF_IRQ_BRS_SCAN_TH_REG_BRS_SCAN_TH_WIDTH 8 +#define HIF_IRQ_BRS_SCAN_TH_REG_BRS_SCAN_TH_MAX_VAL 0xff +#define HIF_IRQ_BRS_SCAN_TH_REG_BRS_SCAN_TH_MIN_VAL 0x0 + +#define HIF_IRQ_BRS_SCAN_EN_REG_ADDR 0xa11000b8 +#define HIF_IRQ_BRS_SCAN_EN_REG_WIDTH 32 +#define HIF_IRQ_BRS_SCAN_EN_REG_LENGTH 32 +#define HIF_IRQ_BRS_SCAN_EN_REG_BRS_SCAN_EN_MASK 0x1 +#define HIF_IRQ_BRS_SCAN_EN_REG_BRS_SCAN_EN_SHIFT 0 +#define HIF_IRQ_BRS_SCAN_EN_REG_BRS_SCAN_EN_WIDTH 1 +#define HIF_IRQ_BRS_SCAN_EN_REG_BRS_SCAN_EN_MAX_VAL 0x1 +#define HIF_IRQ_BRS_SCAN_EN_REG_BRS_SCAN_EN_MIN_VAL 0x0 + +#define HIF_IRQ_BRS_SCAN_MODE_REG_ADDR 0xa11000bc +#define HIF_IRQ_BRS_SCAN_MODE_REG_WIDTH 32 +#define HIF_IRQ_BRS_SCAN_MODE_REG_LENGTH 32 +#define HIF_IRQ_BRS_SCAN_MODE_REG_BRS_SCAN_MODE_MASK 0x1 +#define HIF_IRQ_BRS_SCAN_MODE_REG_BRS_SCAN_MODE_SHIFT 0 +#define HIF_IRQ_BRS_SCAN_MODE_REG_BRS_SCAN_MODE_WIDTH 1 +#define HIF_IRQ_BRS_SCAN_MODE_REG_BRS_SCAN_MODE_MAX_VAL 0x1 +#define HIF_IRQ_BRS_SCAN_MODE_REG_BRS_SCAN_MODE_MIN_VAL 0x0 + +#define HIF_IRQ_INT_STATE_REG_ADDR 0xa11000c0 +#define HIF_IRQ_INT_STATE_REG_WIDTH 32 +#define HIF_IRQ_INT_STATE_REG_LENGTH 32 +#define HIF_IRQ_INT_STATE_REG_INT_DB_FIFO_OVFL_MASK 0x1 +#define HIF_IRQ_INT_STATE_REG_INT_DB_FIFO_OVFL_SHIFT 0 +#define HIF_IRQ_INT_STATE_REG_INT_DB_FIFO_OVFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_INT_DB_FIFO_UNFL_MASK 0x2 +#define HIF_IRQ_INT_STATE_REG_INT_DB_FIFO_UNFL_SHIFT 1 +#define HIF_IRQ_INT_STATE_REG_INT_DB_FIFO_UNFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_TPE2HIF_MSIX_CBR_OVFL_MASK 0x4 +#define HIF_IRQ_INT_STATE_REG_TPE2HIF_MSIX_CBR_OVFL_SHIFT 2 +#define HIF_IRQ_INT_STATE_REG_TPE2HIF_MSIX_CBR_OVFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_TPE2HIF_MSIX_CBR_UNFL_MASK 0x8 +#define HIF_IRQ_INT_STATE_REG_TPE2HIF_MSIX_CBR_UNFL_SHIFT 3 +#define HIF_IRQ_INT_STATE_REG_TPE2HIF_MSIX_CBR_UNFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_CMDQM2IRQ_MSIX_CBR_OVFL_MASK 0x10 +#define HIF_IRQ_INT_STATE_REG_CMDQM2IRQ_MSIX_CBR_OVFL_SHIFT 4 +#define HIF_IRQ_INT_STATE_REG_CMDQM2IRQ_MSIX_CBR_OVFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_CMDQM2IRQ_MSIX_CBR_UNFL_MASK 0x20 +#define HIF_IRQ_INT_STATE_REG_CMDQM2IRQ_MSIX_CBR_UNFL_SHIFT 5 +#define HIF_IRQ_INT_STATE_REG_CMDQM2IRQ_MSIX_CBR_UNFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_WRITE_FAIL_MSIX_FIFO_OVFL_MASK 0x40 +#define HIF_IRQ_INT_STATE_REG_WRITE_FAIL_MSIX_FIFO_OVFL_SHIFT 6 +#define HIF_IRQ_INT_STATE_REG_WRITE_FAIL_MSIX_FIFO_OVFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_WRITE_FAIL_MSIX_FIFO_UNFL_MASK 0x80 +#define HIF_IRQ_INT_STATE_REG_WRITE_FAIL_MSIX_FIFO_UNFL_SHIFT 7 +#define HIF_IRQ_INT_STATE_REG_WRITE_FAIL_MSIX_FIFO_UNFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_WRITE_BACK_MSIX_FIFO_OVFL_MASK 0x100 +#define HIF_IRQ_INT_STATE_REG_WRITE_BACK_MSIX_FIFO_OVFL_SHIFT 8 +#define HIF_IRQ_INT_STATE_REG_WRITE_BACK_MSIX_FIFO_OVFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_WRITE_BACK_MSIX_FIFO_UNFL_MASK 0x200 +#define HIF_IRQ_INT_STATE_REG_WRITE_BACK_MSIX_FIFO_UNFL_SHIFT 9 +#define HIF_IRQ_INT_STATE_REG_WRITE_BACK_MSIX_FIFO_UNFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_DIST2GEN_MSIX_FIFO_OVFL_MASK 0x400 +#define HIF_IRQ_INT_STATE_REG_DIST2GEN_MSIX_FIFO_OVFL_SHIFT 10 +#define HIF_IRQ_INT_STATE_REG_DIST2GEN_MSIX_FIFO_OVFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_DIST2GEN_MSIX_FIFO_UNFL_MASK 0x800 +#define HIF_IRQ_INT_STATE_REG_DIST2GEN_MSIX_FIFO_UNFL_SHIFT 11 +#define HIF_IRQ_INT_STATE_REG_DIST2GEN_MSIX_FIFO_UNFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_IRQ2ADPT_CREDIT_OVFL_MASK 0x3000 +#define HIF_IRQ_INT_STATE_REG_IRQ2ADPT_CREDIT_OVFL_SHIFT 12 +#define HIF_IRQ_INT_STATE_REG_IRQ2ADPT_CREDIT_OVFL_WIDTH 2 +#define HIF_IRQ_INT_STATE_REG_IRQ2ADPT_CREDIT_UNFL_MASK 0xc000 +#define HIF_IRQ_INT_STATE_REG_IRQ2ADPT_CREDIT_UNFL_SHIFT 14 +#define HIF_IRQ_INT_STATE_REG_IRQ2ADPT_CREDIT_UNFL_WIDTH 2 +#define HIF_IRQ_INT_STATE_REG_REQ_INFO_FIFO_OVFL_MASK 0x30000 +#define HIF_IRQ_INT_STATE_REG_REQ_INFO_FIFO_OVFL_SHIFT 16 +#define HIF_IRQ_INT_STATE_REG_REQ_INFO_FIFO_OVFL_WIDTH 2 +#define HIF_IRQ_INT_STATE_REG_REQ_INFO_FIFO_UNFL_MASK 0xc0000 +#define HIF_IRQ_INT_STATE_REG_REQ_INFO_FIFO_UNFL_SHIFT 18 +#define HIF_IRQ_INT_STATE_REG_REQ_INFO_FIFO_UNFL_WIDTH 2 +#define HIF_IRQ_INT_STATE_REG_ACK_INFO_FIFO_OVFL_MASK 0x300000 +#define HIF_IRQ_INT_STATE_REG_ACK_INFO_FIFO_OVFL_SHIFT 20 +#define HIF_IRQ_INT_STATE_REG_ACK_INFO_FIFO_OVFL_WIDTH 2 +#define HIF_IRQ_INT_STATE_REG_ACK_INFO_FIFO_UNFL_MASK 0xc00000 +#define HIF_IRQ_INT_STATE_REG_ACK_INFO_FIFO_UNFL_SHIFT 22 +#define HIF_IRQ_INT_STATE_REG_ACK_INFO_FIFO_UNFL_WIDTH 2 +#define HIF_IRQ_INT_STATE_REG_BRS_SCAN_FIFO_OVFL_MASK 0x1000000 +#define HIF_IRQ_INT_STATE_REG_BRS_SCAN_FIFO_OVFL_SHIFT 24 +#define HIF_IRQ_INT_STATE_REG_BRS_SCAN_FIFO_OVFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_BRS_SCAN_FIFO_UNFL_MASK 0x2000000 +#define HIF_IRQ_INT_STATE_REG_BRS_SCAN_FIFO_UNFL_SHIFT 25 +#define HIF_IRQ_INT_STATE_REG_BRS_SCAN_FIFO_UNFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_BRS_RSLT_FIFO_OVFL_MASK 0x4000000 +#define HIF_IRQ_INT_STATE_REG_BRS_RSLT_FIFO_OVFL_SHIFT 26 +#define HIF_IRQ_INT_STATE_REG_BRS_RSLT_FIFO_OVFL_WIDTH 1 +#define HIF_IRQ_INT_STATE_REG_BRS_RSLT_FIFO_UNFL_MASK 0x8000000 +#define HIF_IRQ_INT_STATE_REG_BRS_RSLT_FIFO_UNFL_SHIFT 27 +#define HIF_IRQ_INT_STATE_REG_BRS_RSLT_FIFO_UNFL_WIDTH 1 + +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ADDR 0xa11000c4 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WIDTH 32 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_LENGTH 32 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_OVFL_INT_MASK_MASK 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_OVFL_INT_MASK_SHIFT 0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_OVFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_UNFL_INT_MASK_MASK 0x2 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_UNFL_INT_MASK_SHIFT 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_UNFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_INT_DB_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_OVFL_INT_MASK_MASK 0x4 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_OVFL_INT_MASK_SHIFT 2 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_OVFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_UNFL_INT_MASK_MASK 0x8 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_UNFL_INT_MASK_SHIFT 3 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_UNFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_TPE2HIF_MSIX_CBR_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_OVFL_INT_MASK_MASK 0x10 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_OVFL_INT_MASK_SHIFT 4 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_OVFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_UNFL_INT_MASK_MASK 0x20 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_UNFL_INT_MASK_SHIFT 5 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_UNFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_CMDQM2IRQ_MSIX_CBR_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_OVFL_INT_MASK_MASK 0x40 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_OVFL_INT_MASK_SHIFT 6 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_OVFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_UNFL_INT_MASK_MASK 0x80 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_UNFL_INT_MASK_SHIFT 7 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_UNFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_FAIL_MSIX_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_OVFL_INT_MASK_MASK 0x100 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_OVFL_INT_MASK_SHIFT 8 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_OVFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_UNFL_INT_MASK_MASK 0x200 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_UNFL_INT_MASK_SHIFT 9 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_UNFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_WRITE_BACK_MSIX_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_OVFL_INT_MASK_MASK 0x400 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_OVFL_INT_MASK_SHIFT 10 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_OVFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_UNFL_INT_MASK_MASK 0x800 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_UNFL_INT_MASK_SHIFT 11 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_UNFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_DIST2GEN_MSIX_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_OVFL_INT_MASK_MASK 0x3000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_OVFL_INT_MASK_SHIFT 12 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_OVFL_INT_MASK_WIDTH 2 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_OVFL_INT_MASK_MAX_VAL 0x3 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_UNFL_INT_MASK_MASK 0xc000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_UNFL_INT_MASK_SHIFT 14 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_UNFL_INT_MASK_WIDTH 2 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_UNFL_INT_MASK_MAX_VAL 0x3 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_IRQ2ADPT_CREDIT_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_OVFL_INT_MASK_MASK 0x30000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_OVFL_INT_MASK_SHIFT 16 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_OVFL_INT_MASK_WIDTH 2 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_OVFL_INT_MASK_MAX_VAL 0x3 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_UNFL_INT_MASK_MASK 0xc0000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_UNFL_INT_MASK_SHIFT 18 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_UNFL_INT_MASK_WIDTH 2 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_UNFL_INT_MASK_MAX_VAL 0x3 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_REQ_INFO_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_OVFL_INT_MASK_MASK 0x300000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_OVFL_INT_MASK_SHIFT 20 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_OVFL_INT_MASK_WIDTH 2 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_OVFL_INT_MASK_MAX_VAL 0x3 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_UNFL_INT_MASK_MASK 0xc00000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_UNFL_INT_MASK_SHIFT 22 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_UNFL_INT_MASK_WIDTH 2 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_UNFL_INT_MASK_MAX_VAL 0x3 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_ACK_INFO_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_OVFL_INT_MASK_MASK 0x1000000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_OVFL_INT_MASK_SHIFT 24 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_OVFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_UNFL_INT_MASK_MASK 0x2000000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_UNFL_INT_MASK_SHIFT 25 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_UNFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_SCAN_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_OVFL_INT_MASK_MASK 0x4000000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_OVFL_INT_MASK_SHIFT 26 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_OVFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_UNFL_INT_MASK_MASK 0x8000000 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_UNFL_INT_MASK_SHIFT 27 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_UNFL_INT_MASK_WIDTH 1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_INT_STATE_INT_MASK_REG_BRS_RSLT_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_CFG_VECTOR_TABLE_START_REG_ADDR 0xa11000c8 +#define HIF_IRQ_CFG_VECTOR_TABLE_START_REG_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_START_REG_LENGTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_START_REG_CFG_VTR_TBL_START_MASK 0x1 +#define HIF_IRQ_CFG_VECTOR_TABLE_START_REG_CFG_VTR_TBL_START_SHIFT 0 +#define HIF_IRQ_CFG_VECTOR_TABLE_START_REG_CFG_VTR_TBL_START_WIDTH 1 +#define HIF_IRQ_CFG_VECTOR_TABLE_START_REG_CFG_VTR_TBL_START_MAX_VAL 0x1 +#define HIF_IRQ_CFG_VECTOR_TABLE_START_REG_CFG_VTR_TBL_START_MIN_VAL 0x0 + +#define HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_ADDR 0xa11000cc +#define HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_LENGTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_CFG_VTR_TBL_CMD_MASK 0x1 +#define HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_CFG_VTR_TBL_CMD_SHIFT 0 +#define HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_CFG_VTR_TBL_CMD_WIDTH 1 +#define HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_CFG_VTR_TBL_CMD_MAX_VAL 0x1 +#define HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_CFG_VTR_TBL_CMD_MIN_VAL 0x0 + +#define HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_ADDR 0xa11000d0 +#define HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_LENGTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_CFG_VTR_TBL_ADDR_MASK 0xfff +#define HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_CFG_VTR_TBL_ADDR_SHIFT 0 +#define HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_CFG_VTR_TBL_ADDR_WIDTH 12 +#define HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_CFG_VTR_TBL_ADDR_MAX_VAL 0xfff +#define HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_CFG_VTR_TBL_ADDR_MIN_VAL 0x0 + +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_ADDR 0xa11000d4 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_LENGTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_CFG_VTR_TBL_MSG_LADDR_MASK 0xffffffff +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_CFG_VTR_TBL_MSG_LADDR_SHIFT 0 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_CFG_VTR_TBL_MSG_LADDR_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_CFG_VTR_TBL_MSG_LADDR_MAX_VAL 0xffffffff +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_CFG_VTR_TBL_MSG_LADDR_MIN_VAL 0x0 + +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_ADDR 0xa11000d8 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_LENGTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_CFG_VTR_TBL_MSG_UADDR_MASK 0xffffffff +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_CFG_VTR_TBL_MSG_UADDR_SHIFT 0 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_CFG_VTR_TBL_MSG_UADDR_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_CFG_VTR_TBL_MSG_UADDR_MAX_VAL 0xffffffff +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_CFG_VTR_TBL_MSG_UADDR_MIN_VAL 0x0 + +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_ADDR 0xa11000dc +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_LENGTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_CFG_VTR_TBL_MSG_DATA_MASK 0xffffffff +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_CFG_VTR_TBL_MSG_DATA_SHIFT 0 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_CFG_VTR_TBL_MSG_DATA_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_CFG_VTR_TBL_MSG_DATA_MAX_VAL 0xffffffff +#define HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_CFG_VTR_TBL_MSG_DATA_MIN_VAL 0x0 + +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_ADDR 0xa11000e0 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_LENGTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_FUNC_ID_MASK 0x7ff +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_FUNC_ID_SHIFT 0 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_FUNC_ID_WIDTH 11 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_FUNC_ID_MAX_VAL 0x7ff +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_FUNC_ID_MIN_VAL 0x0 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_VECTOR_EN_MASK 0x800 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_VECTOR_EN_SHIFT 11 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_VECTOR_EN_WIDTH 1 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_VECTOR_EN_MAX_VAL 0x1 +#define HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_CFG_VTR_TBL_VECTOR_EN_MIN_VAL 0x0 + +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_LADDR_REG_ADDR 0xa11000e4 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_LADDR_REG_WIDTH 32 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_LADDR_REG_LENGTH 32 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_LADDR_REG_RD_VTR_TBL_MSG_LADDR_MASK 0xffffffff +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_LADDR_REG_RD_VTR_TBL_MSG_LADDR_SHIFT 0 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_LADDR_REG_RD_VTR_TBL_MSG_LADDR_WIDTH 32 + +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_UADDR_REG_ADDR 0xa11000e8 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_UADDR_REG_WIDTH 32 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_UADDR_REG_LENGTH 32 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_UADDR_REG_RD_VTR_TBL_MSG_UADDR_MASK 0xffffffff +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_UADDR_REG_RD_VTR_TBL_MSG_UADDR_SHIFT 0 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_UADDR_REG_RD_VTR_TBL_MSG_UADDR_WIDTH 32 + +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_DATA_REG_ADDR 0xa11000ec +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_DATA_REG_WIDTH 32 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_DATA_REG_LENGTH 32 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_DATA_REG_RD_VTR_TBL_MSG_DATA_MASK 0xffffffff +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_DATA_REG_RD_VTR_TBL_MSG_DATA_SHIFT 0 +#define HIF_IRQ_RD_VECTOR_TABLE_MSG_DATA_REG_RD_VTR_TBL_MSG_DATA_WIDTH 32 + +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_ADDR 0xa11000f0 +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_WIDTH 32 +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_LENGTH 32 +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_RD_VTR_TBL_FUNC_ID_MASK 0x7ff +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_RD_VTR_TBL_FUNC_ID_SHIFT 0 +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_RD_VTR_TBL_FUNC_ID_WIDTH 11 +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_RD_VTR_TBL_VECTOR_EN_MASK 0x800 +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_RD_VTR_TBL_VECTOR_EN_SHIFT 11 +#define HIF_IRQ_RD_VECTOR_TABLE_CTRL_REG_RD_VTR_TBL_VECTOR_EN_WIDTH 1 + +#define HIF_IRQ_CFG_VECTOR_TABLE_BUSY_REG_ADDR 0xa11000f4 +#define HIF_IRQ_CFG_VECTOR_TABLE_BUSY_REG_WIDTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_BUSY_REG_LENGTH 32 +#define HIF_IRQ_CFG_VECTOR_TABLE_BUSY_REG_CFG_VTR_TBL_BUSY_MASK 0x1 +#define HIF_IRQ_CFG_VECTOR_TABLE_BUSY_REG_CFG_VTR_TBL_BUSY_SHIFT 0 +#define HIF_IRQ_CFG_VECTOR_TABLE_BUSY_REG_CFG_VTR_TBL_BUSY_WIDTH 1 + +#define HIF_IRQ_RD_PBA_MASK_TABLE_START_REG_ADDR 0xa11000f8 +#define HIF_IRQ_RD_PBA_MASK_TABLE_START_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_START_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_START_REG_RD_PBA_MASK_TBL_START_MASK 0x1 +#define HIF_IRQ_RD_PBA_MASK_TABLE_START_REG_RD_PBA_MASK_TBL_START_SHIFT 0 +#define HIF_IRQ_RD_PBA_MASK_TABLE_START_REG_RD_PBA_MASK_TBL_START_WIDTH 1 +#define HIF_IRQ_RD_PBA_MASK_TABLE_START_REG_RD_PBA_MASK_TBL_START_MAX_VAL 0x1 +#define HIF_IRQ_RD_PBA_MASK_TABLE_START_REG_RD_PBA_MASK_TBL_START_MIN_VAL 0x0 + +#define HIF_IRQ_RD_PBA_MASK_TABLE_CMD_REG_ADDR 0xa11000fc +#define HIF_IRQ_RD_PBA_MASK_TABLE_CMD_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_CMD_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_CMD_REG_RD_PBA_MASK_TBL_CMD_MASK 0x1 +#define HIF_IRQ_RD_PBA_MASK_TABLE_CMD_REG_RD_PBA_MASK_TBL_CMD_SHIFT 0 +#define HIF_IRQ_RD_PBA_MASK_TABLE_CMD_REG_RD_PBA_MASK_TBL_CMD_WIDTH 1 +#define HIF_IRQ_RD_PBA_MASK_TABLE_CMD_REG_RD_PBA_MASK_TBL_CMD_MAX_VAL 0x1 +#define HIF_IRQ_RD_PBA_MASK_TABLE_CMD_REG_RD_PBA_MASK_TBL_CMD_MIN_VAL 0x0 + +#define HIF_IRQ_RD_PBA_MASK_TABLE_ADDR_REG_ADDR 0xa1100100 +#define HIF_IRQ_RD_PBA_MASK_TABLE_ADDR_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_ADDR_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_ADDR_REG_RD_PBA_MASK_TBL_ADDR_MASK 0x3f +#define HIF_IRQ_RD_PBA_MASK_TABLE_ADDR_REG_RD_PBA_MASK_TBL_ADDR_SHIFT 0 +#define HIF_IRQ_RD_PBA_MASK_TABLE_ADDR_REG_RD_PBA_MASK_TBL_ADDR_WIDTH 6 +#define HIF_IRQ_RD_PBA_MASK_TABLE_ADDR_REG_RD_PBA_MASK_TBL_ADDR_MAX_VAL 0x3f +#define HIF_IRQ_RD_PBA_MASK_TABLE_ADDR_REG_RD_PBA_MASK_TBL_ADDR_MIN_VAL 0x0 + +#define HIF_IRQ_RD_PBA_MASK_TABLE_LDATA_REG_ADDR 0xa1100104 +#define HIF_IRQ_RD_PBA_MASK_TABLE_LDATA_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_LDATA_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_LDATA_REG_RD_PBA_MASK_TBL_LDATA_MASK 0xffffffff +#define HIF_IRQ_RD_PBA_MASK_TABLE_LDATA_REG_RD_PBA_MASK_TBL_LDATA_SHIFT 0 +#define HIF_IRQ_RD_PBA_MASK_TABLE_LDATA_REG_RD_PBA_MASK_TBL_LDATA_WIDTH 32 + +#define HIF_IRQ_RD_PBA_MASK_TABLE_UDATA_REG_ADDR 0xa1100108 +#define HIF_IRQ_RD_PBA_MASK_TABLE_UDATA_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_UDATA_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_UDATA_REG_RD_PBA_MASK_TBL_UDATA_MASK 0xffffffff +#define HIF_IRQ_RD_PBA_MASK_TABLE_UDATA_REG_RD_PBA_MASK_TBL_UDATA_SHIFT 0 +#define HIF_IRQ_RD_PBA_MASK_TABLE_UDATA_REG_RD_PBA_MASK_TBL_UDATA_WIDTH 32 + +#define HIF_IRQ_RD_PBA_MASK_TABLE_BUSY_REG_ADDR 0xa110010c +#define HIF_IRQ_RD_PBA_MASK_TABLE_BUSY_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_BUSY_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_MASK_TABLE_BUSY_REG_RD_PBA_MASK_TBL_BUSY_MASK 0x1 +#define HIF_IRQ_RD_PBA_MASK_TABLE_BUSY_REG_RD_PBA_MASK_TBL_BUSY_SHIFT 0 +#define HIF_IRQ_RD_PBA_MASK_TABLE_BUSY_REG_RD_PBA_MASK_TBL_BUSY_WIDTH 1 + +#define HIF_IRQ_DIST_INT_STATE_REG_ADDR 0xa110012c +#define HIF_IRQ_DIST_INT_STATE_REG_WIDTH 32 +#define HIF_IRQ_DIST_INT_STATE_REG_LENGTH 32 +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_DMA_RX_INT_STS_MASK 0x3 +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_DMA_RX_INT_STS_SHIFT 0 +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_DMA_RX_INT_STS_WIDTH 2 +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_DMA_TX_INT_STS_MASK 0xc +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_DMA_TX_INT_STS_SHIFT 2 +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_DMA_TX_INT_STS_WIDTH 2 +#define HIF_IRQ_DIST_INT_STATE_REG_PIO2IRQ_PIO_REQ_INT_STS_MASK 0x30 +#define HIF_IRQ_DIST_INT_STATE_REG_PIO2IRQ_PIO_REQ_INT_STS_SHIFT 4 +#define HIF_IRQ_DIST_INT_STATE_REG_PIO2IRQ_PIO_REQ_INT_STS_WIDTH 2 +#define HIF_IRQ_DIST_INT_STATE_REG_PCIE2HIF_PCIE_MSG_INT_STS_MASK 0xc0 +#define HIF_IRQ_DIST_INT_STATE_REG_PCIE2HIF_PCIE_MSG_INT_STS_SHIFT 6 +#define HIF_IRQ_DIST_INT_STATE_REG_PCIE2HIF_PCIE_MSG_INT_STS_WIDTH 2 +#define HIF_IRQ_DIST_INT_STATE_REG_TOP_INT_SUM_STS_MASK 0x100 +#define HIF_IRQ_DIST_INT_STATE_REG_TOP_INT_SUM_STS_SHIFT 8 +#define HIF_IRQ_DIST_INT_STATE_REG_TOP_INT_SUM_STS_WIDTH 1 +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_CMDQ_REQ_INT_STS_MASK 0x200 +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_CMDQ_REQ_INT_STS_SHIFT 9 +#define HIF_IRQ_DIST_INT_STATE_REG_CMDQM2IRQ_CMDQ_REQ_INT_STS_WIDTH 1 +#define HIF_IRQ_DIST_INT_STATE_REG_TBL2IRQ_TBL_RD_DONE_INT_STS_MASK 0x400 +#define HIF_IRQ_DIST_INT_STATE_REG_TBL2IRQ_TBL_RD_DONE_INT_STS_SHIFT 10 +#define HIF_IRQ_DIST_INT_STATE_REG_TBL2IRQ_TBL_RD_DONE_INT_STS_WIDTH 1 + +#define HIF_IRQ_INT_DB_PBA0_REQ_CNT_REG_ADDR 0xa1100164 +#define HIF_IRQ_INT_DB_PBA0_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_INT_DB_PBA0_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_INT_DB_PBA0_REQ_CNT_REG_INT_DB_PBA0_REQ_CNT_MASK 0xff +#define HIF_IRQ_INT_DB_PBA0_REQ_CNT_REG_INT_DB_PBA0_REQ_CNT_SHIFT 0 +#define HIF_IRQ_INT_DB_PBA0_REQ_CNT_REG_INT_DB_PBA0_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_INT_DB_PBA1_REQ_CNT_REG_ADDR 0xa1100168 +#define HIF_IRQ_INT_DB_PBA1_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_INT_DB_PBA1_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_INT_DB_PBA1_REQ_CNT_REG_INT_DB_PBA1_REQ_CNT_MASK 0xff +#define HIF_IRQ_INT_DB_PBA1_REQ_CNT_REG_INT_DB_PBA1_REQ_CNT_SHIFT 0 +#define HIF_IRQ_INT_DB_PBA1_REQ_CNT_REG_INT_DB_PBA1_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_INT_DB_MASK0_REQ_CNT_REG_ADDR 0xa110016c +#define HIF_IRQ_INT_DB_MASK0_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_INT_DB_MASK0_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_INT_DB_MASK0_REQ_CNT_REG_INT_DB_MASK0_REQ_CNT_MASK 0xff +#define HIF_IRQ_INT_DB_MASK0_REQ_CNT_REG_INT_DB_MASK0_REQ_CNT_SHIFT 0 +#define HIF_IRQ_INT_DB_MASK0_REQ_CNT_REG_INT_DB_MASK0_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_INT_DB_MASK1_REQ_CNT_REG_ADDR 0xa1100170 +#define HIF_IRQ_INT_DB_MASK1_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_INT_DB_MASK1_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_INT_DB_MASK1_REQ_CNT_REG_INT_DB_MASK1_REQ_CNT_MASK 0xff +#define HIF_IRQ_INT_DB_MASK1_REQ_CNT_REG_INT_DB_MASK1_REQ_CNT_SHIFT 0 +#define HIF_IRQ_INT_DB_MASK1_REQ_CNT_REG_INT_DB_MASK1_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_INT_DB_MSIX_REQ_CNT_REG_ADDR 0xa1100174 +#define HIF_IRQ_INT_DB_MSIX_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_INT_DB_MSIX_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_INT_DB_MSIX_REQ_CNT_REG_INT_DB_MSIX_REQ_CNT_MASK 0xff +#define HIF_IRQ_INT_DB_MSIX_REQ_CNT_REG_INT_DB_MSIX_REQ_CNT_SHIFT 0 +#define HIF_IRQ_INT_DB_MSIX_REQ_CNT_REG_INT_DB_MSIX_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_CMDQM2IRQ_MSIX_REQ_CNT_REG_ADDR 0xa1100178 +#define HIF_IRQ_CMDQM2IRQ_MSIX_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_CMDQM2IRQ_MSIX_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_CMDQM2IRQ_MSIX_REQ_CNT_REG_CMDQM2IRQ_MSIX_REQ_CNT_MASK 0xff +#define HIF_IRQ_CMDQM2IRQ_MSIX_REQ_CNT_REG_CMDQM2IRQ_MSIX_REQ_CNT_SHIFT 0 +#define HIF_IRQ_CMDQM2IRQ_MSIX_REQ_CNT_REG_CMDQM2IRQ_MSIX_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_TPE2HIF_MSIX_REQ_CNT_REG_ADDR 0xa110017c +#define HIF_IRQ_TPE2HIF_MSIX_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE2HIF_MSIX_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE2HIF_MSIX_REQ_CNT_REG_TPE2HIF_MSIX_REQ_CNT_MASK 0xff +#define HIF_IRQ_TPE2HIF_MSIX_REQ_CNT_REG_TPE2HIF_MSIX_REQ_CNT_SHIFT 0 +#define HIF_IRQ_TPE2HIF_MSIX_REQ_CNT_REG_TPE2HIF_MSIX_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_WR_BACK_MSIX_REQ_CNT_REG_ADDR 0xa1100180 +#define HIF_IRQ_WR_BACK_MSIX_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_WR_BACK_MSIX_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_WR_BACK_MSIX_REQ_CNT_REG_WR_BACK_MSIX_REQ_CNT_MASK 0xff +#define HIF_IRQ_WR_BACK_MSIX_REQ_CNT_REG_WR_BACK_MSIX_REQ_CNT_SHIFT 0 +#define HIF_IRQ_WR_BACK_MSIX_REQ_CNT_REG_WR_BACK_MSIX_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_WR_FAIL_MSIX_REQ_CNT_REG_ADDR 0xa1100184 +#define HIF_IRQ_WR_FAIL_MSIX_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_WR_FAIL_MSIX_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_WR_FAIL_MSIX_REQ_CNT_REG_WR_FAIL_MSIX_REQ_CNT_MASK 0xff +#define HIF_IRQ_WR_FAIL_MSIX_REQ_CNT_REG_WR_FAIL_MSIX_REQ_CNT_SHIFT 0 +#define HIF_IRQ_WR_FAIL_MSIX_REQ_CNT_REG_WR_FAIL_MSIX_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_DIST2GEN_MSIX_REQ_CNT_REG_ADDR 0xa1100188 +#define HIF_IRQ_DIST2GEN_MSIX_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_DIST2GEN_MSIX_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_DIST2GEN_MSIX_REQ_CNT_REG_DIST2GEN_MSIX_REQ_CNT_MASK 0xff +#define HIF_IRQ_DIST2GEN_MSIX_REQ_CNT_REG_DIST2GEN_MSIX_REQ_CNT_SHIFT 0 +#define HIF_IRQ_DIST2GEN_MSIX_REQ_CNT_REG_DIST2GEN_MSIX_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_BRS2SCH_MSIX_REQ_CNT_REG_ADDR 0xa110018c +#define HIF_IRQ_BRS2SCH_MSIX_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_BRS2SCH_MSIX_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_BRS2SCH_MSIX_REQ_CNT_REG_BRS2SCH_MSIX_REQ_CNT_MASK 0xff +#define HIF_IRQ_BRS2SCH_MSIX_REQ_CNT_REG_BRS2SCH_MSIX_REQ_CNT_SHIFT 0 +#define HIF_IRQ_BRS2SCH_MSIX_REQ_CNT_REG_BRS2SCH_MSIX_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_VECTOR_EN_DROP_CNT_REG_ADDR 0xa1100190 +#define HIF_IRQ_VECTOR_EN_DROP_CNT_REG_WIDTH 32 +#define HIF_IRQ_VECTOR_EN_DROP_CNT_REG_LENGTH 32 +#define HIF_IRQ_VECTOR_EN_DROP_CNT_REG_VECTOR_EN_DROP_CNT_MASK 0xff +#define HIF_IRQ_VECTOR_EN_DROP_CNT_REG_VECTOR_EN_DROP_CNT_SHIFT 0 +#define HIF_IRQ_VECTOR_EN_DROP_CNT_REG_VECTOR_EN_DROP_CNT_WIDTH 8 + +#define HIF_IRQ_IRQ2ADPT_PCIE_REQ_CNT_REG_ADDR 0xa1100198 +#define HIF_IRQ_IRQ2ADPT_PCIE_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_IRQ2ADPT_PCIE_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_IRQ2ADPT_PCIE_REQ_CNT_REG_SIZE 2 +#define HIF_IRQ_IRQ2ADPT_PCIE_REQ_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_IRQ2ADPT_PCIE_REQ_CNT_REG_IRQ2ADPT_PCIE_REQ_CNT_MASK 0xff +#define HIF_IRQ_IRQ2ADPT_PCIE_REQ_CNT_REG_IRQ2ADPT_PCIE_REQ_CNT_SHIFT 0 +#define HIF_IRQ_IRQ2ADPT_PCIE_REQ_CNT_REG_IRQ2ADPT_PCIE_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_CNT_REG_ADDR 0xa11001a0 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_CNT_REG_WIDTH 32 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_CNT_REG_LENGTH 32 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_CNT_REG_SIZE 2 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_CNT_REG_ADPT2IRQ_PCIE_ACK_CNT_MASK 0xff +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_CNT_REG_ADPT2IRQ_PCIE_ACK_CNT_SHIFT 0 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_CNT_REG_ADPT2IRQ_PCIE_ACK_CNT_WIDTH 8 + +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_ERR_CNT_REG_ADDR 0xa11001a8 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_ERR_CNT_REG_WIDTH 32 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_ERR_CNT_REG_LENGTH 32 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_ERR_CNT_REG_SIZE 2 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_ERR_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_ERR_CNT_REG_ADPT2IRQ_PCIE_ACK_ERR_CNT_MASK 0xff +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_ERR_CNT_REG_ADPT2IRQ_PCIE_ACK_ERR_CNT_SHIFT 0 +#define HIF_IRQ_ADPT2IRQ_PCIE_ACK_ERR_CNT_REG_ADPT2IRQ_PCIE_ACK_ERR_CNT_WIDTH 8 + +#define HIF_IRQ_CFG_VTR_TBL_SUM_CNT_REG_ADDR 0xa11001b0 +#define HIF_IRQ_CFG_VTR_TBL_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_CFG_VTR_TBL_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_CFG_VTR_TBL_SUM_CNT_REG_CFG_VTR_TBL_SUM_CNT_MASK 0xff +#define HIF_IRQ_CFG_VTR_TBL_SUM_CNT_REG_CFG_VTR_TBL_SUM_CNT_SHIFT 0 +#define HIF_IRQ_CFG_VTR_TBL_SUM_CNT_REG_CFG_VTR_TBL_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_CFG_VTR_TBL_NRDY_CNT_REG_ADDR 0xa11001b4 +#define HIF_IRQ_CFG_VTR_TBL_NRDY_CNT_REG_WIDTH 32 +#define HIF_IRQ_CFG_VTR_TBL_NRDY_CNT_REG_LENGTH 32 +#define HIF_IRQ_CFG_VTR_TBL_NRDY_CNT_REG_CFG_VTR_TBL_NRDY_CNT_MASK 0xff +#define HIF_IRQ_CFG_VTR_TBL_NRDY_CNT_REG_CFG_VTR_TBL_NRDY_CNT_SHIFT 0 +#define HIF_IRQ_CFG_VTR_TBL_NRDY_CNT_REG_CFG_VTR_TBL_NRDY_CNT_WIDTH 8 + +#define HIF_IRQ_CFG_VTR_TBL_WR_REQ_CNT_REG_ADDR 0xa11001b8 +#define HIF_IRQ_CFG_VTR_TBL_WR_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_CFG_VTR_TBL_WR_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_CFG_VTR_TBL_WR_REQ_CNT_REG_CFG_VTR_TBL_WR_REQ_CNT_MASK 0xff +#define HIF_IRQ_CFG_VTR_TBL_WR_REQ_CNT_REG_CFG_VTR_TBL_WR_REQ_CNT_SHIFT 0 +#define HIF_IRQ_CFG_VTR_TBL_WR_REQ_CNT_REG_CFG_VTR_TBL_WR_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_CFG_VTR_TBL_WR_ACK_CNT_REG_ADDR 0xa11001bc +#define HIF_IRQ_CFG_VTR_TBL_WR_ACK_CNT_REG_WIDTH 32 +#define HIF_IRQ_CFG_VTR_TBL_WR_ACK_CNT_REG_LENGTH 32 +#define HIF_IRQ_CFG_VTR_TBL_WR_ACK_CNT_REG_CFG_VTR_TBL_WR_ACK_CNT_MASK 0xff +#define HIF_IRQ_CFG_VTR_TBL_WR_ACK_CNT_REG_CFG_VTR_TBL_WR_ACK_CNT_SHIFT 0 +#define HIF_IRQ_CFG_VTR_TBL_WR_ACK_CNT_REG_CFG_VTR_TBL_WR_ACK_CNT_WIDTH 8 + +#define HIF_IRQ_CFG_VTR_TBL_RD_REQ_CNT_REG_ADDR 0xa11001c0 +#define HIF_IRQ_CFG_VTR_TBL_RD_REQ_CNT_REG_WIDTH 32 +#define HIF_IRQ_CFG_VTR_TBL_RD_REQ_CNT_REG_LENGTH 32 +#define HIF_IRQ_CFG_VTR_TBL_RD_REQ_CNT_REG_CFG_VTR_TBL_RD_REQ_CNT_MASK 0xff +#define HIF_IRQ_CFG_VTR_TBL_RD_REQ_CNT_REG_CFG_VTR_TBL_RD_REQ_CNT_SHIFT 0 +#define HIF_IRQ_CFG_VTR_TBL_RD_REQ_CNT_REG_CFG_VTR_TBL_RD_REQ_CNT_WIDTH 8 + +#define HIF_IRQ_CFG_VTR_TBL_RD_ACK_CNT_REG_ADDR 0xa11001c4 +#define HIF_IRQ_CFG_VTR_TBL_RD_ACK_CNT_REG_WIDTH 32 +#define HIF_IRQ_CFG_VTR_TBL_RD_ACK_CNT_REG_LENGTH 32 +#define HIF_IRQ_CFG_VTR_TBL_RD_ACK_CNT_REG_CFG_VTR_TBL_RD_ACK_CNT_MASK 0xff +#define HIF_IRQ_CFG_VTR_TBL_RD_ACK_CNT_REG_CFG_VTR_TBL_RD_ACK_CNT_SHIFT 0 +#define HIF_IRQ_CFG_VTR_TBL_RD_ACK_CNT_REG_CFG_VTR_TBL_RD_ACK_CNT_WIDTH 8 + +#define HIF_IRQ_RD_PBA_TBL_CNT_REG_ADDR 0xa11001c8 +#define HIF_IRQ_RD_PBA_TBL_CNT_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_TBL_CNT_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_TBL_CNT_REG_RD_PBA_TBL_CNT_MASK 0xff +#define HIF_IRQ_RD_PBA_TBL_CNT_REG_RD_PBA_TBL_CNT_SHIFT 0 +#define HIF_IRQ_RD_PBA_TBL_CNT_REG_RD_PBA_TBL_CNT_WIDTH 8 + +#define HIF_IRQ_RD_PBA_TBL_ACK_CNT_REG_ADDR 0xa11001cc +#define HIF_IRQ_RD_PBA_TBL_ACK_CNT_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_TBL_ACK_CNT_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_TBL_ACK_CNT_REG_RD_PBA_TBL_ACK_CNT_MASK 0xff +#define HIF_IRQ_RD_PBA_TBL_ACK_CNT_REG_RD_PBA_TBL_ACK_CNT_SHIFT 0 +#define HIF_IRQ_RD_PBA_TBL_ACK_CNT_REG_RD_PBA_TBL_ACK_CNT_WIDTH 8 + +#define HIF_IRQ_RD_MASK_TBL_CNT_REG_ADDR 0xa11001d0 +#define HIF_IRQ_RD_MASK_TBL_CNT_REG_WIDTH 32 +#define HIF_IRQ_RD_MASK_TBL_CNT_REG_LENGTH 32 +#define HIF_IRQ_RD_MASK_TBL_CNT_REG_RD_MASK_TBL_CNT_MASK 0xff +#define HIF_IRQ_RD_MASK_TBL_CNT_REG_RD_MASK_TBL_CNT_SHIFT 0 +#define HIF_IRQ_RD_MASK_TBL_CNT_REG_RD_MASK_TBL_CNT_WIDTH 8 + +#define HIF_IRQ_RD_MASK_TBL_ACK_CNT_REG_ADDR 0xa11001d4 +#define HIF_IRQ_RD_MASK_TBL_ACK_CNT_REG_WIDTH 32 +#define HIF_IRQ_RD_MASK_TBL_ACK_CNT_REG_LENGTH 32 +#define HIF_IRQ_RD_MASK_TBL_ACK_CNT_REG_RD_MASK_TBL_ACK_CNT_MASK 0xff +#define HIF_IRQ_RD_MASK_TBL_ACK_CNT_REG_RD_MASK_TBL_ACK_CNT_SHIFT 0 +#define HIF_IRQ_RD_MASK_TBL_ACK_CNT_REG_RD_MASK_TBL_ACK_CNT_WIDTH 8 + +#define HIF_IRQ_RD_PBA_MASK_TBL_NRDY_CNT_REG_ADDR 0xa11001d8 +#define HIF_IRQ_RD_PBA_MASK_TBL_NRDY_CNT_REG_WIDTH 32 +#define HIF_IRQ_RD_PBA_MASK_TBL_NRDY_CNT_REG_LENGTH 32 +#define HIF_IRQ_RD_PBA_MASK_TBL_NRDY_CNT_REG_RD_PBA_MASK_TBL_NRDY_CNT_MASK 0xff +#define HIF_IRQ_RD_PBA_MASK_TBL_NRDY_CNT_REG_RD_PBA_MASK_TBL_NRDY_CNT_SHIFT 0 +#define HIF_IRQ_RD_PBA_MASK_TBL_NRDY_CNT_REG_RD_PBA_MASK_TBL_NRDY_CNT_WIDTH 8 + +#define HIF_IRQ_HIF_INT_SUM_CNT_REG_ADDR 0xa11001dc +#define HIF_IRQ_HIF_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF_INT_SUM_CNT_REG_HIF_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_HIF_INT_SUM_CNT_REG_HIF_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_HIF_INT_SUM_CNT_REG_HIF_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TM_INT_SUM_CNT_REG_ADDR 0xa11001e0 +#define HIF_IRQ_TM_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TM_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TM_INT_SUM_CNT_REG_TM_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TM_INT_SUM_CNT_REG_TM_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TM_INT_SUM_CNT_REG_TM_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_INT_SUM_CNT_REG_ADDR 0xa11001e4 +#define HIF_IRQ_TPE_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_INT_SUM_CNT_REG_TPE_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_INT_SUM_CNT_REG_TPE_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_INT_SUM_CNT_REG_TPE_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_IPP_INT_SUM_CNT_REG_ADDR 0xa11001e8 +#define HIF_IRQ_IPP_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_IPP_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_IPP_INT_SUM_CNT_REG_IPP_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_IPP_INT_SUM_CNT_REG_IPP_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_IPP_INT_SUM_CNT_REG_IPP_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_EPP_INT_SUM_CNT_REG_ADDR 0xa11001ec +#define HIF_IRQ_EPP_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_EPP_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_EPP_INT_SUM_CNT_REG_EPP_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_EPP_INT_SUM_CNT_REG_EPP_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_EPP_INT_SUM_CNT_REG_EPP_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_NIF_INT_SUM_CNT_REG_ADDR 0xa11001f0 +#define HIF_IRQ_NIF_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_NIF_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_NIF_INT_SUM_CNT_REG_NIF_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_NIF_INT_SUM_CNT_REG_NIF_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_NIF_INT_SUM_CNT_REG_NIF_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TOP_INT_SUM_CNT_REG_ADDR 0xa11001f4 +#define HIF_IRQ_TOP_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TOP_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TOP_INT_SUM_CNT_REG_TOP_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TOP_INT_SUM_CNT_REG_TOP_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TOP_INT_SUM_CNT_REG_TOP_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_HIF_CPM_INT_SUM_CNT_REG_ADDR 0xa11001f8 +#define HIF_IRQ_HIF_CPM_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF_CPM_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF_CPM_INT_SUM_CNT_REG_HIF_CPM_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_HIF_CPM_INT_SUM_CNT_REG_HIF_CPM_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_HIF_CPM_INT_SUM_CNT_REG_HIF_CPM_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_HIF_PIO_INT_SUM_CNT_REG_ADDR 0xa11001fc +#define HIF_IRQ_HIF_PIO_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF_PIO_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF_PIO_INT_SUM_CNT_REG_HIF_PIO_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_HIF_PIO_INT_SUM_CNT_REG_HIF_PIO_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_HIF_PIO_INT_SUM_CNT_REG_HIF_PIO_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_HIF_ADPT_INT_SUM_CNT_REG_ADDR 0xa1100200 +#define HIF_IRQ_HIF_ADPT_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF_ADPT_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF_ADPT_INT_SUM_CNT_REG_HIF_ADPT_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_HIF_ADPT_INT_SUM_CNT_REG_HIF_ADPT_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_HIF_ADPT_INT_SUM_CNT_REG_HIF_ADPT_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_HIF_TBL_INT_SUM_CNT_REG_ADDR 0xa1100204 +#define HIF_IRQ_HIF_TBL_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF_TBL_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF_TBL_INT_SUM_CNT_REG_HIF_TBL_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_HIF_TBL_INT_SUM_CNT_REG_HIF_TBL_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_HIF_TBL_INT_SUM_CNT_REG_HIF_TBL_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_HIF_CMDQM_INT_SUM_CNT_REG_ADDR 0xa1100208 +#define HIF_IRQ_HIF_CMDQM_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF_CMDQM_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF_CMDQM_INT_SUM_CNT_REG_HIF_CMDQM_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_HIF_CMDQM_INT_SUM_CNT_REG_HIF_CMDQM_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_HIF_CMDQM_INT_SUM_CNT_REG_HIF_CMDQM_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_HIF_IRQ_INT_SUM_CNT_REG_ADDR 0xa110020c +#define HIF_IRQ_HIF_IRQ_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF_IRQ_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF_IRQ_INT_SUM_CNT_REG_HIF_IRQ_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_HIF_IRQ_INT_SUM_CNT_REG_HIF_IRQ_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_HIF_IRQ_INT_SUM_CNT_REG_HIF_IRQ_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_HIF_NOC_INT_SUM_CNT_REG_ADDR 0xa1100210 +#define HIF_IRQ_HIF_NOC_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF_NOC_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF_NOC_INT_SUM_CNT_REG_HIF_NOC_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_HIF_NOC_INT_SUM_CNT_REG_HIF_NOC_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_HIF_NOC_INT_SUM_CNT_REG_HIF_NOC_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TM_MTR_INT_SUM_CNT_REG_ADDR 0xa1100214 +#define HIF_IRQ_TM_MTR_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TM_MTR_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TM_MTR_INT_SUM_CNT_REG_TM_MTR_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TM_MTR_INT_SUM_CNT_REG_TM_MTR_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TM_MTR_INT_SUM_CNT_REG_TM_MTR_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TM_QMU_INT_SUM_CNT_REG_ADDR 0xa1100218 +#define HIF_IRQ_TM_QMU_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TM_QMU_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TM_QMU_INT_SUM_CNT_REG_TM_QMU_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TM_QMU_INT_SUM_CNT_REG_TM_QMU_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TM_QMU_INT_SUM_CNT_REG_TM_QMU_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TM_FRG_INT_SUM_CNT_REG_ADDR 0xa110021c +#define HIF_IRQ_TM_FRG_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TM_FRG_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TM_FRG_INT_SUM_CNT_REG_TM_FRG_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TM_FRG_INT_SUM_CNT_REG_TM_FRG_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TM_FRG_INT_SUM_CNT_REG_TM_FRG_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TM_PDB_INT_SUM_CNT_REG_ADDR 0xa1100220 +#define HIF_IRQ_TM_PDB_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TM_PDB_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TM_PDB_INT_SUM_CNT_REG_TM_PDB_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TM_PDB_INT_SUM_CNT_REG_TM_PDB_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TM_PDB_INT_SUM_CNT_REG_TM_PDB_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TM_PRA_INT_SUM_CNT_REG_ADDR 0xa1100224 +#define HIF_IRQ_TM_PRA_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TM_PRA_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TM_PRA_INT_SUM_CNT_REG_TM_PRA_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TM_PRA_INT_SUM_CNT_REG_TM_PRA_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TM_PRA_INT_SUM_CNT_REG_TM_PRA_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TM_SCA_INT_SUM_CNT_REG_ADDR 0xa1100228 +#define HIF_IRQ_TM_SCA_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TM_SCA_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TM_SCA_INT_SUM_CNT_REG_TM_SCA_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TM_SCA_INT_SUM_CNT_REG_TM_SCA_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TM_SCA_INT_SUM_CNT_REG_TM_SCA_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TM_OCA_INT_SUM_CNT_REG_ADDR 0xa110022c +#define HIF_IRQ_TM_OCA_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TM_OCA_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TM_OCA_INT_SUM_CNT_REG_TM_OCA_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TM_OCA_INT_SUM_CNT_REG_TM_OCA_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TM_OCA_INT_SUM_CNT_REG_TM_OCA_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_MMC_INT_SUM_CNT_REG_ADDR 0xa1100230 +#define HIF_IRQ_TPE_MMC_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_MMC_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_MMC_INT_SUM_CNT_REG_TPE_MMC_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_MMC_INT_SUM_CNT_REG_TPE_MMC_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_MMC_INT_SUM_CNT_REG_TPE_MMC_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_QPM_INT_SUM_CNT_REG_ADDR 0xa1100234 +#define HIF_IRQ_TPE_QPM_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_QPM_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_QPM_INT_SUM_CNT_REG_TPE_QPM_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_QPM_INT_SUM_CNT_REG_TPE_QPM_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_QPM_INT_SUM_CNT_REG_TPE_QPM_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_SV_INT_SUM_CNT_REG_ADDR 0xa1100238 +#define HIF_IRQ_TPE_SV_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_SV_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_SV_INT_SUM_CNT_REG_TPE_SV_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_SV_INT_SUM_CNT_REG_TPE_SV_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_SV_INT_SUM_CNT_REG_TPE_SV_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_MER_INT_SUM_CNT_REG_ADDR 0xa110023c +#define HIF_IRQ_TPE_MER_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_MER_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_MER_INT_SUM_CNT_REG_TPE_MER_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_MER_INT_SUM_CNT_REG_TPE_MER_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_MER_INT_SUM_CNT_REG_TPE_MER_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_TOP_INT_SUM_CNT_REG_ADDR 0xa1100240 +#define HIF_IRQ_TPE_TOP_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_TOP_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_TOP_INT_SUM_CNT_REG_TPE_TOP_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_TOP_INT_SUM_CNT_REG_TPE_TOP_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_TOP_INT_SUM_CNT_REG_TPE_TOP_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_CRDT_INT_SUM_CNT_REG_ADDR 0xa1100244 +#define HIF_IRQ_TPE_CRDT_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_CRDT_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_CRDT_INT_SUM_CNT_REG_TPE_CRDT_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_CRDT_INT_SUM_CNT_REG_TPE_CRDT_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_CRDT_INT_SUM_CNT_REG_TPE_CRDT_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_MET_INT_SUM_CNT_REG_ADDR 0xa1100248 +#define HIF_IRQ_TPE_MET_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_MET_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_MET_INT_SUM_CNT_REG_TPE_MET_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_MET_INT_SUM_CNT_REG_TPE_MET_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_MET_INT_SUM_CNT_REG_TPE_MET_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_SHAP_INT_SUM_CNT_REG_ADDR 0xa110024c +#define HIF_IRQ_TPE_SHAP_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_SHAP_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_SHAP_INT_SUM_CNT_REG_TPE_SHAP_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_SHAP_INT_SUM_CNT_REG_TPE_SHAP_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_SHAP_INT_SUM_CNT_REG_TPE_SHAP_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_MA_INT_SUM_CNT_REG_ADDR 0xa1100250 +#define HIF_IRQ_TPE_MA_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_MA_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_MA_INT_SUM_CNT_REG_TPE_MA_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_MA_INT_SUM_CNT_REG_TPE_MA_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_MA_INT_SUM_CNT_REG_TPE_MA_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_CEM_INT_SUM_CNT_REG_ADDR 0xa1100254 +#define HIF_IRQ_TPE_CEM_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_CEM_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_CEM_INT_SUM_CNT_REG_TPE_CEM_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_CEM_INT_SUM_CNT_REG_TPE_CEM_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_CEM_INT_SUM_CNT_REG_TPE_CEM_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_PET_INT_SUM_CNT_REG_ADDR 0xa1100258 +#define HIF_IRQ_TPE_PET_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_PET_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_PET_INT_SUM_CNT_REG_TPE_PET_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_PET_INT_SUM_CNT_REG_TPE_PET_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_PET_INT_SUM_CNT_REG_TPE_PET_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_TPE_PG_INT_SUM_CNT_REG_ADDR 0xa110025c +#define HIF_IRQ_TPE_PG_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_TPE_PG_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_TPE_PG_INT_SUM_CNT_REG_TPE_PG_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_TPE_PG_INT_SUM_CNT_REG_TPE_PG_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_TPE_PG_INT_SUM_CNT_REG_TPE_PG_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_IPP_EM_INT_SUM_CNT_REG_ADDR 0xa1100260 +#define HIF_IRQ_IPP_EM_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_IPP_EM_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_IPP_EM_INT_SUM_CNT_REG_IPP_EM_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_IPP_EM_INT_SUM_CNT_REG_IPP_EM_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_IPP_EM_INT_SUM_CNT_REG_IPP_EM_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_IPP_STAT_INT_SUM_CNT_REG_ADDR 0xa1100264 +#define HIF_IRQ_IPP_STAT_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_IPP_STAT_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_IPP_STAT_INT_SUM_CNT_REG_IPP_STAT_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_IPP_STAT_INT_SUM_CNT_REG_IPP_STAT_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_IPP_STAT_INT_SUM_CNT_REG_IPP_STAT_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_IPP_CLSF_DMA_INT_SUM_CNT_REG_ADDR 0xa1100268 +#define HIF_IRQ_IPP_CLSF_DMA_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_IPP_CLSF_DMA_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_IPP_CLSF_DMA_INT_SUM_CNT_REG_IPP_CLSF_DMA_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_IPP_CLSF_DMA_INT_SUM_CNT_REG_IPP_CLSF_DMA_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_IPP_CLSF_DMA_INT_SUM_CNT_REG_IPP_CLSF_DMA_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_IPP_DIR_INT_SUM_CNT_REG_ADDR 0xa110026c +#define HIF_IRQ_IPP_DIR_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_IPP_DIR_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_IPP_DIR_INT_SUM_CNT_REG_IPP_DIR_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_IPP_DIR_INT_SUM_CNT_REG_IPP_DIR_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_IPP_DIR_INT_SUM_CNT_REG_IPP_DIR_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_IPP_TCAM_INT_SUM_CNT_REG_ADDR 0xa1100270 +#define HIF_IRQ_IPP_TCAM_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_IPP_TCAM_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_IPP_TCAM_INT_SUM_CNT_REG_IPP_TCAM_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_IPP_TCAM_INT_SUM_CNT_REG_IPP_TCAM_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_IPP_TCAM_INT_SUM_CNT_REG_IPP_TCAM_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_IPP_CLSF_CTRL_INT_SUM_CNT_REG_ADDR 0xa1100274 +#define HIF_IRQ_IPP_CLSF_CTRL_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_IPP_CLSF_CTRL_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_IPP_CLSF_CTRL_INT_SUM_CNT_REG_IPP_CLSF_CTRL_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_IPP_CLSF_CTRL_INT_SUM_CNT_REG_IPP_CLSF_CTRL_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_IPP_CLSF_CTRL_INT_SUM_CNT_REG_IPP_CLSF_CTRL_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_IPP_PRS_INT_SUM_CNT_REG_ADDR 0xa1100278 +#define HIF_IRQ_IPP_PRS_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_IPP_PRS_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_IPP_PRS_INT_SUM_CNT_REG_IPP_PRS_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_IPP_PRS_INT_SUM_CNT_REG_IPP_PRS_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_IPP_PRS_INT_SUM_CNT_REG_IPP_PRS_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_EPP_PRS_INT_SUM_CNT_REG_ADDR 0xa110027c +#define HIF_IRQ_EPP_PRS_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_EPP_PRS_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_EPP_PRS_INT_SUM_CNT_REG_EPP_RWE_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_EPP_PRS_INT_SUM_CNT_REG_EPP_RWE_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_EPP_PRS_INT_SUM_CNT_REG_EPP_RWE_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_NIF_NDP_INT_SUM_CNT_REG_ADDR 0xa1100280 +#define HIF_IRQ_NIF_NDP_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_NIF_NDP_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_NIF_NDP_INT_SUM_CNT_REG_NIF_NDP_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_NIF_NDP_INT_SUM_CNT_REG_NIF_NDP_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_NIF_NDP_INT_SUM_CNT_REG_NIF_NDP_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_NIF_NEA_INT_SUM_CNT_REG_ADDR 0xa1100284 +#define HIF_IRQ_NIF_NEA_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_NIF_NEA_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_NIF_NEA_INT_SUM_CNT_REG_NIF_NEA_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_NIF_NEA_INT_SUM_CNT_REG_NIF_NEA_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_NIF_NEA_INT_SUM_CNT_REG_NIF_NEA_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_NIF_NEM_INT_SUM_CNT_REG_ADDR 0xa1100288 +#define HIF_IRQ_NIF_NEM_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_NIF_NEM_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_NIF_NEM_INT_SUM_CNT_REG_NIF_NEM_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_NIF_NEM_INT_SUM_CNT_REG_NIF_NEM_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_NIF_NEM_INT_SUM_CNT_REG_NIF_NEM_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_PCIE0_INT_SUM_CNT_REG_ADDR 0xa110028c +#define HIF_IRQ_PCIE0_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_PCIE0_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_PCIE0_INT_SUM_CNT_REG_PCIE0_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_PCIE0_INT_SUM_CNT_REG_PCIE0_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_PCIE0_INT_SUM_CNT_REG_PCIE0_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_PCIE1_INT_SUM_CNT_REG_ADDR 0xa1100290 +#define HIF_IRQ_PCIE1_INT_SUM_CNT_REG_WIDTH 32 +#define HIF_IRQ_PCIE1_INT_SUM_CNT_REG_LENGTH 32 +#define HIF_IRQ_PCIE1_INT_SUM_CNT_REG_PCIE1_INT_SUM_CNT_MASK 0xff +#define HIF_IRQ_PCIE1_INT_SUM_CNT_REG_PCIE1_INT_SUM_CNT_SHIFT 0 +#define HIF_IRQ_PCIE1_INT_SUM_CNT_REG_PCIE1_INT_SUM_CNT_WIDTH 8 + +#define HIF_IRQ_PCIE2HIF_PCIE_MSG_INT_CNT_REG_ADDR 0xa1100298 +#define HIF_IRQ_PCIE2HIF_PCIE_MSG_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_PCIE2HIF_PCIE_MSG_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_PCIE2HIF_PCIE_MSG_INT_CNT_REG_SIZE 2 +#define HIF_IRQ_PCIE2HIF_PCIE_MSG_INT_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_PCIE2HIF_PCIE_MSG_INT_CNT_REG_PCIE2HIF_PCIE_MSG_INT_CNT_MASK 0xff +#define HIF_IRQ_PCIE2HIF_PCIE_MSG_INT_CNT_REG_PCIE2HIF_PCIE_MSG_INT_CNT_SHIFT 0 +#define HIF_IRQ_PCIE2HIF_PCIE_MSG_INT_CNT_REG_PCIE2HIF_PCIE_MSG_INT_CNT_WIDTH 8 + +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_CNT_REG_ADDR 0xa11002a0 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_CNT_REG_SIZE 2 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_CNT_REG_PIO2IRQ_PIO_REQ_INT_CNT_MASK 0xff +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_CNT_REG_PIO2IRQ_PIO_REQ_INT_CNT_SHIFT 0 +#define HIF_IRQ_PIO2IRQ_PIO_REQ_INT_CNT_REG_PIO2IRQ_PIO_REQ_INT_CNT_WIDTH 8 + +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_CNT_REG_ADDR 0xa11002a8 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_CNT_REG_CMDQM2IRQ_CMDQ_REQ_INT_CNT_MASK 0xff +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_CNT_REG_CMDQM2IRQ_CMDQ_REQ_INT_CNT_SHIFT 0 +#define HIF_IRQ_CMDQM2IRQ_CMDQ_REQ_INT_CNT_REG_CMDQM2IRQ_CMDQ_REQ_INT_CNT_WIDTH 8 + +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_CNT_REG_ADDR 0xa11002b0 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_CNT_REG_SIZE 2 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_CNT_REG_CMDQM2IRQ_DMA_RX_INT_CNT_MASK 0xff +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_CNT_REG_CMDQM2IRQ_DMA_RX_INT_CNT_SHIFT 0 +#define HIF_IRQ_CMDQM2IRQ_DMA_RX_INT_CNT_REG_CMDQM2IRQ_DMA_RX_INT_CNT_WIDTH 8 + +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_CNT_REG_ADDR 0xa11002b8 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_CNT_REG_SIZE 2 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_CNT_REG_CMDQM2IRQ_DMA_TX_INT_CNT_MASK 0xff +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_CNT_REG_CMDQM2IRQ_DMA_TX_INT_CNT_SHIFT 0 +#define HIF_IRQ_CMDQM2IRQ_DMA_TX_INT_CNT_REG_CMDQM2IRQ_DMA_TX_INT_CNT_WIDTH 8 + +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_CNT_REG_ADDR 0xa11002c0 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_CNT_REG_TBL2IRQ_TBL_RD_DONE_INT_CNT_MASK 0xff +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_CNT_REG_TBL2IRQ_TBL_RD_DONE_INT_CNT_SHIFT 0 +#define HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_CNT_REG_TBL2IRQ_TBL_RD_DONE_INT_CNT_WIDTH 8 + +#define HIF_IRQ_HIF2APS_IRQ_SUM_INT_CNT_REG_ADDR 0xa11002c4 +#define HIF_IRQ_HIF2APS_IRQ_SUM_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF2APS_IRQ_SUM_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF2APS_IRQ_SUM_INT_CNT_REG_HIF2APS_IRQ_SUM_INT_CNT_MASK 0xff +#define HIF_IRQ_HIF2APS_IRQ_SUM_INT_CNT_REG_HIF2APS_IRQ_SUM_INT_CNT_SHIFT 0 +#define HIF_IRQ_HIF2APS_IRQ_SUM_INT_CNT_REG_HIF2APS_IRQ_SUM_INT_CNT_WIDTH 8 + +#define HIF_IRQ_HIF2APS_PCIE_MSG_INT_CNT_REG_ADDR 0xa11002c8 +#define HIF_IRQ_HIF2APS_PCIE_MSG_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF2APS_PCIE_MSG_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF2APS_PCIE_MSG_INT_CNT_REG_SIZE 2 +#define HIF_IRQ_HIF2APS_PCIE_MSG_INT_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_HIF2APS_PCIE_MSG_INT_CNT_REG_HIF2APS_PCIE_MSG_INT_CNT_MASK 0xff +#define HIF_IRQ_HIF2APS_PCIE_MSG_INT_CNT_REG_HIF2APS_PCIE_MSG_INT_CNT_SHIFT 0 +#define HIF_IRQ_HIF2APS_PCIE_MSG_INT_CNT_REG_HIF2APS_PCIE_MSG_INT_CNT_WIDTH 8 + +#define HIF_IRQ_HIF2APS_PIO_REQ_INT_CNT_REG_ADDR 0xa11002d0 +#define HIF_IRQ_HIF2APS_PIO_REQ_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF2APS_PIO_REQ_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF2APS_PIO_REQ_INT_CNT_REG_SIZE 2 +#define HIF_IRQ_HIF2APS_PIO_REQ_INT_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_HIF2APS_PIO_REQ_INT_CNT_REG_HIF2APS_PIO_REQ_INT_CNT_MASK 0xff +#define HIF_IRQ_HIF2APS_PIO_REQ_INT_CNT_REG_HIF2APS_PIO_REQ_INT_CNT_SHIFT 0 +#define HIF_IRQ_HIF2APS_PIO_REQ_INT_CNT_REG_HIF2APS_PIO_REQ_INT_CNT_WIDTH 8 + +#define HIF_IRQ_HIF2APS_CMDQ_REQ_INT_CNT_REG_ADDR 0xa11002d8 +#define HIF_IRQ_HIF2APS_CMDQ_REQ_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF2APS_CMDQ_REQ_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF2APS_CMDQ_REQ_INT_CNT_REG_HIF2APS_CMDQ_REQ_INT_CNT_MASK 0xff +#define HIF_IRQ_HIF2APS_CMDQ_REQ_INT_CNT_REG_HIF2APS_CMDQ_REQ_INT_CNT_SHIFT 0 +#define HIF_IRQ_HIF2APS_CMDQ_REQ_INT_CNT_REG_HIF2APS_CMDQ_REQ_INT_CNT_WIDTH 8 + +#define HIF_IRQ_HIF2APS_DMA_RX_INT_CNT_REG_ADDR 0xa11002e0 +#define HIF_IRQ_HIF2APS_DMA_RX_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF2APS_DMA_RX_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF2APS_DMA_RX_INT_CNT_REG_SIZE 2 +#define HIF_IRQ_HIF2APS_DMA_RX_INT_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_HIF2APS_DMA_RX_INT_CNT_REG_HIF2APS_DMA_RX_INT_CNT_MASK 0xff +#define HIF_IRQ_HIF2APS_DMA_RX_INT_CNT_REG_HIF2APS_DMA_RX_INT_CNT_SHIFT 0 +#define HIF_IRQ_HIF2APS_DMA_RX_INT_CNT_REG_HIF2APS_DMA_RX_INT_CNT_WIDTH 8 + +#define HIF_IRQ_HIF2APS_DMA_TX_INT_CNT_REG_ADDR 0xa11002e8 +#define HIF_IRQ_HIF2APS_DMA_TX_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF2APS_DMA_TX_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF2APS_DMA_TX_INT_CNT_REG_SIZE 2 +#define HIF_IRQ_HIF2APS_DMA_TX_INT_CNT_REG_STRIDE 0x4 +#define HIF_IRQ_HIF2APS_DMA_TX_INT_CNT_REG_HIF2APS_DMA_TX_INT_CNT_MASK 0xff +#define HIF_IRQ_HIF2APS_DMA_TX_INT_CNT_REG_HIF2APS_DMA_TX_INT_CNT_SHIFT 0 +#define HIF_IRQ_HIF2APS_DMA_TX_INT_CNT_REG_HIF2APS_DMA_TX_INT_CNT_WIDTH 8 + +#define HIF_IRQ_HIF2APS_TBL_RD_DONE_INT_CNT_REG_ADDR 0xa11002f0 +#define HIF_IRQ_HIF2APS_TBL_RD_DONE_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_HIF2APS_TBL_RD_DONE_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_HIF2APS_TBL_RD_DONE_INT_CNT_REG_HIF2APS_TBL_RD_DONE_INT_CNT_MASK 0xff +#define HIF_IRQ_HIF2APS_TBL_RD_DONE_INT_CNT_REG_HIF2APS_TBL_RD_DONE_INT_CNT_SHIFT 0 +#define HIF_IRQ_HIF2APS_TBL_RD_DONE_INT_CNT_REG_HIF2APS_TBL_RD_DONE_INT_CNT_WIDTH 8 + +#define HIF_IRQ_DIST2GEN_MSIX_INT_CNT_REG_ADDR 0xa11002f4 +#define HIF_IRQ_DIST2GEN_MSIX_INT_CNT_REG_WIDTH 32 +#define HIF_IRQ_DIST2GEN_MSIX_INT_CNT_REG_LENGTH 32 +#define HIF_IRQ_DIST2GEN_MSIX_INT_CNT_REG_DIST2GEN_MSIX_INT_CNT_MASK 0xff +#define HIF_IRQ_DIST2GEN_MSIX_INT_CNT_REG_DIST2GEN_MSIX_INT_CNT_SHIFT 0 +#define HIF_IRQ_DIST2GEN_MSIX_INT_CNT_REG_DIST2GEN_MSIX_INT_CNT_WIDTH 8 + +#define HIF_IRQ_CONTROL_TBL_MEM_ADDR 0xa1102000 +#define HIF_IRQ_CONTROL_TBL_MEM_WIDTH 32 +#define HIF_IRQ_CONTROL_TBL_MEM_LENGTH 32 +#define HIF_IRQ_CONTROL_TBL_MEM_DEPTH 1152 +#define HIF_IRQ_CONTROL_TBL_MEM_FUNC_MASK_MASK 0x1 +#define HIF_IRQ_CONTROL_TBL_MEM_FUNC_MASK_SHIFT 0 +#define HIF_IRQ_CONTROL_TBL_MEM_FUNC_MASK_WIDTH 1 + +#define HIF_IRQ_MEM_ERROR_INT_ADDR 0xa1104000 +#define HIF_IRQ_MEM_ERROR_INT_WIDTH 32 +#define HIF_IRQ_MEM_ERROR_INT_LENGTH 32 +#define HIF_IRQ_MEM_ERROR_INT_CMDQM2IRQ_MSIX_CBR_FIFO_MEM_SB_ERR_MASK 0x1 +#define HIF_IRQ_MEM_ERROR_INT_CMDQM2IRQ_MSIX_CBR_FIFO_MEM_SB_ERR_SHIFT 0 +#define HIF_IRQ_MEM_ERROR_INT_CMDQM2IRQ_MSIX_CBR_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_CMDQM2IRQ_MSIX_CBR_FIFO_MEM_DB_ERR_MASK 0x2 +#define HIF_IRQ_MEM_ERROR_INT_CMDQM2IRQ_MSIX_CBR_FIFO_MEM_DB_ERR_SHIFT 1 +#define HIF_IRQ_MEM_ERROR_INT_CMDQM2IRQ_MSIX_CBR_FIFO_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_TPE2HIF_MSIX_CBR_FIFO_MEM_SB_ERR_MASK 0x4 +#define HIF_IRQ_MEM_ERROR_INT_TPE2HIF_MSIX_CBR_FIFO_MEM_SB_ERR_SHIFT 2 +#define HIF_IRQ_MEM_ERROR_INT_TPE2HIF_MSIX_CBR_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_TPE2HIF_MSIX_CBR_FIFO_MEM_DB_ERR_MASK 0x8 +#define HIF_IRQ_MEM_ERROR_INT_TPE2HIF_MSIX_CBR_FIFO_MEM_DB_ERR_SHIFT 3 +#define HIF_IRQ_MEM_ERROR_INT_TPE2HIF_MSIX_CBR_FIFO_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_DIST2GEN_MSIX_FIFO_MEM_SB_ERR_MASK 0x10 +#define HIF_IRQ_MEM_ERROR_INT_DIST2GEN_MSIX_FIFO_MEM_SB_ERR_SHIFT 4 +#define HIF_IRQ_MEM_ERROR_INT_DIST2GEN_MSIX_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_DIST2GEN_MSIX_FIFO_MEM_DB_ERR_MASK 0x20 +#define HIF_IRQ_MEM_ERROR_INT_DIST2GEN_MSIX_FIFO_MEM_DB_ERR_SHIFT 5 +#define HIF_IRQ_MEM_ERROR_INT_DIST2GEN_MSIX_FIFO_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_FAIL_MSIX_FIFO_MEM_SB_ERR_MASK 0x40 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_FAIL_MSIX_FIFO_MEM_SB_ERR_SHIFT 6 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_FAIL_MSIX_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_FAIL_MSIX_FIFO_MEM_DB_ERR_MASK 0x80 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_FAIL_MSIX_FIFO_MEM_DB_ERR_SHIFT 7 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_FAIL_MSIX_FIFO_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_BACK_MSIX_FIFO_MEM_SB_ERR_MASK 0x100 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_BACK_MSIX_FIFO_MEM_SB_ERR_SHIFT 8 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_BACK_MSIX_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_BACK_MSIX_FIFO_MEM_DB_ERR_MASK 0x200 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_BACK_MSIX_FIFO_MEM_DB_ERR_SHIFT 9 +#define HIF_IRQ_MEM_ERROR_INT_WRITE_BACK_MSIX_FIFO_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_INT_DB_FIFO_MEM_SB_ERR_MASK 0x400 +#define HIF_IRQ_MEM_ERROR_INT_INT_DB_FIFO_MEM_SB_ERR_SHIFT 10 +#define HIF_IRQ_MEM_ERROR_INT_INT_DB_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_INT_DB_FIFO_MEM_DB_ERR_MASK 0x800 +#define HIF_IRQ_MEM_ERROR_INT_INT_DB_FIFO_MEM_DB_ERR_SHIFT 11 +#define HIF_IRQ_MEM_ERROR_INT_INT_DB_FIFO_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_REQ_INFO_FIFO_MEM_SB_ERR_MASK 0x3000 +#define HIF_IRQ_MEM_ERROR_INT_REQ_INFO_FIFO_MEM_SB_ERR_SHIFT 12 +#define HIF_IRQ_MEM_ERROR_INT_REQ_INFO_FIFO_MEM_SB_ERR_WIDTH 2 +#define HIF_IRQ_MEM_ERROR_INT_REQ_INFO_FIFO_MEM_DB_ERR_MASK 0xc000 +#define HIF_IRQ_MEM_ERROR_INT_REQ_INFO_FIFO_MEM_DB_ERR_SHIFT 14 +#define HIF_IRQ_MEM_ERROR_INT_REQ_INFO_FIFO_MEM_DB_ERR_WIDTH 2 +#define HIF_IRQ_MEM_ERROR_INT_PBA_TBL_MEM_SB_ERR_MASK 0x10000 +#define HIF_IRQ_MEM_ERROR_INT_PBA_TBL_MEM_SB_ERR_SHIFT 16 +#define HIF_IRQ_MEM_ERROR_INT_PBA_TBL_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_PBA_TBL_MEM_DB_ERR_MASK 0x20000 +#define HIF_IRQ_MEM_ERROR_INT_PBA_TBL_MEM_DB_ERR_SHIFT 17 +#define HIF_IRQ_MEM_ERROR_INT_PBA_TBL_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_MASK_TBL_MEM_SB_ERR_MASK 0x40000 +#define HIF_IRQ_MEM_ERROR_INT_MASK_TBL_MEM_SB_ERR_SHIFT 18 +#define HIF_IRQ_MEM_ERROR_INT_MASK_TBL_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_MASK_TBL_MEM_DB_ERR_MASK 0x80000 +#define HIF_IRQ_MEM_ERROR_INT_MASK_TBL_MEM_DB_ERR_SHIFT 19 +#define HIF_IRQ_MEM_ERROR_INT_MASK_TBL_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_VECTOR_TBL_MEM_SB_ERR_MASK 0x100000 +#define HIF_IRQ_MEM_ERROR_INT_VECTOR_TBL_MEM_SB_ERR_SHIFT 20 +#define HIF_IRQ_MEM_ERROR_INT_VECTOR_TBL_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_VECTOR_TBL_MEM_DB_ERR_MASK 0x200000 +#define HIF_IRQ_MEM_ERROR_INT_VECTOR_TBL_MEM_DB_ERR_SHIFT 21 +#define HIF_IRQ_MEM_ERROR_INT_VECTOR_TBL_MEM_DB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_CONTROL_TBL_MEM_SB_ERR_MASK 0x400000 +#define HIF_IRQ_MEM_ERROR_INT_CONTROL_TBL_MEM_SB_ERR_SHIFT 22 +#define HIF_IRQ_MEM_ERROR_INT_CONTROL_TBL_MEM_SB_ERR_WIDTH 1 +#define HIF_IRQ_MEM_ERROR_INT_CONTROL_TBL_MEM_DB_ERR_MASK 0x800000 +#define HIF_IRQ_MEM_ERROR_INT_CONTROL_TBL_MEM_DB_ERR_SHIFT 23 +#define HIF_IRQ_MEM_ERROR_INT_CONTROL_TBL_MEM_DB_ERR_WIDTH 1 + +#define HIF_IRQ_MEM_INIT_CTRL_ADDR 0xa1104008 +#define HIF_IRQ_MEM_INIT_CTRL_WIDTH 32 +#define HIF_IRQ_MEM_INIT_CTRL_LENGTH 32 +#define HIF_IRQ_MEM_INIT_CTRL_PBA_TBL_MEM_INIT_RST_N_MASK 0x1 +#define HIF_IRQ_MEM_INIT_CTRL_PBA_TBL_MEM_INIT_RST_N_SHIFT 0 +#define HIF_IRQ_MEM_INIT_CTRL_PBA_TBL_MEM_INIT_RST_N_WIDTH 1 +#define HIF_IRQ_MEM_INIT_CTRL_PBA_TBL_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_IRQ_MEM_INIT_CTRL_PBA_TBL_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_IRQ_MEM_INIT_CTRL_MASK_TBL_MEM_INIT_RST_N_MASK 0x2 +#define HIF_IRQ_MEM_INIT_CTRL_MASK_TBL_MEM_INIT_RST_N_SHIFT 1 +#define HIF_IRQ_MEM_INIT_CTRL_MASK_TBL_MEM_INIT_RST_N_WIDTH 1 +#define HIF_IRQ_MEM_INIT_CTRL_MASK_TBL_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_IRQ_MEM_INIT_CTRL_MASK_TBL_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_IRQ_MEM_INIT_CTRL_VECTOR_TBL_MEM_INIT_RST_N_MASK 0x4 +#define HIF_IRQ_MEM_INIT_CTRL_VECTOR_TBL_MEM_INIT_RST_N_SHIFT 2 +#define HIF_IRQ_MEM_INIT_CTRL_VECTOR_TBL_MEM_INIT_RST_N_WIDTH 1 +#define HIF_IRQ_MEM_INIT_CTRL_VECTOR_TBL_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_IRQ_MEM_INIT_CTRL_VECTOR_TBL_MEM_INIT_RST_N_MIN_VAL 0x0 +#define HIF_IRQ_MEM_INIT_CTRL_CONTROL_TBL_MEM_INIT_RST_N_MASK 0x8 +#define HIF_IRQ_MEM_INIT_CTRL_CONTROL_TBL_MEM_INIT_RST_N_SHIFT 3 +#define HIF_IRQ_MEM_INIT_CTRL_CONTROL_TBL_MEM_INIT_RST_N_WIDTH 1 +#define HIF_IRQ_MEM_INIT_CTRL_CONTROL_TBL_MEM_INIT_RST_N_MAX_VAL 0x1 +#define HIF_IRQ_MEM_INIT_CTRL_CONTROL_TBL_MEM_INIT_RST_N_MIN_VAL 0x0 + +#define HIF_IRQ_TIMEOUT_INT_REG_ADDR 0xa1104010 +#define HIF_IRQ_TIMEOUT_INT_REG_WIDTH 32 +#define HIF_IRQ_TIMEOUT_INT_REG_LENGTH 32 +#define HIF_IRQ_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_MASK 0x1 +#define HIF_IRQ_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_SHIFT 0 +#define HIF_IRQ_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_WIDTH 1 +#define HIF_IRQ_TIMEOUT_INT_REG_CONTROL_TBL_MEM_TIMEOUT_MASK 0x2 +#define HIF_IRQ_TIMEOUT_INT_REG_CONTROL_TBL_MEM_TIMEOUT_SHIFT 1 +#define HIF_IRQ_TIMEOUT_INT_REG_CONTROL_TBL_MEM_TIMEOUT_WIDTH 1 + +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_ADDR 0xa1104014 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_WIDTH 32 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_LENGTH 32 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MASK 0x1 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_SHIFT 0 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CONTROL_TBL_MEM_TIMEOUT_INT_MASK_MASK 0x2 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CONTROL_TBL_MEM_TIMEOUT_INT_MASK_SHIFT 1 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CONTROL_TBL_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CONTROL_TBL_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_IRQ_TIMEOUT_INT_MASK_REG_CONTROL_TBL_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 + +#define HIF_IRQ_TIMEOUT_CFG_REG_ADDR 0xa1104018 +#define HIF_IRQ_TIMEOUT_CFG_REG_WIDTH 32 +#define HIF_IRQ_TIMEOUT_CFG_REG_LENGTH 32 +#define HIF_IRQ_TIMEOUT_CFG_REG_TIMEOUT_PARA_MASK 0xffff +#define HIF_IRQ_TIMEOUT_CFG_REG_TIMEOUT_PARA_SHIFT 0 +#define HIF_IRQ_TIMEOUT_CFG_REG_TIMEOUT_PARA_WIDTH 16 +#define HIF_IRQ_TIMEOUT_CFG_REG_TIMEOUT_PARA_MAX_VAL 0xffff +#define HIF_IRQ_TIMEOUT_CFG_REG_TIMEOUT_PARA_MIN_VAL 0x0 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_tbl_csr_defines.h b/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_tbl_csr_defines.h new file mode 100644 index 0000000000000000000000000000000000000000..9747b87bf9e858c6f22c5cf6c5281b25ad24f257 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/andes/hif_tbl_csr_defines.h @@ -0,0 +1,474 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _HIF_TBL_CSR_DEFINES_H_ +#define _HIF_TBL_CSR_DEFINES_H_ + +#define HIF_TBL_SOFT_RESET_REG_ADDR 0xa1060000 +#define HIF_TBL_SOFT_RESET_REG_WIDTH 32 +#define HIF_TBL_SOFT_RESET_REG_LENGTH 32 +#define HIF_TBL_SOFT_RESET_REG_SOFT_RESET_MASK 0x1 +#define HIF_TBL_SOFT_RESET_REG_SOFT_RESET_SHIFT 0 +#define HIF_TBL_SOFT_RESET_REG_SOFT_RESET_WIDTH 1 +#define HIF_TBL_SOFT_RESET_REG_SOFT_RESET_MAX_VAL 0x1 +#define HIF_TBL_SOFT_RESET_REG_SOFT_RESET_MIN_VAL 0x0 + +#define HIF_TBL_SCRATCH_PAD_REG_ADDR 0xa106000c +#define HIF_TBL_SCRATCH_PAD_REG_WIDTH 32 +#define HIF_TBL_SCRATCH_PAD_REG_LENGTH 32 +#define HIF_TBL_SCRATCH_PAD_REG_SCRATCH_PAD_MASK 0xffffffff +#define HIF_TBL_SCRATCH_PAD_REG_SCRATCH_PAD_SHIFT 0 +#define HIF_TBL_SCRATCH_PAD_REG_SCRATCH_PAD_WIDTH 32 +#define HIF_TBL_SCRATCH_PAD_REG_SCRATCH_PAD_MAX_VAL 0xffffffff +#define HIF_TBL_SCRATCH_PAD_REG_SCRATCH_PAD_MIN_VAL 0x0 + +#define HIF_TBL_CSR_ERR_FLAG_REG_ADDR 0xa1060010 +#define HIF_TBL_CSR_ERR_FLAG_REG_WIDTH 32 +#define HIF_TBL_CSR_ERR_FLAG_REG_LENGTH 32 +#define HIF_TBL_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_MASK 0x1 +#define HIF_TBL_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_SHIFT 0 +#define HIF_TBL_CSR_ERR_FLAG_REG_CSR_ERR_FLAG_WIDTH 1 + +#define HIF_TBL_CSR_ERR_ADDR_REG_ADDR 0xa1060014 +#define HIF_TBL_CSR_ERR_ADDR_REG_WIDTH 32 +#define HIF_TBL_CSR_ERR_ADDR_REG_LENGTH 32 +#define HIF_TBL_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_MASK 0xffffffff +#define HIF_TBL_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_SHIFT 0 +#define HIF_TBL_CSR_ERR_ADDR_REG_CSR_ERR_ADDR_WIDTH 32 + +#define HIF_TBL_CSR_ERR_LEN_REG_ADDR 0xa1060018 +#define HIF_TBL_CSR_ERR_LEN_REG_WIDTH 32 +#define HIF_TBL_CSR_ERR_LEN_REG_LENGTH 32 +#define HIF_TBL_CSR_ERR_LEN_REG_CSR_ERR_LEN_MASK 0x3ff +#define HIF_TBL_CSR_ERR_LEN_REG_CSR_ERR_LEN_SHIFT 0 +#define HIF_TBL_CSR_ERR_LEN_REG_CSR_ERR_LEN_WIDTH 10 + +#define HIF_TBL_CSR_ERR_TYPE_REG_ADDR 0xa106001c +#define HIF_TBL_CSR_ERR_TYPE_REG_WIDTH 32 +#define HIF_TBL_CSR_ERR_TYPE_REG_LENGTH 32 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_MASK 0x1 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_SHIFT 0 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_CMD_ERR_TYPE_WIDTH 1 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_MASK 0x6 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_SHIFT 1 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_LEN_ERR_TYPE_WIDTH 2 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_MASK 0x8 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_SHIFT 3 +#define HIF_TBL_CSR_ERR_TYPE_REG_CSR_ADDR_ERR_TYPE_WIDTH 1 + +#define HIF_TBL_TBL_DL_REQ_REG_ADDR 0xa1060020 +#define HIF_TBL_TBL_DL_REQ_REG_WIDTH 32 +#define HIF_TBL_TBL_DL_REQ_REG_LENGTH 32 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_FUNC_ID_MASK 0x7ff +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_FUNC_ID_SHIFT 0 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_FUNC_ID_WIDTH 11 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_FUNC_ID_MAX_VAL 0x7ff +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_FUNC_ID_MIN_VAL 0x0 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_HOST_ID_MASK 0x800 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_HOST_ID_SHIFT 11 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_HOST_ID_WIDTH 1 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_HOST_ID_MAX_VAL 0x1 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_HOST_ID_MIN_VAL 0x0 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_LEN_MASK 0x3fff000 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_LEN_SHIFT 12 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_LEN_WIDTH 14 +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_LEN_MAX_VAL 0x3fff +#define HIF_TBL_TBL_DL_REQ_REG_TBL_DL_LEN_MIN_VAL 0x0 + +#define HIF_TBL_TBL_DL_ADDR_L_REG_ADDR 0xa1060024 +#define HIF_TBL_TBL_DL_ADDR_L_REG_WIDTH 32 +#define HIF_TBL_TBL_DL_ADDR_L_REG_LENGTH 32 +#define HIF_TBL_TBL_DL_ADDR_L_REG_TBL_DL_ADDR_L_MASK 0xffffffff +#define HIF_TBL_TBL_DL_ADDR_L_REG_TBL_DL_ADDR_L_SHIFT 0 +#define HIF_TBL_TBL_DL_ADDR_L_REG_TBL_DL_ADDR_L_WIDTH 32 +#define HIF_TBL_TBL_DL_ADDR_L_REG_TBL_DL_ADDR_L_MAX_VAL 0xffffffff +#define HIF_TBL_TBL_DL_ADDR_L_REG_TBL_DL_ADDR_L_MIN_VAL 0x0 + +#define HIF_TBL_TBL_DL_ADDR_H_REG_ADDR 0xa1060028 +#define HIF_TBL_TBL_DL_ADDR_H_REG_WIDTH 32 +#define HIF_TBL_TBL_DL_ADDR_H_REG_LENGTH 32 +#define HIF_TBL_TBL_DL_ADDR_H_REG_TBL_DL_ADDR_H_MASK 0xffffffff +#define HIF_TBL_TBL_DL_ADDR_H_REG_TBL_DL_ADDR_H_SHIFT 0 +#define HIF_TBL_TBL_DL_ADDR_H_REG_TBL_DL_ADDR_H_WIDTH 32 +#define HIF_TBL_TBL_DL_ADDR_H_REG_TBL_DL_ADDR_H_MAX_VAL 0xffffffff +#define HIF_TBL_TBL_DL_ADDR_H_REG_TBL_DL_ADDR_H_MIN_VAL 0x0 + +#define HIF_TBL_TBL_DL_START_REG_ADDR 0xa106002c +#define HIF_TBL_TBL_DL_START_REG_WIDTH 32 +#define HIF_TBL_TBL_DL_START_REG_LENGTH 32 +#define HIF_TBL_TBL_DL_START_REG_TBL_DL_START_MASK 0x1 +#define HIF_TBL_TBL_DL_START_REG_TBL_DL_START_SHIFT 0 +#define HIF_TBL_TBL_DL_START_REG_TBL_DL_START_WIDTH 1 +#define HIF_TBL_TBL_DL_START_REG_TBL_DL_START_MAX_VAL 0x1 +#define HIF_TBL_TBL_DL_START_REG_TBL_DL_START_MIN_VAL 0x0 + +#define HIF_TBL_TBL_DL_BUSY_REG_ADDR 0xa1060030 +#define HIF_TBL_TBL_DL_BUSY_REG_WIDTH 32 +#define HIF_TBL_TBL_DL_BUSY_REG_LENGTH 32 +#define HIF_TBL_TBL_DL_BUSY_REG_TBL_DL_BUSY_MASK 0x1 +#define HIF_TBL_TBL_DL_BUSY_REG_TBL_DL_BUSY_SHIFT 0 +#define HIF_TBL_TBL_DL_BUSY_REG_TBL_DL_BUSY_WIDTH 1 + +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_REG_ADDR 0xa1060034 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_REG_WIDTH 32 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_REG_LENGTH 32 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_REG_DL_RX_PCIE_DATA_ERR_MASK 0x1 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_REG_DL_RX_PCIE_DATA_ERR_SHIFT 0 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_REG_DL_RX_PCIE_DATA_ERR_WIDTH 1 + +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_INT_MASK_REG_ADDR 0xa1060038 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_INT_MASK_REG_DL_RX_PCIE_DATA_ERR_INT_MASK_MASK 0x1 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_INT_MASK_REG_DL_RX_PCIE_DATA_ERR_INT_MASK_SHIFT 0 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_INT_MASK_REG_DL_RX_PCIE_DATA_ERR_INT_MASK_WIDTH 1 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_INT_MASK_REG_DL_RX_PCIE_DATA_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_DL_RX_PCIE_DATA_ERR_INT_MASK_REG_DL_RX_PCIE_DATA_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_TBL_CFG_DL_LEN_ERR_INT_REG_ADDR 0xa106003c +#define HIF_TBL_CFG_DL_LEN_ERR_INT_REG_WIDTH 32 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_REG_LENGTH 32 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_REG_CFG_DL_LEN_ERR_MASK 0x1 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_REG_CFG_DL_LEN_ERR_SHIFT 0 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_REG_CFG_DL_LEN_ERR_WIDTH 1 + +#define HIF_TBL_CFG_DL_LEN_ERR_INT_MASK_REG_ADDR 0xa1060040 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_MASK_REG_CFG_DL_LEN_ERR_INT_MASK_MASK 0x1 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_MASK_REG_CFG_DL_LEN_ERR_INT_MASK_SHIFT 0 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_MASK_REG_CFG_DL_LEN_ERR_INT_MASK_WIDTH 1 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_MASK_REG_CFG_DL_LEN_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_CFG_DL_LEN_ERR_INT_MASK_REG_CFG_DL_LEN_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_TBL_MSG_RDY_REG_ADDR 0xa1060044 +#define HIF_TBL_MSG_RDY_REG_WIDTH 32 +#define HIF_TBL_MSG_RDY_REG_LENGTH 32 +#define HIF_TBL_MSG_RDY_REG_TBL2IRQ_TBL_RD_DONE_MASK 0x1 +#define HIF_TBL_MSG_RDY_REG_TBL2IRQ_TBL_RD_DONE_SHIFT 0 +#define HIF_TBL_MSG_RDY_REG_TBL2IRQ_TBL_RD_DONE_WIDTH 1 + +#define HIF_TBL_MSG_RDY_INT_MASK_REG_ADDR 0xa1060048 +#define HIF_TBL_MSG_RDY_INT_MASK_REG_WIDTH 32 +#define HIF_TBL_MSG_RDY_INT_MASK_REG_LENGTH 32 +#define HIF_TBL_MSG_RDY_INT_MASK_REG_TBL2IRQ_TBL_RD_DONE_INT_MASK_MASK 0x1 +#define HIF_TBL_MSG_RDY_INT_MASK_REG_TBL2IRQ_TBL_RD_DONE_INT_MASK_SHIFT 0 +#define HIF_TBL_MSG_RDY_INT_MASK_REG_TBL2IRQ_TBL_RD_DONE_INT_MASK_WIDTH 1 +#define HIF_TBL_MSG_RDY_INT_MASK_REG_TBL2IRQ_TBL_RD_DONE_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_MSG_RDY_INT_MASK_REG_TBL2IRQ_TBL_RD_DONE_INT_MASK_MIN_VAL 0x0 + +#define HIF_TBL_DL_RX_PCIE_L_CNT_REG_ADDR 0xa106004c +#define HIF_TBL_DL_RX_PCIE_L_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_RX_PCIE_L_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_RX_PCIE_L_CNT_REG_DL_RX_PCIE_L_MASK 0xffff +#define HIF_TBL_DL_RX_PCIE_L_CNT_REG_DL_RX_PCIE_L_SHIFT 0 +#define HIF_TBL_DL_RX_PCIE_L_CNT_REG_DL_RX_PCIE_L_WIDTH 16 + +#define HIF_TBL_DL_RX_PCIE_DATA_CNT_REG_ADDR 0xa1060050 +#define HIF_TBL_DL_RX_PCIE_DATA_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_RX_PCIE_DATA_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_RX_PCIE_DATA_CNT_REG_DL_RX_PCIE_DATA_MASK 0xffff +#define HIF_TBL_DL_RX_PCIE_DATA_CNT_REG_DL_RX_PCIE_DATA_SHIFT 0 +#define HIF_TBL_DL_RX_PCIE_DATA_CNT_REG_DL_RX_PCIE_DATA_WIDTH 16 + +#define HIF_TBL_DL_RX_PCIE_ERR_CNT_REG_ADDR 0xa1060054 +#define HIF_TBL_DL_RX_PCIE_ERR_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_RX_PCIE_ERR_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_RX_PCIE_ERR_CNT_REG_DL_RX_PCIE_ERR_MASK 0xffff +#define HIF_TBL_DL_RX_PCIE_ERR_CNT_REG_DL_RX_PCIE_ERR_SHIFT 0 +#define HIF_TBL_DL_RX_PCIE_ERR_CNT_REG_DL_RX_PCIE_ERR_WIDTH 16 + +#define HIF_TBL_DL_RX_PCIE_LAST_CNT_REG_ADDR 0xa1060058 +#define HIF_TBL_DL_RX_PCIE_LAST_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_RX_PCIE_LAST_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_RX_PCIE_LAST_CNT_REG_DL_RX_PCIE_LAST_MASK 0xffff +#define HIF_TBL_DL_RX_PCIE_LAST_CNT_REG_DL_RX_PCIE_LAST_SHIFT 0 +#define HIF_TBL_DL_RX_PCIE_LAST_CNT_REG_DL_RX_PCIE_LAST_WIDTH 16 + +#define HIF_TBL_DL_TX_REQ_CNT_REG_ADDR 0xa106005c +#define HIF_TBL_DL_TX_REQ_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_TX_REQ_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_TX_REQ_CNT_REG_DL_TX_REQ2PCIE_MASK 0xffff +#define HIF_TBL_DL_TX_REQ_CNT_REG_DL_TX_REQ2PCIE_SHIFT 0 +#define HIF_TBL_DL_TX_REQ_CNT_REG_DL_TX_REQ2PCIE_WIDTH 16 + +#define HIF_TBL_DL_TX_TBL2PP_VLD_CNT_REG_ADDR 0xa1060060 +#define HIF_TBL_DL_TX_TBL2PP_VLD_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_TX_TBL2PP_VLD_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_TX_TBL2PP_VLD_CNT_REG_DL_TX_TBL2PP_VLD_MASK 0xffff +#define HIF_TBL_DL_TX_TBL2PP_VLD_CNT_REG_DL_TX_TBL2PP_VLD_SHIFT 0 +#define HIF_TBL_DL_TX_TBL2PP_VLD_CNT_REG_DL_TX_TBL2PP_VLD_WIDTH 16 + +#define HIF_TBL_DL_TX_TBL2PP_SOF_CNT_REG_ADDR 0xa1060064 +#define HIF_TBL_DL_TX_TBL2PP_SOF_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_TX_TBL2PP_SOF_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_TX_TBL2PP_SOF_CNT_REG_DL_TX_TBL2PP_SOF_MASK 0xffff +#define HIF_TBL_DL_TX_TBL2PP_SOF_CNT_REG_DL_TX_TBL2PP_SOF_SHIFT 0 +#define HIF_TBL_DL_TX_TBL2PP_SOF_CNT_REG_DL_TX_TBL2PP_SOF_WIDTH 16 + +#define HIF_TBL_DL_TX_TBL2PP_EOF_CNT_REG_ADDR 0xa1060068 +#define HIF_TBL_DL_TX_TBL2PP_EOF_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_TX_TBL2PP_EOF_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_TX_TBL2PP_EOF_CNT_REG_DL_TX_TBL2PP_EOF_MASK 0xffff +#define HIF_TBL_DL_TX_TBL2PP_EOF_CNT_REG_DL_TX_TBL2PP_EOF_SHIFT 0 +#define HIF_TBL_DL_TX_TBL2PP_EOF_CNT_REG_DL_TX_TBL2PP_EOF_WIDTH 16 + +#define HIF_TBL_DL_TX_TBL2PP_SOB_CNT_REG_ADDR 0xa106006c +#define HIF_TBL_DL_TX_TBL2PP_SOB_CNT_REG_WIDTH 32 +#define HIF_TBL_DL_TX_TBL2PP_SOB_CNT_REG_LENGTH 32 +#define HIF_TBL_DL_TX_TBL2PP_SOB_CNT_REG_DL_TX_TBL2PP_SOB_MASK 0xffff +#define HIF_TBL_DL_TX_TBL2PP_SOB_CNT_REG_DL_TX_TBL2PP_SOB_SHIFT 0 +#define HIF_TBL_DL_TX_TBL2PP_SOB_CNT_REG_DL_TX_TBL2PP_SOB_WIDTH 16 + +#define HIF_TBL_RX_IPP2HIF_DL_FC_CNT_REG_ADDR 0xa1060070 +#define HIF_TBL_RX_IPP2HIF_DL_FC_CNT_REG_WIDTH 32 +#define HIF_TBL_RX_IPP2HIF_DL_FC_CNT_REG_LENGTH 32 +#define HIF_TBL_RX_IPP2HIF_DL_FC_CNT_REG_RX_IPP2HIF_DL_FC_CNT_MASK 0xffff +#define HIF_TBL_RX_IPP2HIF_DL_FC_CNT_REG_RX_IPP2HIF_DL_FC_CNT_SHIFT 0 +#define HIF_TBL_RX_IPP2HIF_DL_FC_CNT_REG_RX_IPP2HIF_DL_FC_CNT_WIDTH 16 + +#define HIF_TBL_TX_UL_CREDIT_FC_CNT_REG_ADDR 0xa1060074 +#define HIF_TBL_TX_UL_CREDIT_FC_CNT_REG_WIDTH 32 +#define HIF_TBL_TX_UL_CREDIT_FC_CNT_REG_LENGTH 32 +#define HIF_TBL_TX_UL_CREDIT_FC_CNT_REG_TX_UL_CREDIT_FC_CNT_MASK 0xffff +#define HIF_TBL_TX_UL_CREDIT_FC_CNT_REG_TX_UL_CREDIT_FC_CNT_SHIFT 0 +#define HIF_TBL_TX_UL_CREDIT_FC_CNT_REG_TX_UL_CREDIT_FC_CNT_WIDTH 16 + +#define HIF_TBL_LAST_FLAG_FC_CNT_REG_ADDR 0xa1060078 +#define HIF_TBL_LAST_FLAG_FC_CNT_REG_WIDTH 32 +#define HIF_TBL_LAST_FLAG_FC_CNT_REG_LENGTH 32 +#define HIF_TBL_LAST_FLAG_FC_CNT_REG_LAST_FLAG_FC_CNT_MASK 0xffff +#define HIF_TBL_LAST_FLAG_FC_CNT_REG_LAST_FLAG_FC_CNT_SHIFT 0 +#define HIF_TBL_LAST_FLAG_FC_CNT_REG_LAST_FLAG_FC_CNT_WIDTH 16 + +#define HIF_TBL_TBL_UL_REQ_REG_ADDR 0xa106007c +#define HIF_TBL_TBL_UL_REQ_REG_WIDTH 32 +#define HIF_TBL_TBL_UL_REQ_REG_LENGTH 32 +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_FUNC_ID_MASK 0x7ff +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_FUNC_ID_SHIFT 0 +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_FUNC_ID_WIDTH 11 +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_FUNC_ID_MAX_VAL 0x7ff +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_FUNC_ID_MIN_VAL 0x0 +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_HOST_ID_MASK 0x800 +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_HOST_ID_SHIFT 11 +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_HOST_ID_WIDTH 1 +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_HOST_ID_MAX_VAL 0x1 +#define HIF_TBL_TBL_UL_REQ_REG_TBL_UL_HOST_ID_MIN_VAL 0x0 + +#define HIF_TBL_TBL_UL_ADDR_L_REG_ADDR 0xa1060080 +#define HIF_TBL_TBL_UL_ADDR_L_REG_WIDTH 32 +#define HIF_TBL_TBL_UL_ADDR_L_REG_LENGTH 32 +#define HIF_TBL_TBL_UL_ADDR_L_REG_TBL_UL_ADDR_L_MASK 0xffffffff +#define HIF_TBL_TBL_UL_ADDR_L_REG_TBL_UL_ADDR_L_SHIFT 0 +#define HIF_TBL_TBL_UL_ADDR_L_REG_TBL_UL_ADDR_L_WIDTH 32 +#define HIF_TBL_TBL_UL_ADDR_L_REG_TBL_UL_ADDR_L_MAX_VAL 0xffffffff +#define HIF_TBL_TBL_UL_ADDR_L_REG_TBL_UL_ADDR_L_MIN_VAL 0x0 + +#define HIF_TBL_TBL_UL_ADDR_H_REG_ADDR 0xa1060084 +#define HIF_TBL_TBL_UL_ADDR_H_REG_WIDTH 32 +#define HIF_TBL_TBL_UL_ADDR_H_REG_LENGTH 32 +#define HIF_TBL_TBL_UL_ADDR_H_REG_TBL_UL_ADDR_H_MASK 0xffffffff +#define HIF_TBL_TBL_UL_ADDR_H_REG_TBL_UL_ADDR_H_SHIFT 0 +#define HIF_TBL_TBL_UL_ADDR_H_REG_TBL_UL_ADDR_H_WIDTH 32 +#define HIF_TBL_TBL_UL_ADDR_H_REG_TBL_UL_ADDR_H_MAX_VAL 0xffffffff +#define HIF_TBL_TBL_UL_ADDR_H_REG_TBL_UL_ADDR_H_MIN_VAL 0x0 + +#define HIF_TBL_TBL_UL_START_REG_ADDR 0xa1060088 +#define HIF_TBL_TBL_UL_START_REG_WIDTH 32 +#define HIF_TBL_TBL_UL_START_REG_LENGTH 32 +#define HIF_TBL_TBL_UL_START_REG_TBL_UL_START_MASK 0x1 +#define HIF_TBL_TBL_UL_START_REG_TBL_UL_START_SHIFT 0 +#define HIF_TBL_TBL_UL_START_REG_TBL_UL_START_WIDTH 1 +#define HIF_TBL_TBL_UL_START_REG_TBL_UL_START_MAX_VAL 0x1 +#define HIF_TBL_TBL_UL_START_REG_TBL_UL_START_MIN_VAL 0x0 + +#define HIF_TBL_UL_CBT_CFG_RDY_ALLOWANCE_REG_ADDR 0xa106008c +#define HIF_TBL_UL_CBT_CFG_RDY_ALLOWANCE_REG_WIDTH 32 +#define HIF_TBL_UL_CBT_CFG_RDY_ALLOWANCE_REG_LENGTH 32 +#define HIF_TBL_UL_CBT_CFG_RDY_ALLOWANCE_REG_CFG_RDY_ALLOWANCE_MASK 0x1f +#define HIF_TBL_UL_CBT_CFG_RDY_ALLOWANCE_REG_CFG_RDY_ALLOWANCE_SHIFT 0 +#define HIF_TBL_UL_CBT_CFG_RDY_ALLOWANCE_REG_CFG_RDY_ALLOWANCE_WIDTH 5 +#define HIF_TBL_UL_CBT_CFG_RDY_ALLOWANCE_REG_CFG_RDY_ALLOWANCE_MAX_VAL 0x1f +#define HIF_TBL_UL_CBT_CFG_RDY_ALLOWANCE_REG_CFG_RDY_ALLOWANCE_MIN_VAL 0x0 + +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_REG_ADDR 0xa1060090 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_REG_WIDTH 32 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_REG_LENGTH 32 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_REG_UL_RX_PCIE_RSP_ERR_MASK 0x1 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_REG_UL_RX_PCIE_RSP_ERR_SHIFT 0 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_REG_UL_RX_PCIE_RSP_ERR_WIDTH 1 + +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_INT_MASK_REG_ADDR 0xa1060094 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_INT_MASK_REG_WIDTH 32 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_INT_MASK_REG_LENGTH 32 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_INT_MASK_REG_UL_RX_PCIE_RSP_ERR_INT_MASK_MASK 0x1 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_INT_MASK_REG_UL_RX_PCIE_RSP_ERR_INT_MASK_SHIFT 0 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_INT_MASK_REG_UL_RX_PCIE_RSP_ERR_INT_MASK_WIDTH 1 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_INT_MASK_REG_UL_RX_PCIE_RSP_ERR_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_UL_RX_PCIE_RSP_ERR_INT_MASK_REG_UL_RX_PCIE_RSP_ERR_INT_MASK_MIN_VAL 0x0 + +#define HIF_TBL_UL_RX_PP2TBL_VLD_CNT_REG_ADDR 0xa1060098 +#define HIF_TBL_UL_RX_PP2TBL_VLD_CNT_REG_WIDTH 32 +#define HIF_TBL_UL_RX_PP2TBL_VLD_CNT_REG_LENGTH 32 +#define HIF_TBL_UL_RX_PP2TBL_VLD_CNT_REG_UL_RX_PP2TBL_VLD_MASK 0xffff +#define HIF_TBL_UL_RX_PP2TBL_VLD_CNT_REG_UL_RX_PP2TBL_VLD_SHIFT 0 +#define HIF_TBL_UL_RX_PP2TBL_VLD_CNT_REG_UL_RX_PP2TBL_VLD_WIDTH 16 + +#define HIF_TBL_UL_RX_PP2TBL_SOF_CNT_REG_ADDR 0xa106009c +#define HIF_TBL_UL_RX_PP2TBL_SOF_CNT_REG_WIDTH 32 +#define HIF_TBL_UL_RX_PP2TBL_SOF_CNT_REG_LENGTH 32 +#define HIF_TBL_UL_RX_PP2TBL_SOF_CNT_REG_UL_RX_PP2TBL_SOF_MASK 0xffff +#define HIF_TBL_UL_RX_PP2TBL_SOF_CNT_REG_UL_RX_PP2TBL_SOF_SHIFT 0 +#define HIF_TBL_UL_RX_PP2TBL_SOF_CNT_REG_UL_RX_PP2TBL_SOF_WIDTH 16 + +#define HIF_TBL_UL_RX_PP2TBL_EOF_CNT_REG_ADDR 0xa10600a0 +#define HIF_TBL_UL_RX_PP2TBL_EOF_CNT_REG_WIDTH 32 +#define HIF_TBL_UL_RX_PP2TBL_EOF_CNT_REG_LENGTH 32 +#define HIF_TBL_UL_RX_PP2TBL_EOF_CNT_REG_UL_RX_PP2TBL_EOF_MASK 0xffff +#define HIF_TBL_UL_RX_PP2TBL_EOF_CNT_REG_UL_RX_PP2TBL_EOF_SHIFT 0 +#define HIF_TBL_UL_RX_PP2TBL_EOF_CNT_REG_UL_RX_PP2TBL_EOF_WIDTH 16 + +#define HIF_TBL_UL_TX_PCIE_ENTRY_CNT_REG_ADDR 0xa10600a4 +#define HIF_TBL_UL_TX_PCIE_ENTRY_CNT_REG_WIDTH 32 +#define HIF_TBL_UL_TX_PCIE_ENTRY_CNT_REG_LENGTH 32 +#define HIF_TBL_UL_TX_PCIE_ENTRY_CNT_REG_UL_TX_PCIE_ENTRY_MASK 0xffff +#define HIF_TBL_UL_TX_PCIE_ENTRY_CNT_REG_UL_TX_PCIE_ENTRY_SHIFT 0 +#define HIF_TBL_UL_TX_PCIE_ENTRY_CNT_REG_UL_TX_PCIE_ENTRY_WIDTH 16 + +#define HIF_TBL_UL_TX_PCIE_LAST_CNT_REG_ADDR 0xa10600a8 +#define HIF_TBL_UL_TX_PCIE_LAST_CNT_REG_WIDTH 32 +#define HIF_TBL_UL_TX_PCIE_LAST_CNT_REG_LENGTH 32 +#define HIF_TBL_UL_TX_PCIE_LAST_CNT_REG_UL_TX_PCIE_LAST_MASK 0xffff +#define HIF_TBL_UL_TX_PCIE_LAST_CNT_REG_UL_TX_PCIE_LAST_SHIFT 0 +#define HIF_TBL_UL_TX_PCIE_LAST_CNT_REG_UL_TX_PCIE_LAST_WIDTH 16 + +#define HIF_TBL_UL_RX_PCIE_ACK_CNT_REG_ADDR 0xa10600ac +#define HIF_TBL_UL_RX_PCIE_ACK_CNT_REG_WIDTH 32 +#define HIF_TBL_UL_RX_PCIE_ACK_CNT_REG_LENGTH 32 +#define HIF_TBL_UL_RX_PCIE_ACK_CNT_REG_UL_RX_PCIE_ACK_MASK 0xffff +#define HIF_TBL_UL_RX_PCIE_ACK_CNT_REG_UL_RX_PCIE_ACK_SHIFT 0 +#define HIF_TBL_UL_RX_PCIE_ACK_CNT_REG_UL_RX_PCIE_ACK_WIDTH 16 + +#define HIF_TBL_UL_RX_PP2TBL_LAST_CNT_REG_ADDR 0xa10600b4 +#define HIF_TBL_UL_RX_PP2TBL_LAST_CNT_REG_WIDTH 32 +#define HIF_TBL_UL_RX_PP2TBL_LAST_CNT_REG_LENGTH 32 +#define HIF_TBL_UL_RX_PP2TBL_LAST_CNT_REG_UL_RX_PP2TBL_LAST_MASK 0xffff +#define HIF_TBL_UL_RX_PP2TBL_LAST_CNT_REG_UL_RX_PP2TBL_LAST_SHIFT 0 +#define HIF_TBL_UL_RX_PP2TBL_LAST_CNT_REG_UL_RX_PP2TBL_LAST_WIDTH 16 + +#define HIF_TBL_UL_TX_TBL2IRQ_RD_DONE_CNT_REG_ADDR 0xa10600b8 +#define HIF_TBL_UL_TX_TBL2IRQ_RD_DONE_CNT_REG_WIDTH 32 +#define HIF_TBL_UL_TX_TBL2IRQ_RD_DONE_CNT_REG_LENGTH 32 +#define HIF_TBL_UL_TX_TBL2IRQ_RD_DONE_CNT_REG_UL_TX_TBL2IRQ_RD_DONE_MASK 0xffff +#define HIF_TBL_UL_TX_TBL2IRQ_RD_DONE_CNT_REG_UL_TX_TBL2IRQ_RD_DONE_SHIFT 0 +#define HIF_TBL_UL_TX_TBL2IRQ_RD_DONE_CNT_REG_UL_TX_TBL2IRQ_RD_DONE_WIDTH 16 + +#define HIF_TBL_FIFO_AFUL_TH_REG_ADDR 0xa10600bc +#define HIF_TBL_FIFO_AFUL_TH_REG_WIDTH 32 +#define HIF_TBL_FIFO_AFUL_TH_REG_LENGTH 32 +#define HIF_TBL_FIFO_AFUL_TH_REG_DL_FIFO_AFUL_TH_MASK 0xff +#define HIF_TBL_FIFO_AFUL_TH_REG_DL_FIFO_AFUL_TH_SHIFT 0 +#define HIF_TBL_FIFO_AFUL_TH_REG_DL_FIFO_AFUL_TH_WIDTH 8 +#define HIF_TBL_FIFO_AFUL_TH_REG_DL_FIFO_AFUL_TH_MAX_VAL 0xff +#define HIF_TBL_FIFO_AFUL_TH_REG_DL_FIFO_AFUL_TH_MIN_VAL 0x0 +#define HIF_TBL_FIFO_AFUL_TH_REG_TBL_LAST_FLAG_DL_FIFO_AFUL_TH_MASK 0x3f00 +#define HIF_TBL_FIFO_AFUL_TH_REG_TBL_LAST_FLAG_DL_FIFO_AFUL_TH_SHIFT 8 +#define HIF_TBL_FIFO_AFUL_TH_REG_TBL_LAST_FLAG_DL_FIFO_AFUL_TH_WIDTH 6 +#define HIF_TBL_FIFO_AFUL_TH_REG_TBL_LAST_FLAG_DL_FIFO_AFUL_TH_MAX_VAL 0x3f +#define HIF_TBL_FIFO_AFUL_TH_REG_TBL_LAST_FLAG_DL_FIFO_AFUL_TH_MIN_VAL 0x0 + +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_ADDR 0xa10600c4 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_WIDTH 32 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_LENGTH 32 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_DL_FIFO_OVFL_MASK 0x1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_DL_FIFO_OVFL_SHIFT 0 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_DL_FIFO_OVFL_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL2ADPT_CREDIT_OVFL_MASK 0x2 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL2ADPT_CREDIT_OVFL_SHIFT 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL2ADPT_CREDIT_OVFL_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL_LAST_FLAG_FIFO_OVFL_MASK 0x4 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL_LAST_FLAG_FIFO_OVFL_SHIFT 2 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL_LAST_FLAG_FIFO_OVFL_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_DL_FIFO_UNFL_MASK 0x8 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_DL_FIFO_UNFL_SHIFT 3 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_DL_FIFO_UNFL_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL2ADPT_CREDIT_UNFL_MASK 0x10 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL2ADPT_CREDIT_UNFL_SHIFT 4 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL2ADPT_CREDIT_UNFL_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL_LAST_FLAG_FIFO_UNFL_MASK 0x20 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL_LAST_FLAG_FIFO_UNFL_SHIFT 5 +#define HIF_TBL_TBL2ADPT_FIFO_INT_REG_TBL_LAST_FLAG_FIFO_UNFL_WIDTH 1 + +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_ADDR 0xa10600c8 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_WIDTH 32 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_LENGTH 32 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_OVFL_INT_MASK_MASK 0x1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_OVFL_INT_MASK_SHIFT 0 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_OVFL_INT_MASK_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_OVFL_INT_MASK_MASK 0x2 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_OVFL_INT_MASK_SHIFT 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_OVFL_INT_MASK_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_OVFL_INT_MASK_MASK 0x4 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_OVFL_INT_MASK_SHIFT 2 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_OVFL_INT_MASK_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_OVFL_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_OVFL_INT_MASK_MIN_VAL 0x0 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_UNFL_INT_MASK_MASK 0x8 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_UNFL_INT_MASK_SHIFT 3 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_UNFL_INT_MASK_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_DL_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_UNFL_INT_MASK_MASK 0x10 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_UNFL_INT_MASK_SHIFT 4 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_UNFL_INT_MASK_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL2ADPT_CREDIT_UNFL_INT_MASK_MIN_VAL 0x0 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_UNFL_INT_MASK_MASK 0x20 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_UNFL_INT_MASK_SHIFT 5 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_UNFL_INT_MASK_WIDTH 1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_UNFL_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_TBL2ADPT_FIFO_INT_MASK_REG_TBL_LAST_FLAG_FIFO_UNFL_INT_MASK_MIN_VAL 0x0 + +#define HIF_TBL_MEM_ERROR_INT_ADDR 0xa10600d0 +#define HIF_TBL_MEM_ERROR_INT_WIDTH 32 +#define HIF_TBL_MEM_ERROR_INT_LENGTH 32 +#define HIF_TBL_MEM_ERROR_INT_DL_FIFO_MEM_SB_ERR_MASK 0x1 +#define HIF_TBL_MEM_ERROR_INT_DL_FIFO_MEM_SB_ERR_SHIFT 0 +#define HIF_TBL_MEM_ERROR_INT_DL_FIFO_MEM_SB_ERR_WIDTH 1 +#define HIF_TBL_MEM_ERROR_INT_DL_FIFO_MEM_DB_ERR_MASK 0x2 +#define HIF_TBL_MEM_ERROR_INT_DL_FIFO_MEM_DB_ERR_SHIFT 1 +#define HIF_TBL_MEM_ERROR_INT_DL_FIFO_MEM_DB_ERR_WIDTH 1 + +#define HIF_TBL_TIMEOUT_INT_REG_ADDR 0xa10600d8 +#define HIF_TBL_TIMEOUT_INT_REG_WIDTH 32 +#define HIF_TBL_TIMEOUT_INT_REG_LENGTH 32 +#define HIF_TBL_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_MASK 0x1 +#define HIF_TBL_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_SHIFT 0 +#define HIF_TBL_TIMEOUT_INT_REG_CSR_ERR_FLAG_REG_TIMEOUT_WIDTH 1 + +#define HIF_TBL_TIMEOUT_INT_MASK_REG_ADDR 0xa10600dc +#define HIF_TBL_TIMEOUT_INT_MASK_REG_WIDTH 32 +#define HIF_TBL_TIMEOUT_INT_MASK_REG_LENGTH 32 +#define HIF_TBL_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MASK 0x1 +#define HIF_TBL_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_SHIFT 0 +#define HIF_TBL_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_WIDTH 1 +#define HIF_TBL_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define HIF_TBL_TIMEOUT_INT_MASK_REG_CSR_ERR_FLAG_REG_TIMEOUT_INT_MASK_MIN_VAL 0x0 + +#define HIF_TBL_TIMEOUT_CFG_REG_ADDR 0xa10600e0 +#define HIF_TBL_TIMEOUT_CFG_REG_WIDTH 32 +#define HIF_TBL_TIMEOUT_CFG_REG_LENGTH 32 +#define HIF_TBL_TIMEOUT_CFG_REG_TIMEOUT_PARA_MASK 0xffff +#define HIF_TBL_TIMEOUT_CFG_REG_TIMEOUT_PARA_SHIFT 0 +#define HIF_TBL_TIMEOUT_CFG_REG_TIMEOUT_PARA_WIDTH 16 +#define HIF_TBL_TIMEOUT_CFG_REG_TIMEOUT_PARA_MAX_VAL 0xffff +#define HIF_TBL_TIMEOUT_CFG_REG_TIMEOUT_PARA_MIN_VAL 0x0 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/andes/mmc_csr_defines.h b/drivers/net/ethernet/yunsilicon/xsc/common/andes/mmc_csr_defines.h new file mode 100644 index 0000000000000000000000000000000000000000..e4bb3fe49d99c88c0cf4e591bb6680a5b2f9fa20 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/andes/mmc_csr_defines.h @@ -0,0 +1,551 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef _MMC_CSR_DEFINES_H_ +#define _MMC_CSR_DEFINES_H_ + +#define MMC_SOFT_RESET_REG_ADDR 0xa4000000 +#define MMC_SOFT_RESET_REG_WIDTH 32 +#define MMC_SOFT_RESET_REG_LENGTH 32 +#define MMC_SOFT_RESET_REG_SOFT_RESET_MASK 0x1 +#define MMC_SOFT_RESET_REG_SOFT_RESET_SHIFT 0 +#define MMC_SOFT_RESET_REG_SOFT_RESET_WIDTH 1 +#define MMC_SOFT_RESET_REG_SOFT_RESET_MAX_VAL 0x1 +#define MMC_SOFT_RESET_REG_SOFT_RESET_MIN_VAL 0x0 + +#define MMC_SCRATCH_PAD_REG_ADDR 0xa400000c +#define MMC_SCRATCH_PAD_REG_WIDTH 32 +#define MMC_SCRATCH_PAD_REG_LENGTH 32 +#define MMC_SCRATCH_PAD_REG_SCRATCH_PAD_MASK 0xffffffff +#define MMC_SCRATCH_PAD_REG_SCRATCH_PAD_SHIFT 0 +#define MMC_SCRATCH_PAD_REG_SCRATCH_PAD_WIDTH 32 +#define MMC_SCRATCH_PAD_REG_SCRATCH_PAD_MAX_VAL 0xffffffff +#define MMC_SCRATCH_PAD_REG_SCRATCH_PAD_MIN_VAL 0x0 + +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_ADDR 0xa4000010 +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_WIDTH 32 +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_LENGTH 32 +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_MET2MMC_IN_FIFO_EMPTY_MASK 0x1 +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_MET2MMC_IN_FIFO_EMPTY_SHIFT 0 +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_MET2MMC_IN_FIFO_EMPTY_WIDTH 1 +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_MET2MMC_IN_FIFO_USED_CNT_MASK 0x7e +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_MET2MMC_IN_FIFO_USED_CNT_SHIFT 1 +#define MMC_MET2MMC_IN_FIFO_STATUS_REG_MET2MMC_IN_FIFO_USED_CNT_WIDTH 6 + +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_ADDR 0xa4000014 +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_WIDTH 32 +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_LENGTH 32 +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_MER2MMC_IN_FIFO_EMPTY_MASK 0x1 +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_MER2MMC_IN_FIFO_EMPTY_SHIFT 0 +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_MER2MMC_IN_FIFO_EMPTY_WIDTH 1 +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_MER2MMC_IN_FIFO_USED_CNT_MASK 0x7e +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_MER2MMC_IN_FIFO_USED_CNT_SHIFT 1 +#define MMC_MER2MMC_IN_FIFO_STATUS_REG_MER2MMC_IN_FIFO_USED_CNT_WIDTH 6 + +#define MMC_FIFO_FATAL_ERR_INT_REG_ADDR 0xa400003c +#define MMC_FIFO_FATAL_ERR_INT_REG_WIDTH 32 +#define MMC_FIFO_FATAL_ERR_INT_REG_LENGTH 32 +#define MMC_FIFO_FATAL_ERR_INT_REG_MET2MMC_IN_FIFO_OVFL_MASK 0x1 +#define MMC_FIFO_FATAL_ERR_INT_REG_MET2MMC_IN_FIFO_OVFL_SHIFT 0 +#define MMC_FIFO_FATAL_ERR_INT_REG_MET2MMC_IN_FIFO_OVFL_WIDTH 1 +#define MMC_FIFO_FATAL_ERR_INT_REG_MET2MMC_IN_FIFO_UNFL_MASK 0x2 +#define MMC_FIFO_FATAL_ERR_INT_REG_MET2MMC_IN_FIFO_UNFL_SHIFT 1 +#define MMC_FIFO_FATAL_ERR_INT_REG_MET2MMC_IN_FIFO_UNFL_WIDTH 1 +#define MMC_FIFO_FATAL_ERR_INT_REG_MER2MMC_IN_FIFO_OVFL_MASK 0x4 +#define MMC_FIFO_FATAL_ERR_INT_REG_MER2MMC_IN_FIFO_OVFL_SHIFT 2 +#define MMC_FIFO_FATAL_ERR_INT_REG_MER2MMC_IN_FIFO_OVFL_WIDTH 1 +#define MMC_FIFO_FATAL_ERR_INT_REG_MER2MMC_IN_FIFO_UNFL_MASK 0x8 +#define MMC_FIFO_FATAL_ERR_INT_REG_MER2MMC_IN_FIFO_UNFL_SHIFT 3 +#define MMC_FIFO_FATAL_ERR_INT_REG_MER2MMC_IN_FIFO_UNFL_WIDTH 1 +#define MMC_FIFO_FATAL_ERR_INT_REG_MPT2MTT_IN_FIFO_OVFL_MASK 0x10 +#define MMC_FIFO_FATAL_ERR_INT_REG_MPT2MTT_IN_FIFO_OVFL_SHIFT 4 +#define MMC_FIFO_FATAL_ERR_INT_REG_MPT2MTT_IN_FIFO_OVFL_WIDTH 1 +#define MMC_FIFO_FATAL_ERR_INT_REG_MPT2MTT_IN_FIFO_UNFL_MASK 0x20 +#define MMC_FIFO_FATAL_ERR_INT_REG_MPT2MTT_IN_FIFO_UNFL_SHIFT 5 +#define MMC_FIFO_FATAL_ERR_INT_REG_MPT2MTT_IN_FIFO_UNFL_WIDTH 1 + +#define MMC_MET2MMC_IN_FIFO_CFG_REG_ADDR 0xa4000050 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_WIDTH 32 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_LENGTH 32 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AFUL_TH_MASK 0x3f +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AFUL_TH_SHIFT 0 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AFUL_TH_WIDTH 6 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AFUL_TH_MAX_VAL 0x3f +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AFUL_TH_MIN_VAL 0x0 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AMTY_TH_MASK 0xfc0 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AMTY_TH_SHIFT 6 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AMTY_TH_WIDTH 6 +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AMTY_TH_MAX_VAL 0x3f +#define MMC_MET2MMC_IN_FIFO_CFG_REG_MET2MMC_IN_FIFO_AMTY_TH_MIN_VAL 0x0 + +#define MMC_MER2MMC_IN_FIFO_CFG_REG_ADDR 0xa4000054 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_WIDTH 32 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_LENGTH 32 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AFUL_TH_MASK 0x3f +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AFUL_TH_SHIFT 0 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AFUL_TH_WIDTH 6 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AFUL_TH_MAX_VAL 0x3f +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AFUL_TH_MIN_VAL 0x0 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AMTY_TH_MASK 0xfc0 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AMTY_TH_SHIFT 6 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AMTY_TH_WIDTH 6 +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AMTY_TH_MAX_VAL 0x3f +#define MMC_MER2MMC_IN_FIFO_CFG_REG_MER2MMC_IN_FIFO_AMTY_TH_MIN_VAL 0x0 + +#define MMC_MET2MMC_RD_CNT_REG_ADDR 0xa4000060 +#define MMC_MET2MMC_RD_CNT_REG_WIDTH 32 +#define MMC_MET2MMC_RD_CNT_REG_LENGTH 32 +#define MMC_MET2MMC_RD_CNT_REG_MET2MMC_RD_CNT_MASK 0xffff +#define MMC_MET2MMC_RD_CNT_REG_MET2MMC_RD_CNT_SHIFT 0 +#define MMC_MET2MMC_RD_CNT_REG_MET2MMC_RD_CNT_WIDTH 16 + +#define MMC_MER2MMC_RD_CNT_REG_ADDR 0xa4000064 +#define MMC_MER2MMC_RD_CNT_REG_WIDTH 32 +#define MMC_MER2MMC_RD_CNT_REG_LENGTH 32 +#define MMC_MER2MMC_RD_CNT_REG_MER2MMC_RD_CNT_MASK 0xffff +#define MMC_MER2MMC_RD_CNT_REG_MER2MMC_RD_CNT_SHIFT 0 +#define MMC_MER2MMC_RD_CNT_REG_MER2MMC_RD_CNT_WIDTH 16 + +#define MMC_MET_KEY_MATCH_FAIL_CNT_REG_ADDR 0xa4000068 +#define MMC_MET_KEY_MATCH_FAIL_CNT_REG_WIDTH 32 +#define MMC_MET_KEY_MATCH_FAIL_CNT_REG_LENGTH 32 +#define MMC_MET_KEY_MATCH_FAIL_CNT_REG_MET_KEY_MACTH_FAIL_CNT_MASK 0xffff +#define MMC_MET_KEY_MATCH_FAIL_CNT_REG_MET_KEY_MACTH_FAIL_CNT_SHIFT 0 +#define MMC_MET_KEY_MATCH_FAIL_CNT_REG_MET_KEY_MACTH_FAIL_CNT_WIDTH 16 + +#define MMC_MER_KEY_MATCH_FAIL_CNT_REG_ADDR 0xa400006c +#define MMC_MER_KEY_MATCH_FAIL_CNT_REG_WIDTH 32 +#define MMC_MER_KEY_MATCH_FAIL_CNT_REG_LENGTH 32 +#define MMC_MER_KEY_MATCH_FAIL_CNT_REG_MER_KEY_MACTH_FAIL_CNT_MASK 0xffff +#define MMC_MER_KEY_MATCH_FAIL_CNT_REG_MER_KEY_MACTH_FAIL_CNT_SHIFT 0 +#define MMC_MER_KEY_MATCH_FAIL_CNT_REG_MER_KEY_MACTH_FAIL_CNT_WIDTH 16 + +#define MMC_MET_PD_MATCH_FAIL_CNT_REG_ADDR 0xa4000070 +#define MMC_MET_PD_MATCH_FAIL_CNT_REG_WIDTH 32 +#define MMC_MET_PD_MATCH_FAIL_CNT_REG_LENGTH 32 +#define MMC_MET_PD_MATCH_FAIL_CNT_REG_MET_PD_MACTH_FAIL_CNT_MASK 0xffff +#define MMC_MET_PD_MATCH_FAIL_CNT_REG_MET_PD_MACTH_FAIL_CNT_SHIFT 0 +#define MMC_MET_PD_MATCH_FAIL_CNT_REG_MET_PD_MACTH_FAIL_CNT_WIDTH 16 + +#define MMC_MER_PD_MATCH_FAIL_CNT_REG_ADDR 0xa4000074 +#define MMC_MER_PD_MATCH_FAIL_CNT_REG_WIDTH 32 +#define MMC_MER_PD_MATCH_FAIL_CNT_REG_LENGTH 32 +#define MMC_MER_PD_MATCH_FAIL_CNT_REG_MER_PD_MACTH_FAIL_CNT_MASK 0xffff +#define MMC_MER_PD_MATCH_FAIL_CNT_REG_MER_PD_MACTH_FAIL_CNT_SHIFT 0 +#define MMC_MER_PD_MATCH_FAIL_CNT_REG_MER_PD_MACTH_FAIL_CNT_WIDTH 16 + +#define MMC_MET_ACC_MATCH_FAIL_CNT_REG_ADDR 0xa4000078 +#define MMC_MET_ACC_MATCH_FAIL_CNT_REG_WIDTH 32 +#define MMC_MET_ACC_MATCH_FAIL_CNT_REG_LENGTH 32 +#define MMC_MET_ACC_MATCH_FAIL_CNT_REG_MET_ACC_MACTH_FAIL_CNT_MASK 0xffff +#define MMC_MET_ACC_MATCH_FAIL_CNT_REG_MET_ACC_MACTH_FAIL_CNT_SHIFT 0 +#define MMC_MET_ACC_MATCH_FAIL_CNT_REG_MET_ACC_MACTH_FAIL_CNT_WIDTH 16 + +#define MMC_MER_ACC_MATCH_FAIL_CNT_REG_ADDR 0xa400007c +#define MMC_MER_ACC_MATCH_FAIL_CNT_REG_WIDTH 32 +#define MMC_MER_ACC_MATCH_FAIL_CNT_REG_LENGTH 32 +#define MMC_MER_ACC_MATCH_FAIL_CNT_REG_MER_ACC_MACTH_FAIL_CNT_MASK 0xffff +#define MMC_MER_ACC_MATCH_FAIL_CNT_REG_MER_ACC_MACTH_FAIL_CNT_SHIFT 0 +#define MMC_MER_ACC_MATCH_FAIL_CNT_REG_MER_ACC_MACTH_FAIL_CNT_WIDTH 16 + +#define MMC_MET_BOUNDARY_MATCH_FAIL_CNT_REG_ADDR 0xa4000080 +#define MMC_MET_BOUNDARY_MATCH_FAIL_CNT_REG_WIDTH 32 +#define MMC_MET_BOUNDARY_MATCH_FAIL_CNT_REG_LENGTH 32 +#define MMC_MET_BOUNDARY_MATCH_FAIL_CNT_REG_MET_BOUNDARY_MACTH_FAIL_CNT_MASK 0xffff +#define MMC_MET_BOUNDARY_MATCH_FAIL_CNT_REG_MET_BOUNDARY_MACTH_FAIL_CNT_SHIFT 0 +#define MMC_MET_BOUNDARY_MATCH_FAIL_CNT_REG_MET_BOUNDARY_MACTH_FAIL_CNT_WIDTH 16 + +#define MMC_MER_BOUNDARY_MATCH_FAIL_CNT_REG_ADDR 0xa4000084 +#define MMC_MER_BOUNDARY_MATCH_FAIL_CNT_REG_WIDTH 32 +#define MMC_MER_BOUNDARY_MATCH_FAIL_CNT_REG_LENGTH 32 +#define MMC_MER_BOUNDARY_MATCH_FAIL_CNT_REG_MER_BOUNDARY_MACTH_FAIL_CNT_MASK 0xffff +#define MMC_MER_BOUNDARY_MATCH_FAIL_CNT_REG_MER_BOUNDARY_MACTH_FAIL_CNT_SHIFT 0 +#define MMC_MER_BOUNDARY_MATCH_FAIL_CNT_REG_MER_BOUNDARY_MACTH_FAIL_CNT_WIDTH 16 + +#define MMC_MMC2MET_EOP_CNT_REG_ADDR 0xa4000090 +#define MMC_MMC2MET_EOP_CNT_REG_WIDTH 32 +#define MMC_MMC2MET_EOP_CNT_REG_LENGTH 32 +#define MMC_MMC2MET_EOP_CNT_REG_MMC2MET_EOP_CNT_MASK 0xffff +#define MMC_MMC2MET_EOP_CNT_REG_MMC2MET_EOP_CNT_SHIFT 0 +#define MMC_MMC2MET_EOP_CNT_REG_MMC2MET_EOP_CNT_WIDTH 16 + +#define MMC_MMC2MER_EOP_CNT_REG_ADDR 0xa4000094 +#define MMC_MMC2MER_EOP_CNT_REG_WIDTH 32 +#define MMC_MMC2MER_EOP_CNT_REG_LENGTH 32 +#define MMC_MMC2MER_EOP_CNT_REG_MMC2MER_EOP_CNT_MASK 0xffff +#define MMC_MMC2MER_EOP_CNT_REG_MMC2MER_EOP_CNT_SHIFT 0 +#define MMC_MMC2MER_EOP_CNT_REG_MMC2MER_EOP_CNT_WIDTH 16 + +#define MMC_MMC2MET_VLD_CNT_REG_ADDR 0xa4000098 +#define MMC_MMC2MET_VLD_CNT_REG_WIDTH 32 +#define MMC_MMC2MET_VLD_CNT_REG_LENGTH 32 +#define MMC_MMC2MET_VLD_CNT_REG_MMC2MET_VLD_CNT_MASK 0xffff +#define MMC_MMC2MET_VLD_CNT_REG_MMC2MET_VLD_CNT_SHIFT 0 +#define MMC_MMC2MET_VLD_CNT_REG_MMC2MET_VLD_CNT_WIDTH 16 + +#define MMC_MMC2MER_VLD_CNT_REG_ADDR 0xa400009c +#define MMC_MMC2MER_VLD_CNT_REG_WIDTH 32 +#define MMC_MMC2MER_VLD_CNT_REG_LENGTH 32 +#define MMC_MMC2MER_VLD_CNT_REG_MMC2MER_VLD_CNT_MASK 0xffff +#define MMC_MMC2MER_VLD_CNT_REG_MMC2MER_VLD_CNT_SHIFT 0 +#define MMC_MMC2MER_VLD_CNT_REG_MMC2MER_VLD_CNT_WIDTH 16 + +#define MMC_MER_NO_MAP_CNT_REG_ADDR 0xa40000a0 +#define MMC_MER_NO_MAP_CNT_REG_WIDTH 32 +#define MMC_MER_NO_MAP_CNT_REG_LENGTH 32 +#define MMC_MER_NO_MAP_CNT_REG_MER_NO_MAP_CNT_MASK 0xffff +#define MMC_MER_NO_MAP_CNT_REG_MER_NO_MAP_CNT_SHIFT 0 +#define MMC_MER_NO_MAP_CNT_REG_MER_NO_MAP_CNT_WIDTH 16 + +#define MMC_MET_NO_MAP_CNT_REG_ADDR 0xa40000a4 +#define MMC_MET_NO_MAP_CNT_REG_WIDTH 32 +#define MMC_MET_NO_MAP_CNT_REG_LENGTH 32 +#define MMC_MET_NO_MAP_CNT_REG_MET_NO_MAP_CNT_MASK 0xffff +#define MMC_MET_NO_MAP_CNT_REG_MET_NO_MAP_CNT_SHIFT 0 +#define MMC_MET_NO_MAP_CNT_REG_MET_NO_MAP_CNT_WIDTH 16 + +#define MMC_MMC2MET_FC_CNT_REG_ADDR 0xa40000a8 +#define MMC_MMC2MET_FC_CNT_REG_WIDTH 32 +#define MMC_MMC2MET_FC_CNT_REG_LENGTH 32 +#define MMC_MMC2MET_FC_CNT_REG_MMC2MET_FC_CNT_MASK 0xffffffff +#define MMC_MMC2MET_FC_CNT_REG_MMC2MET_FC_CNT_SHIFT 0 +#define MMC_MMC2MET_FC_CNT_REG_MMC2MET_FC_CNT_WIDTH 32 + +#define MMC_MMC2MER_FC_CNT_REG_ADDR 0xa40000ac +#define MMC_MMC2MER_FC_CNT_REG_WIDTH 32 +#define MMC_MMC2MER_FC_CNT_REG_LENGTH 32 +#define MMC_MMC2MER_FC_CNT_REG_MMC2MER_FC_CNT_MASK 0xffffffff +#define MMC_MMC2MER_FC_CNT_REG_MMC2MER_FC_CNT_SHIFT 0 +#define MMC_MMC2MER_FC_CNT_REG_MMC2MER_FC_CNT_WIDTH 32 + +#define MMC_MPT2MTT_FC_CNT_REG_ADDR 0xa40000b0 +#define MMC_MPT2MTT_FC_CNT_REG_WIDTH 32 +#define MMC_MPT2MTT_FC_CNT_REG_LENGTH 32 +#define MMC_MPT2MTT_FC_CNT_REG_MPT2MTT_FC_CNT_MASK 0xffffffff +#define MMC_MPT2MTT_FC_CNT_REG_MPT2MTT_FC_CNT_SHIFT 0 +#define MMC_MPT2MTT_FC_CNT_REG_MPT2MTT_FC_CNT_WIDTH 32 + +#define MMC_MET_STG1_ERR_QP_ID_STS_REG_ADDR 0xa40000b4 +#define MMC_MET_STG1_ERR_QP_ID_STS_REG_WIDTH 32 +#define MMC_MET_STG1_ERR_QP_ID_STS_REG_LENGTH 32 +#define MMC_MET_STG1_ERR_QP_ID_STS_REG_MET_STG1_ERR_QP_ID_STS_MASK 0x7fff +#define MMC_MET_STG1_ERR_QP_ID_STS_REG_MET_STG1_ERR_QP_ID_STS_SHIFT 0 +#define MMC_MET_STG1_ERR_QP_ID_STS_REG_MET_STG1_ERR_QP_ID_STS_WIDTH 15 + +#define MMC_MET_STG1_ERR_MPT_ADDR_STS_REG_ADDR 0xa40000b8 +#define MMC_MET_STG1_ERR_MPT_ADDR_STS_REG_WIDTH 32 +#define MMC_MET_STG1_ERR_MPT_ADDR_STS_REG_LENGTH 32 +#define MMC_MET_STG1_ERR_MPT_ADDR_STS_REG_MET_STG1_ERR_MPT_ADDR_STS_MASK 0x7fff +#define MMC_MET_STG1_ERR_MPT_ADDR_STS_REG_MET_STG1_ERR_MPT_ADDR_STS_SHIFT 0 +#define MMC_MET_STG1_ERR_MPT_ADDR_STS_REG_MET_STG1_ERR_MPT_ADDR_STS_WIDTH 15 + +#define MMC_MET_STG1_ERR_MKEY_STS_REG_ADDR 0xa40000bc +#define MMC_MET_STG1_ERR_MKEY_STS_REG_WIDTH 32 +#define MMC_MET_STG1_ERR_MKEY_STS_REG_LENGTH 32 +#define MMC_MET_STG1_ERR_MKEY_STS_REG_MET_STG1_ERR_MKEY_STS_MASK 0xff +#define MMC_MET_STG1_ERR_MKEY_STS_REG_MET_STG1_ERR_MKEY_STS_SHIFT 0 +#define MMC_MET_STG1_ERR_MKEY_STS_REG_MET_STG1_ERR_MKEY_STS_WIDTH 8 + +#define MMC_MET_STG1_ERR_VA_STS_REG_ADDR 0xa40000c0 +#define MMC_MET_STG1_ERR_VA_STS_REG_WIDTH 64 +#define MMC_MET_STG1_ERR_VA_STS_REG_LENGTH 64 +#define MMC_MET_STG1_ERR_VA_STS_REG_MET_STG1_ERR_VA_STS_MASK 0xffffffffffffffff +#define MMC_MET_STG1_ERR_VA_STS_REG_MET_STG1_ERR_VA_STS_SHIFT 0 +#define MMC_MET_STG1_ERR_VA_STS_REG_MET_STG1_ERR_VA_STS_WIDTH 64 + +#define MMC_MET_STG1_ERR_LEN_STS_REG_ADDR 0xa40000c8 +#define MMC_MET_STG1_ERR_LEN_STS_REG_WIDTH 32 +#define MMC_MET_STG1_ERR_LEN_STS_REG_LENGTH 32 +#define MMC_MET_STG1_ERR_LEN_STS_REG_MET_STG1_ERR_LEN_STS_MASK 0xffffffff +#define MMC_MET_STG1_ERR_LEN_STS_REG_MET_STG1_ERR_LEN_STS_SHIFT 0 +#define MMC_MET_STG1_ERR_LEN_STS_REG_MET_STG1_ERR_LEN_STS_WIDTH 32 + +#define MMC_MET_STG1_ERR_ACC_TYPE_STS_REG_ADDR 0xa40000cc +#define MMC_MET_STG1_ERR_ACC_TYPE_STS_REG_WIDTH 32 +#define MMC_MET_STG1_ERR_ACC_TYPE_STS_REG_LENGTH 32 +#define MMC_MET_STG1_ERR_ACC_TYPE_STS_REG_MET_STG1_ERR_ACC_TYPE_STS_MASK 0xf +#define MMC_MET_STG1_ERR_ACC_TYPE_STS_REG_MET_STG1_ERR_ACC_TYPE_STS_SHIFT 0 +#define MMC_MET_STG1_ERR_ACC_TYPE_STS_REG_MET_STG1_ERR_ACC_TYPE_STS_WIDTH 4 + +#define MMC_MET_STG1_ERR_MPT_STS_REG_ADDR 0xa40000e0 +#define MMC_MET_STG1_ERR_MPT_STS_REG_WIDTH 256 +#define MMC_MET_STG1_ERR_MPT_STS_REG_LENGTH 160 +#define MMC_MET_STG1_ERR_MPT_STS_REG_MET_STG1_ERR_MPT_STS_MASK 0xffffffffffffffff +#define MMC_MET_STG1_ERR_MPT_STS_REG_MET_STG1_ERR_MPT_STS_SHIFT 0 +#define MMC_MET_STG1_ERR_MPT_STS_REG_MET_STG1_ERR_MPT_STS_WIDTH 153 + +#define MMC_MET_STG1_ERR_QPMCT_STS_REG_ADDR 0xa4000100 +#define MMC_MET_STG1_ERR_QPMCT_STS_REG_WIDTH 32 +#define MMC_MET_STG1_ERR_QPMCT_STS_REG_LENGTH 32 +#define MMC_MET_STG1_ERR_QPMCT_STS_REG_MET_STG1_ERR_QPMCT_STS_MASK 0xffffff +#define MMC_MET_STG1_ERR_QPMCT_STS_REG_MET_STG1_ERR_QPMCT_STS_SHIFT 0 +#define MMC_MET_STG1_ERR_QPMCT_STS_REG_MET_STG1_ERR_QPMCT_STS_WIDTH 24 + +#define MMC_MER_STG1_ERR_QP_ID_STS_REG_ADDR 0xa4000104 +#define MMC_MER_STG1_ERR_QP_ID_STS_REG_WIDTH 32 +#define MMC_MER_STG1_ERR_QP_ID_STS_REG_LENGTH 32 +#define MMC_MER_STG1_ERR_QP_ID_STS_REG_MER_STG1_ERR_QP_ID_STS_MASK 0x7fff +#define MMC_MER_STG1_ERR_QP_ID_STS_REG_MER_STG1_ERR_QP_ID_STS_SHIFT 0 +#define MMC_MER_STG1_ERR_QP_ID_STS_REG_MER_STG1_ERR_QP_ID_STS_WIDTH 15 + +#define MMC_MER_STG1_ERR_MPT_ADDR_STS_REG_ADDR 0xa4000108 +#define MMC_MER_STG1_ERR_MPT_ADDR_STS_REG_WIDTH 32 +#define MMC_MER_STG1_ERR_MPT_ADDR_STS_REG_LENGTH 32 +#define MMC_MER_STG1_ERR_MPT_ADDR_STS_REG_MER_STG1_ERR_MPT_ADDR_STS_MASK 0x7fff +#define MMC_MER_STG1_ERR_MPT_ADDR_STS_REG_MER_STG1_ERR_MPT_ADDR_STS_SHIFT 0 +#define MMC_MER_STG1_ERR_MPT_ADDR_STS_REG_MER_STG1_ERR_MPT_ADDR_STS_WIDTH 15 + +#define MMC_MER_STG1_ERR_MKEY_STS_REG_ADDR 0xa400010c +#define MMC_MER_STG1_ERR_MKEY_STS_REG_WIDTH 32 +#define MMC_MER_STG1_ERR_MKEY_STS_REG_LENGTH 32 +#define MMC_MER_STG1_ERR_MKEY_STS_REG_MER_STG1_ERR_MKEY_STS_MASK 0xff +#define MMC_MER_STG1_ERR_MKEY_STS_REG_MER_STG1_ERR_MKEY_STS_SHIFT 0 +#define MMC_MER_STG1_ERR_MKEY_STS_REG_MER_STG1_ERR_MKEY_STS_WIDTH 8 + +#define MMC_MER_STG1_ERR_VA_STS_REG_ADDR 0xa4000110 +#define MMC_MER_STG1_ERR_VA_STS_REG_WIDTH 64 +#define MMC_MER_STG1_ERR_VA_STS_REG_LENGTH 64 +#define MMC_MER_STG1_ERR_VA_STS_REG_MER_STG1_ERR_VA_STS_MASK 0xffffffffffffffff +#define MMC_MER_STG1_ERR_VA_STS_REG_MER_STG1_ERR_VA_STS_SHIFT 0 +#define MMC_MER_STG1_ERR_VA_STS_REG_MER_STG1_ERR_VA_STS_WIDTH 64 + +#define MMC_MER_STG1_ERR_LEN_STS_REG_ADDR 0xa4000118 +#define MMC_MER_STG1_ERR_LEN_STS_REG_WIDTH 32 +#define MMC_MER_STG1_ERR_LEN_STS_REG_LENGTH 32 +#define MMC_MER_STG1_ERR_LEN_STS_REG_MER_STG1_ERR_LEN_STS_MASK 0xffffffff +#define MMC_MER_STG1_ERR_LEN_STS_REG_MER_STG1_ERR_LEN_STS_SHIFT 0 +#define MMC_MER_STG1_ERR_LEN_STS_REG_MER_STG1_ERR_LEN_STS_WIDTH 32 + +#define MMC_MER_STG1_ERR_ACC_TYPE_STS_REG_ADDR 0xa400011c +#define MMC_MER_STG1_ERR_ACC_TYPE_STS_REG_WIDTH 32 +#define MMC_MER_STG1_ERR_ACC_TYPE_STS_REG_LENGTH 32 +#define MMC_MER_STG1_ERR_ACC_TYPE_STS_REG_MER_STG1_ERR_ACC_TYPE_STS_MASK 0xf +#define MMC_MER_STG1_ERR_ACC_TYPE_STS_REG_MER_STG1_ERR_ACC_TYPE_STS_SHIFT 0 +#define MMC_MER_STG1_ERR_ACC_TYPE_STS_REG_MER_STG1_ERR_ACC_TYPE_STS_WIDTH 4 + +#define MMC_MER_STG1_ERR_MPT_STS_REG_ADDR 0xa4000120 +#define MMC_MER_STG1_ERR_MPT_STS_REG_WIDTH 256 +#define MMC_MER_STG1_ERR_MPT_STS_REG_LENGTH 160 +#define MMC_MER_STG1_ERR_MPT_STS_REG_MER_STG1_ERR_MPT_STS_MASK 0xffffffffffffffff +#define MMC_MER_STG1_ERR_MPT_STS_REG_MER_STG1_ERR_MPT_STS_SHIFT 0 +#define MMC_MER_STG1_ERR_MPT_STS_REG_MER_STG1_ERR_MPT_STS_WIDTH 153 + +#define MMC_MER_STG1_ERR_QPMCT_STS_REG_ADDR 0xa4000140 +#define MMC_MER_STG1_ERR_QPMCT_STS_REG_WIDTH 32 +#define MMC_MER_STG1_ERR_QPMCT_STS_REG_LENGTH 32 +#define MMC_MER_STG1_ERR_QPMCT_STS_REG_MER_STG1_ERR_QPMCT_STS_MASK 0xffffff +#define MMC_MER_STG1_ERR_QPMCT_STS_REG_MER_STG1_ERR_QPMCT_STS_SHIFT 0 +#define MMC_MER_STG1_ERR_QPMCT_STS_REG_MER_STG1_ERR_QPMCT_STS_WIDTH 24 + +#define MMC_MET_STG2_ERR_MPT_PAGE_MODE_REG_ADDR 0xa4000144 +#define MMC_MET_STG2_ERR_MPT_PAGE_MODE_REG_WIDTH 32 +#define MMC_MET_STG2_ERR_MPT_PAGE_MODE_REG_LENGTH 32 +#define MMC_MET_STG2_ERR_MPT_PAGE_MODE_REG_MET_STG2_ERR_MPT_PAGE_MODE_MASK 0x3 +#define MMC_MET_STG2_ERR_MPT_PAGE_MODE_REG_MET_STG2_ERR_MPT_PAGE_MODE_SHIFT 0 +#define MMC_MET_STG2_ERR_MPT_PAGE_MODE_REG_MET_STG2_ERR_MPT_PAGE_MODE_WIDTH 2 + +#define MMC_MET_STG2_ERR_MPT_MTT_BASE_REG_ADDR 0xa4000148 +#define MMC_MET_STG2_ERR_MPT_MTT_BASE_REG_WIDTH 32 +#define MMC_MET_STG2_ERR_MPT_MTT_BASE_REG_LENGTH 32 +#define MMC_MET_STG2_ERR_MPT_MTT_BASE_REG_MET_STG2_ERR_MPT_MTT_BASE_MASK 0x3ffff +#define MMC_MET_STG2_ERR_MPT_MTT_BASE_REG_MET_STG2_ERR_MPT_MTT_BASE_SHIFT 0 +#define MMC_MET_STG2_ERR_MPT_MTT_BASE_REG_MET_STG2_ERR_MPT_MTT_BASE_WIDTH 18 + +#define MMC_MET_STG2_ERR_MPT_MEM_SZ_REG_ADDR 0xa400014c +#define MMC_MET_STG2_ERR_MPT_MEM_SZ_REG_WIDTH 32 +#define MMC_MET_STG2_ERR_MPT_MEM_SZ_REG_LENGTH 32 +#define MMC_MET_STG2_ERR_MPT_MEM_SZ_REG_MET_STG2_ERR_MPT_MEM_SZ_MASK 0xffffffff +#define MMC_MET_STG2_ERR_MPT_MEM_SZ_REG_MET_STG2_ERR_MPT_MEM_SZ_SHIFT 0 +#define MMC_MET_STG2_ERR_MPT_MEM_SZ_REG_MET_STG2_ERR_MPT_MEM_SZ_WIDTH 32 + +#define MMC_MET_STG2_ERR_MPT_VA_BASE_REG_ADDR 0xa4000150 +#define MMC_MET_STG2_ERR_MPT_VA_BASE_REG_WIDTH 64 +#define MMC_MET_STG2_ERR_MPT_VA_BASE_REG_LENGTH 64 +#define MMC_MET_STG2_ERR_MPT_VA_BASE_REG_MET_STG2_ERR_MPT_VA_BASE_MASK 0xffffffffffffffff +#define MMC_MET_STG2_ERR_MPT_VA_BASE_REG_MET_STG2_ERR_MPT_VA_BASE_SHIFT 0 +#define MMC_MET_STG2_ERR_MPT_VA_BASE_REG_MET_STG2_ERR_MPT_VA_BASE_WIDTH 64 + +#define MMC_MET_STG2_ERR_VA_STS_REG_ADDR 0xa4000158 +#define MMC_MET_STG2_ERR_VA_STS_REG_WIDTH 64 +#define MMC_MET_STG2_ERR_VA_STS_REG_LENGTH 64 +#define MMC_MET_STG2_ERR_VA_STS_REG_MET_STG2_ERR_VA_STS_MASK 0xffffffffffffffff +#define MMC_MET_STG2_ERR_VA_STS_REG_MET_STG2_ERR_VA_STS_SHIFT 0 +#define MMC_MET_STG2_ERR_VA_STS_REG_MET_STG2_ERR_VA_STS_WIDTH 64 + +#define MMC_MET_STG2_ERR_LEN_STS_REG_ADDR 0xa4000160 +#define MMC_MET_STG2_ERR_LEN_STS_REG_WIDTH 32 +#define MMC_MET_STG2_ERR_LEN_STS_REG_LENGTH 32 +#define MMC_MET_STG2_ERR_LEN_STS_REG_MET_STG2_ERR_LEN_STS_MASK 0xffffffff +#define MMC_MET_STG2_ERR_LEN_STS_REG_MET_STG2_ERR_LEN_STS_SHIFT 0 +#define MMC_MET_STG2_ERR_LEN_STS_REG_MET_STG2_ERR_LEN_STS_WIDTH 32 + +#define MMC_MER_STG2_ERR_MPT_PAGE_MODE_REG_ADDR 0xa4000164 +#define MMC_MER_STG2_ERR_MPT_PAGE_MODE_REG_WIDTH 32 +#define MMC_MER_STG2_ERR_MPT_PAGE_MODE_REG_LENGTH 32 +#define MMC_MER_STG2_ERR_MPT_PAGE_MODE_REG_MER_STG2_ERR_MPT_PAGE_MODE_MASK 0x3 +#define MMC_MER_STG2_ERR_MPT_PAGE_MODE_REG_MER_STG2_ERR_MPT_PAGE_MODE_SHIFT 0 +#define MMC_MER_STG2_ERR_MPT_PAGE_MODE_REG_MER_STG2_ERR_MPT_PAGE_MODE_WIDTH 2 + +#define MMC_MER_STG2_ERR_MPT_MTT_BASE_REG_ADDR 0xa4000168 +#define MMC_MER_STG2_ERR_MPT_MTT_BASE_REG_WIDTH 32 +#define MMC_MER_STG2_ERR_MPT_MTT_BASE_REG_LENGTH 32 +#define MMC_MER_STG2_ERR_MPT_MTT_BASE_REG_MER_STG2_ERR_MPT_MTT_BASE_MASK 0x3ffff +#define MMC_MER_STG2_ERR_MPT_MTT_BASE_REG_MER_STG2_ERR_MPT_MTT_BASE_SHIFT 0 +#define MMC_MER_STG2_ERR_MPT_MTT_BASE_REG_MER_STG2_ERR_MPT_MTT_BASE_WIDTH 18 + +#define MMC_MER_STG2_ERR_MPT_MEM_SZ_REG_ADDR 0xa400016c +#define MMC_MER_STG2_ERR_MPT_MEM_SZ_REG_WIDTH 32 +#define MMC_MER_STG2_ERR_MPT_MEM_SZ_REG_LENGTH 32 +#define MMC_MER_STG2_ERR_MPT_MEM_SZ_REG_MER_STG2_ERR_MPT_MEM_SZ_MASK 0xffffffff +#define MMC_MER_STG2_ERR_MPT_MEM_SZ_REG_MER_STG2_ERR_MPT_MEM_SZ_SHIFT 0 +#define MMC_MER_STG2_ERR_MPT_MEM_SZ_REG_MER_STG2_ERR_MPT_MEM_SZ_WIDTH 32 + +#define MMC_MER_STG2_ERR_MPT_VA_BASE_REG_ADDR 0xa4000170 +#define MMC_MER_STG2_ERR_MPT_VA_BASE_REG_WIDTH 64 +#define MMC_MER_STG2_ERR_MPT_VA_BASE_REG_LENGTH 64 +#define MMC_MER_STG2_ERR_MPT_VA_BASE_REG_MER_STG2_ERR_MPT_VA_BASE_MASK 0xffffffffffffffff +#define MMC_MER_STG2_ERR_MPT_VA_BASE_REG_MER_STG2_ERR_MPT_VA_BASE_SHIFT 0 +#define MMC_MER_STG2_ERR_MPT_VA_BASE_REG_MER_STG2_ERR_MPT_VA_BASE_WIDTH 64 + +#define MMC_MER_STG2_ERR_VA_STS_REG_ADDR 0xa4000178 +#define MMC_MER_STG2_ERR_VA_STS_REG_WIDTH 64 +#define MMC_MER_STG2_ERR_VA_STS_REG_LENGTH 64 +#define MMC_MER_STG2_ERR_VA_STS_REG_MER_STG2_ERR_VA_STS_MASK 0xffffffffffffffff +#define MMC_MER_STG2_ERR_VA_STS_REG_MER_STG2_ERR_VA_STS_SHIFT 0 +#define MMC_MER_STG2_ERR_VA_STS_REG_MER_STG2_ERR_VA_STS_WIDTH 64 + +#define MMC_MER_STG2_ERR_LEN_STS_REG_ADDR 0xa4000180 +#define MMC_MER_STG2_ERR_LEN_STS_REG_WIDTH 32 +#define MMC_MER_STG2_ERR_LEN_STS_REG_LENGTH 32 +#define MMC_MER_STG2_ERR_LEN_STS_REG_MER_STG2_ERR_LEN_STS_MASK 0xffffffff +#define MMC_MER_STG2_ERR_LEN_STS_REG_MER_STG2_ERR_LEN_STS_SHIFT 0 +#define MMC_MER_STG2_ERR_LEN_STS_REG_MER_STG2_ERR_LEN_STS_WIDTH 32 + +#define MMC_MPT_TBL_MEM_ADDR 0xa4100000 +#define MMC_MPT_TBL_MEM_WIDTH 256 +#define MMC_MPT_TBL_MEM_LENGTH 160 +#define MMC_MPT_TBL_MEM_DEPTH 32768 +#define MMC_MPT_TBL_MEM_MPT_CONTENT_MASK 0xffffffffffffffff +#define MMC_MPT_TBL_MEM_MPT_CONTENT_SHIFT 0 +#define MMC_MPT_TBL_MEM_MPT_CONTENT_WIDTH 153 + +#define MMC_MTT_TBL_MEM_ADDR 0xa4200000 +#define MMC_MTT_TBL_MEM_WIDTH 64 +#define MMC_MTT_TBL_MEM_LENGTH 64 +#define MMC_MTT_TBL_MEM_DEPTH 65536 +#define MMC_MTT_TBL_MEM_MTT_CONTENT_MASK 0xfffffffffffff +#define MMC_MTT_TBL_MEM_MTT_CONTENT_SHIFT 0 +#define MMC_MTT_TBL_MEM_MTT_CONTENT_WIDTH 52 + +#define MMC_QPMCT_TBL_MEM_ADDR 0xa4280000 +#define MMC_QPMCT_TBL_MEM_WIDTH 32 +#define MMC_QPMCT_TBL_MEM_LENGTH 32 +#define MMC_QPMCT_TBL_MEM_DEPTH 32768 +#define MMC_QPMCT_TBL_MEM_QPMCT_CONTENT_MASK 0xffffff +#define MMC_QPMCT_TBL_MEM_QPMCT_CONTENT_SHIFT 0 +#define MMC_QPMCT_TBL_MEM_QPMCT_CONTENT_WIDTH 24 + +#define MMC_MEM_ERROR_INT_ADDR 0xa42a0000 +#define MMC_MEM_ERROR_INT_WIDTH 32 +#define MMC_MEM_ERROR_INT_LENGTH 32 +#define MMC_MEM_ERROR_INT_MPT_TBL_MEM_SB_ERR_MASK 0x1 +#define MMC_MEM_ERROR_INT_MPT_TBL_MEM_SB_ERR_SHIFT 0 +#define MMC_MEM_ERROR_INT_MPT_TBL_MEM_SB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MPT_TBL_MEM_DB_ERR_MASK 0x2 +#define MMC_MEM_ERROR_INT_MPT_TBL_MEM_DB_ERR_SHIFT 1 +#define MMC_MEM_ERROR_INT_MPT_TBL_MEM_DB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MTT_TBL_MEM_SB_ERR_MASK 0x4 +#define MMC_MEM_ERROR_INT_MTT_TBL_MEM_SB_ERR_SHIFT 2 +#define MMC_MEM_ERROR_INT_MTT_TBL_MEM_SB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MTT_TBL_MEM_DB_ERR_MASK 0x8 +#define MMC_MEM_ERROR_INT_MTT_TBL_MEM_DB_ERR_SHIFT 3 +#define MMC_MEM_ERROR_INT_MTT_TBL_MEM_DB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_QPMCT_TBL_MEM_SB_ERR_MASK 0x10 +#define MMC_MEM_ERROR_INT_QPMCT_TBL_MEM_SB_ERR_SHIFT 4 +#define MMC_MEM_ERROR_INT_QPMCT_TBL_MEM_SB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_QPMCT_TBL_MEM_DB_ERR_MASK 0x20 +#define MMC_MEM_ERROR_INT_QPMCT_TBL_MEM_DB_ERR_SHIFT 5 +#define MMC_MEM_ERROR_INT_QPMCT_TBL_MEM_DB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MET2MMC_IN_FIFO_MEM_SB_ERR_MASK 0x40 +#define MMC_MEM_ERROR_INT_MET2MMC_IN_FIFO_MEM_SB_ERR_SHIFT 6 +#define MMC_MEM_ERROR_INT_MET2MMC_IN_FIFO_MEM_SB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MET2MMC_IN_FIFO_MEM_DB_ERR_MASK 0x80 +#define MMC_MEM_ERROR_INT_MET2MMC_IN_FIFO_MEM_DB_ERR_SHIFT 7 +#define MMC_MEM_ERROR_INT_MET2MMC_IN_FIFO_MEM_DB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MER2MMC_IN_FIFO_MEM_SB_ERR_MASK 0x100 +#define MMC_MEM_ERROR_INT_MER2MMC_IN_FIFO_MEM_SB_ERR_SHIFT 8 +#define MMC_MEM_ERROR_INT_MER2MMC_IN_FIFO_MEM_SB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MER2MMC_IN_FIFO_MEM_DB_ERR_MASK 0x200 +#define MMC_MEM_ERROR_INT_MER2MMC_IN_FIFO_MEM_DB_ERR_SHIFT 9 +#define MMC_MEM_ERROR_INT_MER2MMC_IN_FIFO_MEM_DB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MPT2MTT_IN_FIFO_MEM_SB_ERR_MASK 0x400 +#define MMC_MEM_ERROR_INT_MPT2MTT_IN_FIFO_MEM_SB_ERR_SHIFT 10 +#define MMC_MEM_ERROR_INT_MPT2MTT_IN_FIFO_MEM_SB_ERR_WIDTH 1 +#define MMC_MEM_ERROR_INT_MPT2MTT_IN_FIFO_MEM_DB_ERR_MASK 0x800 +#define MMC_MEM_ERROR_INT_MPT2MTT_IN_FIFO_MEM_DB_ERR_SHIFT 11 +#define MMC_MEM_ERROR_INT_MPT2MTT_IN_FIFO_MEM_DB_ERR_WIDTH 1 + +#define MMC_MEM_INIT_CTRL_ADDR 0xa42a0008 +#define MMC_MEM_INIT_CTRL_WIDTH 32 +#define MMC_MEM_INIT_CTRL_LENGTH 32 +#define MMC_MEM_INIT_CTRL_MPT_TBL_MEM_INIT_RST_N_MASK 0x1 +#define MMC_MEM_INIT_CTRL_MPT_TBL_MEM_INIT_RST_N_SHIFT 0 +#define MMC_MEM_INIT_CTRL_MPT_TBL_MEM_INIT_RST_N_WIDTH 1 +#define MMC_MEM_INIT_CTRL_MPT_TBL_MEM_INIT_RST_N_MAX_VAL 0x1 +#define MMC_MEM_INIT_CTRL_MPT_TBL_MEM_INIT_RST_N_MIN_VAL 0x0 +#define MMC_MEM_INIT_CTRL_MTT_TBL_MEM_INIT_RST_N_MASK 0x2 +#define MMC_MEM_INIT_CTRL_MTT_TBL_MEM_INIT_RST_N_SHIFT 1 +#define MMC_MEM_INIT_CTRL_MTT_TBL_MEM_INIT_RST_N_WIDTH 1 +#define MMC_MEM_INIT_CTRL_MTT_TBL_MEM_INIT_RST_N_MAX_VAL 0x1 +#define MMC_MEM_INIT_CTRL_MTT_TBL_MEM_INIT_RST_N_MIN_VAL 0x0 +#define MMC_MEM_INIT_CTRL_QPMCT_TBL_MEM_INIT_RST_N_MASK 0x4 +#define MMC_MEM_INIT_CTRL_QPMCT_TBL_MEM_INIT_RST_N_SHIFT 2 +#define MMC_MEM_INIT_CTRL_QPMCT_TBL_MEM_INIT_RST_N_WIDTH 1 +#define MMC_MEM_INIT_CTRL_QPMCT_TBL_MEM_INIT_RST_N_MAX_VAL 0x1 +#define MMC_MEM_INIT_CTRL_QPMCT_TBL_MEM_INIT_RST_N_MIN_VAL 0x0 + +#define MMC_TIMEOUT_INT_REG_ADDR 0xa42a0010 +#define MMC_TIMEOUT_INT_REG_WIDTH 32 +#define MMC_TIMEOUT_INT_REG_LENGTH 32 +#define MMC_TIMEOUT_INT_REG_MPT_TBL_MEM_TIMEOUT_MASK 0x1 +#define MMC_TIMEOUT_INT_REG_MPT_TBL_MEM_TIMEOUT_SHIFT 0 +#define MMC_TIMEOUT_INT_REG_MPT_TBL_MEM_TIMEOUT_WIDTH 1 +#define MMC_TIMEOUT_INT_REG_MTT_TBL_MEM_TIMEOUT_MASK 0x2 +#define MMC_TIMEOUT_INT_REG_MTT_TBL_MEM_TIMEOUT_SHIFT 1 +#define MMC_TIMEOUT_INT_REG_MTT_TBL_MEM_TIMEOUT_WIDTH 1 +#define MMC_TIMEOUT_INT_REG_QPMCT_TBL_MEM_TIMEOUT_MASK 0x4 +#define MMC_TIMEOUT_INT_REG_QPMCT_TBL_MEM_TIMEOUT_SHIFT 2 +#define MMC_TIMEOUT_INT_REG_QPMCT_TBL_MEM_TIMEOUT_WIDTH 1 + +#define MMC_TIMEOUT_INT_MASK_REG_ADDR 0xa42a0014 +#define MMC_TIMEOUT_INT_MASK_REG_WIDTH 32 +#define MMC_TIMEOUT_INT_MASK_REG_LENGTH 32 +#define MMC_TIMEOUT_INT_MASK_REG_MPT_TBL_MEM_TIMEOUT_INT_MASK_MASK 0x1 +#define MMC_TIMEOUT_INT_MASK_REG_MPT_TBL_MEM_TIMEOUT_INT_MASK_SHIFT 0 +#define MMC_TIMEOUT_INT_MASK_REG_MPT_TBL_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define MMC_TIMEOUT_INT_MASK_REG_MPT_TBL_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define MMC_TIMEOUT_INT_MASK_REG_MPT_TBL_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define MMC_TIMEOUT_INT_MASK_REG_MTT_TBL_MEM_TIMEOUT_INT_MASK_MASK 0x2 +#define MMC_TIMEOUT_INT_MASK_REG_MTT_TBL_MEM_TIMEOUT_INT_MASK_SHIFT 1 +#define MMC_TIMEOUT_INT_MASK_REG_MTT_TBL_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define MMC_TIMEOUT_INT_MASK_REG_MTT_TBL_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define MMC_TIMEOUT_INT_MASK_REG_MTT_TBL_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 +#define MMC_TIMEOUT_INT_MASK_REG_QPMCT_TBL_MEM_TIMEOUT_INT_MASK_MASK 0x4 +#define MMC_TIMEOUT_INT_MASK_REG_QPMCT_TBL_MEM_TIMEOUT_INT_MASK_SHIFT 2 +#define MMC_TIMEOUT_INT_MASK_REG_QPMCT_TBL_MEM_TIMEOUT_INT_MASK_WIDTH 1 +#define MMC_TIMEOUT_INT_MASK_REG_QPMCT_TBL_MEM_TIMEOUT_INT_MASK_MAX_VAL 0x1 +#define MMC_TIMEOUT_INT_MASK_REG_QPMCT_TBL_MEM_TIMEOUT_INT_MASK_MIN_VAL 0x0 + +#define MMC_TIMEOUT_CFG_REG_ADDR 0xa42a0018 +#define MMC_TIMEOUT_CFG_REG_WIDTH 32 +#define MMC_TIMEOUT_CFG_REG_LENGTH 32 +#define MMC_TIMEOUT_CFG_REG_TIMEOUT_PARA_MASK 0xffff +#define MMC_TIMEOUT_CFG_REG_TIMEOUT_PARA_SHIFT 0 +#define MMC_TIMEOUT_CFG_REG_TIMEOUT_PARA_WIDTH 16 +#define MMC_TIMEOUT_CFG_REG_TIMEOUT_PARA_MAX_VAL 0xffff +#define MMC_TIMEOUT_CFG_REG_TIMEOUT_PARA_MIN_VAL 0x0 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/cq.h b/drivers/net/ethernet/yunsilicon/xsc/common/cq.h new file mode 100644 index 0000000000000000000000000000000000000000..1b68667e00699d7502ab1b16690c0c689b1a09b9 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/cq.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_CORE_CQ_H +#define XSC_CORE_CQ_H + +#include +#include +#include +#include + +struct xsc_core_cq { + u32 cqn; + int cqe_sz; + u64 arm_db; + u64 ci_db; + struct xsc_core_device *dev; + atomic_t refcount; + struct completion free; + unsigned int vector; + int irqn; + void (*comp)(struct xsc_core_cq *cq); + void (*event)(struct xsc_core_cq *cq, enum xsc_event); + u32 cons_index; + unsigned int arm_sn; + struct xsc_rsc_debug *dbg; + int pid; + u32 reg_next_cid; + u32 reg_done_pid; +}; + +enum { + XSC_CQE_OWNER_MASK = 1, +}; + +enum { + CQE_SIZE_64 = 0, + CQE_SIZE_128 = 1, +}; + +enum { + XSC_CQ_DB_REQ_NOT_SOL = 1, + XSC_CQ_DB_REQ_NOT = 0, +}; + +static inline void xsc_cq_arm(struct xsc_core_cq *cq, u8 solicited) +{ + union xsc_cq_doorbell db; + + db.val = 0; + db.cq_next_cid = cq->cons_index; + db.cq_id = cq->cqn; + db.arm = solicited; + + /* Make sure that the doorbell record in host memory is + * written before ringing the doorbell via PCI MMIO. + */ + wmb(); + writel(db.val, REG_ADDR(cq->dev, cq->arm_db)); +} + +static inline void xsc_cq_set_ci(struct xsc_core_cq *cq) +{ + struct xsc_core_device *xdev = cq->dev; + union xsc_cq_doorbell db; + + db.cq_next_cid = cq->cons_index; + db.cq_id = cq->cqn; + /* ensure write val visable before doorbell */ + wmb(); + + writel(db.val, REG_ADDR(xdev, cq->ci_db)); +} + +int xsc_core_create_cq(struct xsc_core_device *dev, struct xsc_core_cq *cq, + struct xsc_create_cq_mbox_in *in, int inlen); +int xsc_core_destroy_cq(struct xsc_core_device *dev, struct xsc_core_cq *cq); +int xsc_core_query_cq(struct xsc_core_device *dev, struct xsc_core_cq *cq, + struct xsc_query_cq_mbox_out *out); +int xsc_debug_cq_add(struct xsc_core_device *dev, struct xsc_core_cq *cq); +void xsc_debug_cq_remove(struct xsc_core_device *dev, struct xsc_core_cq *cq); + +void xsc_init_cq_table(struct xsc_core_device *dev); +void xsc_cleanup_cq_table(struct xsc_core_device *dev); +#endif /* XSC_CORE_CQ_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/device.h b/drivers/net/ethernet/yunsilicon/xsc/common/device.h new file mode 100644 index 0000000000000000000000000000000000000000..6c8c39097ddf7bcaae2ff1f3474a94081aabd4aa --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/device.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_DEVICE_H +#define XSC_DEVICE_H + +#include +#include + +enum { + XSC_MAX_COMMANDS = 32, + XSC_CMD_DATA_BLOCK_SIZE = 512, + XSC_PCI_CMD_XPORT = 7, +}; + +enum { + XSC_PERM_LOCAL_READ = 1 << 0, + XSC_PERM_LOCAL_WRITE = 1 << 1, + XSC_PERM_REMOTE_READ = 1 << 2, + XSC_PERM_REMOTE_WRITE = 1 << 3, + XSC_PERM_ATOMIC = 1 << 6, + XSC_PERM_UMR_EN = 1 << 7, +}; + +enum { + XSC_ACCESS_MODE_PA = 0, + XSC_ACCESS_MODE_MTT = 1, + XSC_ACCESS_MODE_KLM = 2 +}; + +enum { + XSC_MKEY_REMOTE_INVAL = 1 << 24, + XSC_MKEY_FLAG_SYNC_UMR = 1 << 29, + XSC_MKEY_BSF_EN = 1 << 30, + XSC_MKEY_LEN64 = 1 << 31, +}; + +enum { + XSC_BF_REGS_PER_PAGE = 4, + XSC_MAX_UAR_PAGES = 1 << 8, + XSC_MAX_UUARS = XSC_MAX_UAR_PAGES * XSC_BF_REGS_PER_PAGE, +}; + +enum { + XSC_DEV_CAP_FLAG_RC = 1LL << 0, + XSC_DEV_CAP_FLAG_UC = 1LL << 1, + XSC_DEV_CAP_FLAG_UD = 1LL << 2, + XSC_DEV_CAP_FLAG_XRC = 1LL << 3, + XSC_DEV_CAP_FLAG_SRQ = 1LL << 6, + XSC_DEV_CAP_FLAG_BAD_PKEY_CNTR = 1LL << 8, + XSC_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1LL << 9, + XSC_DEV_CAP_FLAG_APM = 1LL << 17, + XSC_DEV_CAP_FLAG_ATOMIC = 1LL << 18, + XSC_DEV_CAP_FLAG_ON_DMND_PG = 1LL << 24, + XSC_DEV_CAP_FLAG_RESIZE_SRQ = 1LL << 32, + XSC_DEV_CAP_FLAG_REMOTE_FENCE = 1LL << 38, + XSC_DEV_CAP_FLAG_TLP_HINTS = 1LL << 39, + XSC_DEV_CAP_FLAG_SIG_HAND_OVER = 1LL << 40, + XSC_DEV_CAP_FLAG_DCT = 1LL << 41, + XSC_DEV_CAP_FLAG_CMDIF_CSUM = 1LL << 46, +}; + +enum xsc_event { + XSC_EVENT_TYPE_COMP = 0x0, + XSC_EVENT_TYPE_COMM_EST = 0x02,//mad + XSC_EVENT_TYPE_CQ_ERROR = 0x04, + XSC_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, + XSC_EVENT_TYPE_INTERNAL_ERROR = 0x08,//tpe私有err，无IB event对应 + XSC_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10,//IBV_EVENT_QP_REQ_ERR + XSC_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11,//IBV_EVENT_QP_ACCESS_ERR +}; + +struct xsc_cmd_prot_block { + u8 data[XSC_CMD_DATA_BLOCK_SIZE]; + u8 rsvd0[48]; + __be64 next; + __be32 block_num; + u8 owner_status; //init to 0, dma user should change this val to 1 + u8 token; + u8 ctrl_sig; + u8 sig; +}; + +#define XSC_ATTR_EXTENDED_PORT_INFO cpu_to_be16(0xff90) + +enum xsc_traffic_types { + XSC_TT_IPV4, + XSC_TT_IPV4_TCP, + XSC_TT_IPV4_UDP, + XSC_TT_IPV6, + XSC_TT_IPV6_TCP, + XSC_TT_IPV6_UDP, + XSC_TT_IPV4_IPSEC_AH, + XSC_TT_IPV6_IPSEC_AH, + XSC_TT_IPV4_IPSEC_ESP, + XSC_TT_IPV6_IPSEC_ESP, + XSC_TT_ANY, + XSC_NUM_TT, +}; + +#define XSC_NUM_INDIR_TIRS XSC_NUM_TT + +enum { + XSC_HASH_FUNC_XOR = 0, + XSC_HASH_FUNC_TOP = 1, + XSC_HASH_FUNC_TOP_SYM = 2, + XSC_HASH_FUNC_RSV = 3, +}; + +enum { + XSC_L3_PROT_TYPE_IPV4 = 1 << 0, + XSC_L3_PROT_TYPE_IPV6 = 1 << 1, +}; + +enum { + XSC_L4_PROT_TYPE_TCP = 1 << 0, + XSC_L4_PROT_TYPE_UDP = 1 << 1, +}; + +struct xsc_tirc_config { + u8 l3_prot_type; + u8 l4_prot_type; + u32 rx_hash_fields; +}; + +static inline u8 hash_func_type(u8 hash_func) +{ + switch (hash_func) { + case ETH_RSS_HASH_TOP: + return XSC_HASH_FUNC_TOP; + case ETH_RSS_HASH_XOR: + return XSC_HASH_FUNC_XOR; + default: + return XSC_HASH_FUNC_TOP; + } +} + +#endif /* XSC_DEVICE_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/doorbell.h b/drivers/net/ethernet/yunsilicon/xsc/common/doorbell.h new file mode 100644 index 0000000000000000000000000000000000000000..ecbeeeb4c8f5b6460263fa55e82d4c8c5830eb88 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/doorbell.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_DOORBELL_H +#define XSC_DOORBELL_H + +#if BITS_PER_LONG == 64 +/* Assume that we can just write a 64-bit doorbell atomically. s390 + * actually doesn't have writeq() but S/390 systems don't even have + * PCI so we won't worry about it. + */ + +#define XSC_DECLARE_DOORBELL_LOCK(name) +#define XSC_INIT_DOORBELL_LOCK(ptr) do { } while (0) +#define XSC_GET_DOORBELL_LOCK(ptr) (NULL) + +static inline void xsc_write64(__be32 val[2], void __iomem *dest, + spinlock_t *doorbell_lock) +{ + __raw_writeq(*(u64 *)val, dest); +} + +#else + +/* Just fall back to a spinlock to protect the doorbell if + * BITS_PER_LONG is 32 -- there's no portable way to do atomic 64-bit + * MMIO writes. + */ + +#define XSC_DECLARE_DOORBELL_LOCK(name) spinlock_t name +#define XSC_INIT_DOORBELL_LOCK(ptr) spin_lock_init(ptr) +#define XSC_GET_DOORBELL_LOCK(ptr) (ptr) + +static inline void xsc_write64(__be32 val[2], void __iomem *dest, + spinlock_t *doorbell_lock) +{ + unsigned long flags; + + spin_lock_irqsave(doorbell_lock, flags); + __raw_writel((__force u32) val[0], dest); + __raw_writel((__force u32) val[1], dest + 4); + spin_unlock_irqrestore(doorbell_lock, flags); +} + +#endif + +#endif /* XSC_DOORBELL_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/driver.h b/drivers/net/ethernet/yunsilicon/xsc/common/driver.h new file mode 100644 index 0000000000000000000000000000000000000000..1fd8029eb1f9b9df26c9fe3e1b5ecf2ce1228f15 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/driver.h @@ -0,0 +1,374 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_DRIVER_H +#define XSC_DRIVER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LS_64(val, field) (((u64)val << field ## _SHIFT) & (field ## _MASK)) +#define RS_64(val, field) ((u64)(val & field ## _MASK) >> field ## _SHIFT) +#define LS_32(val, field) ((val << field ## _SHIFT) & (field ## _MASK)) +#define RS_32(val, field) ((val & field ## _MASK) >> field ## _SHIFT) + +enum { + CMD_OWNER_SW = 0x0, + CMD_OWNER_HW = 0x1, + CMD_STATUS_SUCCESS = 0, +}; + +enum { + XSC_MAX_FW_PORTS = 1, +}; + +enum { + XSC_MAX_IRQ_NAME = 32 +}; + +enum { + XSC_MAX_EQ_NAME = 20 +}; + +enum { + XSC_REG_PCAP = 0x5001, + XSC_REG_PMTU = 0x5003, + XSC_REG_PTYS = 0x5004, + XSC_REG_PAOS = 0x5006, + XSC_REG_PMAOS = 0x5012, + XSC_REG_PUDE = 0x5009, + XSC_REG_PMPE = 0x5010, + XSC_REG_PELC = 0x500e, + XSC_REG_PMLP = 0, /* TBD */ + XSC_REG_NODE_DESC = 0x6001, + XSC_REG_HOST_ENDIANNESS = 0x7004, + XSC_REG_MCIA = 0x9014, +}; + +enum dbg_rsc_type { + XSC_DBG_RSC_QP, + XSC_DBG_RSC_EQ, + XSC_DBG_RSC_CQ, +}; + +struct xsc_field_desc { + struct dentry *dent; + int i; +}; + +struct xsc_rsc_debug { + struct xsc_core_device *xdev; + void *object; + enum dbg_rsc_type type; + struct dentry *root; + struct xsc_field_desc fields[0]; +}; + +struct __packed xsc_qp_trace { + u16 main_ver; + u16 sub_ver; + u32 pid; + u16 qp_type; + u16 af_type; + union { + u32 s_addr4; + u8 s_addr6[16]; + } s_addr; + union { + u32 d_addr4; + u8 d_addr6[16]; + } d_addr; + u16 s_port; + u16 d_port; + u32 affinity_idx; + u64 timestamp; + u32 lqpn; + u32 rqpn; +}; + +struct xsc_buf_list { + void *buf; + dma_addr_t map; +}; + +struct xsc_buf { + struct xsc_buf_list direct; + struct xsc_buf_list *page_list; + int nbufs; + int npages; + int page_shift; + int size; +}; + +struct xsc_frag_buf { + struct xsc_buf_list *frags; + int npages; + int size; + u8 page_shift; +}; + +struct xsc_frag_buf_ctrl { + struct xsc_buf_list *frags; + u32 sz_m1; + u16 frag_sz_m1; + u16 strides_offset; + u8 log_sz; + u8 log_stride; + u8 log_frag_strides; +}; + +struct xsc_cq_table { + /* protect radix tree + */ + spinlock_t lock; + struct radix_tree_root tree; +}; + +struct xsc_eq { + struct xsc_core_device *dev; + struct xsc_cq_table cq_table; + u32 doorbell;//offset from bar0/2 space start + u32 cons_index; + struct xsc_buf buf; + int size; + unsigned int irqn; + u16 eqn; + int nent; + cpumask_var_t mask; + char name[XSC_MAX_EQ_NAME]; + struct list_head list; + int index; + struct xsc_rsc_debug *dbg; +}; + +struct xsc_core_mr { + u64 iova; + u64 size; + u32 key; + u32 pd; + u32 access; +}; + +struct xsc_eq_table { + void __iomem *update_ci; + void __iomem *update_arm_ci; + struct list_head comp_eqs_list; + struct xsc_eq pages_eq; + struct xsc_eq async_eq; + struct xsc_eq cmd_eq; + int num_comp_vectors; + int eq_vec_comp_base; + /* protect EQs list + */ + spinlock_t lock; +}; + +struct xsc_irq_info { + cpumask_var_t mask; + char name[XSC_MAX_IRQ_NAME]; +}; + +struct xsc_qp_table { + /* protect radix tree + */ + spinlock_t lock; + struct radix_tree_root tree; +}; + +struct counter_name_map { + int index; + const char *reg_name; +}; + +struct counter_reg_map { + int index; + int reg_addr; +}; + +struct xsc_dev_resource { + struct xsc_qp_table qp_table; + struct xsc_cq_table cq_table; + struct xsc_eq_table eq_table; + struct xsc_irq_info *irq_info; + spinlock_t mkey_lock; + u8 mkey_key; + struct mutex alloc_mutex; + int numa_node; + struct workqueue_struct *pg_wq; + struct rb_root page_root; + int fw_pages; + int reg_pages; + struct mutex pgdir_mutex; + struct list_head pgdir_list; + struct dentry *qp_debugfs; + struct dentry *eq_debugfs; + struct dentry *cq_debugfs; + struct dentry *cmdif_debugfs; + struct dentry *qptrace_debugfs; + struct dentry *dbg_root; +}; + +struct xsc_db { + __be32 *db; + union { + struct xsc_db_pgdir *pgdir; + struct xsc_ib_user_db_page *user_page; + } u; + dma_addr_t dma; + int index; +}; + +enum { + XSC_COMP_EQ_SIZE = 1024, +}; + +/*replace by struct define in ofed*/ +struct xsc_db_pgdir { + struct list_head list; + unsigned long *bitmap; + __be32 *db_page; + dma_addr_t db_dma; +}; + +static inline void *xsc_buf_offset(struct xsc_buf *buf, int offset) +{ + if (likely(BITS_PER_LONG == 64 || buf->nbufs == 1)) + return buf->direct.buf + offset; + else + return buf->page_list[offset >> PAGE_SHIFT].buf + + (offset & (PAGE_SIZE - 1)); +} + +static inline struct xsc_core_device *pci2xdev(struct pci_dev *pdev) +{ + return pci_get_drvdata(pdev); +} + +extern struct dentry *xsc_debugfs_root; + +static inline void *xsc_vzalloc(unsigned long size) +{ + void *rtn; + + rtn = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); + if (!rtn) + rtn = vzalloc(size); + return rtn; +} + +static inline void xsc_vfree(const void *addr) +{ + if (addr && is_vmalloc_addr(addr)) + vfree(addr); + else + kfree(addr); +} + +int xsc_dev_init(struct xsc_core_device *xdev); +void xsc_dev_cleanup(struct xsc_core_device *xdev); +int xsc_cmd_init(struct xsc_core_device *xdev); +void xsc_cmd_cleanup(struct xsc_core_device *xdev); +void xsc_cmd_use_events(struct xsc_core_device *xdev); +void xsc_cmd_use_polling(struct xsc_core_device *xdev); +int xsc_cmd_err_handler(struct xsc_core_device *xdev); +void xsc_cmd_resp_handler(struct xsc_core_device *xdev); +int xsc_cmd_status_to_err(struct xsc_outbox_hdr *hdr); +int _xsc_cmd_exec(struct xsc_core_device *xdev, void *in, int in_size, void *out, + int out_size); +int xsc_buf_alloc(struct xsc_core_device *xdev, int size, int max_direct, + struct xsc_buf *buf); +void xsc_buf_free(struct xsc_core_device *dev, struct xsc_buf *buf); +int xsc_core_create_mkey(struct xsc_core_device *dev, struct xsc_core_mr *mr); +int xsc_core_destroy_mkey(struct xsc_core_device *dev, struct xsc_core_mr *mr); +int xsc_core_register_mr(struct xsc_core_device *dev, struct xsc_core_mr *mr, + struct xsc_register_mr_mbox_in *in, int inlen); +int xsc_core_dereg_mr(struct xsc_core_device *dev, struct xsc_core_mr *mr); +void xsc_reg_local_dma_mr(struct xsc_core_device *dev); +int xsc_core_alloc_pd(struct xsc_core_device *xdev, u32 *pdn); +int xsc_core_dealloc_pd(struct xsc_core_device *xdev, u32 pdn); +int xsc_core_mad_ifc(struct xsc_core_device *xdev, void *inb, void *outb, + u16 opmod, int port); +int xsc_pagealloc_init(struct xsc_core_device *xdev); +void xsc_pagealloc_cleanup(struct xsc_core_device *xdev); +int xsc_pagealloc_start(struct xsc_core_device *xdev); +void xsc_pagealloc_stop(struct xsc_core_device *xdev); +void xsc_core_req_pages_handler(struct xsc_core_device *xdev, u16 func_id, + s16 npages); +int xsc_satisfy_startup_pages(struct xsc_core_device *xdev); +int xsc_reclaim_startup_pages(struct xsc_core_device *xdev); +void xsc_register_debugfs(void); +void xsc_unregister_debugfs(void); +int xsc_eq_init(struct xsc_core_device *dev); +void xsc_eq_cleanup(struct xsc_core_device *dev); +void xsc_fill_page_array(struct xsc_buf *buf, __be64 *pas, int npages); +void xsc_fill_page_frag_array(struct xsc_frag_buf *buf, __be64 *pas, int npages); +void xsc_qp_event(struct xsc_core_device *xdev, u32 qpn, int event_type); +int xsc_vector2eqn(struct xsc_core_device *dev, int vector, int *eqn, + unsigned int *irqn); +void xsc_cq_event(struct xsc_core_device *xdev, u32 cqn, int event_type); +int xsc_create_map_eq(struct xsc_core_device *dev, struct xsc_eq *eq, u8 vecidx, + int nent, const char *name); +int xsc_destroy_unmap_eq(struct xsc_core_device *dev, struct xsc_eq *eq); +int xsc_start_eqs(struct xsc_core_device *dev); +void xsc_stop_eqs(struct xsc_core_device *dev); + +int xsc_qp_debugfs_init(struct xsc_core_device *dev); +void xsc_qp_debugfs_cleanup(struct xsc_core_device *dev); +int xsc_core_access_reg(struct xsc_core_device *xdev, void *data_in, + int size_in, void *data_out, int size_out, + u16 reg_num, int arg, int write); +int xsc_set_port_caps(struct xsc_core_device *xdev, int port_num, u32 caps); + +int xsc_debug_eq_add(struct xsc_core_device *xdev, struct xsc_eq *eq); +void xsc_debug_eq_remove(struct xsc_core_device *xdev, struct xsc_eq *eq); +int xsc_core_eq_query(struct xsc_core_device *dev, struct xsc_eq *eq, + struct xsc_query_eq_mbox_out *out, int outlen); +int xsc_eq_debugfs_init(struct xsc_core_device *dev); +void xsc_eq_debugfs_cleanup(struct xsc_core_device *dev); +int xsc_cq_debugfs_init(struct xsc_core_device *dev); +void xsc_cq_debugfs_cleanup(struct xsc_core_device *dev); + +const char *xsc_command_str(int command); +int xsc_cmdif_debugfs_init(struct xsc_core_device *xdev); +void xsc_cmdif_debugfs_cleanup(struct xsc_core_device *xdev); + +int xsc_qptrace_debugfs_init(struct xsc_core_device *dev); +void xsc_qptrace_debugfs_cleanup(struct xsc_core_device *dev); + +int xsc_db_alloc_node(struct xsc_core_device *xdev, struct xsc_db *db, int node); +int xsc_frag_buf_alloc_node(struct xsc_core_device *xdev, int size, + struct xsc_frag_buf *buf, int node); +void xsc_db_free(struct xsc_core_device *xdev, struct xsc_db *db); +void xsc_frag_buf_free(struct xsc_core_device *xdev, struct xsc_frag_buf *buf); + +static inline u32 xsc_mkey_to_idx(u32 mkey) +{ + return mkey >> ((MMC_MPT_TBL_MEM_DEPTH == 32768) ? 17 : 18); +} + +static inline u32 xsc_idx_to_mkey(u32 mkey_idx) +{ + return mkey_idx << ((MMC_MPT_TBL_MEM_DEPTH == 32768) ? 17 : 18); +} + +enum { + XSC_PROF_MASK_QP_SIZE = (u64)1 << 0, + XSC_PROF_MASK_CMDIF_CSUM = (u64)1 << 1, + XSC_PROF_MASK_MR_CACHE = (u64)1 << 2, +}; + +#endif /* XSC_DRIVER_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/port.h b/drivers/net/ethernet/yunsilicon/xsc/common/port.h new file mode 100644 index 0000000000000000000000000000000000000000..f27cb886cce036fc2c69995d0fc45f967522d082 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/port.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef __XSC_PORT_H__ +#define __XSC_PORT_H__ + +enum xsc_module_id { + XSC_MODULE_ID_SFP = 0x3, + XSC_MODULE_ID_QSFP = 0xC, + XSC_MODULE_ID_QSFP_PLUS = 0xD, + XSC_MODULE_ID_QSFP28 = 0x11, + XSC_MODULE_ID_DSFP = 0x1B, +}; + +#define XSC_EEPROM_MAX_BYTES 32 +#define XSC_EEPROM_IDENTIFIER_BYTE_MASK 0x000000ff +#define XSC_I2C_ADDR_LOW 0x50 +#define XSC_I2C_ADDR_HIGH 0x51 +#define XSC_EEPROM_PAGE_LENGTH 256 +#define XSC_EEPROM_HIGH_PAGE_LENGTH 128 + +struct xsc_module_eeprom_query_params { + u16 size; + u16 offset; + u16 i2c_address; + u32 page; + u32 bank; + u32 module_number; +}; + +int xsc_query_module_eeprom(struct xsc_core_device *dev, + u16 offset, u16 size, u8 *data); +int xsc_query_module_eeprom_by_page(struct xsc_core_device *dev, + struct xsc_module_eeprom_query_params *params, + u8 *data); +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/qp.h b/drivers/net/ethernet/yunsilicon/xsc/common/qp.h new file mode 100644 index 0000000000000000000000000000000000000000..aece803310bdc1c93abf8124f21583eb679db879 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/qp.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_QP_H +#define XSC_QP_H + +#include +#include +#include + +enum { + XSC_QP_PM_MIGRATED = 0x3, + XSC_QP_PM_ARMED = 0x0, + XSC_QP_PM_REARM = 0x1 +}; + +enum { + XSC_WQE_CTRL_CQ_UPDATE = 2 << 2, + XSC_WQE_CTRL_SOLICITED = 1 << 1, +}; + +struct xsc_send_wqe_ctrl_seg { + __le32 msg_opcode:8; + __le32 with_immdt:1; + __le32 csum_en:2; + __le32 ds_data_num:5; + __le32 wqe_id:16; + __le32 msg_len; + union { + __le32 opcode_data; + struct { + u8 has_pph:1; + u8 so_type:1; + __le16 so_data_size:14; + u8 rsvd:8; + u8 so_hdr_len:8; + }; + struct { + __le16 desc_id; + __le16 is_last_wqe:1; + __le16 dst_qp_id:15; + }; + }; + __le32 se:1; + __le32 ce:1; + __le32 rsvd1:30; +}; + +struct xsc_wqe_data_seg { + union { + __le32 in_line:1; + struct { + __le32 rsvd:1; + __le32 seg_len:31; + __le32 mkey; + __le64 va; + }; + struct { + __le32 rsvd1:1; + __le32 len:7; + u8 in_line_data[15]; + }; + }; +}; + +struct xsc_wqe_ctrl_seg_2 { + __be32 opmod_idx_opcode; + __be32 qpn_ds; + u8 signature; + u8 rsvd[2]; + u8 fm_ce_se; + __be32 imm; +}; + +struct xsc_av { + union { + struct { + __be32 qkey; + __be32 reserved; + } qkey; + __be64 dc_key; + } key; + __be32 dqp_dct; + u8 stat_rate_sl; + u8 fl_mlid; + union { + __be16 rlid; + __be16 udp_sport; + }; + u8 reserved0[4]; + u8 rmac[6]; + u8 tclass; + u8 hop_limit; + __be32 grh_gid_fl; + u8 rgid[16]; +}; + +struct xsc_wqe_data_seg_2 { + __be32 byte_count; + __be32 lkey; + __be64 addr; +}; + +struct xsc_core_qp { + void (*event)(struct xsc_core_qp *qp, int event); + int qpn; + atomic_t refcount; + struct completion free; + struct xsc_rsc_debug *dbg; + int pid; + u16 qp_type; + u16 eth_queue_type; + struct dentry *trace; + struct xsc_qp_trace *trace_info; + u16 qp_type_internal; + u16 grp_id; +}; + +struct xsc_qp_path { + u8 fl; + u8 rsvd3; + u8 free_ar; + u8 pkey_index; + u8 rsvd0; + u8 grh_mlid; + __be16 rlid; + u8 ackto_lt; + u8 mgid_index; + u8 static_rate; + u8 hop_limit; + __be32 tclass_flowlabel; + u8 rgid[16]; + u8 rsvd1[4]; + u8 sl; + u8 port; + u8 rsvd2[6]; + u8 dmac[6]; + u8 smac[6]; + __be16 af_type; + __be32 sip[4]; + __be32 dip[4]; + __be16 sport; + u8 ecn_dscp; + u8 vlan_valid; + __be16 vlan_id; + u8 dci_cfi_prio_sl; //not left moved yet. +}; + +static inline struct xsc_core_qp *__xsc_qp_lookup(struct xsc_core_device *xdev, u32 qpn) +{ + return radix_tree_lookup(&xdev->dev_res->qp_table.tree, qpn); +} + +int create_resource_common(struct xsc_core_device *xdev, + struct xsc_core_qp *qp); +void destroy_resource_common(struct xsc_core_device *xdev, + struct xsc_core_qp *qp); + +int xsc_core_create_qp(struct xsc_core_device *xdev, + struct xsc_core_qp *qp, + struct xsc_create_qp_mbox_in *in, + int inlen); +int xsc_core_qp_modify(struct xsc_core_device *xdev, enum xsc_qp_state cur_state, + enum xsc_qp_state new_state, + struct xsc_modify_qp_mbox_in *in, int sqd_event, + struct xsc_core_qp *qp); +int xsc_core_destroy_qp(struct xsc_core_device *xdev, + struct xsc_core_qp *qp); +int xsc_core_qp_query(struct xsc_core_device *xdev, struct xsc_core_qp *qp, + struct xsc_query_qp_mbox_out *out, int outlen); + +void xsc_init_qp_table(struct xsc_core_device *xdev); +void xsc_cleanup_qp_table(struct xsc_core_device *xdev); +int xsc_debug_qp_add(struct xsc_core_device *xdev, struct xsc_core_qp *qp); +void xsc_debug_qp_remove(struct xsc_core_device *xdev, struct xsc_core_qp *qp); + +int xsc_create_qptrace(struct xsc_core_device *xdev, struct xsc_core_qp *qp); +void xsc_remove_qptrace(struct xsc_core_device *xdev, struct xsc_core_qp *qp); + +#endif /* XSC_QP_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/version.h b/drivers/net/ethernet/yunsilicon/xsc/common/version.h new file mode 100644 index 0000000000000000000000000000000000000000..4c3b27df3864d6c8d13ce879db9497821e2321bd --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/version.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#define BRANCH_VERSION 1 +#define MAJOR_VERSION 1 +#define MINOR_VERSION 0 +#define HOTFIX_NUM 0 +#define BUILD_VERSION 28 diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/vport.h b/drivers/net/ethernet/yunsilicon/xsc/common/vport.h new file mode 100644 index 0000000000000000000000000000000000000000..a5cc5a9ed40f93adf7ae2b9279a5b81507526d9e --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/vport.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_VPORT_H +#define XSC_VPORT_H + +#include +#include + +#define XSC_VPORT_PF_PLACEHOLDER (1u) +#define XSC_VPORT_UPLINK_PLACEHOLDER (1u) +#define XSC_VPORT_ECPF_PLACEHOLDER(dev) (xsc_ecpf_vport_exists(dev) || \ + xsc_core_is_ecpf_esw_manager(dev)) + +#define XSC_SPECIAL_VPORTS(dev) (XSC_VPORT_PF_PLACEHOLDER + \ + XSC_VPORT_UPLINK_PLACEHOLDER + \ + XSC_VPORT_ECPF_PLACEHOLDER(dev)) + +#define XSC_VPORT_MANAGER(dev) (xsc_core_is_vport_manager(dev)) + +enum { + XSC_CAP_INLINE_MODE_L2, + XSC_CAP_INLINE_MODE_VPORT_CONTEXT, + XSC_CAP_INLINE_MODE_NOT_REQUIRED, +}; + +/* Vport number for each function must keep unchanged */ +enum { + XSC_VPORT_PF = 0x0, + XSC_VPORT_FIRST_VF = 0x1, + XSC_VPORT_ECPF = 0xfffe, + XSC_VPORT_UPLINK = 0xffff, +}; + +enum { + XSC_VPORT_ADMIN_STATE_DOWN = 0x0, + XSC_VPORT_ADMIN_STATE_UP = 0x1, + XSC_VPORT_ADMIN_STATE_AUTO = 0x2, +}; + +u8 xsc_query_vport_state(struct xsc_core_device *dev, u8 opmod, u16 vport); +int xsc_modify_vport_admin_state(struct xsc_core_device *dev, u8 opmod, + u16 vport, u8 other_vport, u8 state); +int xsc_query_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr); +int xsc_query_other_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr); +int xsc_query_nic_vport_min_inline(struct xsc_core_device *dev, + u16 vport, u8 *min_inline); +void xsc_query_min_inline(struct xsc_core_device *dev, u8 *min_inline); +int xsc_modify_nic_vport_min_inline(struct xsc_core_device *dev, + u16 vport, u8 min_inline); +int xsc_modify_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr); +int xsc_modify_other_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr); +int xsc_query_nic_vport_mtu(struct xsc_core_device *dev, u16 *mtu); +int xsc_modify_nic_vport_mtu(struct xsc_core_device *dev, u16 mtu); +int xsc_query_nic_vport_system_image_guid(struct xsc_core_device *dev, + u64 *system_image_guid); +int xsc_query_nic_vport_node_guid(struct xsc_core_device *dev, u32 vport, + u64 *node_guid); +int xsc_modify_nic_vport_node_guid(struct xsc_core_device *dev, + u16 vport, u64 node_guid); +int xsc_modify_other_nic_vport_node_guid(struct xsc_core_device *dev, + u16 vport, u64 node_guid); +int xsc_query_nic_vport_qkey_viol_cntr(struct xsc_core_device *dev, + u16 *qkey_viol_cntr); +int xsc_query_hca_vport_gid(struct xsc_core_device *dev, u8 other_vport, + u8 port_num, u16 vf_num, u16 gid_index, + union ib_gid *gid); +int xsc_query_hca_vport_pkey(struct xsc_core_device *dev, u8 other_vport, + u8 port_num, u16 vf_num, u16 pkey_index, + u16 *pkey); +int xsc_query_hca_vport_context(struct xsc_core_device *dev, + u8 other_vport, u8 port_num, + u16 vf_num, + struct xsc_hca_vport_context *rep); +int xsc_query_hca_vport_node_guid(struct xsc_core_device *dev, + u64 *node_guid); +int xsc_query_nic_vport_mac_list(struct xsc_core_device *dev, + u16 vport, + enum xsc_list_type list_type, + u8 addr_list[][ETH_ALEN], + int *list_size); +int xsc_modify_nic_vport_mac_list(struct xsc_core_device *dev, + enum xsc_list_type list_type, + u8 addr_list[][ETH_ALEN], + int list_size); +int xsc_query_nic_vport_promisc(struct xsc_core_device *dev, + u16 vport, + int *promisc_uc, + int *promisc_mc, + int *promisc_all); +int xsc_modify_nic_vport_promisc(struct xsc_core_device *dev, + int promisc_uc, + int promisc_mc, + int promisc_all); +int xsc_query_nic_vport_vlans(struct xsc_core_device *dev, u32 vport, + unsigned long *vlans); +int xsc_modify_nic_vport_vlans(struct xsc_core_device *dev, + u16 vlans[], + int list_size); +int xsc_query_vport_down_stats(struct xsc_core_device *dev, u16 vport, + u8 other_vport, u64 *rx_discard_vport_down, + u64 *tx_discard_vport_down); +int xsc_query_vport_counter(struct xsc_core_device *dev, u8 other_vport, + int vf, u8 port_num, void *out, + size_t out_sz); +int xsc_modify_hca_vport_context(struct xsc_core_device *dev, + u8 other_vport, u8 port_num, + int vf, + struct xsc_hca_vport_context *req); +//int xsc_nic_vport_enable_roce(struct xsc_core_device *dev); +//int xsc_nic_vport_disable_roce(struct xsc_core_device *dev); + +u16 xsc_eswitch_get_total_vports(const struct xsc_core_device *dev); +#endif /* XSC_VPORT_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_cmd.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_cmd.h new file mode 100644 index 0000000000000000000000000000000000000000..56bafdd62d032796a550c9111a14799c8a9c37f7 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_cmd.h @@ -0,0 +1,1823 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_CMD_H +#define XSC_CMD_H + +#define CMDQ_VERSION 0xe + +#define QOS_PRIO_MAX 7 +#define QOS_DSCP_MAX 63 +#define MAC_PORT_DSCP_SHIFT 6 +#define QOS_PCP_MAX 7 +#define DSCP_PCP_UNSET 255 +#define MAC_PORT_PCP_SHIFT 3 +#define XSC_MAX_MAC_NUM 8 +#define XSC_BOARD_SN_LEN 32 +#define MAX_PKT_LEN 9800 +#define XSC_RTT_CFG_QPN_MAX 32 + +enum { + XSC_CMD_OP_QUERY_HCA_CAP = 0x100, + XSC_CMD_OP_QUERY_ADAPTER = 0x101, + XSC_CMD_OP_INIT_HCA = 0x102, + XSC_CMD_OP_TEARDOWN_HCA = 0x103, + XSC_CMD_OP_ENABLE_HCA = 0x104, + XSC_CMD_OP_DISABLE_HCA = 0x105, + XSC_CMD_OP_QUERY_PAGES = 0x107, + XSC_CMD_OP_MANAGE_PAGES = 0x108, + XSC_CMD_OP_SET_HCA_CAP = 0x109, + XSC_CMD_OP_QUERY_CMDQ_VERSION = 0x10a, + XSC_CMD_OP_QUERY_MSIX_TBL_INFO = 0x10b, + XSC_CMD_OP_FUNCTION_RESET = 0x10c, + XSC_CMD_OP_DUMMY = 0x10d, + + XSC_CMD_OP_CREATE_MKEY = 0x200, + XSC_CMD_OP_QUERY_MKEY = 0x201, + XSC_CMD_OP_DESTROY_MKEY = 0x202, + XSC_CMD_OP_QUERY_SPECIAL_CONTEXTS = 0x203, + XSC_CMD_OP_REG_MR = 0x204, + XSC_CMD_OP_DEREG_MR = 0x205, + + XSC_CMD_OP_CREATE_EQ = 0x301, + XSC_CMD_OP_DESTROY_EQ = 0x302, + XSC_CMD_OP_QUERY_EQ = 0x303, + + XSC_CMD_OP_CREATE_CQ = 0x400, + XSC_CMD_OP_DESTROY_CQ = 0x401, + XSC_CMD_OP_QUERY_CQ = 0x402, + XSC_CMD_OP_MODIFY_CQ = 0x403, + XSC_CMD_OP_ALLOC_MULTI_VIRTQ_CQ = 0x404, + XSC_CMD_OP_RELEASE_MULTI_VIRTQ_CQ = 0x405, + + XSC_CMD_OP_CREATE_QP = 0x500, + XSC_CMD_OP_DESTROY_QP = 0x501, + XSC_CMD_OP_RST2INIT_QP = 0x502, + XSC_CMD_OP_INIT2RTR_QP = 0x503, + XSC_CMD_OP_RTR2RTS_QP = 0x504, + XSC_CMD_OP_RTS2RTS_QP = 0x505, + XSC_CMD_OP_SQERR2RTS_QP = 0x506, + XSC_CMD_OP_2ERR_QP = 0x507, + XSC_CMD_OP_RTS2SQD_QP = 0x508, + XSC_CMD_OP_SQD2RTS_QP = 0x509, + XSC_CMD_OP_2RST_QP = 0x50a, + XSC_CMD_OP_QUERY_QP = 0x50b, + XSC_CMD_OP_CONF_SQP = 0x50c, + XSC_CMD_OP_MAD_IFC = 0x50d, + XSC_CMD_OP_INIT2INIT_QP = 0x50e, + XSC_CMD_OP_SUSPEND_QP = 0x50f, + XSC_CMD_OP_UNSUSPEND_QP = 0x510, + XSC_CMD_OP_SQD2SQD_QP = 0x511, + XSC_CMD_OP_ALLOC_QP_COUNTER_SET = 0x512, + XSC_CMD_OP_DEALLOC_QP_COUNTER_SET = 0x513, + XSC_CMD_OP_QUERY_QP_COUNTER_SET = 0x514, + XSC_CMD_OP_CREATE_MULTI_QP = 0x515, + XSC_CMD_OP_ALLOC_MULTI_VIRTQ = 0x516, + XSC_CMD_OP_RELEASE_MULTI_VIRTQ = 0x517, + + XSC_CMD_OP_CREATE_PSV = 0x600, + XSC_CMD_OP_DESTROY_PSV = 0x601, + XSC_CMD_OP_QUERY_PSV = 0x602, + XSC_CMD_OP_QUERY_SIG_RULE_TABLE = 0x603, + XSC_CMD_OP_QUERY_BLOCK_SIZE_TABLE = 0x604, + + XSC_CMD_OP_CREATE_SRQ = 0x700, + XSC_CMD_OP_DESTROY_SRQ = 0x701, + XSC_CMD_OP_QUERY_SRQ = 0x702, + XSC_CMD_OP_ARM_RQ = 0x703, + XSC_CMD_OP_RESIZE_SRQ = 0x704, + + XSC_CMD_OP_ALLOC_PD = 0x800, + XSC_CMD_OP_DEALLOC_PD = 0x801, + XSC_CMD_OP_ALLOC_UAR = 0x802, + XSC_CMD_OP_DEALLOC_UAR = 0x803, + + XSC_CMD_OP_ATTACH_TO_MCG = 0x806, + XSC_CMD_OP_DETACH_FROM_MCG = 0x807, + + XSC_CMD_OP_ALLOC_XRCD = 0x80e, + XSC_CMD_OP_DEALLOC_XRCD = 0x80f, + + XSC_CMD_OP_ACCESS_REG = 0x805, + + XSC_CMD_OP_MODIFY_RAW_QP = 0x81f, + + XSC_CMD_OP_ENABLE_NIC_HCA = 0x810, + XSC_CMD_OP_DISABLE_NIC_HCA = 0x811, + XSC_CMD_OP_MODIFY_NIC_HCA = 0x812, + + XSC_CMD_OP_QUERY_NIC_VPORT_CONTEXT = 0x820, + XSC_CMD_OP_MODIFY_NIC_VPORT_CONTEXT = 0x821, + XSC_CMD_OP_QUERY_VPORT_STATE = 0x822, + XSC_CMD_OP_MODIFY_VPORT_STATE = 0x823, + XSC_CMD_OP_QUERY_HCA_VPORT_CONTEXT = 0x824, + XSC_CMD_OP_MODIFY_HCA_VPORT_CONTEXT = 0x825, + XSC_CMD_OP_QUERY_HCA_VPORT_GID = 0x826, + XSC_CMD_OP_QUERY_HCA_VPORT_PKEY = 0x827, + XSC_CMD_OP_QUERY_VPORT_COUNTER = 0x828, + XSC_CMD_OP_QUERY_PRIO_STATS = 0x829, + XSC_CMD_OP_QUERY_PHYPORT_STATE = 0x830, + XSC_CMD_OP_QUERY_EVENT_TYPE = 0x831, + + XSC_CMD_OP_LAG_CREATE = 0x840, + XSC_CMD_OP_LAG_MODIFY = 0x841, + XSC_CMD_OP_LAG_DESTROY = 0x842, + XSC_CMD_OP_LAG_SET_QOS = 0x848, + XSC_CMD_OP_ENABLE_MSIX = 0x850, + + XSC_CMD_OP_IOCTL_FLOW = 0x900, + XSC_CMD_OP_IOCTL_OTHER = 0x901, + + XSC_CMD_OP_IOCTL_SET_DSCP_PMT = 0x1000, + XSC_CMD_OP_IOCTL_GET_DSCP_PMT = 0x1001, + XSC_CMD_OP_IOCTL_SET_TRUST_MODE = 0x1002, + XSC_CMD_OP_IOCTL_GET_TRUST_MODE = 0x1003, + XSC_CMD_OP_IOCTL_SET_PCP_PMT = 0x1004, + XSC_CMD_OP_IOCTL_GET_PCP_PMT = 0x1005, + XSC_CMD_OP_IOCTL_SET_DEFAULT_PRI = 0x1006, + XSC_CMD_OP_IOCTL_GET_DEFAULT_PRI = 0x1007, + XSC_CMD_OP_IOCTL_SET_PFC = 0x1008, + XSC_CMD_OP_IOCTL_GET_PFC = 0x1009, + XSC_CMD_OP_IOCTL_SET_RATE_LIMIT = 0x100a, + XSC_CMD_OP_IOCTL_GET_RATE_LIMIT = 0x100b, + XSC_CMD_OP_IOCTL_SET_SP = 0x100c, + XSC_CMD_OP_IOCTL_GET_SP = 0x100d, + XSC_CMD_OP_IOCTL_SET_WEIGHT = 0x100e, + XSC_CMD_OP_IOCTL_GET_WEIGHT = 0x100f, + + XSC_CMD_OP_IOCTL_SET_ENABLE_RP = 0x1030, + XSC_CMD_OP_IOCTL_SET_ENABLE_NP = 0x1031, + XSC_CMD_OP_IOCTL_SET_INIT_ALPHA = 0x1032, + XSC_CMD_OP_IOCTL_SET_G = 0x1033, + XSC_CMD_OP_IOCTL_SET_AI = 0x1034, + XSC_CMD_OP_IOCTL_SET_HAI = 0x1035, + XSC_CMD_OP_IOCTL_SET_TH = 0x1036, + XSC_CMD_OP_IOCTL_SET_BC_TH = 0x1037, + XSC_CMD_OP_IOCTL_SET_CNP_OPCODE = 0x1038, + XSC_CMD_OP_IOCTL_SET_CNP_BTH_B = 0x1039, + XSC_CMD_OP_IOCTL_SET_CNP_BTH_F = 0x103a, + XSC_CMD_OP_IOCTL_SET_CNP_ECN = 0x103b, + XSC_CMD_OP_IOCTL_SET_DATA_ECN = 0x103c, + XSC_CMD_OP_IOCTL_SET_CNP_TX_INTERVAL = 0x103d, + XSC_CMD_OP_IOCTL_SET_EVT_PERIOD_RSTTIME = 0x103e, + XSC_CMD_OP_IOCTL_SET_CNP_DSCP = 0x103f, + XSC_CMD_OP_IOCTL_SET_CNP_PCP = 0x1040, + XSC_CMD_OP_IOCTL_SET_EVT_PERIOD_ALPHA = 0x1041, + XSC_CMD_OP_IOCTL_GET_CC_CFG = 0x1042, + XSC_CMD_OP_IOCTL_GET_CC_STAT = 0x104a, + XSC_CMD_OP_IOCTL_SET_CLAMP_TGT_RATE = 0x1052, + + XSC_CMD_OP_IOCTL_SET_HWC = 0x1060, + XSC_CMD_OP_IOCTL_GET_HWC = 0x1061, + + XSC_CMD_OP_SET_MTU = 0x1100, + XSC_CMD_OP_QUERY_ETH_MAC = 0X1101, + + XSC_CMD_OP_QUERY_HW_STATS = 0X1200, + XSC_CMD_OP_QUERY_PAUSE_CNT = 0X1201, + + XSC_CMD_OP_SET_RTT_EN = 0X1220, + XSC_CMD_OP_GET_RTT_EN = 0X1221, + XSC_CMD_OP_SET_RTT_QPN = 0X1222, + XSC_CMD_OP_GET_RTT_QPN = 0X1223, + XSC_CMD_OP_SET_RTT_PERIOD = 0X1224, + XSC_CMD_OP_GET_RTT_PERIOD = 0X1225, + XSC_CMD_OP_GET_RTT_RESULT = 0X1226, + XSC_CMD_OP_GET_RTT_STATS = 0X1227, + + XSC_CMD_OP_SET_LED_STATUS = 0X1228, + + XSC_CMD_OP_MAX +}; + +enum { + XSC_CMD_EVENT_RESP_CHANGE_LINK = 0x0001, +}; + +enum xsc_eth_qp_num_sel { + XSC_ETH_QP_NUM_1K_SEL = 0, + XSC_ETH_QP_NUM_8K_SEL, + XSC_ETH_QP_NUM_8K_8TC_SEL, + XSC_ETH_QP_NUM_SEL_MAX, +}; + +enum xsc_eth_vf_num_sel { + XSC_ETH_VF_NUM_SEL_8 = 0, + XSC_ETH_VF_NUM_SEL_16, + XSC_ETH_VF_NUM_SEL_32, + XSC_ETH_VF_NUM_SEL_64, + XSC_ETH_VF_NUM_SEL_128, + XSC_ETH_VF_NUM_SEL_256, + XSC_ETH_VF_NUM_SEL_512, + XSC_ETH_VF_NUM_SEL_MAX +}; + +enum { + XSC_CMD_RESP_LINKSPEED_MODE_25G = 0x0, + XSC_CMD_RESP_LINKSPEED_MODE_100G = 0x1 +}; + +enum xsc_dma_direct { + DMA_DIR_TO_MAC, + DMA_DIR_READ, + DMA_DIR_WRITE, + DMA_DIR_LOOPBACK, + DMA_DIR_MAX, +}; + +struct xsc_inbox_hdr { + __be16 opcode; + u8 rsvd[4]; + __be16 opmod; +}; + +struct xsc_outbox_hdr { + u8 status; + u8 rsvd[3]; + __be32 syndrome; +}; + +struct xsc_pci_driver_init_params_in { + struct xsc_inbox_hdr hdr; + __be32 s_wqe_mode; + __be32 r_wqe_mode; + __be32 local_timeout_retrans; + u8 mac_lossless_prio[XSC_MAX_MAC_NUM]; + __be32 group_mod; +}; + +struct xsc_pci_driver_init_params_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +/*CQ mbox*/ +struct xsc_cq_context { + __be16 eqn; + __be16 pa_num; + __be16 glb_func_id; + u8 log_cq_sz; + u8 cq_type; +}; + +struct xsc_create_cq_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_cq_context ctx; + __be64 pas[0]; +}; + +struct xsc_create_cq_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 cqn; + u8 rsvd[4]; +}; + +struct xsc_destroy_cq_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 cqn; + u8 rsvd[4]; +}; + +struct xsc_destroy_cq_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +/*QP mbox*/ +struct xsc_create_qp_request { + __be16 input_qpn; + __be16 pa_num; + u8 qp_type; + u8 log_sq_sz; + u8 log_rq_sz; + u8 dma_direct;//0 for dma read, 1 for dma write + __be32 pdn; + __be16 cqn_send; + __be16 cqn_recv; + __be16 glb_funcid; + __be16 rsvd; + __be64 pas[0]; +}; + +struct xsc_create_qp_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_create_qp_request req; +}; +struct xsc_create_qp_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 qpn; + u8 rsvd[4]; +}; + +struct xsc_destroy_qp_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 qpn; + u8 rsvd[4]; + +}; + +struct xsc_destroy_qp_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_qp_context { + __be32 remote_qpn; + __be32 cqn_send; + __be32 cqn_recv; + __be32 next_send_psn; + __be32 next_recv_psn; + __be32 pdn; + __be16 src_udp_port; + __be16 path_id; + u8 mtu_mode; + u8 lag_sel; + u8 lag_sel_en; + u8 retry_cnt; + u8 rnr_retry; + u8 dscp; + u8 state; + u8 hop_limit; + u8 dmac[6]; + u8 smac[6]; + __be32 dip[4]; + __be32 sip[4]; + __be16 ip_type; + __be16 grp_id; + u8 vlan_valid; + u8 dci_cfi_prio_sl; + __be16 vlan_id; + u8 qp_out_port; + u8 pcie_no; + __be16 lag_id; + __be16 func_id; + __be16 rsvd; +}; + +struct xsc_query_qp_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 qpn; + u8 rsvd[4]; +}; + +struct xsc_query_qp_mbox_out { + struct xsc_outbox_hdr hdr; + struct xsc_qp_context ctx; +}; + +struct xsc_modify_qp_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 qpn; + struct xsc_qp_context ctx; +}; + +struct xsc_modify_qp_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_create_multiqp_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 qp_num; + u8 qp_type; + u8 rsvd; + __be32 req_len; + u8 data[0]; +}; + +struct xsc_create_multiqp_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 qpn_base; +}; + +struct xsc_alloc_multi_virtq_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 qp_or_cq_num; + __be16 pa_num; + __be32 rsvd; + __be32 rsvd2; +}; + +struct xsc_alloc_multi_virtq_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 qnum_base; + __be32 pa_list_base; + __be32 rsvd; +}; + +struct xsc_release_multi_virtq_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 qp_or_cq_num; + __be16 pa_num; + __be32 qnum_base; + __be32 pa_list_base; +}; + +struct xsc_release_multi_virtq_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 rsvd; + __be32 rsvd2; + __be32 rsvd3; +}; + +/* MSIX TABLE mbox */ +struct xsc_msix_table_info_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 index; + u8 rsvd[6]; +}; + +struct xsc_msix_table_info_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 addr_lo; + __be32 addr_hi; + __be32 data; +}; + +/*EQ mbox*/ +struct xsc_eq_context { + __be16 vecidx; + __be16 pa_num; + u8 log_eq_sz; + __be16 glb_func_id; + u8 is_async_eq; + u8 rsvd[1]; +}; + +struct xsc_create_eq_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_eq_context ctx; + __be64 pas[0]; +}; + +struct xsc_create_eq_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 eqn; + u8 rsvd[4]; +}; + +struct xsc_destroy_eq_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 eqn; + u8 rsvd[4]; +}; + +struct xsc_destroy_eq_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +/*PD mbox*/ +struct xsc_alloc_pd_request { + u8 rsvd[8]; +}; + +struct xsc_alloc_pd_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_alloc_pd_request req; +}; + +struct xsc_alloc_pd_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 pdn; + u8 rsvd[4]; +}; + +struct xsc_dealloc_pd_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 pdn; + u8 rsvd[4]; +}; + +struct xsc_dealloc_pd_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +/*MR mbox*/ +struct xsc_register_mr_request { + __be32 pdn; + __be32 pa_num; + __be32 len; + __be32 mkey; + u8 rsvd; + u8 acc; + u8 page_mode; + u8 map_en; + __be64 va_base; + __be64 pas[0]; +}; + +struct xsc_register_mr_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_register_mr_request req; +}; + +struct xsc_register_mr_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 mkey; + u8 rsvd[4]; +}; + +struct xsc_unregister_mr_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 mkey; + u8 rsvd[4]; +}; + +struct xsc_unregister_mr_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_create_mkey_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd[4]; +}; + +struct xsc_create_mkey_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 mkey; +}; + +struct xsc_destroy_mkey_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 mkey; +}; + +struct xsc_destroy_mkey_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd; +}; + +struct xsc_access_reg_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd0[2]; + __be16 register_id; + __be32 arg; + __be32 data[0]; +}; + +struct xsc_access_reg_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; + __be32 data[0]; +}; + +struct xsc_mad_ifc_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 remote_lid; + u8 rsvd0; + u8 port; + u8 rsvd1[4]; + u8 data[256]; +}; + +struct xsc_mad_ifc_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; + u8 data[256]; +}; + +struct xsc_query_eq_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd0[3]; + u8 eqn; + u8 rsvd1[4]; +}; + +struct xsc_query_eq_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; + struct xsc_eq_context ctx; +}; + +struct xsc_query_cq_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 cqn; + u8 rsvd0[4]; +}; + +struct xsc_query_cq_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd0[8]; + struct xsc_cq_context ctx; + u8 rsvd6[16]; + __be64 pas[0]; +}; + +struct xsc_cmd_query_cmdq_ver_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_cmd_query_cmdq_ver_mbox_out { + struct xsc_outbox_hdr hdr; + __be16 cmdq_ver; + u8 rsvd[6]; +}; + +struct xsc_cmd_dummy_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_cmd_dummy_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_hca_cap { + u8 rsvd1[12]; + u8 send_seg_num; + u8 send_wqe_shift; + u8 recv_seg_num; + u8 recv_wqe_shift; + u8 log_max_srq_sz; + u8 log_max_qp_sz; + u8 log_max_mtt; + u8 log_max_qp; + u8 log_max_strq_sz; + u8 log_max_srqs; + u8 rsvd4[2]; + u8 log_max_tso; + u8 log_max_cq_sz; + u8 rsvd6; + u8 log_max_cq; + u8 log_max_eq_sz; + u8 log_max_mkey; + u8 log_max_msix; + u8 log_max_eq; + u8 max_indirection; + u8 log_max_mrw_sz; + u8 log_max_bsf_list_sz; + u8 log_max_klm_list_sz; + u8 rsvd_8_0; + u8 log_max_ra_req_dc; + u8 rsvd_8_1; + u8 log_max_ra_res_dc; + u8 rsvd9; + u8 log_max_ra_req_qp; + u8 log_max_qp_depth; + u8 log_max_ra_res_qp; + __be16 max_vfs; + __be16 raweth_qp_id_end; + __be16 raw_tpe_qp_num; + __be16 max_qp_count; + __be16 raweth_qp_id_base; + u8 rsvd13; + u8 local_ca_ack_delay; + u8 max_num_eqs; + u8 num_ports; + u8 log_max_msg; + u8 mac_port; + __be16 raweth_rss_qp_id_base; + __be16 stat_rate_support; + u8 rsvd16[2]; + __be64 flags; + u8 rsvd17; + u8 uar_sz; + u8 rsvd18; + u8 log_pg_sz; + __be16 bf_log_bf_reg_size; + __be16 msix_base; + __be16 msix_num; + __be16 max_desc_sz_sq; + u8 rsvd20[2]; + __be16 max_desc_sz_rq; + u8 rsvd21[2]; + __be16 max_desc_sz_sq_dc; + u8 rsvd22[4]; + __be16 max_qp_mcg; + u8 rsvd23; + u8 log_max_mcg; + u8 rsvd24; + u8 log_max_pd; + u8 rsvd25; + u8 log_max_xrcd; + u8 rsvd26[40]; + __be32 uar_page_sz; + u8 rsvd27[28]; + u8 log_msx_atomic_size_qp; + u8 rsvd28[2]; + u8 log_msx_atomic_size_dc; + u8 board_sn[XSC_BOARD_SN_LEN]; + u8 rsvd29[12]; + __be32 hca_core_clock; + __be32 max_rwq_indirection_tables;/*rss_caps*/ + __be32 max_rwq_indirection_table_size;/*rss_caps*/ + __be32 chip_ver_h; + __be32 chip_ver_m; + __be32 chip_ver_l; + __be32 hotfix_num; + __be32 feature_flag; + __be32 rx_pkt_len_max; + __be32 glb_func_id; + __be64 tx_db; + __be64 rx_db; + __be64 complete_db; + __be64 complete_reg; + __be64 event_db; + __be32 qp_rate_limit_min; + __be32 qp_rate_limit_max; +}; + +struct xsc_cmd_query_hca_cap_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 cpu_num; + u8 rsvd[6]; +}; + +struct xsc_cmd_query_hca_cap_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd0[8]; + struct xsc_hca_cap hca_cap; +}; + +struct xsc_cmd_init_hca_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd0[2]; + __be16 profile; + u8 rsvd1[4]; +}; + +struct xsc_cmd_init_hca_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_cmd_teardown_hca_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd0[2]; + __be16 profile; + u8 rsvd1[4]; +}; + +struct xsc_cmd_teardown_hca_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_cmd_enable_hca_mbox_in { + struct xsc_inbox_hdr hdr; + u8 pf; + u8 pcie; + u8 pf_id; + __be16 vf_id; + u8 rsvd0[3]; +}; + +struct xsc_cmd_enable_hca_mbox_out { + struct xsc_outbox_hdr hdr; + u8 status; + u8 rsvd0[3]; +}; + +struct xsc_cmd_disable_hca_mbox_in { + struct xsc_inbox_hdr hdr; + u8 pf; + u8 pcie; + u8 pf_id; + __be16 vf_id; + u8 rsvd0[3]; +}; + +struct xsc_cmd_disable_hca_mbox_out { + struct xsc_outbox_hdr hdr; + u8 status; + u8 rsvd0[3]; +}; + +struct xsc_query_special_ctxs_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_query_special_ctxs_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 dump_fill_mkey; + __be32 reserved_lkey; +}; + +/* vport mbox */ +struct xsc_nic_vport_context { + u32 min_wqe_inline_mode:3; + u32 disable_mc_local_lb:1; + u32 disable_uc_local_lb:1; + u32 roce_en:1; + + u32 arm_change_event:1; + u32 event_on_mtu:1; + u32 event_on_promisc_change:1; + u32 event_on_vlan_change:1; + u32 event_on_mc_address_change:1; + u32 event_on_uc_address_change:1; + u32 affiliation_criteria:4; + u32 affiliated_vhca_id; + + u16 mtu; + + u64 system_image_guid; + u64 port_guid; + u64 node_guid; + + u32 qkey_violation_counter; + + u16 promisc_uc:1; + u16 promisc_mc:1; + u16 promisc_all:1; + u16 allowed_list_type:3; + u16 allowed_list_size:10; + + u8 permanent_address[6]; + u64 current_uc_mac_address[0]; +}; + +enum { + XSC_HCA_VPORT_SEL_PORT_GUID = 1 << 0, + XSC_HCA_VPORT_SEL_NODE_GUID = 1 << 1, + XSC_HCA_VPORT_SEL_STATE_POLICY = 1 << 2, +}; + +struct xsc_hca_vport_context { + u32 field_select; + u32 port_physical_state:4; + u32 vport_state_policy:4; + u32 port_state:4; + u32 vport_state:4; + u32 rcvd0:16; + + u64 system_image_guid; + u64 port_guid; + u64 node_guid; + + u16 qkey_violation_counter; + u16 pkey_violation_counter; +}; + +struct xsc_query_nic_vport_context_out { + struct xsc_outbox_hdr hdr; + struct xsc_nic_vport_context nic_vport_ctx; +}; + +struct xsc_query_nic_vport_context_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 vport_number:16; + u32 allowed_list_type:3; + u32 rsvd:12; +}; + +struct xsc_modify_nic_vport_context_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[2]; +}; + +struct xsc_modify_nic_vport_field_select { + u32 affiliation:1; + u32 disable_uc_local_lb:1; + u32 disable_mc_local_lb:1; + u32 node_guid:1; + u32 port_guid:1; + u32 min_inline:1; + u32 mtu:1; + u32 change_event:1; + u32 promisc:1; + u32 permanent_address:1; + u32 addresses_list:1; + u32 roce_en:1; + u32 rsvd:20; +}; + +struct xsc_modify_nic_vport_context_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 vport_number:16; + u32 rsvd:15; + + struct xsc_modify_nic_vport_field_select field_select; + struct xsc_nic_vport_context nic_vport_ctx; +}; + +struct xsc_query_hca_vport_context_out { + struct xsc_outbox_hdr hdr; + struct xsc_hca_vport_context hca_vport_ctx; +}; + +struct xsc_query_hca_vport_context_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 port_num:4; + u32 vport_number:16; + u32 rsvd0:11; +}; + +struct xsc_modify_hca_vport_context_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[2]; +}; + +struct xsc_modify_hca_vport_context_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 port_num:4; + u32 vport_number:16; + u32 rsvd0:11; + + struct xsc_hca_vport_context hca_vport_ctx; +}; + +struct xsc_array128 { + u8 array128[16]; +}; + +struct xsc_query_hca_vport_gid_out { + struct xsc_outbox_hdr hdr; + u16 gids_num; + struct xsc_array128 gid[0]; +}; + +struct xsc_query_hca_vport_gid_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 port_num:4; + u32 vport_number:16; + u32 rsvd0:11; + u16 gid_index; +}; + +struct xsc_pkey { + u16 pkey; +}; + +struct xsc_query_hca_vport_pkey_out { + struct xsc_outbox_hdr hdr; + struct xsc_pkey pkey[0]; +}; + +struct xsc_query_hca_vport_pkey_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 port_num:4; + u32 vport_number:16; + u32 rsvd0:11; + u16 pkey_index; +}; + +struct xsc_query_vport_state_out { + struct xsc_outbox_hdr hdr; + u8 admin_state:4; + u8 state:4; +}; + +struct xsc_query_vport_state_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 vport_number:16; + u32 rsvd0:15; +}; + +struct xsc_modify_vport_state_out { + struct xsc_outbox_hdr hdr; +}; + +struct xsc_modify_vport_state_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 vport_number:16; + u32 rsvd0:15; + u8 admin_state:4; + u8 rsvd1:4; +}; + +struct xsc_traffic_counter { + u64 packets; + u64 bytes; +}; + +struct xsc_query_vport_counter_out { + struct xsc_outbox_hdr hdr; + struct xsc_traffic_counter received_errors; + struct xsc_traffic_counter transmit_errors; + struct xsc_traffic_counter received_ib_unicast; + struct xsc_traffic_counter transmitted_ib_unicast; + struct xsc_traffic_counter received_ib_multicast; + struct xsc_traffic_counter transmitted_ib_multicast; + struct xsc_traffic_counter received_eth_broadcast; + struct xsc_traffic_counter transmitted_eth_broadcast; + struct xsc_traffic_counter received_eth_unicast; + struct xsc_traffic_counter transmitted_eth_unicast; + struct xsc_traffic_counter received_eth_multicast; + struct xsc_traffic_counter transmitted_eth_multicast; +}; + +struct xsc_query_vport_counter_in { + struct xsc_inbox_hdr hdr; + u32 other_vport:1; + u32 port_num:4; + u32 vport_number:16; + u32 rsvd0:11; +}; + +/* ioctl mbox */ +struct xsc_ioctl_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 len; + __be16 rsvd; + u8 data[0]; +}; + +struct xsc_ioctl_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 error; + __be16 len; + __be16 rsvd; + u8 data[0]; +}; + +struct xsc_modify_raw_qp_request { + u16 qpn; + u16 lag_id; + u16 func_id; + u8 dma_direct; + u8 prio; + u8 qp_out_port; + u8 rsvd[7]; +}; + +struct xsc_modify_raw_qp_mbox_in { + struct xsc_inbox_hdr hdr; + u8 pcie_no; + u8 rsv[7]; + struct xsc_modify_raw_qp_request req; +}; + +struct xsc_modify_raw_qp_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +#define ETH_ALEN 6 + +struct xsc_logic_port_info { + u8 pf; + u8 pcie; + u8 mac_port; /* mac physic port */ + u8 pcie_port; /* pcie physic port */ + u8 pf_id; + __be16 vf_id; + __be16 glb_func_id; /* function id */ + __be16 logic_port; /* logic port for pp */ + __be16 pf_logic_port; /* pf logic port */ + __be16 mac_logic_port; /* mac logic port */ + __be16 gsi_qpn; /* logic qpn for gsi*/ +}; + +struct xsc_lag_port_info { + struct xsc_logic_port_info info_mac; + u8 netdev_addr[ETH_ALEN]; + u8 gw_dmac[ETH_ALEN]; +}; + +struct xsc_create_lag_request { + struct xsc_lag_port_info info_mac0; + struct xsc_lag_port_info info_mac1; + u8 mp_lag; + u8 roce_lag; + u8 lag_num; + __be16 lag_id; + __be16 lag_start; + u8 lag_sel_mode; + u8 remap_port1; + u8 remap_port2; + u8 rsvd[2]; +}; + +struct xsc_modify_lag_request { + u8 mp_lag; + u8 roce_lag; + __be16 lag_id; + u8 lag_sel_mode; + u8 remap_port1; + u8 remap_port2; + u8 rsvd[2]; +}; + +struct xsc_destroy_lag_request { + __be16 lag_id; + u8 rsvd[2]; +}; + +struct xsc_set_lag_qos_request { + __be16 lag_id; + u8 member_bitmap; + u8 lag_del; + u8 pcie_no; + u8 esw_mode; + u8 resv[2]; +}; + +struct xsc_create_lag_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_create_lag_request req; +}; + +struct xsc_create_lag_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_modify_lag_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_modify_lag_request req; +}; + +struct xsc_modify_lag_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_destroy_lag_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_destroy_lag_request req; +}; + +struct xsc_destroy_lag_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_set_lag_qos_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_set_lag_qos_request req; +}; + +struct xsc_set_lag_qos_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +/*ioctl qos*/ +struct xsc_qos_req_prfx { + u8 mac_port; + u8 rsvd[7]; +}; + +struct xsc_qos_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_qos_req_prfx req_prfx; + u8 data[0]; +}; + +struct xsc_qos_mbox_out { + struct xsc_outbox_hdr hdr; + u8 data[0]; +}; + +struct xsc_prio_stats { + u64 tx_bytes; + u64 rx_bytes; + u64 tx_pkts; + u64 rx_pkts; +}; + +struct xsc_prio_stats_mbox_in { + struct xsc_inbox_hdr hdr; + u8 pport; +}; + +struct xsc_prio_stats_mbox_out { + struct xsc_outbox_hdr hdr; + struct xsc_prio_stats prio_stats[QOS_PRIO_MAX + 1]; +}; + +struct xsc_hw_stats { + /*by mac port*/ + u64 rdma_tx_pkts; + u64 rdma_tx_bytes; + u64 rdma_rx_pkts; + u64 rdma_rx_bytes; + u64 np_cnp_sent; + u64 rp_cnp_handled; + u64 np_ecn_marked_roce_packets; + u64 rp_cnp_ignored; + u64 tx_pause; + u64 rx_pause; + u64 rx_fcs_errors; + u64 rx_discards; + /*by function*/ + u64 out_of_sequence; + u64 packet_seq_err; + u64 out_of_buffer; + u64 rnr_nak_retry_err; + u64 local_ack_timeout_err; + u64 rx_read_requests; + u64 rx_write_requests; + u64 duplicate_requests; + u64 rdma_tx_pkts_func; + u64 rdma_tx_payload_bytes; + u64 rdma_rx_pkts_func; + u64 rdma_rx_payload_bytes; + /*by global*/ + u64 rdma_loopback_pkts; + u64 rdma_loopback_bytes; +}; + +struct xsc_hw_stats_mbox_in { + struct xsc_inbox_hdr hdr; + u8 mac_port; + u8 is_lag; + u8 lag_member_num; + u8 member_port[0]; +}; + +struct xsc_hw_stats_mbox_out { + struct xsc_outbox_hdr hdr; + struct xsc_hw_stats hw_stats; +}; + +struct xsc_dscp_pmt_set { + u8 dscp; + u8 priority; + u8 rsvd[6]; +}; + +struct xsc_dscp_pmt_get { + u8 prio_map[QOS_DSCP_MAX + 1]; + u8 max_prio; + u8 rsvd[7]; +}; + +struct xsc_trust_mode_set { + u8 is_pcp; + u8 rsvd[7]; +}; + +struct xsc_trust_mode_get { + u8 is_pcp; + u8 rsvd[7]; +}; + +struct xsc_pcp_pmt_set { + u8 pcp; + u8 priority; + u8 rsvd[6]; +}; + +struct xsc_pcp_pmt_get { + u8 prio_map[QOS_PCP_MAX + 1]; + u8 max_prio; + u8 rsvd[7]; +}; + +struct xsc_default_pri_set { + u8 priority; + u8 rsvd[7]; +}; + +struct xsc_default_pri_get { + u8 priority; + u8 rsvd[7]; +}; + +struct xsc_event_resp { + u16 resp_cmd_type; /* bitmap:0x0001: link up/down */ +}; + +struct xsc_event_linkstatus_resp { + u8 linkstatus; /*0:down, 1:up*/ + u8 linkspeed; /*0:25G, 1:100G*/ +}; + +struct xsc_event_query_type_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd[2]; +}; + +struct xsc_event_query_type_mbox_out { + struct xsc_outbox_hdr hdr; + struct xsc_event_resp ctx; +}; + +struct xsc_event_query_linkstatus_mbox_in { + struct xsc_inbox_hdr hdr; + u8 rsvd[2]; +}; + +struct xsc_event_query_linkstatus_mbox_out { + struct xsc_outbox_hdr hdr; + struct xsc_event_linkstatus_resp ctx; +}; + +struct xsc_event_set_led_status_mbox_in { + struct xsc_inbox_hdr hdr; + u8 port_id; +}; + +struct xsc_event_set_led_status_mbox_out { + struct xsc_outbox_hdr hdr; + u32 status; +}; + +#define PFC_ON_PG_PRFL_IDX 0 +#define PFC_OFF_PG_PRFL_IDX 1 +#define PFC_ON_QMU_VALUE 0 +#define PFC_OFF_QMU_VALUE 1 + +#define NIF_PFC_EN_ON 1 +#define NIF_PFC_EN_OFF 0 + +struct xsc_pfc_set { + u8 priority; + u8 pfc_on; + u8 rsvd[6]; +}; + +#define PFC_PRIO_MAX 7 +struct xsc_pfc_get { + u8 pfc_on[PFC_PRIO_MAX + 1]; + u8 max_prio; + u8 rsvd[7]; +}; + +struct xsc_rate_limit_set { + u32 rate_cir; + u32 limit_id; + u8 limit_level; + u8 rsvd[7]; +}; + +struct xsc_rate_limit_get { + u32 rate_cir[QOS_PRIO_MAX + 1]; + u32 max_limit_id; + u8 limit_level; + u8 rsvd[3]; +}; + +struct xsc_sp_set { + u8 sp[QOS_PRIO_MAX + 1]; +}; + +struct xsc_sp_get { + u8 sp[QOS_PRIO_MAX + 1]; + u8 max_prio; + u8 rsvd[7]; +}; + +struct xsc_weight_set { + u8 weight[QOS_PRIO_MAX + 1]; +}; + +struct xsc_weight_get { + u8 weight[QOS_PRIO_MAX + 1]; + u8 max_prio; + u8 rsvd[7]; +}; + +struct xsc_cc_mbox_in { + struct xsc_inbox_hdr hdr; + u8 data[0]; +}; + +struct xsc_cc_mbox_out { + struct xsc_outbox_hdr hdr; + u8 data[0]; +}; + +struct xsc_cc_ctrl_cmd { + u16 cmd; + u16 len; + u8 val[0]; +}; + +struct xsc_cc_cmd_enable_rp { + u16 cmd; + u16 len; + u32 enable; + u32 section; +}; + +struct xsc_cc_cmd_enable_np { + u16 cmd; + u16 len; + u32 enable; + u32 section; +}; + +struct xsc_cc_cmd_init_alpha { + u16 cmd; + u16 len; + u32 alpha; + u32 section; +}; + +struct xsc_cc_cmd_g { + u16 cmd; + u16 len; + u32 g; + u32 section; +}; + +struct xsc_cc_cmd_ai { + u16 cmd; + u16 len; + u32 ai; + u32 section; +}; + +struct xsc_cc_cmd_hai { + u16 cmd; + u16 len; + u32 hai; + u32 section; +}; + +struct xsc_cc_cmd_th { + u16 cmd; + u16 len; + u32 threshold; + u32 section; +}; + +struct xsc_cc_cmd_bc { + u16 cmd; + u16 len; + u32 bytecount; + u32 section; +}; + +struct xsc_cc_cmd_cnp_opcode { + u16 cmd; + u16 len; + u32 opcode; +}; + +struct xsc_cc_cmd_cnp_bth_b { + u16 cmd; + u16 len; + u32 bth_b; +}; + +struct xsc_cc_cmd_cnp_bth_f { + u16 cmd; + u16 len; + u32 bth_f; +}; + +struct xsc_cc_cmd_cnp_ecn { + u16 cmd; + u16 len; + u32 ecn; +}; + +struct xsc_cc_cmd_data_ecn { + u16 cmd; + u16 len; + u32 ecn; +}; + +struct xsc_cc_cmd_cnp_tx_interval { + u16 cmd; + u16 len; + u32 interval; // us + u32 section; +}; + +struct xsc_cc_cmd_evt_rsttime { + u16 cmd; + u16 len; + u32 period; +}; + +struct xsc_cc_cmd_cnp_dscp { + u16 cmd; + u16 len; + u32 dscp; + u32 section; +}; + +struct xsc_cc_cmd_cnp_pcp { + u16 cmd; + u16 len; + u32 pcp; + u32 section; +}; + +struct xsc_cc_cmd_evt_period_alpha { + u16 cmd; + u16 len; + u32 period; +}; + +struct xsc_cc_cmd_clamp_tgt_rate { + u16 cmd; + u16 len; + u32 clamp_tgt_rate; + u32 section; +}; + +struct xsc_cc_cmd_get_cfg { + u16 cmd; + u16 len; + u32 enable_rp; + u32 enable_np; + u32 init_alpha; + u32 g; + u32 ai; + u32 hai; + u32 threshold; + u32 bytecount; + u32 opcode; + u32 bth_b; + u32 bth_f; + u32 cnp_ecn; + u32 data_ecn; + u32 cnp_tx_interval; + u32 evt_period_rsttime; + u32 cnp_dscp; + u32 cnp_pcp; + u32 evt_period_alpha; + u32 clamp_tgt_rate; + u32 section; +}; + +struct xsc_cc_cmd_get_stat { + u16 cmd; + u16 len; + u32 section; +}; + +struct xsc_cc_cmd_stat { + u32 cnp_handled; + u32 alpha_recovery; + u32 reset_timeout; + u32 reset_bytecount; +}; + +struct xsc_set_mtu_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 mtu; + u8 mac_port; + u8 rsvd; +}; + +struct xsc_hwc_mbox_in { + struct xsc_inbox_hdr hdr; + u8 data[0]; +}; + +struct xsc_hwc_mbox_out { + struct xsc_outbox_hdr hdr; + u8 data[0]; +}; + +struct hwc_set_t { + u8 type; + u8 model; + u8 s_wqe_mode; + u8 r_wqe_mode; + u8 ack_timeout; + u8 group_mode; + u8 lossless_prio[XSC_MAX_MAC_NUM]; + u8 retry_cnt_th; + u8 adapt_to_other; + u8 rsvd[1]; + u16 max_vf_nums; +}; + +struct hwc_get_t { + u8 cur_model; + u8 next_model; + u8 cur_s_wqe_mode; + u8 next_s_wqe_mode; + u8 cur_r_wqe_mode; + u8 next_r_wqe_mode; + u8 cur_ack_timeout; + u8 next_ack_timeout; + u8 cur_group_mode; + u8 next_group_mode; + u8 cur_lossless_prio[XSC_MAX_MAC_NUM]; + u8 next_lossless_prio[XSC_MAX_MAC_NUM]; + u8 cur_retry_cnt_th; + u8 next_retry_cnt_th; + u8 cur_adapt_to_other; + u8 next_adapt_to_other; + u16 cur_max_vf_nums; + u16 next_max_vf_nums; +}; + +struct xsc_set_mtu_mbox_out { + struct xsc_outbox_hdr hdr; +}; + +struct xsc_query_eth_mac_mbox_in { + struct xsc_inbox_hdr hdr; + u8 index; +}; + +struct xsc_query_eth_mac_mbox_out { + struct xsc_outbox_hdr hdr; + u8 mac[6]; +}; + +struct xsc_query_pause_cnt_mbox_in { + struct xsc_inbox_hdr hdr; + u16 mac_port; + u16 cnt_type; + u32 reg_addr; +}; + +struct xsc_query_pause_cnt_mbox_out { + struct xsc_outbox_hdr hdr; + u64 val; +}; + +enum { + XSC_TBM_CAP_IPAT_BYPASS = 0, + XSC_TBM_CAP_PCT_BYPASS, + XSC_TBM_CAP_HASH_PPH, + XSC_TBM_CAP_RSS, + XSC_TBM_CAP_BC_BYPASS, +}; + +struct xsc_nic_attr { + struct xsc_logic_port_info info; + __be16 caps; + __be16 caps_mask; + u8 mac_addr[6]; +}; + +struct xsc_rss_attr { + u8 rss_en; + u8 hfunc; + __be16 rqn_base; + __be16 rqn_num; + __be32 hash_tmpl; +}; + +struct xsc_cmd_enable_nic_hca_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_nic_attr nic; + struct xsc_rss_attr rss; +}; + +struct xsc_enable_nic_hca_resp { + __be16 bomt_idx; +}; + +struct xsc_cmd_enable_nic_hca_mbox_out { + struct xsc_outbox_hdr hdr; + struct xsc_enable_nic_hca_resp res; + u8 status; + u8 rsvd0; +}; + +struct xsc_nic_dis_attr { + struct xsc_logic_port_info info; + __be16 caps; +}; + +struct xsc_broadcast_dis_attr { + __be16 bomt_idx; +}; + +struct xsc_cmd_disable_nic_hca_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_nic_dis_attr nic; + struct xsc_broadcast_dis_attr bc; +}; + +struct xsc_cmd_disable_nic_hca_mbox_out { + struct xsc_outbox_hdr hdr; + u8 status; + u8 rsvd0[3]; +}; + +enum { + XSC_RSS_HASH_KEY_UPDATE = 0, + XSC_RSS_HASH_TEMP_UPDATE, + XSC_RSS_HASH_FUNC_UPDATE, + XSC_RSS_RXQ_UPDATE, + XSC_RSS_RXQ_DROP, +}; + +struct xsc_rss_modify_attr { + u8 caps_mask; + u8 rss_en; + __be16 rqn_base; + __be16 rqn_num; + u8 hfunc; + __be32 hash_tmpl; + u8 hash_key[52]; +}; + +struct xsc_cmd_modify_nic_hca_mbox_in { + struct xsc_inbox_hdr hdr; + struct xsc_nic_attr nic; + struct xsc_rss_modify_attr rss; +}; + +struct xsc_cmd_modify_nic_hca_mbox_out { + struct xsc_outbox_hdr hdr; + u8 status; + u8 rsvd0[3]; +}; + +struct xsc_function_reset_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 glb_func_id; + u8 rsvd[6]; +}; + +struct xsc_function_reset_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_reg_mcia { + u8 module; + u8 status; + + u8 i2c_device_address; + u8 page_number; + u8 device_address; + + u8 size; + + u8 dword_0[0x20]; + u8 dword_1[0x20]; + u8 dword_2[0x20]; + u8 dword_3[0x20]; + u8 dword_4[0x20]; + u8 dword_5[0x20]; + u8 dword_6[0x20]; + u8 dword_7[0x20]; + u8 dword_8[0x20]; + u8 dword_9[0x20]; + u8 dword_10[0x20]; + u8 dword_11[0x20]; +}; + +struct xsc_rtt_en_mbox_in { + struct xsc_inbox_hdr hdr; + u8 en;//0-disable, 1-enable + u8 rsvd[7]; +}; + +struct xsc_rtt_en_mbox_out { + struct xsc_outbox_hdr hdr; + u8 en;//0-disable, 1-enable + u8 rsvd[7]; +}; + +struct xsc_rtt_qpn_mbox_in { + struct xsc_inbox_hdr hdr; + __be16 qpn[32]; +}; + +struct xsc_rtt_qpn_mbox_out { + struct xsc_outbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_get_rtt_qpn_mbox_out { + struct xsc_outbox_hdr hdr; + __be16 qpn[32]; +}; + +struct xsc_rtt_period_mbox_in { + struct xsc_inbox_hdr hdr; + __be32 period; //ms +}; + +struct xsc_rtt_period_mbox_out { + struct xsc_outbox_hdr hdr; + __be32 period; //ms + u8 rsvd[4]; +}; + +struct xsc_rtt_result_mbox_out { + struct xsc_outbox_hdr hdr; + __be64 result[32]; +}; + +struct rtt_stats { + u64 rtt_succ_snd_req_cnt; + u64 rtt_succ_snd_rsp_cnt; + u64 rtt_fail_snd_req_cnt; + u64 rtt_fail_snd_rsp_cnt; + u64 rtt_rcv_req_cnt; + u64 rtt_rcv_rsp_cnt; + u64 rtt_rcv_unk_cnt; + u64 rtt_grp_invalid_cnt; +}; + +struct xsc_rtt_stats_mbox_out { + struct xsc_outbox_hdr hdr; + struct rtt_stats stats; +}; + +#endif /* XSC_CMD_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_core.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_core.h new file mode 100644 index 0000000000000000000000000000000000000000..f4c89d502c823c03b8112940398644a46967410f --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_core.h @@ -0,0 +1,1132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_CORE_H +#define XSC_CORE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef USE_VIRTIO +#include +#include +#include +#endif /* USE_VIRTIO */ + +#include +#include +#include +#include +#include +#include +#include + +extern uint xsc_debug_mask; + +#ifndef mmiowb +#define mmiowb() +#endif + +#define REG_ADDR(dev, offset) \ + (xsc_core_is_pf(dev) ? ((dev->bar2) + ((offset) - 0xA0000000)) : ((dev->bar2) + (offset))) + +#define REG_WIDTH_TO_STRIDE(width) ((width) / 8) +#define QPM_PAM_TBL_NUM 4 +#define QPM_PAM_TBL_NUM_MASK 3 +#define QPM_PAM_TBL_INDEX_SHIFT 2 +#define QPM_PAM_PAGE_SHIFT 12 + +#define xsc_core_dbg(__dev, format, ...) \ + dev_dbg(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) + +#define xsc_core_dbg_once(__dev, format, ...) \ + dev_dbg_once(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) + +#define xsc_core_dbg_mask(__dev, mask, format, ...) \ +do { \ + if ((mask) & xsc_debug_mask) \ + xsc_core_dbg(__dev, format, ##__VA_ARGS__); \ +} while (0) + +#define xsc_core_err(__dev, format, ...) \ + dev_err(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) + +#define xsc_core_err_rl(__dev, format, ...) \ + dev_err_ratelimited(&(__dev)->pdev->dev, \ + "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) + +#define xsc_core_warn(__dev, format, ...) \ + dev_warn(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) + +#define xsc_core_info(__dev, format, ...) \ + dev_info(&(__dev)->pdev->dev, format, ##__VA_ARGS__) + +#define assert(__dev, expr) \ +do { \ + if (!(expr)) { \ + dev_err(&(__dev)->pdev->dev, \ + "Assertion failed! %s, %s, %s, line %d\n", \ + #expr, __FILE__, __func__, __LINE__); \ + } \ +} while (0) + +#define IS_ALIGNED(x, a) (((x) & ((typeof(x))(a) - 1)) == 0) + +enum xsc_dev_event { + XSC_DEV_EVENT_SYS_ERROR, + XSC_DEV_EVENT_PORT_UP, + XSC_DEV_EVENT_PORT_DOWN, + XSC_DEV_EVENT_PORT_INITIALIZED, + XSC_DEV_EVENT_LID_CHANGE, + XSC_DEV_EVENT_PKEY_CHANGE, + XSC_DEV_EVENT_GUID_CHANGE, + XSC_DEV_EVENT_CLIENT_REREG, +}; + +enum { + /* one minute for the sake of bringup. Generally, commands must always + * complete and we may need to increase this timeout value + */ + XSC_CMD_TIMEOUT_MSEC = 10 * 1000, + XSC_CMD_WQ_MAX_NAME = 32, +}; + +enum { + XSC_MAX_NAME_LEN = 32, +}; + +enum { + XSC_MAX_PORTS = 2, +}; + +enum { + MAX_MR_CACHE_ENTRIES = 16, +}; + +enum { + XSC_CMD_DATA, /* print command payload only */ + XSC_CMD_TIME, /* print command execution time */ +}; + +enum xsc_rdma_driver_id { + RDMA_DRIVER_XSC_UNKNOWN, + RDMA_DRIVER_XSC5, + RDMA_DRIVER_XSC4, +}; + +static LIST_HEAD(intf_list); +static LIST_HEAD(xsc_dev_list); +/* mutex for interface device list */ +static DEFINE_MUTEX(xsc_intf_mutex); + +#define GROUP_REFER_CNT_SIZE 1024 + +struct qp_group_refer { + spinlock_t lock; + u16 refer_cnt[GROUP_REFER_CNT_SIZE]; +}; + +struct xsc_priv_device { + struct class *priv_class; + char device_name[IB_DEVICE_NAME_MAX]; + dev_t devno; + struct cdev cdev; + struct list_head mem_list; + spinlock_t mem_lock; +}; + +struct xsc_dpdk_mem { + struct list_head mem_list; + spinlock_t mem_lock; +}; + +enum xsc_pci_status { + XSC_PCI_STATUS_DISABLED, + XSC_PCI_STATUS_ENABLED, +}; + +enum xsc_device_state { + XSC_DEVICE_STATE_UNINITIALIZED, + XSC_DEVICE_STATE_UP, + XSC_DEVICE_STATE_INTERNAL_ERROR, +}; + +enum xsc_interface_state { + XSC_INTERFACE_STATE_UP = BIT(0), + XSC_INTERFACE_STATE_TEARDOWN = BIT(1), +}; + +enum { + XSC_INTERFACE_PROTOCOL_IB = 0, + XSC_INTERFACE_PROTOCOL_ETH = 1, +}; + +enum { + XSC_INTERFACE_ADDED, + XSC_INTERFACE_ATTACHED, +}; + +#define CONFIG_XSC_SRIOV 1 +#define CONFIG_XSC_ESWITCH 1 + +enum xsc_coredev_type { + XSC_COREDEV_PF, + XSC_COREDEV_VF, + XSC_COREDEV_SF +}; + +enum { + XSC_PCI_DEV_IS_VF = 1 << 0, +}; + +enum port_state_policy { + XSC_POLICY_DOWN = 0, + XSC_POLICY_UP = 1, + XSC_POLICY_FOLLOW = 2, + XSC_POLICY_INVALID = 0xffffffff +}; + +enum { + XSC_CAP_PORT_TYPE_IB = 0x0, + XSC_CAP_PORT_TYPE_ETH = 0x1, +}; + +enum xsc_list_type { + XSC_NVPRT_LIST_TYPE_UC = 0x0, + XSC_NVPRT_LIST_TYPE_MC = 0x1, + XSC_NVPRT_LIST_TYPE_VLAN = 0x2, +}; + +enum xsc_inline_modes { + XSC_INLINE_MODE_NONE, + XSC_INLINE_MODE_L2, + XSC_INLINE_MODE_IP, + XSC_INLINE_MODE_TCP_UDP, +}; + +struct xsc_core_device; + +struct xsc_vf_context { + int enabled; + u64 port_guid; + u64 node_guid; + enum port_state_policy policy; +}; + +struct xsc_sriov_vf { + struct xsc_core_device *dev; + struct kobject kobj; + int vf; +}; + +struct xsc_pci_sriov { + /* standard SRIOV capability fields, mostly for debug */ + int pos; /* capability position */ + int nres; /* number of resources */ + u32 cap; /* SR-IOV Capabilities */ + u16 ctrl; /* SR-IOV Control */ + u16 total_vfs; /* total VFs of PF */ + u16 initial_vfs; /* initial VFs of PF */ + u16 num_vfs; /* number of VFs available */ + u16 offset; /* first VF Routing ID offset */ + u16 stride; /* following VF stride */ + u16 vf_device; /* VF device ID */ + u32 pgsz; /* page size for BAR alignment */ + u8 link; /* Function Dependency Link */ +}; + +struct xsc_core_sriov { + int num_vfs; + u16 max_vfs; + u16 vf_bdf_base; + bool probe_vf; + struct xsc_vf_context *vfs_ctx; + struct kobject *config; + struct kobject *groups_config; + struct kobject node_guid_kobj; + struct xsc_sriov_vf *vfs; + struct xsc_pci_sriov pci_sriov; +}; + +struct xsc_vgroup { + struct xsc_core_device *dev; + u32 group_id; + u32 num_vports; + u32 tsar_ix; + u32 max_rate; + u32 min_rate; + u32 bw_share; + struct kobject kobj; + struct list_head list; +}; + +struct xsc_vf_info { + u16 vf_id; //start from 1, 0 is reserved for pf + u16 phy_port; //pcie0=0, pcie1=1 + u16 pf_id; //pf0=0, pf1=1 + u32 func_id; + u32 logic_port; +}; + +struct xsc_vport_info { + u8 mac[ETH_ALEN]; + u16 vlan; + u8 qos; + __be16 vlan_proto; + u64 node_guid; + int link_state; + u32 min_rate; + u32 max_rate; + bool spoofchk; + bool trusted; + bool roce; + /* the admin approved vlan list */ + DECLARE_BITMAP(vlan_trunk_8021q_bitmap, VLAN_N_VID); + u32 group; +}; + +#define XSC_L2_ADDR_HASH_SIZE 8 + +enum xsc_eswitch_vport_event { + XSC_VPORT_UC_ADDR_CHANGE = BIT(0), + XSC_VPORT_MC_ADDR_CHANGE = BIT(1), + XSC_VPORT_PROMISC_CHANGE = BIT(2), + XSC_VPORT_VLAN_CHANGE = BIT(3), +}; + +struct xsc_vport { + struct xsc_core_device *dev; + u16 vport; + struct hlist_head uc_list[XSC_L2_ADDR_HASH_SIZE]; + struct hlist_head mc_list[XSC_L2_ADDR_HASH_SIZE]; + /* The requested vlan list from the vport side */ + DECLARE_BITMAP(req_vlan_bitmap, VLAN_N_VID); + /* Actual accepted vlans on the acl tables */ + DECLARE_BITMAP(acl_vlan_8021q_bitmap, VLAN_N_VID); +// struct xsc_flow_handle *promisc_rule; +// struct xsc_flow_handle *allmulti_rule; + struct work_struct vport_change_handler; + +// struct vport_ingress ingress; +// struct vport_egress egress; + + struct xsc_vport_info info; + + struct { + bool enabled; + u32 esw_tsar_ix; + u32 bw_share; + u32 min_rate; + u32 max_rate; +// struct xsc_vgroup *group; + } qos; + + bool enabled; + enum xsc_eswitch_vport_event enabled_events; + u16 match_id; + u32 bond_metadata; +}; + +struct xsc_eswitch { + struct xsc_core_device *dev; + u32 flags; + int total_vports; + int enabled_vports; + int num_vfs; + struct xsc_vport *vports; + struct workqueue_struct *work_queue; + + /* Synchronize between vport change events + * and async SRIOV admin state changes + */ + struct mutex state_lock; + + /* Protects eswitch mode changes occurring via sriov + * state change, devlink commands. + */ + struct mutex mode_lock; + int mode; + int nvports; + u16 manager_vport; + u16 first_host_vport; +}; + +struct xsc_core_health { + bool sick; +}; + +struct xsc_priv { + char name[XSC_MAX_NAME_LEN]; + struct list_head dev_list; + struct list_head ctx_list; + spinlock_t ctx_lock; + int numa_node; + struct xsc_core_sriov sriov; + struct xsc_eswitch *eswitch; + struct xsc_lag *lag; + struct xsc_core_health health; +}; + +struct xsc_port_ctrl { + struct list_head node; + dev_t devid; + struct cdev cdev; + struct device *device; +}; + +struct xsc_port_caps { + int gid_table_len; + int pkey_table_len; +}; + +struct xsc_caps { + u8 log_max_eq; + u8 log_max_cq; + u8 log_max_qp; + u8 log_max_mkey; + u8 log_max_pd; + u8 log_max_srq; + u8 log_max_msix; + u32 max_cqes; + u32 max_wqes; + u32 max_sq_desc_sz; + u32 max_rq_desc_sz; + u64 flags; + u16 stat_rate_support; + u32 log_max_msg; + u32 num_ports; + u32 max_ra_res_qp; + u32 max_ra_req_qp; + u32 max_srq_wqes; + u32 bf_reg_size; + u32 bf_regs_per_page; + struct xsc_port_caps port[XSC_MAX_PORTS]; + u8 ext_port_cap[XSC_MAX_PORTS]; + u32 reserved_lkey; + u8 local_ca_ack_delay; + u8 log_max_mcg; + u16 max_qp_mcg; + u32 min_page_sz; + u32 send_ds_num; + u32 send_wqe_shift; + u32 recv_ds_num; + u32 recv_wqe_shift; + u32 rx_pkt_len_max; + + u32 msix_enable:1; + u32 port_type:1; + u32 embedded_cpu:1; + u32 eswitch_manager:1; + u32 ecpf_vport_exists:1; + u32 vport_group_manager:1; + u32 sf:1; + u32 wqe_inline_mode:3; + u32 raweth_qp_id_base:15; + u32 rsvd0:7; + + u16 max_vfs; + u8 log_max_qp_depth; + u8 log_max_current_uc_list; + u8 log_max_current_mc_list; + u16 log_max_vlan_list; + u8 fdb_multi_path_to_table; + u8 log_esw_max_sched_depth; + + u8 max_num_sf_partitions; + u8 log_max_esw_sf; + u16 sf_base_id; + + u32 max_tc:8; + u32 ets:1; + u32 dcbx:1; + u32 dscp:1; + u32 sbcam_reg:1; + u32 qos:1; + u32 port_buf:1; + u32 rsvd1:2; + u32 raw_tpe_qp_num:16; + u32 max_num_eqs:8; + u32 mac_port:8; + u32 raweth_rss_qp_id_base:16; + u16 msix_base; + u16 msix_num; + u8 log_max_mtt; + u8 log_max_tso; + u32 hca_core_clock; + u32 max_rwq_indirection_tables;/*rss_caps*/ + u32 max_rwq_indirection_table_size;/*rss_caps*/ + u16 raweth_qp_id_end; + u32 qp_rate_limit_min; + u32 qp_rate_limit_max; +}; + +struct cache_ent { + /* protect block chain allocations + */ + spinlock_t lock; + struct list_head head; +}; + +struct cmd_msg_cache { + struct cache_ent large; + struct cache_ent med; + +}; + +#define CMD_FIRST_SIZE 8 +struct xsc_cmd_first { + __be32 data[CMD_FIRST_SIZE]; +}; + +struct xsc_cmd_mailbox { + void *buf; + dma_addr_t dma; + struct xsc_cmd_mailbox *next; +}; + +struct xsc_cmd_msg { + struct list_head list; + struct cache_ent *cache; + u32 len; + struct xsc_cmd_first first; + struct xsc_cmd_mailbox *next; +}; + +#define RSP_FIRST_SIZE 14 +struct xsc_rsp_first { + __be32 data[RSP_FIRST_SIZE]; //can be larger, xsc_rsp_layout +}; + +struct xsc_rsp_msg { + struct list_head list; + struct cache_ent *cache; + u32 len; + struct xsc_rsp_first first; + struct xsc_cmd_mailbox *next; +}; + +typedef void (*xsc_cmd_cbk_t)(int status, void *context); + +//hw will use this for some records(e.g. vf_id) +struct cmdq_rsv { + u16 vf_id; + u8 rsv[2]; +}; + +//related with hw, won't change +#define CMDQ_ENTRY_SIZE 64 + +struct xsc_cmd_layout { + struct cmdq_rsv rsv0; + __be32 inlen; + __be64 in_ptr; + __be32 in[CMD_FIRST_SIZE]; + __be64 out_ptr; + __be32 outlen; + u8 token; + u8 sig; + u8 idx; + u8 type: 7; + u8 owner_bit: 1; //rsv for hw, arm will check this bit to make sure mem written +}; + +//static_assert(sizeof(struct xsc_cmd_layout) == CMDQ_ENTRY_SIZE, "xsc_cmd_layout wrong size"); + +struct xsc_rsp_layout { + struct cmdq_rsv rsv0; + __be32 out[RSP_FIRST_SIZE]; + u8 token; + u8 sig; + u8 idx; + u8 type: 7; + u8 owner_bit: 1; //rsv for hw, driver will check this bit to make sure mem written +}; + +//static_assert(sizeof(struct xsc_rsp_layout) == CMDQ_ENTRY_SIZE, "xsc_rsp_layout wrong size"); + +struct xsc_cmd_work_ent { + struct xsc_cmd_msg *in; + struct xsc_rsp_msg *out; + xsc_cmd_cbk_t callback; + void *context; + int idx; + struct completion done; + struct xsc_cmd *cmd; + struct work_struct work; + struct xsc_cmd_layout *lay; + struct xsc_rsp_layout *rsp_lay; + int ret; + int page_queue; + u8 status; + u8 token; + struct timespec64 ts1; + struct timespec64 ts2; +}; + +struct xsc_cmd_debug { + struct dentry *dbg_root; + struct dentry *dbg_in; + struct dentry *dbg_out; + struct dentry *dbg_outlen; + struct dentry *dbg_status; + struct dentry *dbg_run; + void *in_msg; + void *out_msg; + u8 status; + u16 inlen; + u16 outlen; +}; + +struct xsc_cmd_stats { + u64 sum; + u64 n; + struct dentry *root; + struct dentry *avg; + struct dentry *count; + /* protect command average calculations */ + spinlock_t lock; +}; + +struct xsc_cmd_reg { + u32 req_pid_addr; + u32 req_cid_addr; + u32 rsp_pid_addr; + u32 rsp_cid_addr; + u32 req_buf_h_addr; + u32 req_buf_l_addr; + u32 rsp_buf_h_addr; + u32 rsp_buf_l_addr; + u32 msix_vec_addr; + u32 element_sz_addr; + u32 q_depth_addr; + u32 interrupt_stat_addr; +}; + +struct xsc_cmd { + struct xsc_cmd_reg reg; + void *cmd_buf; + void *cq_buf; + dma_addr_t dma; + dma_addr_t cq_dma; + u16 cmd_pid; + u16 cq_cid; + u8 owner_bit; + u8 cmdif_rev; + u8 log_sz; + u8 log_stride; + int max_reg_cmds; + int events; + u32 __iomem *vector; + + /* protect command queue allocations + */ + spinlock_t alloc_lock; + + /* protect token allocations + */ + spinlock_t token_lock; + spinlock_t doorbell_lock; + u8 token; + unsigned long bitmask; + char wq_name[XSC_CMD_WQ_MAX_NAME]; + struct workqueue_struct *wq; + struct task_struct *cq_task; + struct semaphore sem; + struct semaphore pages_sem; + int mode; + struct xsc_cmd_work_ent *ent_arr[XSC_MAX_COMMANDS]; + struct pci_pool *pool; + struct xsc_cmd_debug dbg; + struct cmd_msg_cache cache; + int checksum_disabled; + struct xsc_cmd_stats stats[XSC_CMD_OP_MAX]; + unsigned int irqn; + u8 ownerbit_learned; +}; + +struct xsc_profile { + u64 mask; + u32 log_max_qp; + int cmdif_csum; + struct { + int size; + int limit; + } mr_cache[MAX_MR_CACHE_ENTRIES]; +}; + +struct xsc_lock { + spinlock_t lock; +}; + +struct xsc_reg_addr { + u64 tx_db; + u64 rx_db; + u64 complete_db; + u64 complete_reg; + u64 event_db; + u64 cpm_get_lock; + u64 cpm_put_lock; + u64 cpm_lock_avail; + u64 cpm_data_mem; + u64 cpm_cmd; + u64 cpm_addr; + u64 cpm_busy; +}; + +/* our core device */ +struct xsc_core_device { + struct pci_dev *pdev; + struct device *device; + struct xsc_priv priv; + struct xsc_port_ctrl port_ctrl; + struct xsc_dev_resource *dev_res; +#ifdef USE_VIRTIO + struct device virtio_dev; +#endif /* USE_VIRTIO */ + void *xsc_ib_dev; + void *netdev; + void *eth_priv; + void *ovs_priv; +#ifdef USE_VIRTIO + void __iomem *bar0; +#endif /* USE_VIRTIO */ + void __iomem *bar2; + int bar_num; + + u16 bus_id; + u16 dev_id; + u16 func_id; + + u8 pf; + u8 pcie; + u8 mac_port; /* mac physic port */ + u8 pcie_port; /* pcie physic port */ + u8 pf_id; + u16 vf_id; + u16 glb_func_id; /* function id */ + u16 logic_port; /* logic port for pp */ + u16 pf_logic_port; /* pf logic port */ + u16 mac_logic_port; /* mac logic port */ + u16 gsi_qpn; /* logic qpn for gsi*/ + + u16 bomt_idx; + bool vf_pp_init; + + u16 msix_vec_base; + + struct mutex pci_status_mutex; + enum xsc_pci_status pci_status; + u32 board_id; + char board_sn[XSC_BOARD_SN_LEN]; + struct mutex intf_state_mutex; + unsigned long intf_state; + enum xsc_coredev_type coredev_type; + struct xsc_caps caps; + atomic_t num_qps; + struct xsc_cmd cmd; + struct xsc_lock reg_access_lock; + bool rdma_ready; + struct xsc_profile *profile; + + void *counters_priv; + struct xsc_priv_device priv_device; + struct xsc_dpdk_mem dpdk_mem; + void (*event)(struct xsc_core_device *dev, + enum xsc_dev_event event, unsigned long param); + + void (*event_handler)(void *adapter); + + struct xsc_reg_addr regs; + u32 chip_ver_h; + u32 chip_ver_m; + u32 chip_ver_l; + u32 hotfix_num; + u32 feature_flag; + u16 cmdq_ver; + cpumask_var_t xps_cpumask; + + void *rtt_priv; +}; + +struct xsc_feature_flag { + u8 fpga_type:2; + u8 hps_ddr:2; + u8 onchip_ft:1; + u8 rdma_icrc:1; + u8 ma_xbar:1; + u8 anlt_fec:1; + u8 pp_tbl_dma:1; + u8 pct_exp:1; +}; + +struct xsc_interface { + struct list_head list; + int protocol; + + void *(*add)(struct xsc_core_device *dev); + void (*remove)(struct xsc_core_device *dev, void *context); + int (*attach)(struct xsc_core_device *dev, void *context); + void (*detach)(struct xsc_core_device *dev, void *context); + void (*event)(struct xsc_core_device *dev, void *context, + enum xsc_dev_event event, unsigned long param); + void *(*get_dev)(void *context); +}; + +struct xsc_device_context { + struct list_head list; + struct xsc_interface *intf; + void *context; + unsigned long state; +}; + +struct xsc_mem_entry { + struct list_head list; + char task_name[TASK_COMM_LEN]; + struct xsc_ioctl_mem_info mem_info; +}; + +int xsc_debugfs_init(struct xsc_core_device *dev); +void xsc_debugfs_fini(struct xsc_core_device *dev); +void xsc_register_debugfs(void); +void xsc_unregister_debugfs(void); + +bool xsc_device_registered(struct xsc_core_device *dev); +int xsc_register_device(struct xsc_core_device *dev); +void xsc_unregister_device(struct xsc_core_device *dev); +void xsc_attach_device(struct xsc_core_device *dev); +void xsc_detach_device(struct xsc_core_device *dev); +int xsc_register_interface(struct xsc_interface *intf); +void xsc_unregister_interface(struct xsc_interface *intf); +void xsc_reload_interface(struct xsc_core_device *dev, int protocol); +void xsc_reload_interfaces(struct xsc_core_device *dev, + int protocol1, int protocol2, + bool valid1, bool valid2); +struct xsc_core_device *xsc_get_next_phys_dev(struct xsc_core_device *dev); +void xsc_remove_dev_by_protocol(struct xsc_core_device *dev, int protocol); +void xsc_add_dev_by_protocol(struct xsc_core_device *dev, int protocol); + +int xsc_cmd_write_reg_directly(struct xsc_core_device *dev, void *in, int in_size, void *out, + int out_size, int func_id); +int xsc_cmd_exec(struct xsc_core_device *dev, void *in, int in_size, + void *out, int out_size); +int xsc_create_mkey(struct xsc_core_device *xdev, void *in, void *out); +int xsc_destroy_mkey(struct xsc_core_device *xdev, void *in, void *out); +int xsc_reg_mr(struct xsc_core_device *dev, void *in, void *out); +int xsc_dereg_mr(struct xsc_core_device *dev, void *in, void *out); +int xsc_board_reset(struct xsc_core_device *dev, int pcie_no); +int xsc_eth_reset(struct xsc_core_device *dev); +int xsc_tbm_init(struct xsc_core_device *dev); +int xsc_qos_init(struct xsc_core_device *xdev); + +bool xsc_chk_chip_ver(struct xsc_core_device *dev); + +int xsc_alloc_iae_idx(struct xsc_core_device *dev, int *iae_idx); +void xsc_release_iae_idx(struct xsc_core_device *dev, int *iae_idx); +int xsc_get_iae_idx(struct xsc_core_device *dev); + +int xsc_create_res(struct xsc_core_device *dev); +void xsc_destroy_res(struct xsc_core_device *dev); + +int xsc_counters_init(struct ib_device *ib_dev, + struct xsc_core_device *dev); +void xsc_counters_fini(struct ib_device *ib_dev, + struct xsc_core_device *dev); + +int xsc_priv_dev_init(struct ib_device *ib_dev, struct xsc_core_device *dev); +void xsc_priv_dev_fini(struct ib_device *ib_dev, struct xsc_core_device *dev); + +int xsc_rtt_sysfs_init(struct ib_device *ib_dev, struct xsc_core_device *xdev); +void xsc_rtt_sysfs_fini(struct xsc_core_device *xdev); + +int xsc_cmd_query_hca_cap(struct xsc_core_device *dev, + struct xsc_caps *caps); +int xsc_cmd_init_hca(struct xsc_core_device *dev); +int xsc_cmd_teardown_hca(struct xsc_core_device *dev); +int xsc_cmd_enable_hca(struct xsc_core_device *dev, u16 vf_idx); +int xsc_cmd_disable_hca(struct xsc_core_device *dev, u16 vf_idx); +int xsc_get_board_id(struct xsc_core_device *dev); + +int xsc_irq_eq_create(struct xsc_core_device *dev); +int xsc_irq_eq_destroy(struct xsc_core_device *dev); + +int xsc_sriov_init(struct xsc_core_device *dev); +void xsc_sriov_cleanup(struct xsc_core_device *dev); +int xsc_sriov_attach(struct xsc_core_device *dev); +void xsc_sriov_detach(struct xsc_core_device *dev); +int xsc_core_sriov_configure(struct pci_dev *dev, int num_vfs); +int xsc_sriov_sysfs_init(struct xsc_core_device *dev); +void xsc_sriov_sysfs_cleanup(struct xsc_core_device *dev); +int xsc_create_vfs_sysfs(struct xsc_core_device *dev, int num_vfs); +void xsc_destroy_vfs_sysfs(struct xsc_core_device *dev, int num_vfs); +int xsc_create_vf_group_sysfs(struct xsc_core_device *dev, + u32 group_id, struct kobject *group_kobj); +void xsc_destroy_vf_group_sysfs(struct xsc_core_device *dev, + struct kobject *group_kobj); +void xsc_pci_get_vf_info(struct xsc_core_device *dev, + struct xsc_vf_info *info); +int xsc_get_pcie_no(void); +u32 xsc_eth_pcie_read32_by_mac_port(struct xsc_core_device *xdev, u32 mac_port, + u32 eth_ip_inter_addr); +void xsc_eth_pcie_write32_by_mac_port(struct xsc_core_device *xdev, u32 mac_port, + u32 eth_ip_inter_addr, u32 val); +struct cpumask *xsc_comp_irq_get_affinity_mask(struct xsc_core_device *dev, int vector); +void mask_cpu_by_node(int node, struct cpumask *dstp); + +#define XSC_ESWITCH_MANAGER(dev) (dev->caps.eswitch_manager) + +static inline bool xsc_sriov_is_enabled(struct xsc_core_device *dev) +{ + return pci_num_vf(dev->pdev) ? true : false; +} + +static inline u16 xsc_core_max_vfs(const struct xsc_core_device *dev) +{ + return dev->priv.sriov.max_vfs; +} + +static inline int xsc_core_vfs_num(const struct xsc_core_device *dev) +{ + return dev->priv.sriov.num_vfs; +} + +static inline bool xsc_core_is_pf(const struct xsc_core_device *dev) +{ + return dev->coredev_type == XSC_COREDEV_PF; +} + +static inline bool xsc_core_is_sf(const struct xsc_core_device *dev) +{ + return dev->coredev_type == XSC_COREDEV_SF; +} + +static inline bool xsc_core_is_ecpf(struct xsc_core_device *dev) +{ + return dev->caps.embedded_cpu; +} + +#define XSC_ESWITCH_MANAGER(dev) (dev->caps.eswitch_manager) +#define ESW_ALLOWED(esw) ((esw) && XSC_ESWITCH_MANAGER((esw)->dev)) + +static inline bool +xsc_core_is_ecpf_esw_manager(const struct xsc_core_device *dev) +{ + return dev->caps.embedded_cpu && dev->caps.eswitch_manager; +} + +static inline bool +xsc_ecpf_vport_exists(const struct xsc_core_device *dev) +{ + return xsc_core_is_pf(dev) && dev->caps.ecpf_vport_exists; +} + +static inline bool +xsc_core_is_vport_manager(const struct xsc_core_device *dev) +{ + return dev->caps.vport_group_manager && xsc_core_is_pf(dev); +} + +static inline bool xsc_rl_is_supported(struct xsc_core_device *dev) +{ + //return !!(dev->eth_priv.rl_table.max_size); + return false; +} + +/* define in andes */ +#define HIF_CPM_IDA_DATA_MEM_STRIDE 0x40 + +#define CPM_IAE_CMD_READ 0 +#define CPM_IAE_CMD_WRITE 1 + +#define CPM_IAE_ADDR_REG_STRIDE HIF_CPM_IDA_ADDR_REG_STRIDE + +#define CPM_IAE_DATA_MEM_STRIDE HIF_CPM_IDA_DATA_MEM_STRIDE + +#define CPM_IAE_DATA_MEM_MAX_LEN 16 + +struct iae_cmd { + union { + struct { + u32 iae_idx:HIF_CPM_IDA_CMD_REG_IDA_IDX_WIDTH; + u32 iae_len:HIF_CPM_IDA_CMD_REG_IDA_LEN_WIDTH; + u32 iae_r0w1:HIF_CPM_IDA_CMD_REG_IDA_R0W1_WIDTH; + }; + unsigned int raw_data; + }; +}; + +static inline void acquire_ia_lock(struct xsc_core_device *xdev, int *iae_idx) +{ + int lock_val; + int lock_vld; + + lock_val = readl(REG_ADDR(xdev, xdev->regs.cpm_get_lock)); + lock_vld = lock_val >> HIF_CPM_LOCK_GET_REG_LOCK_VLD_SHIFT; + if (lock_vld) + *iae_idx = lock_val & HIF_CPM_LOCK_GET_REG_LOCK_IDX_MASK; + else + *iae_idx = -1; +} +#define ACQUIRE_IA_LOCK(bp, iae_idx) \ + do { \ + int idx; \ + acquire_ia_lock(bp, &idx); \ + iae_idx = idx; \ + } while (0) + +static inline void release_ia_lock(struct xsc_core_device *xdev, int lock_idx) +{ + writel(lock_idx, REG_ADDR(xdev, xdev->regs.cpm_put_lock)); +} + +#define RELEASE_IA_LOCK(bp, iae_idx) release_ia_lock(bp, iae_idx) + +static inline void ia_write_data(struct xsc_core_device *xdev, u32 *ptr, int n, int iae_idx) +{ + int i; + int offset = xdev->regs.cpm_data_mem + (iae_idx) * CPM_IAE_DATA_MEM_STRIDE; + + for (i = 0; i < n; i++) { + writel(*(ptr++), REG_ADDR(xdev, offset)); + offset += sizeof(*ptr); + } +} + +static inline void ia_read_data(struct xsc_core_device *xdev, u32 *ptr, int n, int iae_idx) +{ + int i; + int offset = xdev->regs.cpm_data_mem + (iae_idx) * CPM_IAE_DATA_MEM_STRIDE; + u32 *pptr = ptr; + + for (i = 0; i < n; i++) { + *(pptr) = readl(REG_ADDR(xdev, offset)); + offset += sizeof(*ptr); + pptr = pptr + 1; + } +} + +static inline void ia_write_reg_addr(struct xsc_core_device *xdev, u32 reg, int iae_idx) +{ + int offset = xdev->regs.cpm_addr + (iae_idx) * CPM_IAE_ADDR_REG_STRIDE; + + writel(reg, REG_ADDR(xdev, offset)); +} + +static inline void initiate_ia_cmd(struct xsc_core_device *xdev, int iae_idx, int length, int r0w1) +{ + struct iae_cmd cmd; + int addr = xdev->regs.cpm_cmd; + + cmd.iae_r0w1 = r0w1; + cmd.iae_len = length - 1; + cmd.iae_idx = iae_idx; + writel(cmd.raw_data, REG_ADDR(xdev, addr)); +} + +static inline void initiate_ia_write_cmd(struct xsc_core_device *xdev, int iae_idx, int length) +{ + initiate_ia_cmd(xdev, iae_idx, length, CPM_IAE_CMD_WRITE); +} + +static inline void initiate_ia_read_cmd(struct xsc_core_device *xdev, int iae_idx, int length) +{ + initiate_ia_cmd(xdev, iae_idx, length, CPM_IAE_CMD_READ); +} + +static inline void wait_for_complete(struct xsc_core_device *xdev, int iae_idx) +{ + while ((readl(REG_ADDR(xdev, xdev->regs.cpm_busy)) & (1 << iae_idx))) + ; +} + +static inline void ia_write_reg_mr(struct xsc_core_device *xdev, u32 reg, + u32 *ptr, int n, int idx) +{ + ia_write_data(xdev, ptr, n, idx); + ia_write_reg_addr(xdev, reg, idx); + initiate_ia_write_cmd(xdev, idx, n); +} + +#define IA_WRITE_REG_MR(bp, reg, ptr, n, idx) ia_write_reg_mr(bp, reg, ptr, n, idx) + +static inline void ia_write(struct xsc_core_device *xdev, u32 reg, u32 *ptr, int n) +{ + int iae_idx; + + acquire_ia_lock(xdev, &iae_idx); + ia_write_data(xdev, ptr, n, iae_idx); + ia_write_reg_addr(xdev, reg, iae_idx); + initiate_ia_write_cmd(xdev, iae_idx, n); + release_ia_lock(xdev, iae_idx); +} + +#define IA_WRITE(bp, reg, ptr, n) ia_write(bp, reg, ptr, n) + +static inline void ia_read(struct xsc_core_device *xdev, u32 reg, u32 *ptr, int n) +{ + int iae_idx; + + acquire_ia_lock(xdev, &iae_idx); + ia_write_reg_addr(xdev, reg, iae_idx); + initiate_ia_read_cmd(xdev, iae_idx, n); + wait_for_complete(xdev, iae_idx); + ia_read_data(xdev, ptr, n, iae_idx); + release_ia_lock(xdev, iae_idx); +} + +#define IA_READ(bp, reg, ptr, n) ia_read(bp, reg, ptr, n) + +static inline u32 reg_read32(struct xsc_core_device *dev, u32 offset) +{ + u32 val = 0; + + if (xsc_core_is_pf(dev)) + val = readl(REG_ADDR(dev, offset)); + else + IA_READ(dev, offset, &val, 1); + + return val; +} + +static inline void reg_write32(struct xsc_core_device *dev, u32 offset, u32 val) +{ + u32 *ptr = &val; + + if (xsc_core_is_pf(dev)) + writel(val, REG_ADDR(dev, offset)); + else + IA_WRITE(dev, offset, ptr, 1); +} + +#define REG_RD32(dev, offset) reg_read32(dev, offset) +#define REG_WR32(dev, offset, val) reg_write32(dev, offset, val) + +#endif /* XSC_CORE_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_eswitch.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_eswitch.h new file mode 100644 index 0000000000000000000000000000000000000000..f2ab499e6f4a0429e7551e5407a8b53b51d236d7 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_eswitch.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ESWITCH_H +#define XSC_ESWITCH_H + +//#define XSC_ESWITCH_QOS_ENABLE 1 + +enum { + XSC_ESWITCH_NONE, + XSC_ESWITCH_LEGACY, + XSC_ESWITCH_OFFLOADS +}; + +enum { + REP_ETH, + REP_IB, + NUM_REP_TYPES, +}; + +enum { + REP_UNREGISTERED, + REP_REGISTERED, + REP_LOADED, +}; + +enum xsc_switchdev_event { + XSC_SWITCHDEV_EVENT_PAIR, + XSC_SWITCHDEV_EVENT_UNPAIR, +}; + +enum { + SET_VLAN_STRIP = BIT(0), + SET_VLAN_INSERT = BIT(1), + CLR_VLAN_STRIP = BIT(2), + CLR_VLAN_INSERT = BIT(3), +}; + +#endif /* XSC_ESWITCH_H */ + diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_hsi.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_hsi.h new file mode 100644 index 0000000000000000000000000000000000000000..d2ab84a0c0e67d22b35d0852e324972d1f83e14c --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_hsi.h @@ -0,0 +1,691 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_HSI_H +#define XSC_HSI_H + +#include + +#include +#include +#include +#include +#include +#include +#include + +#define MSIX_SUPPORT +#define XSC_BQL_SUPPORT +#define XSC_RSS_SUPPORT + +#ifdef MSIX_SUPPORT +#else +#define NEED_CREATE_RX_THREAD +#endif + +/*define product macro*/ +#if ((CHIP_VERSION_L == 0xa) || \ + (CHIP_VERSION_L == 0xa10)) +#define XSC_CHIP_ANDES +#elif ((CHIP_VERSION_L == 0xb) || \ + (CHIP_VERSION_L == 0xb20) || \ + (CHIP_VERSION_L == 0xb11) || \ + (CHIP_VERSION_L == 0xb13)) +#define XSC_CHIP_BERYL +#if (CHIP_VERSION_L == 0xb20) +#define XSC_CHIP_BERYL_100 +#endif +#if (CHIP_VERSION_L == 0xb11) +#define XSC_CHIP_BERYL_50 +#endif +#if (CHIP_VERSION_L == 0xb13) +#define XSC_CHIP_BERYL_50S +#endif +#if (CHIP_VERSION_L == 0xb12) +#define XSC_CHIP_BERYL_50R +#endif +#elif ((CHIP_VERSION_L == 0xc) || \ + (CHIP_VERSION_L == 0xc11) || \ + (CHIP_VERSION_L == 0xc12) || \ + (CHIP_VERSION_L == 0xc13)) +#define XSC_CHIP_CRYSTAL +#if (CHIP_VERSION_L == 0xc13) +#define XSC_CHIP_CRYSTAL_A +#endif +#if (CHIP_VERSION_L == 0xc12) +#define XSC_CHIP_CRYSTAL_B +#endif +#if (CHIP_VERSION_L == 0xc11) +#define XSC_CHIP_CRYSTAL_C +#endif +#endif + +#if (CHIP_HOTFIX_NUM > 0x22) +#if ((CHIP_VERSION_L == 0xb13) || \ + (CHIP_VERSION_L == 0xc11) || \ + (CHIP_VERSION_L == 0xc12) || \ + (CHIP_VERSION_L == 0xc13)) +#define XSC_CHIP_RDMA_UNSUPPORTED 1 +#endif +#endif + +#ifndef RUN_WITH_PSV +#if (CHIP_HOTFIX_NUM >= 0x23) +#define XSC_MSIX_BAR_EMUL +#endif +#endif + +#define PAGE_SHIFT_4K 12 +#define PAGE_SIZE_4K (_AC(1, UL) << PAGE_SHIFT_4K) +#define PAGE_MASK_4K (~(PAGE_SIZE_4K-1)) + +#ifndef EQ_NUM_MAX +#define EQ_NUM_MAX 1024 +#endif +#ifndef EQ_SIZE_MAX +#define EQ_SIZE_MAX 1024 +#endif + +#if defined XSC_CHIP_ANDES +#define XSC_RSS_INDIR_TBL_N 1032 /*8PFs+1024VFs*/ +#elif defined XSC_CHIP_BERYL +#define XSC_RSS_INDIR_TBL_N 514 /*2PFs+512VFs*/ +#elif defined XSC_CHIP_BERYL_100 +#define XSC_RSS_INDIR_TBL_N 514 /*2PFs+512VFs*/ +#elif defined XSC_CHIP_CRYSTAL +#define XSC_RSS_INDIR_TBL_N 514 /*2PFs+512VFs*/ +#endif + +#define XSC_RSS_INDIR_TBL_S 256 +#define XSC_MAX_TSO_PAYLOAD 0x10000/*64kb*/ + +#define MAX_BOARD_NUM 8 + +#define DMA_LO_LE(x) __cpu_to_le32(lower_32_bits(x)) +#define DMA_HI_LE(x) __cpu_to_le32(upper_32_bits(x)) +#define DMA_REGPAIR_LE(x, val) do { \ + (x).hi = DMA_HI_LE((val)); \ + (x).lo = DMA_LO_LE((val)); \ + } while (0) + +#define WR_LE_16(x, val) (x = __cpu_to_le16(val)) +#define WR_LE_32(x, val) (x = __cpu_to_le32(val)) +#define WR_LE_64(x, val) (x = __cpu_to_le64(val)) +#define WR_LE_R64(x, val) (DMA_REGPAIR_LE(x, val)) +#define WR_BE_32(x, val) (x = __cpu_to_be32(val)) + +#define RD_LE_16(x) __le16_to_cpu(x) +#define RD_LE_32(x) __le32_to_cpu(x) +#define RD_BE_32(x) __be32_to_cpu(x) + +#define WR_REG(addr, val) mmio_write64_le(addr, val) +#define RD_REG(addr) mmio_read64_le(addr) + +#define XSC_MPT_MAP_EN 0 + +/* FIXME: 32-byte alignment for SW descriptors for Amber for now */ +#define XSC_DESC_ALIGNMENT 32 + +/* each ds holds one fragment in skb */ +#define XSC_MAX_RX_FRAGS 4 +#define XSC_RX_FRAG_SZ_ORDER 0 +#define XSC_RX_FRAG_SZ (PAGE_SIZE << XSC_RX_FRAG_SZ_ORDER) + +/* message opcode */ +enum { + XSC_MSG_OPCODE_SEND = 0, + XSC_MSG_OPCODE_RDMA_WRITE = 1, + XSC_MSG_OPCODE_RDMA_READ = 2, + XSC_MSG_OPCODE_MAD = 3, + XSC_MSG_OPCODE_RDMA_ACK = 4, + XSC_MSG_OPCODE_RDMA_ACK_READ = 5, + XSC_MSG_OPCODE_RDMA_CNP = 6, + XSC_MSG_OPCODE_RAW = 7, + XSC_MSG_OPCODE_VIRTIO_NET = 8, + XSC_MSG_OPCODE_VIRTIO_BLK = 9, + XSC_MSG_OPCODE_RAW_TPE = 10, + XSC_MSG_OPCODE_INIT_QP_REQ = 11, + XSC_MSG_OPCODE_INIT_QP_RSP = 12, + XSC_MSG_OPCODE_INIT_PATH_REQ = 13, + XSC_MSG_OPCODE_INIT_PATH_RSP = 14, +}; + +/* TODO: sw cqe opcode*/ +enum { + XSC_OPCODE_RDMA_REQ_SEND = 0, + XSC_OPCODE_RDMA_REQ_SEND_IMMDT = 1, + XSC_OPCODE_RDMA_RSP_RECV = 2, + XSC_OPCODE_RDMA_RSP_RECV_IMMDT = 3, + XSC_OPCODE_RDMA_REQ_WRITE = 4, + XSC_OPCODE_RDMA_REQ_WRITE_IMMDT = 5, + XSC_OPCODE_RDMA_RSP_WRITE_IMMDT = 6, + XSC_OPCODE_RDMA_REQ_READ = 7, + XSC_OPCODE_RDMA_REQ_ERROR = 8, + XSC_OPCODE_RDMA_RSP_ERROR = 9, + XSC_OPCODE_RDMA_CQE_ERROR = 10, + XSC_OPCODE_RDMA_MAD_REQ_SEND, + XSC_OPCODE_RDMA_MAD_RSP_RECV, +}; + +enum { + XSC_REQ = 0, + XSC_RSP = 1, +}; + +enum { + XSC_WITHOUT_IMMDT = 0, + XSC_WITH_IMMDT = 1, +}; + +enum { + XSC_ERR_CODE_NAK_RETRY = 0x40, + XSC_ERR_CODE_NAK_OPCODE = 0x41, + XSC_ERR_CODE_NAK_MR = 0x42, + XSC_ERR_CODE_NAK_OPERATION = 0x43, + XSC_ERR_CODE_NAK_RNR = 0x44, + XSC_ERR_CODE_LOCAL_MR = 0x45, + XSC_ERR_CODE_LOCAL_LEN = 0x46, + XSC_ERR_CODE_LOCAL_OPCODE = 0x47, + XSC_ERR_CODE_CQ_OVER_FLOW = 0x48, + XSC_ERR_CODE_STRG_ACC_GEN_CQE = 0x4c, + XSC_ERR_CODE_CQE_ACC = 0x4d, + XSC_ERR_CODE_FLUSH = 0x4e, + XSC_ERR_CODE_MALF_WQE_HOST = 0x50, + XSC_ERR_CODE_MALF_WQE_INFO = 0x51, + XSC_ERR_CODE_MR_NON_NAK = 0x52, + XSC_ERR_CODE_OPCODE_GEN_CQE = 0x61, + XSC_ERR_CODE_MANY_READ = 0x62, + XSC_ERR_CODE_LEN_GEN_CQE = 0x63, + XSC_ERR_CODE_MR = 0x65, + XSC_ERR_CODE_MR_GEN_CQE = 0x66, + XSC_ERR_CODE_OPERATION = 0x67, + XSC_ERR_CODE_MALF_WQE_INFO_GEN_NAK = 0x68, +}; + +/* QP type */ +enum { + XSC_QUEUE_TYPE_RDMA_RC = 0, + XSC_QUEUE_TYPE_RDMA_MAD = 1, + XSC_QUEUE_TYPE_RAW = 2, + XSC_QUEUE_TYPE_VIRTIO_NET = 3, + XSC_QUEUE_TYPE_VIRTIO_BLK = 4, + XSC_QUEUE_TYPE_RAW_TPE = 5, + XSC_QUEUE_TYPE_RAW_TSO = 6, + XSC_QUEUE_TYPE_INVALID = 0xFF, +}; + +/* CQ type */ +enum { + XSC_CQ_TYPE_NORMAL = 0, + XSC_CQ_TYPE_VIRTIO = 1, +}; + +enum xsc_qp_state { + XSC_QP_STATE_RST = 0, + XSC_QP_STATE_INIT = 1, + XSC_QP_STATE_RTR = 2, + XSC_QP_STATE_RTS = 3, + XSC_QP_STATE_SQER = 4, + XSC_QP_STATE_SQD = 5, + XSC_QP_STATE_ERR = 6, + XSC_QP_STATE_SQ_DRAINING = 7, + XSC_QP_STATE_SUSPENDED = 9, + XSC_QP_NUM_STATE +}; + +enum { + XSC_SEND_SEG_MAX = 32, + XSC_BASE_WQE_SHIFT = 4, + XSC_SEND_SEG_NUM = 4, + XSC_SEND_WQE_SHIFT = 6, +}; + +enum { + XSC_RECV_SEG_MAX = 4, + XSC_RECV_SEG_NUM = 1, + XSC_RECV_WQE_SHIFT = 4, +}; + +enum { + XSC_INLINE_SIZE_MAX = 15, +}; + +/* Descriptors that are allocated by SW and accessed by HW, 32-byte aligned + */ +/* this is to keep descriptor structures packed */ +struct regpair { + __le32 lo; + __le32 hi; +}; + +struct xsc_cqe { + union { + u8 msg_opcode; + struct { + u8 error_code:7; + u8 is_error:1; + }; + }; + __le32 qp_id:15; + u8 rsv1:1; + u8 se:1; + u8 has_pph:1; + u8 type:1; + u8 with_immdt:1; + u8 csum_err:4; + __le32 imm_data; + __le32 msg_len; + __le32 vni; + __le64 ts:48; + __le16 wqe_id; + __le16 rsv[3]; + __le16 rsv2:15; + u8 owner:1; +}; + +struct xsc_cqe64 { + union { + u8 msg_opcode; + struct { + u8 error_code:7; + u8 is_error:1; + }; + }; + __le32 qp_id:15; + u8 rsv1:1; + u8 se:1; + u8 has_pph:1; + u8 type:1; + u8 with_immdt:1; + u8 csum_err:4; + __le32 imm_data; + __le32 msg_len; + __le32 vni; //rx hash + __le64 ts:48; + __le16 wqe_id; + __le16 rsv[3]; + __le16 rsv2:15; + u8 owner:1; +}; + +/* CQ doorbell */ +union xsc_cq_doorbell { + struct{ + u32 cq_next_cid:16; + u32 cq_id:15; + u32 arm:1; + }; + u32 val; +}; + +/* EQE TBD */ +struct xsc_eqe { + u8 type; + u8 sub_type; + __le16 queue_id:15; + u8 rsv1:1; + u8 err_code; + u8 rsvd[2]; + u8 rsv2:7; + u8 owner:1; +}; + +/* EQ doorbell */ +union xsc_eq_doorbell { + struct{ + u32 eq_next_cid : 11; + u32 eq_id : 11; + u32 arm : 1; + }; + u32 val; +}; + +/*for beryl tcam table .begin*/ +#define XSC_TBM_PCT_DW_SIZE_MAX 20 +#define XSC_TCAM_REG_ADDR_STRIDE 4 + +enum xsc_tbm_tcam_type { + XSC_TBM_TCAM_PCT = 0, + XSC_TBM_TCAM_PRS_STAGE0, + XSC_TBM_TCAM_PRS_STAGE1, + XSC_TBM_TCAM_PRS_STAGE2, +}; + +enum xsc_tbm_tcam_oper { + XSC_TCAM_OP_X_WRITE = 0, + XSC_TCAM_OP_Y_WRITE, + XSC_TCAM_OP_ACTION_WRITE, + XSC_TCAM_OP_X_READ, + XSC_TCAM_OP_Y_READ, + XSC_TCAM_OP_ACTION_READ, + XSC_TCAM_OP_TCAM_FLUSH, + XSC_TCAM_OP_ACTION_FLUSH, + XSC_TCAM_OP_CPU_SEARCH, + XSC_TCAM_OP_LONG_X_WRT, + XSC_TCAM_OP_LONG_Y_WRT +}; + +enum xsc_tbm_prs_stage_encode { + XSC_PRS_STAGE0_HDR_TYPE_NONE = 0x00, + XSC_PRS_STAGE0_HDR_TYPE_ETH0 = 0x01, + XSC_PRS_STAGE1_HDR_TYPE_NONE = 0x10, + XSC_PRS_STAGE1_HDR_TYPE_RSV = 0x11, + XSC_PRS_STAGE1_HDR_TYPE_IPV4 = 0x12, + XSC_PRS_STAGE1_HDR_TYPE_IPV6 = 0x13, + XSC_PRS_STAGE2_HDR_TYPE_NONE = 0x20, + XSC_PRS_STAGE2_HDR_TYPE_TCP = 0x21, + XSC_PRS_STAGE2_HDR_TYPE_UDP = 0x22, + XSC_PRS_STAGE2_HDR_TYPE_GRE = 0x23, + XSC_PRS_STAGE2_HDR_TYPE_RSV = 0x24, + XSC_PRS_STAGE2_HDR_TYPE_IFA_TCP = 0x25, + XSC_PRS_STAGE2_HDR_TYPE_IFA_UDP = 0x26, + XSC_PRS_STAGE2_HDR_TYPE_IFA_GRE = 0x27, + XSC_PRS_STAGE6_HDR_TYPE_ICMP = 0x63, + XSC_PRS_STAGEX_HDR_TYPE_PAYLOAD = 0xa0, + XSC_PRS_STAGEX_HDR_TYPE_BTH = 0xa1, +}; + +enum xsc_tbm_prs_eth_hdr_type_encode { + ETH_HDR_TYPE_MAC0 = 0x0, + ETH_HDR_TYPE_MAC0_VLANA = 0x2, + ETH_HDR_TYPE_MAC0_VLANA_VLANB = 0x3, +}; + +enum xsc_tbm_pct_pkttype { + XSC_PCT_RDMA_NORMAL = 0x0, + XSC_PCT_RDMA_CNP, + XSC_PCT_RDMA_MAD, + XSC_PCT_RAW, + XSC_PCT_RAW_TPE, + XSC_PCT_VIRTIO_NET_TO_HOST, + XSC_PCT_SOC_WITH_PPH, +}; + +enum xsc_tbm_pct_inport { + XSC_PCT_PORT_NIF0 = 0x0, + XSC_PCT_PORT_NIF1, + XSC_PCT_PORT_PCIE0_PF0, + XSC_PCT_PORT_PCIE0_PF1, + XSC_PCT_PORT_PCIE1_PF0, +}; + +/*for beryl tcam table .end*/ + +/* Size of WQE */ +#define XSC_SEND_WQE_SIZE (1 << XSC_SEND_WQE_SHIFT) +#define XSC_RECV_WQE_SIZE (1 << XSC_RECV_WQE_SHIFT) + +union xsc_db_data { + struct { + __le32 sq_next_pid:16; + __le32 sqn:15; + __le32:1; + }; + struct { + __le32 rq_next_pid:13; + __le32 rqn:15; + __le32:4; + }; + struct { + __le32 cq_next_cid:16; + __le32 cqn:15; + __le32 solicited:1; + + }; + __le32 raw_data; +}; + +enum xsc_port_type_encode { + XSC_PHY_PORT_MAC_0 = 0x0, + XSC_PHY_PORT_MAC_1 = 0x1, + XSC_PHY_PORT_MAC_2 = 0x2, + XSC_PHY_PORT_MAC_3 = 0x3, + XSC_PHY_PORT_MAC_4 = 0x4, + XSC_PHY_PORT_MAC_5 = 0x5, + XSC_PHY_PORT_MAC_6 = 0x6, + XSC_PHY_PORT_MAC_7 = 0x7, + XSC_PHY_PORT_PCIE_0 = 0x8, + XSC_PHY_PORT_PCIE_1 = 0x9, + + XSC_LAG_PORT_START = 15, + XSC_LAG_PORT_END = 62, + + XSC_PORT_FUNC_ID_START = 63, + XSC_PORT_FUNC_ID_END = 1214, +}; + +#define XSC_PHY_PORT_MAC_NUM 8 + +#ifndef XSC_CHIP_ANDES +#ifdef XSC_CHIP_BERYL_50S +#define XSC_PCIE0_VF_LOGIC_PORT_NUM 512 +#else +#define XSC_PCIE0_VF_LOGIC_PORT_NUM 256 +#endif +#define XSC_PCIE1_VF_LOGIC_PORT_NUM 0 +#define XSC_PCIE0_PF_LOGIC_PORT_NUM 2 +#define XSC_PCIE1_PF_LOGIC_PORT_NUM 8 + +#ifndef PCIE0_PF1_VF_NUM +#define PCIE0_PF1_VF_NUM 0 +#endif + +enum xsc_port_func_id_encode { + XSC_PCIE0_VF0_FUNC_ID = 0, + XSC_PCIE0_PF0_VF0_FUNC_ID = XSC_PCIE0_VF0_FUNC_ID, +#if (CHIP_HOTFIX_NUM >= 0x23) + XSC_PCIE0_PF1_VF0_FUNC_ID = (XSC_PCIE0_PF0_VF0_FUNC_ID + + PCIE0_PF0_VF_NUM), + XSC_PCIE0_VF_FUNC_ID_END = (XSC_PCIE0_PF1_VF0_FUNC_ID + + PCIE0_PF1_VF_NUM - 1), +#else + XSC_PCIE0_PF1_VF0_FUNC_ID = (XSC_PCIE0_PF0_VF0_FUNC_ID + + XSC_PCIE0_VF_LOGIC_PORT_NUM), + XSC_PCIE0_VF_FUNC_ID_END = (XSC_PCIE0_PF0_VF0_FUNC_ID + + 2*XSC_PCIE0_VF_LOGIC_PORT_NUM - 1),//511 +#endif + XSC_PCIE1_VF0_FUNC_ID = XSC_PCIE0_VF_FUNC_ID_END, + XSC_PCIE1_PF0_VF0_FUNC_ID = XSC_PCIE0_VF_FUNC_ID_END, + XSC_PCIE1_PF1_VF0_FUNC_ID = XSC_PCIE0_VF_FUNC_ID_END, + XSC_PCIE1_VF_FUNC_ID_END = XSC_PCIE0_VF_FUNC_ID_END, + +#if (CHIP_HOTFIX_NUM >= 0x23) + XSC_PCIE0_PF0_FUNC_ID = (XSC_PCIE0_VF_FUNC_ID_END + 1), + XSC_PCIE0_PF_FUNC_ID_END = (XSC_PCIE0_PF0_FUNC_ID + PCIE0_PF_NUM - 1), + XSC_PCIE1_PF0_FUNC_ID = (XSC_PCIE0_PF_FUNC_ID_END + 1), + XSC_PCIE1_PF_FUNC_ID_END = (XSC_PCIE1_PF0_FUNC_ID + PCIE1_PF_NUM - 1),//521 +#else + XSC_PCIE0_PF0_FUNC_ID = (XSC_PCIE1_PF0_VF0_FUNC_ID + 1),//512 + XSC_PCIE0_PF_FUNC_ID_END = (XSC_PCIE0_PF0_FUNC_ID + + XSC_PCIE0_PF_LOGIC_PORT_NUM - 1),//513 + XSC_PCIE1_PF0_FUNC_ID = (XSC_PCIE0_PF0_FUNC_ID + + XSC_PCIE0_PF_LOGIC_PORT_NUM),//514 + XSC_PCIE1_PF_FUNC_ID_END = (XSC_PCIE1_PF0_FUNC_ID + + XSC_PCIE1_PF_LOGIC_PORT_NUM - 1),//521 +#endif + XSC_FUNC_ID_END, +}; +#else +#define XSC_PCIE0_VF_LOGIC_PORT_NUM 512 +#define XSC_PCIE1_VF_LOGIC_PORT_NUM 0 +#define XSC_PCIE0_PF_LOGIC_PORT_NUM 2 +#define XSC_PCIE1_PF_LOGIC_PORT_NUM 8 +enum xsc_port_func_id_encode { + XSC_PCIE0_VF0_FUNC_ID = 0, + XSC_PCIE0_PF0_VF0_FUNC_ID = XSC_PCIE0_VF0_FUNC_ID, + XSC_PCIE0_PF1_VF0_FUNC_ID = (XSC_PCIE0_PF0_VF0_FUNC_ID + + XSC_PCIE0_VF_LOGIC_PORT_NUM),//512 + XSC_PCIE0_VF_FUNC_ID_END = (XSC_PCIE0_VF0_FUNC_ID + + 2*XSC_PCIE0_VF_LOGIC_PORT_NUM - 1),//1023 + + XSC_PCIE1_VF0_FUNC_ID = XSC_PCIE0_VF_FUNC_ID_END, + XSC_PCIE1_PF0_VF0_FUNC_ID = XSC_PCIE0_VF_FUNC_ID_END, + XSC_PCIE1_PF1_VF0_FUNC_ID = XSC_PCIE0_VF_FUNC_ID_END, + XSC_PCIE1_VF_FUNC_ID_END = XSC_PCIE0_VF_FUNC_ID_END, + + XSC_PCIE0_PF0_FUNC_ID = (XSC_PCIE0_VF0_FUNC_ID + + 2*XSC_PCIE0_VF_LOGIC_PORT_NUM),//1024 + XSC_PCIE0_PF_FUNC_ID_END = (XSC_PCIE0_PF0_FUNC_ID + + XSC_PCIE0_PF_LOGIC_PORT_NUM - 1),//1025 + + XSC_PCIE1_PF0_FUNC_ID = 1032,//from program manual + XSC_PCIE1_PF_FUNC_ID_END = (XSC_PCIE1_PF0_FUNC_ID + + XSC_PCIE1_PF_LOGIC_PORT_NUM - 1),//1039 + XSC_FUNC_ID_END, +}; +#endif + +#define XSC_PHY_PORT_MAC_N(mac_id) \ + (XSC_PHY_PORT_MAC_0 + mac_id) +#define XSC_PHY_PORT_PCIE_N(pcie_id) \ + (XSC_PHY_PORT_PCIE_0 + pcie_id) +#define XSC_PHY_PORT_TO_PCIE0_PF_ID(pcie_port) \ + (pcie_port - XSC_PHY_PORT_PCIE_0) +#define XSC_PHY_PORT_TO_PCIE1_PF_ID(pcie_port) \ + (pcie_port - XSC_PHY_PORT_PCIE_1 - 1) + +#define XSC_GLB_FUNC_TO_PCIE0_PF_ID(glb_func) \ + (glb_func - XSC_PCIE0_PF0_FUNC_ID) +#define XSC_GLB_FUNC_TO_PCIE1_PF_ID(glb_func) \ + (glb_func - XSC_PCIE1_PF0_FUNC_ID) + +#define XSC_PCIE0_PF0_VF_N_FUNC_ID(vf_id) \ + (XSC_PCIE0_PF0_VF0_FUNC_ID + vf_id) +#define XSC_PCIE0_PF1_VF_N_FUNC_ID(vf_id) \ + (XSC_PCIE0_PF1_VF0_FUNC_ID + vf_id) + +#define XSC_PCIE1_PF0_VF_N_FUNC_ID(vf_id) \ + (XSC_PCIE1_PF0_VF0_FUNC_ID + vf_id) +#define XSC_PCIE1_PF1_VF_N_FUNC_ID(vf_id) \ + (XSC_PCIE1_PF1_VF0_FUNC_ID + vf_id) + +#define XSC_PCIE0_PF_N_FUNC_ID(pf_id) \ + (XSC_PCIE0_PF0_FUNC_ID + pf_id) +#define XSC_PCIE1_PF_N_FUNC_ID(pf_id) \ + (XSC_PCIE1_PF0_FUNC_ID + pf_id) + +#define XSC_PCIE0_PF_N_LOGIC_PORT(pf_id) \ + (XSC_PORT_FUNC_ID_START + XSC_PCIE0_PF0_FUNC_ID + pf_id) +#define XSC_PCIE1_PF_N_LOGIC_PORT(pf_id) \ + (XSC_PORT_FUNC_ID_START + XSC_PCIE1_PF0_FUNC_ID + pf_id) + +#define XSC_PCIE0_PF0_VF_N_LOGIC_PORT(vf_id) \ + (XSC_PORT_FUNC_ID_START + XSC_PCIE0_PF0_VF_N_FUNC_ID(vf_id)) +#define XSC_PCIE0_PF1_VF_N_LOGIC_PORT(vf_id) \ + (XSC_PORT_FUNC_ID_START + XSC_PCIE0_PF1_VF_N_FUNC_ID(vf_id)) + +#define XSC_PCIE1_PF0_VF_N_LOGIC_PORT(vf_id) \ + (XSC_PORT_FUNC_ID_START + XSC_PCIE1_PF0_VF_N_FUNC_ID(vf_id)) +#define XSC_PCIE1_PF1_VF_N_LOGIC_PORT(vf_id) \ + (XSC_PORT_FUNC_ID_START + XSC_PCIE1_PF1_VF_N_FUNC_ID(vf_id)) + +#define XSC_BROADCASTID_MAX 2 +#define XSC_TBM_BOMT_DESTINFO_SHIFT (XSC_BROADCASTID_MAX/2) +#define XSC_TBM_BOMT_BROADCASTID_MASK (XSC_BROADCASTID_MAX - 1) + +static inline bool xsc_cal_pf_vf_id(int func_id, u8 *pf_id, u8 *pcie_no, u16 *vf_id) +{ + bool is_pf = true; + + if (func_id >= XSC_PCIE0_PF0_FUNC_ID && + func_id <= XSC_PCIE0_PF_FUNC_ID_END) { + is_pf = true; + *pf_id = func_id - XSC_PCIE0_PF0_FUNC_ID; + *pcie_no = 0; + } else if (func_id >= XSC_PCIE1_PF0_FUNC_ID && + func_id <= XSC_PCIE1_PF_FUNC_ID_END) { + is_pf = true; + *pf_id = func_id - XSC_PCIE1_PF0_FUNC_ID; + *pcie_no = 1; + } else if (func_id >= XSC_PCIE0_PF0_VF0_FUNC_ID && + func_id < XSC_PCIE0_PF1_VF0_FUNC_ID) { + is_pf = false; + *pf_id = 0; + *vf_id = func_id - XSC_PCIE0_PF0_VF0_FUNC_ID; + *pcie_no = 0; + } else if (func_id >= XSC_PCIE0_PF1_VF0_FUNC_ID && + func_id <= XSC_PCIE0_VF_FUNC_ID_END) { + is_pf = false; + *pf_id = 1; + *vf_id = func_id - XSC_PCIE0_PF1_VF0_FUNC_ID; + *pcie_no = 0; + } + return is_pf; +} + +#define XSC_IS_VF(glb_func) (((glb_func) >= XSC_PCIE0_VF0_FUNC_ID && \ + (glb_func) <= XSC_PCIE0_VF_FUNC_ID_END) || \ + ((glb_func) >= XSC_PCIE1_VF0_FUNC_ID && \ + (glb_func) <= XSC_PCIE1_VF_FUNC_ID_END)) + +#define XSC_IS_PF0_VF(glb_func) (((glb_func) >= XSC_PCIE0_PF0_VF0_FUNC_ID && \ + (glb_func) < XSC_PCIE0_PF1_VF0_FUNC_ID) || \ + ((glb_func) >= XSC_PCIE1_PF0_VF0_FUNC_ID && \ + (glb_func) < XSC_PCIE1_PF1_VF0_FUNC_ID)) + +#define XSC_IS_PCIE0_PF(glb_func) ((glb_func) >= XSC_PCIE0_PF0_FUNC_ID && \ + (glb_func) <= XSC_PCIE0_PF_FUNC_ID_END) + +#define XSC_PF_GET_PF_ID(glb_func) (XSC_IS_PCIE0_PF(glb_func) ? \ + (glb_func) - XSC_PCIE0_PF0_FUNC_ID : \ + (glb_func) - XSC_PCIE1_PF0_FUNC_ID) + +#define XSC_PF_VF_GET_PF_ID(glb_func) (XSC_IS_VF(glb_func) ? \ + (XSC_IS_PF0_VF(glb_func) ? 0 : 1) : \ + XSC_PF_GET_PF_ID(glb_func)) + +#define XSC_IS_PCIE0(glb_func) (((glb_func) >= XSC_PCIE0_VF0_FUNC_ID && \ + (glb_func) <= XSC_PCIE0_VF_FUNC_ID_END) || \ + ((glb_func) >= XSC_PCIE0_PF0_FUNC_ID && \ + (glb_func) <= XSC_PCIE0_PF_FUNC_ID_END)) + +#define XSC_GET_PCIE_NO(glb_func) (XSC_IS_PCIE0(glb_func) ? 0 : 1) + +#define XSC_IS_PF(func_id) \ + (((func_id) >= XSC_PCIE0_PF0_FUNC_ID && (func_id) <= XSC_PCIE0_PF_FUNC_ID_END) || \ + ((func_id) >= XSC_PCIE1_PF0_FUNC_ID && (func_id) <= XSC_PCIE1_PF_FUNC_ID_END)) + +/* Doorbell registers */ +// +//#define SQM_DB_NEXT_PID_OFFSET 0 +//#define RQM_DB_NEXT_PID_OFFSET 0 +//#define CQM_DB_NEXT_CID_OFFSET 0 +// +//#define XSC_SEND_WQE_RING_DEPTH_MIN 16 +//#define XSC_RECV_WQE_RING_DEPTH_MIN 2 +//#define XSC_CQE_RING_DEPTH_MIN 2 +//#define XSC_SEND_WQE_RING_DEPTH_MAX 128 +//#define XSC_RECV_WQE_RING_DEPTH_MAX 128 +//#define XSC_CQE_RING_DEPTH_MAX 128 + +enum { + XSC_EQ_VEC_ASYNC = 0, + XSC_VEC_CMD = 1, + XSC_VEC_CMD_EVENT = 2, + XSC_DMA_READ_DONE_VEC = 3, + XSC_EQ_VEC_COMP_BASE, +}; + +struct rxe_bth { + u8 opcode; + u8 flags; + __be16 pkey; + __be32 qpn; + __be32 apsn; +}; + +struct rxe_deth { + __be32 qkey; + __be32 sqp; +}; + +struct xsc_broadcast_attr { + u16 broadcast_id; + u16 bc_membernum; + u16 vf_num; +}; + +#endif /* XSC_HSI_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_ioctl.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_ioctl.h new file mode 100644 index 0000000000000000000000000000000000000000..387250d4f5255112dfaeae8d870b2fd24b4f85bd --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_ioctl.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_IOCTL_H +#define XSC_IOCTL_H + +#include +#include + +/* Documentation/ioctl/ioctl-number.txt */ +#define XSC_IOCTL_MAGIC (0x1b) /* TBD */ +#define XSC_IOCTL_CMDQ \ + _IOWR(XSC_IOCTL_MAGIC, 1, struct xsc_ioctl_hdr) +#define XSC_IOCTL_DRV_GET \ + _IOR(XSC_IOCTL_MAGIC, 2, struct xsc_ioctl_hdr) +#define XSC_IOCTL_DRV_SET \ + _IOWR(XSC_IOCTL_MAGIC, 3, struct xsc_ioctl_hdr) +#define XSC_IOCTL_MEM \ + _IOWR(XSC_IOCTL_MAGIC, 4, struct xsc_ioctl_hdr) + +#define XSC_IOCTL_CHECK_FILED 0x01234567 +enum { + XSC_IOCTL_OP_GET_LOCAL, + XSC_IOCTL_OP_GET_VF_INFO, + XSC_IOCTL_OP_GET_CONTEXT, + XSC_IOCTL_OP_GET_MAX +}; + +enum { + XSC_IOCTL_GET_PHY_INFO = 0x100, + XSC_IOCTL_GET_GLOBAL_PCP = 0x101, + XSC_IOCTL_GET_GLOBAL_DSCP = 0x102, + XSC_IOCTL_GET_CMA_PCP = 0x103, + XSC_IOCTL_GET_CMA_DSCP = 0x104, + XSC_IOCTL_GET_CONTEXT = 0x105, + XSC_IOCTL_GAT_MAX +}; + +enum { + XSC_IOCTL_SET_QP_STATUS = 0x200, + XSC_IOCTL_SET_GLOBAL_PCP = 0x201, + XSC_IOCTL_SET_GLOBAL_DSCP = 0x202, + XSC_IOCTL_SET_CMA_PCP = 0x203, + XSC_IOCTL_SET_CMA_DSCP = 0x204, + XSC_IOCTL_SET_MAX +}; + +enum { + XSC_IOCTL_MEM_ALLOC = 0x300, + XSC_IOCTL_MEM_FREE, + XSC_IOCTL_MEM_MAX +}; + +enum xsc_flow_tbl_id { + XSC_FLOW_TBL_IPAT, //IN_PORT_ATTR + XSC_FLOW_TBL_IPVLANMT, //IN_PORT_VLAN_MEMBER + XSC_FLOW_TBL_IN_VLAN_M, //IN_VLAN_MAPPING + XSC_FLOW_TBL_HOST_VLAN_M, //HOST_VLAN_MAPPING + XSC_FLOW_TBL_PCT_V4, //PACKET_CLASSIFIER_V4 + XSC_FLOW_TBL_PCT_V6, //PACKET_CLASSIFIER_V6 + XSC_FLOW_TBL_WCT_KP, //WCT_KEY_PROFILE + XSC_FLOW_TBL_WCT, //WILDCARD_TBL + XSC_FLOW_TBL_FKP, //FLOW_KEY_PROFILE + XSC_FLOW_TBL_EM, //EXACT_MATCH + XSC_FLOW_TBL_FAT, //FLOW_ACTION + XSC_FLOW_TBL_TNL_ECP, //TUNNEL_ENCAP + XSC_FLOW_TBL_ERP_HDR, //ERSPAN_HDR_INFO + XSC_FLOW_TBL_MIR_IDX, //MIRROR_INDEX + XSC_FLOW_TBL_MIR, //MIRROR_TBL + XSC_FLOW_TBL_MIR_HDR, //ENCAP_MIRROR_HDR + XSC_FLOW_TBL_VER, //VERSION_TBL + XSC_FLOW_TBL_LCMT, //LCMT_TBL + XSC_FLOW_TBL_CT, //CONN_TRACK + XSC_FLOW_TBL_EPAT, //EG_PORT_ATTR + XSC_FLOW_TBL_OPVLANMT, //OUT_PORT_VLAN_MEMBER + XSC_FLOW_TBL_RSS_HASH, //RSS_HASH + XSC_FLOW_TBL_MDF_MAC, //MODIFY_MAC + XSC_FLOW_TBL_MDF_IP, //MODIFY_IP + XSC_FLOW_TBL_MDF_TPID, //MODIFY_TPID + XSC_FLOW_TBL_ECP_HDR, //ENCAP_HDR + XSC_FLOW_TBL_ECP_MAC, //ENCAP_MAC + XSC_FLOW_TBL_ECP_IP, //ENCAP_IP + XSC_FLOW_TBL_ECP_TPID, //ENCAP_TPID + XSC_FLOW_TBL_ECP_TP_TNL, //ENCAP_TP_TUNNEL + XSC_FLOW_TBL_ECP_DPORT, //ENCAP_DPORT + XSC_FLOW_TBL_VFSO, //VF_START_OFST + XSC_FLOW_TBL_IACL, //INGRESS_ACL + XSC_FLOW_TBL_IACL_CNT, //INGRESS_ACL_COUNTER + XSC_FLOW_TBL_EACL, //EGRESS_ACL + XSC_FLOW_TBL_EACL_CNT, //EGRESS_ACL_COUNTER + XSC_FLOW_TBL_EM_EXT, //EXACT_MATCH_EXT + XSC_FLOW_TBL_EM_EXT_2M_HASH_ADR, //EM_EXT_2M_HASH_ADDR + XSC_FLOW_TBL_EM_EXT_1G_HASH_ADR, //EM_EXT_1G_HASH_ADDR + XSC_FLOW_TBL_EM_EXT_2M_KEY_ADR, //EM_EXT_2M_KEY_ADDR + XSC_FLOW_TBL_EM_EXT_1G_KEY_ADR, //EM_EXT_1G_KEY_ADDR + XSC_FLOW_TBL_PG_QP_SET_ID, //PG_QP_SET_ID + XSC_FLOW_DIR_REGISTER, //DIR_REGISTER + XSC_FLOW_INDIR_REGISTER, //INDIR_REGISTER + XSC_FLOW_TBL_BM_PCT_V4, //BIM MATCH PACKET_CLASSIFIER_V4 + XSC_FLOW_TBL_BM_PCT_V6, //BIM MATCH PACKET_CLASSIFIER_V6 + XSC_FLOW_TBL_BM_WCT, //BIM MATCH WILDCARD_TBL + XSC_FLOW_TBL_BM_IACL, //BIM MATCH INGRESS_ACL + XSC_FLOW_TBL_BMT, //BROADCAST MEMBER + XSC_FLOW_TBL_BOMT, //BROADCAST OUTPUT + XSC_FLOW_TBL_PST, //pst + XSC_FLOW_DMA_WR, //DMA WRITE + XSC_FLOW_DMA_RD, //DMA READ + XSC_FLOW_PARSER_TBL, //PARSER_TBL + XSC_FLOW_UDF_AWARE_TBL, //UDF_AWARE_TBL + XSC_FLOW_UDF_UNAWARE_TBL, //UDF_UNAWARE_TBL + XSC_FLOW_MTR_CTRL_TBL, //MTR_CTRL_TBL + XSC_FLOW_MTR_FLOW_PD, //MTR_FLOW_PD + XSC_FLOW_MTR_VPORT_PD, //MTR_VPORT_PD + XSC_FLOW_MTR_VPG_PD, //MTR_VPG_PD + XSC_FLOW_MTR_FLOW_SCAN, //MTR_FLOW_SCAN + XSC_FLOW_MTR_VPORT_SCAN, //MTR_VPORT_SCAN + XSC_FLOW_MTR_VPG_SCAN, //MTR_VPG_SCAN + XSC_FLOW_MTR_MAPPING, //MTR_MAPPING + XSC_FLOW_PRG_ACT_IDX, //PRG_ACT_INDEX + XSC_FLOW_PRG_ACT0, //PRG_ACT0 + XSC_FLOW_PRG_ACT1, //PRG_ACT1 + XSC_FLOW_PRG_ACT2, //PRG_ACT2 + XSC_FLOW_NIF_PRI_CNT, //NIF_PRI_CNT + XSC_FLOW_PRS2CLSF_SRC_PORT_CNT, //PRS2CLSF_SRC_PORT_CNT + XSC_FLOW_QUEUE_RX_CNT, //QUEUE_TX_CNT + XSC_FLOW_QUEUE_TX_CNT, //QUEUE_TX_CNT + XSC_FLOW_MAC_LAG_PORT_SEL, //MAC_LAG_PORT_SEL + XSC_FLOW_EXT_CT_CLR, //EXT_CT_CLR + XSC_FLOW_IP_TBL_CFG, //IP_TBL_CFG + XSC_FLOW_RSS_HASH_INIT_KEY_CFG, //SS_HASH_INIT_KEY_CFG + XSC_FLOW_QP_ID_BASE_CFG, //QP_ID_BASE_CFG + XSC_FLOW_PSS_INFO, //CLSF_CTRL_PSS_INFO + XSC_FLOW_SNAPSHOT, //SNAPSHOT + XSC_FLOW_PSS_MATCH_KEY, //PSS_MATCH_KEY + XSC_FLOW_PSS_CLR, //PSS_CLEAR + XSC_FLOW_PSS_START, //PSS_START + XSC_FLOW_PSS_DONE, //PSS_DONE + XSC_FLOW_MAC_PORT_MTU, //MAC_PORT_MTU + XSC_FLOW_ECP_PKT_LEN_INC, //ECP_PKT_LEN_INC + XSC_FLOW_TCP_FLAGS_CFG, //TCP_FLAGS_CFG + XSC_FLOW_TBL_MAX +}; + +enum xsc_other_tbl_id { + XSC_OTHER_TBL_MAX +}; + +enum xsc_ioctl_op { + XSC_IOCTL_OP_ADD, + XSC_IOCTL_OP_DEL, + XSC_IOCTL_OP_GET, + XSC_IOCTL_OP_CLR, + XSC_IOCTL_OP_MOD, + XSC_IOCTL_OP_MAX +}; + +struct xsc_ioctl_mem_info { + u32 mem_num; + u32 size; + u64 vir_addr; + u64 phy_addr; +}; + +/* get phy info */ +struct xsc_ioctl_get_phy_info_attr { + u16 bdf; + u16 rsvd; +}; + +struct xsc_ioctl_qp_range { + u16 opcode; + int num; + u32 qpn; +}; + +struct xsc_ioctl_get_phy_info_res { + u32 phy_port; //local pf pcie funcid xdev->pcie_port + u32 func_id; //local pf port funcid xdev->glb_func_id ? + u32 logic_in_port; //local pf port logical_in_port xdev->logic_port + u32 mac_phy_port; // xdev->mac_port + u32 mac_logic_in_port;// xdev->logic_port + u16 lag_id; + u16 raw_qp_id_base; + u16 raw_rss_qp_id_base; + u16 funcid_encode[8]; + u16 lag_port_start; + u16 raw_tpe_qp_num; + int send_seg_num; + int recv_seg_num; + u8 on_chip_tbl_vld; + u8 dma_rw_tbl_vld; + u8 pct_compress_vld; + u32 chip_version; +}; + +struct xsc_ioctl_get_vf_info_res { + u16 vf_id; //start from 1, 0 is reserved for pf + u16 phy_port; //pcie0=0, pcie1=1 + u16 pf_id; //pf0=0, pf1=1 + u32 func_id; + u32 logic_port; +}; + +struct xsc_ioctl_global_pcp { + int pcp; +}; + +struct xsc_ioctl_global_dscp { + int dscp; +}; + +struct xsc_alloc_ucontext_resp { + int max_cq; + int max_qp; + u32 max_rwq_indirection_table_size; + u64 qpm_tx_db; + u64 qpm_rx_db; + u64 cqm_next_cid_reg; + u64 cqm_armdb; + u32 send_ds_num; + u32 recv_ds_num; + u32 send_ds_shift; + u32 recv_ds_shift; + u32 glb_func_id; + u32 max_wqes; +}; + +struct xsc_ioctl_cma_pcp { + int pcp; +}; + +struct xsc_ioctl_cma_dscp { + int dscp; +}; + +/* type-value */ +struct xsc_ioctl_data_tl { + u16 table; /* table id */ + u16 opmod; /* add/del/mod */ + u16 length; + u16 rsvd; +}; + +/* public header */ +struct xsc_ioctl_attr { + u16 opcode; /* ioctl cmd */ + u16 length; /* data length */ + u32 error; /* ioctl error info */ + u8 data[0]; /* specific table info */ +}; + +struct xsc_ioctl_hdr { + u32 check_filed; /* Validity verification fileds */ + struct xsc_ioctl_attr attr; +}; + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_lag.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_lag.h new file mode 100644 index 0000000000000000000000000000000000000000..1f12bec9a680c8c937904adde67e20a25ef5910d --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_lag.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_LAG_H +#define XSC_LAG_H + +struct lag_func { + struct xsc_core_device *xdev; + struct net_device *netdev; +}; + +/* Used for collection of netdev event info. */ +struct lag_tracker { + enum netdev_lag_hash hash_type; + enum netdev_lag_hash old_hash_type; + enum netdev_lag_tx_type tx_type; + struct netdev_lag_lower_state_info netdev_state[XSC_MAX_PORTS]; + struct net_device *ndev[XSC_MAX_PORTS]; + unsigned int is_bonded:1; + unsigned int lag_disable:1; + u8 gw_dmac0[6]; + u8 gw_dmac1[6]; +}; + +/* used in tracking fib events */ +struct lag_mp { + struct notifier_block fib_nb; + struct fib_info *mfi; + struct workqueue_struct *wq; +}; + +struct xsc_lag { + u8 flags; + struct kref ref; + u8 v2p_map[XSC_MAX_PORTS]; + struct lag_func pf[XSC_MAX_PORTS]; + struct lag_tracker tracker; + struct workqueue_struct *wq; + struct delayed_work bond_work; + struct notifier_block nb; + struct lag_mp lag_mp; + u16 lag_id; +}; + +struct xsc_fib_event_work { + struct work_struct work; + struct xsc_lag *ldev; + unsigned long event; + union { + struct fib_entry_notifier_info fen_info; + struct fib_nh_notifier_info fnh_info; + }; +}; + +enum { + XSC_LAG_FLAG_ROCE = 1 << 0, + XSC_LAG_FLAG_SRIOV = 1 << 1, + XSC_LAG_FLAG_MULTIPATH = 1 << 2, +}; + +enum xsc_lag_hash { + XSC_LAG_HASH_L23, + XSC_LAG_HASH_L34, + XSC_LAG_HASH_E23, + XSC_LAG_HASH_E34 +}; + +#define MAC_SHIFT 1 +#define MAC_0_1_LOGIC 0 +#define MAC_0_LOGIC (XSC_PHY_PORT_MAC_0 + MAC_SHIFT) +#define MAC_1_LOGIC (XSC_PHY_PORT_MAC_1 + MAC_SHIFT) +#define MAC_INVALID 0xff + +#define XSC_LAG_MODE_FLAGS (XSC_LAG_FLAG_ROCE | XSC_LAG_FLAG_SRIOV |\ + XSC_LAG_FLAG_MULTIPATH) + +#define GET_LAG_MEMBER_BITMAP(remap_port1, remap_port2) \ + (((remap_port1 != MAC_INVALID) ? BIT(remap_port1 - MAC_SHIFT) : 0) | \ + ((remap_port2 != MAC_INVALID) ? BIT(remap_port2 - MAC_SHIFT) : 0)) + +static inline bool __xsc_lag_is_active(struct xsc_lag *ldev) +{ + return !!(ldev->flags & XSC_LAG_MODE_FLAGS); +} + +static bool __attribute__((unused)) __xsc_lag_is_multipath(struct xsc_lag *ldev) +{ + return !!(ldev->flags & XSC_LAG_FLAG_MULTIPATH); +} + +static bool __attribute__((unused)) __xsc_lag_is_roce(struct xsc_lag *ldev) +{ + return !!(ldev->flags & XSC_LAG_FLAG_ROCE); +} + +static inline struct xsc_lag *xsc_lag_dev_get(struct xsc_core_device *xdev) +{ + return xdev->priv.lag; +} + +void xsc_lag_add(struct xsc_core_device *xdev, struct net_device *netdev); +void xsc_lag_remove(struct xsc_core_device *xdev); +void xsc_lag_add_xdev(struct xsc_core_device *xdev); +void xsc_lag_remove_xdev(struct xsc_core_device *xdev); +void xsc_lag_enable(struct xsc_core_device *xdev); +void xsc_lag_disable(struct xsc_core_device *xdev); + +#endif /* XSC_LAG_H */ + diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_port_ctrl.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_port_ctrl.h new file mode 100644 index 0000000000000000000000000000000000000000..fa476590d4cfa50b7cc4e9db9fddf788b5f62a09 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_port_ctrl.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_PORT_CTRL_H +#define XSC_PORT_CTRL_H + +typedef void (*port_ctrl_cb)(struct xsc_core_device *xdev, unsigned int cmd, + struct xsc_ioctl_hdr __user *user_hdr, void *data); + +void xsc_port_ctrl_remove(struct xsc_core_device *dev); +int xsc_port_ctrl_probe(struct xsc_core_device *dev); +int xsc_port_ctrl_cb_reg(const char *name, port_ctrl_cb cb, void *data); +void xsc_port_ctrl_cb_dereg(const char *name); + +void xsc_port_ctrl_fini(void); +int xsc_port_ctrl_init(void); + +#endif + diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_pph.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_pph.h new file mode 100644 index 0000000000000000000000000000000000000000..f05b4509e2a752d4612e52fa5a1f98f199ee5aaf --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_pph.h @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_PPH_H +#define XSC_PPH_H + +#define XSC_PPH_HEAD_LEN 64 + +enum { + L4_PROTO_NONE = 0, + L4_PROTO_TCP = 1, + L4_PROTO_UDP = 2, + L4_PROTO_ICMP = 3, + L4_PROTO_GRE = 4, +}; + +enum { + L3_PROTO_NONE = 0, + L3_PROTO_IP = 2, + L3_PROTO_IP6 = 3, +}; + +struct epp_pph { + u16 outer_eth_type; //2 bytes + u16 inner_eth_type; //4 bytes + + u16 rsv1:1; + u16 outer_vlan_flag:2; + u16 outer_ip_type:2; + u16 outer_ip_ofst:5; + u16 outer_ip_len:6; //6 bytes + + u16 rsv2:1; + u16 outer_tp_type:3; + u16 outer_tp_csum_flag:1; + u16 outer_tp_ofst:7; + u16 ext_tunnel_type:4; //8 bytes + + u8 tunnel_ofst; //9 bytes + u8 inner_mac_ofst; //10 bytes + + u32 rsv3:2; + u32 inner_mac_flag:1; + u32 inner_vlan_flag:2; + u32 inner_ip_type:2; + u32 inner_ip_ofst:8; + u32 inner_ip_len:6; + u32 inner_tp_type:2; + u32 inner_tp_csum_flag:1; + u32 inner_tp_ofst:8; //14 bytees + + u16 rsv4:1; + u16 payload_type:4; + u16 payload_ofst:8; + u16 pkt_type:3; //16 bytes + + u16 rsv5:2; + u16 pri:3; + u16 logical_in_port:11; + u16 vlan_info; + u8 error_bitmap:8; //21 bytes + + u8 rsv6:7; + u8 recirc_id_vld:1; + u16 recirc_id; //24 bytes + + u8 rsv7:7; + u8 recirc_data_vld:1; + u32 recirc_data; //29 bytes + + u8 rsv8:6; + u8 mark_tag_vld:2; + u16 mark_tag; //32 bytes + + u8 rsv9:4; + u8 upa_to_soc:1; + u8 upa_from_soc:1; + u8 upa_re_up_call:1; + u8 upa_pkt_drop:1; //33 bytes + + u8 ucdv; + u16 rsv10:2; + u16 pkt_len:14; //36 bytes + + u16 rsv11:2; + u16 pkt_hdr_ptr:14; //38 bytes + + u64 rsv12:5; + u64 csum_ofst:8; + u64 csum_val:29; + u64 csum_plen:14; + u64 rsv11_0:8; //46 bytes + + u64 rsv11_1; + u64 rsv11_2; + u16 rsv11_3; +}; + +#define OUTER_L3_BIT BIT(3) +#define OUTER_L4_BIT BIT(2) +#define INNER_L3_BIT BIT(1) +#define INNER_L4_BIT BIT(0) +#define OUTER_BIT (OUTER_L3_BIT | OUTER_L4_BIT) +#define INNER_BIT (INNER_L3_BIT | INNER_L4_BIT) +#define OUTER_AND_INNER (OUTER_BIT | INNER_BIT) + +#define PACKET_UNKNOWN BIT(4) + +#define EPP2SOC_PPH_EXT_TUNNEL_TYPE_OFFSET (6UL) +#define EPP2SOC_PPH_EXT_TUNNEL_TYPE_BIT_MASK (0XF00) +#define EPP2SOC_PPH_EXT_TUNNEL_TYPE_BIT_OFFSET (8) + +#define EPP2SOC_PPH_EXT_ERROR_BITMAP_OFFSET (20UL) +#define EPP2SOC_PPH_EXT_ERROR_BITMAP_BIT_MASK (0XFF) +#define EPP2SOC_PPH_EXT_ERROR_BITMAP_BIT_OFFSET (0) + +#define XSC_GET_EPP2SOC_PPH_EXT_TUNNEL_TYPE(PPH_BASE_ADDR) \ + ((*(u16 *)((u8 *)PPH_BASE_ADDR + EPP2SOC_PPH_EXT_TUNNEL_TYPE_OFFSET) & \ + EPP2SOC_PPH_EXT_TUNNEL_TYPE_BIT_MASK) >> EPP2SOC_PPH_EXT_TUNNEL_TYPE_BIT_OFFSET) + +#define XSC_GET_EPP2SOC_PPH_ERROR_BITMAP(PPH_BASE_ADDR) \ + ((*(u8 *)((u8 *)PPH_BASE_ADDR + EPP2SOC_PPH_EXT_ERROR_BITMAP_OFFSET) & \ + EPP2SOC_PPH_EXT_ERROR_BITMAP_BIT_MASK) >> EPP2SOC_PPH_EXT_ERROR_BITMAP_BIT_OFFSET) + +#define PPH_OUTER_IP_TYPE_OFF (4UL) +#define PPH_OUTER_IP_TYPE_MASK (0x3) +#define PPH_OUTER_IP_TYPE_SHIFT (11) +#define PPH_OUTER_IP_TYPE(base) \ + ((ntohs(*(u16 *)((u8 *)base + PPH_OUTER_IP_TYPE_OFF)) >> \ + PPH_OUTER_IP_TYPE_SHIFT) & PPH_OUTER_IP_TYPE_MASK) + +#define PPH_OUTER_IP_OFST_OFF (4UL) +#define PPH_OUTER_IP_OFST_MASK (0x1f) +#define PPH_OUTER_IP_OFST_SHIFT (6) +#define PPH_OUTER_IP_OFST(base) \ + ((ntohs(*(u16 *)((u8 *)base + PPH_OUTER_IP_OFST_OFF)) >> \ + PPH_OUTER_IP_OFST_SHIFT) & PPH_OUTER_IP_OFST_MASK) + +#define PPH_OUTER_IP_LEN_OFF (4UL) +#define PPH_OUTER_IP_LEN_MASK (0x3f) +#define PPH_OUTER_IP_LEN_SHIFT (0) +#define PPH_OUTER_IP_LEN(base) \ + ((ntohs(*(u16 *)((u8 *)base + PPH_OUTER_IP_LEN_OFF)) >> \ + PPH_OUTER_IP_LEN_SHIFT) & PPH_OUTER_IP_LEN_MASK) + +#define PPH_OUTER_TP_TYPE_OFF (6UL) +#define PPH_OUTER_TP_TYPE_MASK (0x7) +#define PPH_OUTER_TP_TYPE_SHIFT (12) +#define PPH_OUTER_TP_TYPE(base) \ + ((ntohs(*(u16 *)((u8 *)base + PPH_OUTER_TP_TYPE_OFF)) >> \ + PPH_OUTER_TP_TYPE_SHIFT) & PPH_OUTER_TP_TYPE_MASK) + +#define PPH_PAYLOAD_OFST_OFF (14UL) +#define PPH_PAYLOAD_OFST_MASK (0xff) +#define PPH_PAYLOAD_OFST_SHIFT (3) +#define PPH_PAYLOAD_OFST(base) \ + ((ntohs(*(u16 *)((u8 *)base + PPH_PAYLOAD_OFST_OFF)) >> \ + PPH_PAYLOAD_OFST_SHIFT) & PPH_PAYLOAD_OFST_MASK) + +#define PPH_CSUM_OFST_OFF (38UL) +#define PPH_CSUM_OFST_MASK (0xff) +#define PPH_CSUM_OFST_SHIFT (51) +#define PPH_CSUM_OFST(base) \ + ((be64_to_cpu(*(u64 *)((u8 *)base + PPH_CSUM_OFST_OFF)) >> \ + PPH_CSUM_OFST_SHIFT) & PPH_CSUM_OFST_MASK) + +#define PPH_CSUM_VAL_OFF (38UL) +#define PPH_CSUM_VAL_MASK (0xeffffff) +#define PPH_CSUM_VAL_SHIFT (22) +#define PPH_CSUM_VAL(base) \ + ((be64_to_cpu(*(u64 *)((u8 *)base + PPH_CSUM_VAL_OFF)) >> \ + PPH_CSUM_VAL_SHIFT) & PPH_CSUM_VAL_MASK) +#endif /* XSC_TBM_H */ + diff --git a/drivers/net/ethernet/yunsilicon/xsc/common/xsc_reg.h b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_reg.h new file mode 100644 index 0000000000000000000000000000000000000000..4e5a3fc6a7a7b2cdcf916f1e03385ddf21fe9179 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/common/xsc_reg.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_REG_H +#define XSC_REG_H +#define CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_ADDR 0x0 +#define CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_ADDR 0x4 +#define CMDQM_HOST_REQ_PID_MEM_ADDR 0x8 +#define CMDQM_HOST_REQ_CID_MEM_ADDR 0xc +#define CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_ADDR 0x10 +#define CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_ADDR 0x14 +#define CMDQM_HOST_RSP_PID_MEM_ADDR 0x18 +#define CMDQM_HOST_RSP_CID_MEM_ADDR 0x1c +#define CMDQM_HOST_VF_ERR_STS_MEM_ADDR 0x20 +#define CMDQM_VECTOR_ID_MEM_ADDR 0x24 +#define CMDQM_Q_ELEMENT_SZ_REG_ADDR 0x28 +#define CMDQM_HOST_Q_DEPTH_REG_ADDR 0x2c + +#define CPM_LOCK_GET_REG_ADDR 0x30 +#define CPM_LOCK_PUT_REG_ADDR 0x34 +#define CPM_LOCK_AVAIL_REG_ADDR 0x38 +#define CPM_IDA_DATA_MEM_ADDR 0x3c +#define CPM_IDA_CMD_REG_ADDR 0x83c +#define CPM_IDA_ADDR_REG_ADDR 0x840 +#define CPM_IDA_BUSY_REG_ADDR 0x8c0 + +#define DB_CQ_FUNC_MEM_ADDR 0x8c4 +#define DB_EQ_FUNC_MEM_ADDR 0x8c8 +#define DB_CQ_CID_DIRECT_MEM_ADDR 0x8cc +#define TX_DB_FUNC_MEM_ADDR 0x8d0 +#define RX_DB_FUNC_MEM_ADDR 0x8d4 + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/Kconfig b/drivers/net/ethernet/yunsilicon/xsc/net/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..341c7be739f6f3ca18a9ea20de2c26adc968fba2 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/Kconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. +# All rights reserved. +# Yunsilicon driver configuration +# + +config YUNSILICON_XSC_ETH + tristate "Yunsilicon XSC ethernet driver" + default n + depends on YUNSILICON_XSC_PCI + help + This driver provides ethernet support for + Yunsilicon XSC devices. diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/Makefile b/drivers/net/ethernet/yunsilicon/xsc/net/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..cd0430ee128315caa4d49c21edb7171f6a18022a --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. +# All rights reserved. +# +# Makefile for the Yunsilicon xsc ethernet driver +# +ccflags-y := -I $(srctree)/drivers/net/ethernet/yunsilicon/xsc/ +obj-$(CONFIG_YUNSILICON_XSC_ETH) += xsc_net.o + +xsc_net-y := main.o xsc_eth_ctrl.o xsc_eth_tx.o xsc_eth_rx.o \ + xsc_eth_txrx.o ut_main.o xsc_eth_ethtool.o \ + xsc_eth_stats.o xsc_dcbnl.o + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/main.c b/drivers/net/ethernet/yunsilicon/xsc/net/main.c new file mode 100644 index 0000000000000000000000000000000000000000..70aba07b5b3d3f0fc246ddacd68dba994f1748c6 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/main.c @@ -0,0 +1,3169 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "../pci/fw/xsc_tbm.h" + +#include "xsc_eth.h" +#include "xsc_eth_txrx.h" +#include "xsc_eth_ethtool.h" +#include "xsc_eth_common.h" +#include "xsc_eth_stats.h" +#include "xsc_accel.h" +#include "xsc_eth_ctrl.h" + +static void xsc_eth_close_channel(struct xsc_channel *c, bool free_rq); +static void xsc_eth_remove(struct xsc_core_device *xdev, void *context); + +static int xsc_eth_open(struct net_device *netdev); +static int xsc_eth_close(struct net_device *netdev); + +#ifdef NEED_CREATE_RX_THREAD +extern u32 xsc_eth_rx_thread_create(struct xsc_adapter *adapter); +#endif + +#define XSC_SET_FEATURE(features, feature, enable) \ + do { \ + if (enable) \ + *features |= feature; \ + else \ + *features &= ~feature; \ + } while (0) + +typedef int (*xsc_feature_handler)(struct net_device *netdev, bool enable); + +int xsc_eth_modify_qp_status(struct xsc_core_device *xdev, + u32 qpn, u16 status) +{ + int ret = -1; + int insize; + struct xsc_modify_qp_mbox_in *in; + struct xsc_modify_qp_mbox_out out; + + insize = sizeof(struct xsc_modify_qp_mbox_in); + + in = kvzalloc(insize, GFP_KERNEL); + if (!in) + return -ENOMEM; + + /*eth: only set status according to cmd,ignore other fields*/ + in->hdr.opcode = cpu_to_be16(status); + in->qpn = cpu_to_be32(qpn); + + ret = xsc_cmd_exec(xdev, in, insize, &out, sizeof(out)); + if (ret) { + xsc_core_warn(xdev, "failed to modify qp%d status=%d, err=%d\n", + qpn, status, ret); + goto exit; + } + + if (out.hdr.status != 0) { + xsc_core_warn(xdev, "return error status %d\n", out.hdr.status); + ret = -ENOEXEC; + } + +exit: + kvfree(in); + return ret; +} + +static void xsc_eth_build_queue_param(struct xsc_adapter *adapter, + struct xsc_queue_attr *attr, u8 type) +{ + struct xsc_core_device *xdev = adapter->xdev; + + if (adapter->nic_param.sq_size == 0) + adapter->nic_param.sq_size = BIT(xdev->caps.log_max_qp_depth); + if (adapter->nic_param.rq_size == 0) + adapter->nic_param.rq_size = BIT(xdev->caps.log_max_qp_depth); + + if (type == XSC_QUEUE_TYPE_EQ) { + attr->q_type = XSC_QUEUE_TYPE_EQ; + attr->ele_num = XSC_EQ_ELE_NUM; + attr->ele_size = XSC_EQ_ELE_SZ; + attr->ele_log_size = order_base_2(XSC_EQ_ELE_SZ); + attr->q_log_size = order_base_2(XSC_EQ_ELE_NUM); + } else if (type == XSC_QUEUE_TYPE_RQCQ) { + attr->q_type = XSC_QUEUE_TYPE_RQCQ; + attr->ele_num = adapter->nic_param.rq_size; + attr->ele_size = XSC_RQCQ_ELE_SZ; + attr->ele_log_size = order_base_2(XSC_RQCQ_ELE_SZ); + attr->q_log_size = order_base_2(attr->ele_num); + } else if (type == XSC_QUEUE_TYPE_SQCQ) { + attr->q_type = XSC_QUEUE_TYPE_SQCQ; + attr->ele_num = adapter->nic_param.sq_size; + attr->ele_size = XSC_SQCQ_ELE_SZ; + attr->ele_log_size = order_base_2(XSC_SQCQ_ELE_SZ); + attr->q_log_size = order_base_2(attr->ele_num); + } else if (type == XSC_QUEUE_TYPE_RQ) { + attr->q_type = XSC_QUEUE_TYPE_RQ; + attr->ele_num = adapter->nic_param.rq_size; + attr->ele_size = xdev->caps.recv_ds_num * XSC_RECV_WQE_DS; + attr->ele_log_size = order_base_2(attr->ele_size); + attr->q_log_size = order_base_2(attr->ele_num); + } else if (type == XSC_QUEUE_TYPE_SQ) { + attr->q_type = XSC_QUEUE_TYPE_SQ; + attr->ele_num = adapter->nic_param.sq_size; + attr->ele_size = xdev->caps.send_ds_num * XSC_SEND_WQE_DS; + attr->ele_log_size = order_base_2(attr->ele_size); + attr->q_log_size = order_base_2(attr->ele_num); + } +} + +static void xsc_eth_init_frags_partition(struct xsc_rq *rq) +{ + struct xsc_wqe_frag_info next_frag, *prev; + int i; + + next_frag.di = &rq->wqe.di[0]; + next_frag.offset = 0; + prev = NULL; + + for (i = 0; i < xsc_wq_cyc_get_size(&rq->wqe.wq); i++) { + struct xsc_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; + struct xsc_wqe_frag_info *frag = + &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; + int f; + + for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { + if (next_frag.offset + frag_info[f].frag_stride > + XSC_RX_FRAG_SZ) { + next_frag.di++; + next_frag.offset = 0; + if (prev) + prev->last_in_page = true; + } + *frag = next_frag; + + /* prepare next */ + next_frag.offset += frag_info[f].frag_stride; + prev = frag; + } + } + + if (prev) + prev->last_in_page = true; +} + +static int xsc_eth_init_di_list(struct xsc_rq *rq, int wq_sz, int cpu) +{ + int len = wq_sz << rq->wqe.info.log_num_frags; + + rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), + GFP_KERNEL, cpu_to_node(cpu)); + if (!rq->wqe.di) + return -ENOMEM; + + xsc_eth_init_frags_partition(rq); + + return 0; +} + +static void xsc_eth_free_di_list(struct xsc_rq *rq) +{ + kvfree(rq->wqe.di); +} + +static void xsc_rx_cache_reduce_clean_pending(struct xsc_rq *rq) +{ + struct xsc_page_cache_reduce *reduce = &rq->page_cache.reduce; + int i; + + if (!test_bit(XSC_ETH_RQ_STATE_CACHE_REDUCE_PENDING, &rq->state)) + return; + + for (i = 0; i < reduce->npages; i++) + xsc_page_release_dynamic(rq, &reduce->pending[i], false); + reduce->npages = 0; + + clear_bit(XSC_ETH_RQ_STATE_CACHE_REDUCE_PENDING, &rq->state); +} + +static void xsc_rx_cache_reduce_work(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct xsc_page_cache_reduce *reduce = + container_of(dwork, struct xsc_page_cache_reduce, reduce_work); + struct xsc_page_cache *cache = + container_of(reduce, struct xsc_page_cache, reduce); + struct xsc_rq *rq = container_of(cache, struct xsc_rq, page_cache); + + local_bh_disable(); + napi_schedule(rq->cq.napi); + local_bh_enable(); + xsc_rx_cache_reduce_clean_pending(rq); + + if (ilog2(cache->sz) > cache->log_min_sz) + schedule_delayed_work_on(smp_processor_id(), + dwork, reduce->delay); +} + +int xsc_rx_alloc_page_cache(struct xsc_rq *rq, int node, u8 log_init_sz) +{ + struct xsc_page_cache *cache = &rq->page_cache; + struct xsc_page_cache_reduce *reduce = &cache->reduce; + u32 max_sz; + + cache->log_max_sz = log_init_sz + XSC_PAGE_CACHE_LOG_MAX_RQ_MULT; + cache->log_min_sz = log_init_sz; + max_sz = 1 << cache->log_max_sz; + + cache->page_cache = kvzalloc_node(max_sz * sizeof(*cache->page_cache), + GFP_KERNEL, node); + if (!cache->page_cache) + return -ENOMEM; + + reduce->pending = kvzalloc_node(max_sz * sizeof(*reduce->pending), + GFP_KERNEL, node); + if (!reduce->pending) + goto err_free_cache; + + cache->sz = 1 << cache->log_min_sz; + cache->head = -1; + INIT_DELAYED_WORK(&reduce->reduce_work, xsc_rx_cache_reduce_work); + reduce->delay = msecs_to_jiffies(XSC_PAGE_CACHE_REDUCE_WORK_INTERVAL); + reduce->grace_period = msecs_to_jiffies(XSC_PAGE_CACHE_REDUCE_GRACE_PERIOD); + reduce->next_ts = MAX_JIFFY_OFFSET; /* in init, no reduce is needed */ + + return 0; + +err_free_cache: + kvfree(cache->page_cache); + + return -ENOMEM; +} + +void xsc_rx_free_page_cache(struct xsc_rq *rq) +{ + struct xsc_page_cache *cache = &rq->page_cache; + struct xsc_page_cache_reduce *reduce = &cache->reduce; + int i; + + cancel_delayed_work_sync(&reduce->reduce_work); + xsc_rx_cache_reduce_clean_pending(rq); + kvfree(reduce->pending); + + for (i = 0; i <= cache->head; i++) { + struct xsc_dma_info *dma_info = &cache->page_cache[i]; + + xsc_page_release_dynamic(rq, dma_info, false); + } + kvfree(cache->page_cache); +} + +int xsc_eth_reset(struct xsc_core_device *dev) +{ + return 0; +} + +void xsc_eth_cq_error_event(struct xsc_core_cq *xcq, enum xsc_event event) +{ + struct xsc_cq *xsc_cq = container_of(xcq, struct xsc_cq, xcq); + struct xsc_core_device *xdev = xsc_cq->xdev; + + if (event != XSC_EVENT_TYPE_CQ_ERROR) { + xsc_core_err(xdev, "Unexpected event type %d on CQ %06x\n", + event, xcq->cqn); + return; + } + + xsc_core_err(xdev, "Eth catch CQ ERROR：%x, cqn: %d\n", event, xcq->cqn); +} + +void xsc_eth_completion_event(struct xsc_core_cq *xcq) +{ + struct xsc_cq *cq = container_of(xcq, struct xsc_cq, xcq); + struct xsc_core_device *xdev = cq->xdev; + struct xsc_rq *rq = &cq->channel->qp.rq[0]; + + set_bit(XSC_CHANNEL_NAPI_SCHED, &cq->channel->flags); + cq->channel->stats->events++; + cq->channel->stats->poll = 0; + if (!test_bit(XSC_ETH_RQ_STATE_ENABLED, &rq->state)) + xsc_core_warn(xdev, "ch%d_cq%d, napi_flag=0x%lx\n", + cq->channel->chl_idx, xcq->cqn, cq->napi->state); + + napi_schedule(cq->napi); +} + +inline int xsc_cmd_destroy_cq(struct xsc_core_device *dev, struct xsc_core_cq *xcq) +{ + struct xsc_destroy_cq_mbox_in in; + struct xsc_destroy_cq_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_CQ); + in.cqn = cpu_to_be32(xcq->cqn); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + return -1; + + xcq->cqn = 0; + return 0; +} + +int xsc_eth_create_cq(struct xsc_core_device *xdev, struct xsc_core_cq *xcq, + struct xsc_create_cq_mbox_in *in, int insize) +{ + int err, ret = -1; + struct xsc_cq_table *table = &xdev->dev_res->cq_table; + struct xsc_create_cq_mbox_out out; + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_CREATE_CQ); + ret = xsc_cmd_exec(xdev, in, insize, &out, sizeof(out)); + if (ret) + return ret; + + xcq->cqn = be32_to_cpu(out.cqn) & 0xffffff; + xcq->cons_index = 0; + xcq->arm_sn = 0; + atomic_set(&xcq->refcount, 1); + init_completion(&xcq->free); + + spin_lock_irq(&table->lock); + ret = radix_tree_insert(&table->tree, xcq->cqn, xcq); + spin_unlock_irq(&table->lock); + if (ret) + goto err_insert_cq; + return 0; + +err_insert_cq: + err = xsc_cmd_destroy_cq(xdev, xcq); + if (err) + xsc_core_warn(xdev, "failed to destroy cqn=%d, err=%d\n", xcq->cqn, err); + return ret; +} + +int xsc_eth_destroy_cq(struct xsc_core_device *xdev, struct xsc_cq *cq) +{ + struct xsc_cq_table *table = &xdev->dev_res->cq_table; + struct xsc_core_cq *tmp; + int err; + + spin_lock_irq(&table->lock); + tmp = radix_tree_delete(&table->tree, cq->xcq.cqn); + spin_unlock_irq(&table->lock); + if (!tmp) { + err = -ENOENT; + goto err_delete_cq; + } + + if (tmp != &cq->xcq) { + err = -EINVAL; + goto err_delete_cq; + } + + err = xsc_cmd_destroy_cq(xdev, &cq->xcq); + if (err) + goto err_destroy_cq; + + if (atomic_dec_and_test(&cq->xcq.refcount)) + complete(&cq->xcq.free); + wait_for_completion(&cq->xcq.free); + return 0; + +err_destroy_cq: + xsc_core_warn(xdev, "failed to destroy cqn=%d, err=%d\n", + cq->xcq.cqn, err); + return err; +err_delete_cq: + xsc_core_warn(xdev, "cqn=%d not found in tree, err=%d\n", + cq->xcq.cqn, err); + return err; +} + +void xsc_eth_free_cq(struct xsc_cq *cq) +{ + xsc_eth_wq_destroy(&cq->wq_ctrl); +} + +int xsc_eth_create_rss_qp_rqs(struct xsc_core_device *xdev, + struct xsc_create_multiqp_mbox_in *in, + int insize, + int *prqn_base) +{ + int ret; + struct xsc_create_multiqp_mbox_out out; + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_CREATE_MULTI_QP); + ret = xsc_cmd_exec(xdev, in, insize, &out, sizeof(out)); + if (ret) { + xsc_core_warn(xdev, + "failed to create rss rq, qp_num=%d, type=%d, err=%d\n", + in->qp_num, in->qp_type, ret); + return ret; + } + + if (out.hdr.status != 0) { + xsc_core_warn(xdev, "create rss qp(num=%d) return err status=%d\n", + in->qp_num, out.hdr.status); + return -ENOEXEC; + } + + *prqn_base = be32_to_cpu(out.qpn_base) & 0xffffff; + return 0; +} + +void xsc_eth_qp_event(struct xsc_core_qp *qp, int type) +{ + struct xsc_rq *rq; + struct xsc_sq *sq; + struct xsc_core_device *xdev; + + if (qp->eth_queue_type == XSC_RES_RQ) { + rq = container_of(qp, struct xsc_rq, cqp); + xdev = rq->cq.xdev; + } else if (qp->eth_queue_type == XSC_RES_SQ) { + sq = container_of(qp, struct xsc_sq, cqp); + xdev = sq->cq.xdev; + } else { + pr_err("%s:Unknown eth qp type %d\n", __func__, type); + return; + } + + switch (type) { + case XSC_EVENT_TYPE_WQ_CATAS_ERROR: + case XSC_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case XSC_EVENT_TYPE_WQ_ACCESS_ERROR: + xsc_core_err(xdev, "%s:Async event %x on QP %d\n", + __func__, type, qp->qpn); + break; + default: + xsc_core_err(xdev, "%s: Unexpected event type %d on QP %d\n", + __func__, type, qp->qpn); + return; + } + + // TODO: add eth event handler + +} + +int xsc_eth_create_qp_rq(struct xsc_core_device *xdev, struct xsc_rq *prq, + struct xsc_create_qp_mbox_in *in, int insize) +{ + int ret = -1; + struct xsc_create_qp_mbox_out out; + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_CREATE_QP); + ret = xsc_cmd_exec(xdev, in, insize, &out, sizeof(out)); + if (ret) { + xsc_core_warn(xdev, "failed to create rq, err=%d\n", ret); + return ret; + } + + prq->rqn = be32_to_cpu(out.qpn) & 0xffffff; + prq->cqp.event = xsc_eth_qp_event; + prq->cqp.eth_queue_type = XSC_RES_RQ; + + ret = create_resource_common(xdev, &prq->cqp); + if (ret) { + xsc_core_err(xdev, "%s:error qp:%d errno:%d\n", + __func__, prq->rqn, ret); + return ret; + } + + return 0; +} + +int xsc_eth_destroy_qp_rq(struct xsc_core_device *xdev, struct xsc_rq *prq) +{ + struct xsc_destroy_qp_mbox_in in; + struct xsc_destroy_qp_mbox_out out; + int err; + + err = xsc_eth_modify_qp_status(xdev, prq->rqn, + XSC_CMD_OP_2RST_QP); + if (err) { + xsc_core_warn(xdev, "failed to set rq%d status=rst, err=%d\n", + prq->rqn, err); + return err; + } + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_QP); + in.qpn = cpu_to_be32(prq->rqn); + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) { + xsc_core_warn(xdev, "failed to destroy rq%d, err=%d\n", + prq->rqn, err); + return err; + } + + if (out.hdr.status) { + xsc_core_warn(xdev, "cmdq destroy rq%d return err status=%d\n", + prq->rqn, out.hdr.status); + return -ENOEXEC; + } + + return 0; +} + +static void xsc_free_qp_rq(struct xsc_rq *rq) +{ + if (rq->dealloc_wqes) + rq->dealloc_wqes(rq); + + if (rq->page_cache.page_cache) + xsc_rx_free_page_cache(rq); + + kvfree(rq->wqe.frags); + kvfree(rq->wqe.di); + + if (rq->page_pool) + page_pool_destroy(rq->page_pool); + + xsc_eth_wq_destroy(&rq->wq_ctrl); +} + +int xsc_eth_create_qp_sq(struct xsc_core_device *xdev, struct xsc_sq *psq, + struct xsc_create_qp_mbox_in *in, int insize) +{ + int ret = -1; + struct xsc_create_qp_mbox_out out; + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_CREATE_QP); + ret = xsc_cmd_exec(xdev, in, insize, &out, sizeof(out)); + if (ret) { + xsc_core_warn(xdev, "failed to create sq, err=%d\n", ret); + return ret; + } + + psq->sqn = be32_to_cpu(out.qpn) & 0xffffff; + + return 0; +} + +int xsc_eth_modify_qp_sq(struct xsc_core_device *xdev, struct xsc_modify_raw_qp_mbox_in *in) +{ + struct xsc_modify_raw_qp_mbox_out out; + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_MODIFY_RAW_QP); + + xsc_cmd_exec(xdev, in, sizeof(struct xsc_modify_raw_qp_mbox_in), + &out, sizeof(struct xsc_modify_raw_qp_mbox_out)); + if (out.hdr.status) + xsc_core_warn(xdev, "failed to modify sq, err=%d\n", out.hdr.status); + + return out.hdr.status; +} + +int xsc_eth_destroy_qp_sq(struct xsc_core_device *xdev, struct xsc_sq *psq) +{ + struct xsc_destroy_qp_mbox_in in; + struct xsc_destroy_qp_mbox_out out; + int err; + + err = xsc_eth_modify_qp_status(xdev, psq->sqn, XSC_CMD_OP_2RST_QP); + if (err) { + xsc_core_warn(xdev, "failed to set sq%d status=rst, err=%d\n", + psq->sqn, err); + return err; + } + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_QP); + in.qpn = cpu_to_be32(psq->sqn); + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) { + xsc_core_warn(xdev, "failed to destroy sq%d, err=%d\n", + psq->sqn, err); + return err; + } + + if (out.hdr.status) { + xsc_core_warn(xdev, "destroy sq%d return err status=%d\n", + psq->sqn, out.hdr.status); + return -ENOEXEC; + } + + return 0; +} + +static void xsc_free_qp_sq_db(struct xsc_sq *sq) +{ + kvfree(sq->db.wqe_info); + kvfree(sq->db.dma_fifo); +} + +static void xsc_free_qp_sq(struct xsc_sq *sq) +{ + xsc_free_qp_sq_db(sq); + xsc_eth_wq_destroy(&sq->wq_ctrl); +} + +static int xsc_eth_alloc_qp_sq_db(struct xsc_sq *sq, int numa) +{ + int wq_sz = xsc_wq_cyc_get_size(&sq->wq); + struct xsc_core_device *xdev = sq->cq.xdev; + int df_sz = wq_sz * xdev->caps.send_ds_num; + + sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, + sizeof(*sq->db.dma_fifo)), + GFP_KERNEL, numa); + sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, + sizeof(*sq->db.wqe_info)), + GFP_KERNEL, numa); + + if (!sq->db.dma_fifo || !sq->db.wqe_info) { + xsc_free_qp_sq_db(sq); + return -ENOMEM; + } + + sq->dma_fifo_mask = df_sz - 1; + + return 0; +} + +static int xsc_eth_alloc_cq(struct xsc_channel *c, + struct xsc_cq *pcq, + struct xsc_cq_param *pcq_param) +{ + int ret; + struct xsc_core_device *xdev = c->adapter->xdev; + struct xsc_core_cq *core_cq = &pcq->xcq; + u32 i; + u8 q_log_size = pcq_param->cq_attr.q_log_size; + u8 ele_log_size = pcq_param->cq_attr.ele_log_size; + + /*xdev params is not match with function defination*/ +// ret = xsc_vector2eqn(xdev, c->chl_idx, &eqn, &irqn); +// if (ret) +// return ret; + + ret = xsc_eth_cqwq_create(xdev, &pcq_param->wq, + q_log_size, ele_log_size, &pcq->wq, + &pcq->wq_ctrl); + if (ret) + return ret; + + core_cq->cqe_sz = pcq_param->cq_attr.ele_num; + core_cq->comp = xsc_eth_completion_event; + core_cq->event = xsc_eth_cq_error_event; + core_cq->vector = c->chl_idx; +// core_cq->irqn = irqn; + + for (i = 0; i < xsc_cqwq_get_size(&pcq->wq); i++) { + struct xsc_cqe64 *cqe = xsc_cqwq_get_wqe(&pcq->wq, i); + + cqe->owner = 1; + } + pcq->xdev = xdev; + + return ret; +} + +#ifdef NEED_CREATE_RX_THREAD +static int xsc_eth_set_cq(struct xsc_channel *c, + struct xsc_cq *pcq, + struct xsc_cq_param *pcq_param) +{ + int ret = XSCALE_RET_SUCCESS; + struct xsc_create_cq_mbox_in *in; + int inlen; + int hw_npages; + + hw_npages = DIV_ROUND_UP(pcq->wq_ctrl.buf.size, PAGE_SIZE_4K); + /*mbox size + pas size*/ + inlen = sizeof(struct xsc_create_cq_mbox_in) + + sizeof(__be64) * hw_npages; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + /*construct param of in struct*/ + in->ctx.log_cq_sz = pcq_param->cq_attr.q_log_size; + in->ctx.pa_num = cpu_to_be16(hw_npages); + in->ctx.glb_func_id = cpu_to_be16(c->adapter->xdev->glb_func_id); + + xsc_fill_page_frag_array(&pcq->wq_ctrl.buf, + &in->pas[0], hw_npages); + + ret = xsc_eth_create_cq(c->adapter->xdev, &pcq->xcq, in, inlen); + + kfree(in); + xsc_core_info(c->adapter->xdev, "%s: create cqn%d, func_id=%d, ret=%d\n", + __func__, pcq->xcq.cqn, c->adapter->xdev->glb_func_id, ret); + return ret; +} +#else +static int xsc_eth_set_cq(struct xsc_channel *c, + struct xsc_cq *pcq, + struct xsc_cq_param *pcq_param) +{ + int ret = XSCALE_RET_SUCCESS; + struct xsc_core_device *xdev = c->adapter->xdev; + struct xsc_create_cq_mbox_in *in; + int inlen; + int eqn, irqn; + int hw_npages; + + hw_npages = DIV_ROUND_UP(pcq->wq_ctrl.buf.size, PAGE_SIZE_4K); + /*mbox size + pas size*/ + inlen = sizeof(struct xsc_create_cq_mbox_in) + + sizeof(__be64) * hw_npages; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + /*construct param of in struct*/ + ret = xsc_vector2eqn(xdev, c->chl_idx, &eqn, &irqn); + if (ret) + goto err; + + in->ctx.eqn = eqn; + in->ctx.eqn = cpu_to_be16(in->ctx.eqn); + in->ctx.log_cq_sz = pcq_param->cq_attr.q_log_size; + in->ctx.pa_num = cpu_to_be16(hw_npages); + in->ctx.glb_func_id = cpu_to_be16(xdev->glb_func_id); + + xsc_fill_page_frag_array(&pcq->wq_ctrl.buf, &in->pas[0], hw_npages); + + ret = xsc_eth_create_cq(c->adapter->xdev, &pcq->xcq, in, inlen); + +err: + kvfree(in); + xsc_core_info(c->adapter->xdev, "%s: create ch%d cqn%d, eqn=%d, func_id=%d, ret=%d\n", + __func__, c->chl_idx, pcq->xcq.cqn, eqn, xdev->glb_func_id, ret); + return ret; +} +#endif + +static int xsc_eth_open_cq(struct xsc_channel *c, + struct xsc_cq *pcq, + struct xsc_cq_param *pcq_param) +{ + int ret = XSCALE_RET_SUCCESS; + + ret = xsc_eth_alloc_cq(c, pcq, pcq_param); + if (ret) + return ret; + + ret = xsc_eth_set_cq(c, pcq, pcq_param); + if (ret) + goto err_set_cq; + + xsc_cq_notify_hw_rearm(pcq); + + pcq->napi = &c->napi; + pcq->channel = c; + + return 0; + +err_set_cq: + xsc_eth_free_cq(pcq); + return ret; +} + +static int xsc_eth_close_cq(struct xsc_channel *c, struct xsc_cq *pcq) +{ + int ret; + struct xsc_core_device *xdev = c->adapter->xdev; + + ret = xsc_eth_destroy_cq(xdev, pcq); + if (ret) { + xsc_core_warn(xdev, "failed to close ch%d cq%d, ret=%d\n", + c->chl_idx, pcq->xcq.cqn, ret); + return ret; + } + + xsc_eth_free_cq(pcq); + + return 0; +} + +int xsc_eth_set_hw_mtu(struct xsc_core_device *dev, u16 mtu) +{ + int ret; + struct xsc_set_mtu_mbox_in in; + struct xsc_set_mtu_mbox_out out; + + if (!xsc_core_is_pf(dev)) + return 0; + + memset(&in, 0, sizeof(struct xsc_set_mtu_mbox_in)); + memset(&out, 0, sizeof(struct xsc_set_mtu_mbox_out)); + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_SET_MTU); + in.mtu = cpu_to_be16(mtu); + in.mac_port = dev->mac_port; + + ret = xsc_cmd_exec(dev, &in, sizeof(struct xsc_set_mtu_mbox_in), &out, + sizeof(struct xsc_set_mtu_mbox_out)); + if (ret || out.hdr.status) { + xsc_core_warn(dev, "%s: set hw mtu %u failed!\n", __func__, mtu); + ret = -ENOEXEC; + } + + return ret; +} + +int xsc_eth_get_mac(struct xsc_core_device *dev, u8 index, char *mac) +{ + struct xsc_query_eth_mac_mbox_out *out; + struct xsc_query_eth_mac_mbox_in in; + int err; + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) + return -ENOMEM; + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_ETH_MAC); + in.hdr.opmod = cpu_to_be16(0x1); + in.index = xsc_core_is_pf(dev) ? index : (index | BIT(7)); + err = xsc_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); + if (err) { + xsc_core_warn(dev, "get mac fail: %d\n", err); + goto exit; + } + + err = xsc_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); + if (err || out->hdr.status) { + xsc_core_warn(dev, "failed! err=%d, status=%d\n", err, out->hdr.status); + err = -ENOEXEC; + goto exit; + } + + memcpy(mac, out->mac, 6); + xsc_core_dbg(dev, "get mac %02x:%02x:%02x:%02x:%02x:%02x\n", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + +exit: + kfree(out); + + return err; +} + +int xsc_eth_modify_qps_channel(struct xsc_adapter *adapter, struct xsc_channel *c) +{ + int ret = 0; + int i; + + for (i = 0; i < c->qp.rq_num; i++) { + ret = xsc_eth_modify_qp_status(adapter->xdev, c->qp.rq[i].rqn, + XSC_CMD_OP_RTR2RTS_QP); + if (ret) + return ret; + c->qp.rq[i].post_wqes(&c->qp.rq[i]); + } + + for (i = 0; i < c->qp.sq_num; i++) { + ret = xsc_eth_modify_qp_status(adapter->xdev, c->qp.sq[i].sqn, + XSC_CMD_OP_RTR2RTS_QP); + if (ret) + return ret; + } + return 0; +} + +int xsc_eth_modify_qps(struct xsc_adapter *adapter, + struct xsc_eth_channels *chls) +{ + int ret; + int i; + + for (i = 0; i < chls->num_chl; i++) { + struct xsc_channel *c = &chls->c[i]; + + ret = xsc_eth_modify_qps_channel(adapter, c); + if (ret) + return ret; + } + + return 0; +} + +#ifdef XSC_RSS_SUPPORT +static int xsc_eth_open_rss_qp_rq(struct xsc_channel *c, + struct xsc_rq *prq, + struct xsc_rq_param *prq_param) +{ + struct xsc_adapter *adapter = c->adapter; + struct page_pool_params pagepool_params = { 0 }; + u8 q_log_size = prq_param->rq_attr.q_log_size; + u8 ele_log_size = prq_param->rq_attr.ele_log_size; + u32 pool_size = 1 << q_log_size; + int wq_sz; + int i, f; + struct xsc_wq_param wq_param; + struct xsc_stats *stats = c->adapter->stats; + struct xsc_channel_stats *channel_stats = &stats->channel_stats[c->chl_idx]; + int cache_init_sz = 0; + int frag_used_num = adapter->nic_param.rq_frags_size/XSC_RX_FRAG_SZ; + int ret = 0; + + prq->stats = &channel_stats->rq; + + wq_param.buf_numa_node = dev_to_node(c->adapter->dev); + wq_param.db_numa_node = cpu_to_node(c->cpu); + + ret = xsc_eth_wq_cyc_create(c->adapter->xdev, &wq_param, + q_log_size, ele_log_size, &prq->wqe.wq, + &prq->wq_ctrl); + if (ret) + return ret; + + wq_sz = xsc_wq_cyc_get_size(&prq->wqe.wq); + + prq->wqe.info = prq_param->frags_info; + prq->wqe.frags = kvzalloc_node(array_size((wq_sz << prq->wqe.info.log_num_frags), + sizeof(*prq->wqe.frags)), + GFP_KERNEL, + cpu_to_node(c->cpu)); + if (!prq->wqe.frags) { + ret = -ENOMEM; + goto err_alloc_frags; + } + + ret = xsc_eth_init_di_list(prq, wq_sz, c->cpu); + if (ret) + goto err_init_di; + + prq->buff.map_dir = DMA_FROM_DEVICE; +#ifdef XSC_PAGE_CACHE + cache_init_sz = wq_sz << prq->wqe.info.log_num_frags; + ret = xsc_rx_alloc_page_cache(prq, cpu_to_node(c->cpu), ilog2(cache_init_sz)); + if (ret) + goto err_create_pool; +#endif + + /* Create a page_pool and register it with rxq */ + pool_size = wq_sz * frag_used_num; + pagepool_params.order = XSC_RX_FRAG_SZ_ORDER; + pagepool_params.flags = 0; /* No-internal DMA mapping in page_pool */ + pagepool_params.pool_size = pool_size; + pagepool_params.nid = cpu_to_node(c->cpu); + pagepool_params.dev = c->adapter->dev; + pagepool_params.dma_dir = prq->buff.map_dir; + + prq->page_pool = page_pool_create(&pagepool_params); + if (IS_ERR(prq->page_pool)) { + ret = PTR_ERR(prq->page_pool); + prq->page_pool = NULL; + goto err_create_pool; + } + if (c->chl_idx == 0) + xsc_core_dbg(adapter->xdev, + "page pool: size=%d, cpu=%d, node=%d, cache_size=%d, frags_size=%d, mtu=%d\n", + pool_size, c->cpu, pagepool_params.nid, + cache_init_sz, adapter->nic_param.rq_frags_size, + adapter->nic_param.mtu); + + for (i = 0; i < wq_sz; i++) { + struct xsc_eth_rx_wqe_cyc *wqe = + xsc_wq_cyc_get_wqe(&prq->wqe.wq, i); + + for (f = 0; f < prq->wqe.info.num_frags; f++) { + u32 frag_size = prq->wqe.info.arr[f].frag_size; + + wqe->data[f].seg_len = cpu_to_le32(frag_size); + wqe->data[f].mkey = cpu_to_le32(XSC_INVALID_LKEY); + } + /* check if num_frags is not a pow of two */ + if (prq->wqe.info.num_frags < (1 << prq->wqe.info.log_num_frags)) { + wqe->data[f].seg_len = 0; + wqe->data[f].mkey = cpu_to_le32(XSC_INVALID_LKEY); + wqe->data[f].va = 0; + } + } + + prq->post_wqes = xsc_eth_post_rx_wqes; + prq->handle_rx_cqe = xsc_eth_handle_rx_cqe; + prq->dealloc_wqes = xsc_eth_free_rx_wqes; + prq->wqe.skb_from_cqe = xsc_skb_from_cqe_nonlinear; + prq->ix = c->chl_idx; + prq->hw_mtu = XSC_SW2HW_FRAG_SIZE(adapter->nic_param.mtu); + prq->frags_reuse_num = adapter->nic_param.rq_frags_size/XSC_RX_FRAG_SZ; + + return 0; + +err_create_pool: + xsc_eth_free_di_list(prq); +err_init_di: + kvfree(prq->wqe.frags); +err_alloc_frags: + xsc_eth_wq_destroy(&prq->wq_ctrl); + return ret; +} + +static int xsc_eth_open_rss_qp_rqs(struct xsc_adapter *adapter, + struct xsc_rq_param *prq_param, + struct xsc_eth_channels *chls, + unsigned int num_chl) +{ + int ret = 0, err = XSCALE_RET_SUCCESS; + struct xsc_create_multiqp_mbox_in *in; + struct xsc_create_qp_request *req; + u8 q_log_size = prq_param->rq_attr.q_log_size; + int paslen = 0; + struct xsc_rq *prq; + struct xsc_channel *c; + int rqn_base; + int inlen; + int entry_len; + int i, j, n; + int hw_npages; + + for (i = 0; i < num_chl; i++) { + c = &chls->c[i]; + + for (j = 0; j < c->qp.rq_num; j++) { + prq = &c->qp.rq[j]; + ret = xsc_eth_open_rss_qp_rq(c, prq, prq_param); + if (ret) + goto err_open_rss_rq; + + hw_npages = DIV_ROUND_UP(prq->wq_ctrl.buf.size, PAGE_SIZE_4K); + /*support different npages number smoothly*/ + entry_len = sizeof(struct xsc_create_qp_request) + + sizeof(__be64) * hw_npages; + + paslen += entry_len; + } + } + + inlen = sizeof(struct xsc_create_multiqp_mbox_in) + paslen; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + ret = -ENOMEM; + goto err_create_rss_rqs; + } + + in->qp_num = cpu_to_be16(num_chl); + in->qp_type = XSC_QUEUE_TYPE_RAW; + in->req_len = cpu_to_be32(inlen); + + req = (struct xsc_create_qp_request *)&in->data[0]; + n = 0; + for (i = 0; i < num_chl; i++) { + c = &chls->c[i]; + for (j = 0; j < c->qp.rq_num; j++) { + prq = &c->qp.rq[j]; + + hw_npages = DIV_ROUND_UP(prq->wq_ctrl.buf.size, PAGE_SIZE_4K); + /* no use for eth */ + req->input_qpn = cpu_to_be16(0); + req->qp_type = XSC_QUEUE_TYPE_RAW; + req->log_rq_sz = ilog2(adapter->xdev->caps.recv_ds_num) + + q_log_size; + req->pa_num = cpu_to_be16(hw_npages); + req->cqn_recv = cpu_to_be16(prq->cq.xcq.cqn); + req->cqn_send = req->cqn_recv; + req->glb_funcid = cpu_to_be16(adapter->xdev->glb_func_id); + + xsc_fill_page_frag_array(&prq->wq_ctrl.buf, &req->pas[0], hw_npages); + n++; + req = (struct xsc_create_qp_request *)(&in->data[0] + + entry_len*n); + } + } + + ret = xsc_eth_create_rss_qp_rqs(adapter->xdev, in, inlen, &rqn_base); + kvfree(in); + if (ret) + goto err_create_rss_rqs; + + n = 0; + for (i = 0; i < num_chl; i++) { + c = &chls->c[i]; + for (j = 0; j < c->qp.rq_num; j++) { + prq = &c->qp.rq[j]; + prq->cqp.qpn = prq->rqn = rqn_base + n; + prq->cqp.event = xsc_eth_qp_event; + prq->cqp.eth_queue_type = XSC_RES_RQ; + ret = create_resource_common(adapter->xdev, &prq->cqp); + if (ret) { + err = XSCALE_RET_ERROR; + xsc_core_err(adapter->xdev, + "create resource common error qp:%d errno:%d\n", + prq->rqn, ret); + continue; + } + n++; + } + } + if (err != XSCALE_RET_SUCCESS) + return XSCALE_RET_ERROR; + + adapter->channels.rqn_base = rqn_base; + xsc_core_info(adapter->xdev, "%s: rqn_base=%d, ch_num=%d\n", + __func__, rqn_base, num_chl); + return 0; + +err_create_rss_rqs: + i = num_chl; +err_open_rss_rq: + for (--i; i >= 0; i--) { + c = &chls->c[i]; + for (j = 0; j < c->qp.rq_num; j++) { + prq = &c->qp.rq[j]; + xsc_free_qp_rq(prq); + } + } + return ret; +} + +#else +static int xsc_eth_open_qp_rq(struct xsc_channel *c, + struct xsc_rq *prq, + struct xsc_rq_param *prq_param, + u32 rq_idx) +{ + int ret = 0; + struct page_pool_params pp_params = { 0 }; + struct xsc_adapter *adapter = c->adapter; + struct xsc_core_device *xdev = adapter->xdev; + u8 q_log_size = prq_param->rq_attr.q_log_size; + u8 ele_log_size = prq_param->rq_attr.ele_log_size; + u32 pool_size = 1 << q_log_size; + int wq_sz; + int inlen; + int i, f; + int hw_npages; + struct xsc_create_qp_mbox_in *in; + struct xsc_wq_param wq_param; + struct xsc_stats *stats = adapter->stats; + struct xsc_channel_stats *channel_stats = &stats->channel_stats[c->chl_idx]; + + prq->stats = &channel_stats->rq; + + wq_param.buf_numa_node = dev_to_node(c->adapter->dev); + wq_param.db_numa_node = cpu_to_node(c->cpu); + + ret = xsc_eth_wq_cyc_create(xdev, &wq_param, + q_log_size, ele_log_size, &prq->wqe.wq, + &prq->wq_ctrl); + if (ret) + return ret; + + wq_sz = xsc_wq_cyc_get_size(&prq->wqe.wq); + + prq->wqe.info = prq_param->frags_info; + + prq->wqe.frags = kvzalloc_node(array_size((wq_sz << prq->wqe.info.log_num_frags), + sizeof(*prq->wqe.frags)), + GFP_KERNEL, + cpu_to_node(c->cpu)); + if (!prq->wqe.frags) { + ret = -ENOMEM; + goto err_rq_wq_destroy; + } + + ret = xsc_eth_init_di_list(prq, wq_sz, c->cpu); + if (ret) + goto err_frags_destroy; + + /* Create a page_pool and register it with rxq */ + pp_params.order = XSC_RX_FRAG_SZ_ORDER; + pp_params.flags = 0; /* No-internal DMA mapping in page_pool */ + pp_params.pool_size = pool_size; + pp_params.nid = cpu_to_node(c->cpu); + pp_params.dev = c->adapter->dev; + pp_params.dma_dir = prq->buff.map_dir; + + /* page_pool can be used even when there is no rq->xdp_prog, + * given page_pool does not handle DMA mapping there is no + * required state to clear. And page_pool gracefully handle + * elevated refcnt. + */ + prq->page_pool = page_pool_create(&pp_params); + if (IS_ERR(prq->page_pool)) { + ret = PTR_ERR(prq->page_pool); + prq->page_pool = NULL; + goto err_di_destroy; + } + + for (i = 0; i < wq_sz; i++) { + + struct xsc_eth_rx_wqe_cyc *wqe = + xsc_wq_cyc_get_wqe(&prq->wqe.wq, i); + + for (f = 0; f < prq->wqe.info.num_frags; f++) { + u32 frag_size = prq->wqe.info.arr[f].frag_size; + + wqe->data[f].seg_len = cpu_to_le32(frag_size); + wqe->data[f].mkey = cpu_to_le32(XSC_INVALID_LKEY); + } + /* check if num_frags is not a pow of two */ + if (prq->wqe.info.num_frags < (1 << prq->wqe.info.log_num_frags)) { + wqe->data[f].seg_len = 0; + wqe->data[f].mkey = cpu_to_le32(XSC_INVALID_LKEY); + wqe->data[f].va = 0; + } + } + + hw_npages = DIV_ROUND_UP(prq->wq_ctrl.buf.size, PAGE_SIZE_4K); + inlen = sizeof(struct xsc_create_qp_mbox_in) + + sizeof(__be64) * hw_npages; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + ret = -ENOMEM; + goto err_rq_pagepool_destroy; + } + + in->req.input_qpn = cpu_to_be16(XSC_QPN_RQN_STUB); /*no use for eth*/ + in->req.qp_type = XSC_QUEUE_TYPE_RAW; + in->req.log_rq_sz = ilog2(xdev->caps.recv_ds_num) + q_log_size; + in->req.pa_num = cpu_to_be16(hw_npages); + in->req.cqn_recv = cpu_to_be16(prq->cq.xcq.cqn); + in->req.cqn_send = in->req.cqn_recv; + in->req.glb_funcid = cpu_to_be16(xdev->glb_func_id); + + xsc_fill_page_frag_array(&prq->wq_ctrl.buf, &in->req.pas[0], hw_npages); + + ret = xsc_eth_create_qp_rq(xdev, prq, in, inlen); + if (ret) + goto err_rq_in_destroy; + + prq->cqp.qpn = prq->rqn; + prq->cqp.event = xsc_eth_qp_event; + prq->cqp.eth_queue_type = XSC_RES_RQ; + + ret = create_resource_common(xdev, &prq->cqp); + if (ret) { + xsc_core_err(xdev, "%s:error qp:%d errno:%d\n", + __func__, prq->rqn, ret); + goto err_rq_destroy; + } + + prq->post_wqes = xsc_eth_post_rx_wqes; + prq->handle_rx_cqe = xsc_eth_handle_rx_cqe; + prq->dealloc_wqes = xsc_eth_free_rx_wqes; + prq->wqe.skb_from_cqe = xsc_skb_from_cqe_nonlinear; + + prq->ix = c->chl_idx; + + prq->hw_mtu = XSC_ETH_HW_MTU_RECV; + + xsc_core_info(c->adapter->xdev, "%s ok\n", __func__); + + kvfree(in); + + return XSCALE_RET_SUCCESS; + +err_rq_destroy: + xsc_eth_destroy_qp_rq(xdev, prq); + +err_rq_pagepool_destroy: + if (prq->page_pool) + page_pool_destroy(prq->page_pool); + +err_rq_in_destroy: + kvfree(in); + +err_di_destroy: + if (prq->wqe.di) + kvfree(prq->wqe.di); + +err_frags_destroy: + if (prq->wqe.frags) + kvfree(prq->wqe.frags); + +err_rq_wq_destroy: + xsc_eth_wq_destroy(&prq->wq_ctrl); + + return ret; +} +#endif + +static int xsc_eth_close_qp_rq(struct xsc_channel *c, struct xsc_rq *prq) +{ + int ret; + struct xsc_core_device *xdev = c->adapter->xdev; + + destroy_resource_common(xdev, &prq->cqp); + + ret = xsc_eth_destroy_qp_rq(xdev, prq); + if (ret) + return ret; + + xsc_free_qp_rq(prq); + + return 0; +} + +static int xsc_eth_open_qp_sq(struct xsc_channel *c, + struct xsc_sq *psq, + struct xsc_sq_param *psq_param, + u32 sq_idx) +{ + int ret; + struct xsc_adapter *adapter = c->adapter; + struct xsc_core_device *xdev = adapter->xdev; + u8 q_log_size = psq_param->sq_attr.q_log_size; + u8 ele_log_size = psq_param->sq_attr.ele_log_size; + struct xsc_create_qp_mbox_in *in; + struct xsc_modify_raw_qp_mbox_in *modify_in; + int inlen; + int hw_npages; + struct xsc_wq_param wq_param; + struct xsc_stats *stats = adapter->stats; + struct xsc_channel_stats *channel_stats = &stats->channel_stats[c->chl_idx]; + + psq->stats = &channel_stats->sq[sq_idx]; + + wq_param.buf_numa_node = 0; + wq_param.db_numa_node = 0; + + ret = xsc_eth_wq_cyc_create(xdev, &wq_param, + q_log_size, ele_log_size, &psq->wq, + &psq->wq_ctrl); + if (ret) + return ret; + + hw_npages = DIV_ROUND_UP(psq->wq_ctrl.buf.size, PAGE_SIZE_4K); + inlen = sizeof(struct xsc_create_qp_mbox_in) + + sizeof(__be64) * hw_npages; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + ret = -ENOMEM; + goto err_sq_wq_destroy; + } + in->req.input_qpn = cpu_to_be16(XSC_QPN_SQN_STUB); /*no use for eth*/ + in->req.qp_type = XSC_QUEUE_TYPE_RAW_TSO; /*default sq is tso qp*/ + in->req.log_sq_sz = ilog2(xdev->caps.send_ds_num) + q_log_size; + in->req.pa_num = cpu_to_be16(hw_npages); + in->req.cqn_send = cpu_to_be16(psq->cq.xcq.cqn); + in->req.cqn_recv = in->req.cqn_send; + in->req.glb_funcid = cpu_to_be16(xdev->glb_func_id); + + xsc_fill_page_frag_array(&psq->wq_ctrl.buf, + &in->req.pas[0], hw_npages); + + ret = xsc_eth_create_qp_sq(xdev, psq, in, inlen); + if (ret) + goto err_sq_in_destroy; + + psq->cqp.qpn = psq->sqn; + psq->cqp.event = xsc_eth_qp_event; + psq->cqp.eth_queue_type = XSC_RES_SQ; + + ret = create_resource_common(xdev, &psq->cqp); + if (ret) { + xsc_core_err(xdev, "%s:error qp:%d errno:%d\n", + __func__, psq->sqn, ret); + goto err_sq_destroy; + } + + psq->channel = c; + psq->ch_ix = c->chl_idx; + psq->txq_ix = psq->ch_ix * c->num_tc + sq_idx; + + /*need to querify from hardware*/ + psq->hw_mtu = XSC_ETH_HW_MTU_SEND; + psq->stop_room = 1; + + ret = xsc_eth_alloc_qp_sq_db(psq, cpu_to_node(c->cpu)); + if (ret) + goto err_sq_common_destroy; + + xsc_core_info(c->adapter->xdev, "%s ok, ch%d sqn=%d\n", + __func__, c->chl_idx, psq->sqn); + + inlen = sizeof(struct xsc_modify_raw_qp_mbox_in); + modify_in = kvzalloc(inlen, GFP_KERNEL); + if (!modify_in) { + ret = -ENOMEM; + goto err_sq_common_destroy; + } + + modify_in->req.qp_out_port = XSC_PF_VF_GET_PF_ID(xdev->glb_func_id); + modify_in->pcie_no = xsc_get_pcie_no(); + modify_in->req.qpn = cpu_to_be16((u16)(psq->sqn)); + modify_in->req.func_id = cpu_to_be16(xdev->glb_func_id); + modify_in->req.dma_direct = DMA_DIR_TO_MAC; + modify_in->req.prio = sq_idx; + ret = xsc_eth_modify_qp_sq(xdev, modify_in); + if (ret) + goto err_sq_modify_in_destroy; + + kvfree(modify_in); + kvfree(in); + return 0; + +err_sq_modify_in_destroy: + kvfree(modify_in); + +err_sq_common_destroy: + destroy_resource_common(xdev, &psq->cqp); + +err_sq_destroy: + xsc_eth_destroy_qp_sq(xdev, psq); + +err_sq_in_destroy: + kvfree(in); + +err_sq_wq_destroy: + xsc_eth_wq_destroy(&psq->wq_ctrl); + return ret; + +} + +static int xsc_eth_close_qp_sq(struct xsc_channel *c, struct xsc_sq *psq) +{ + struct xsc_core_device *xdev = c->adapter->xdev; + int ret; + + destroy_resource_common(xdev, &psq->cqp); + + ret = xsc_eth_destroy_qp_sq(xdev, psq); + if (ret) + return ret; + + xsc_free_tx_wqe(c->adapter->dev, psq); + xsc_free_qp_sq(psq); + + return 0; +} + +int xsc_eth_open_channel(struct xsc_adapter *adapter, + int idx, + struct xsc_channel *c, + struct xsc_channel_param *chl_param) +{ + int ret = 0; + struct net_device *netdev = adapter->netdev; + struct xsc_stats *stats = adapter->stats; + struct xsc_core_device *xdev = adapter->xdev; + int i, j, eqn, irqn; + struct cpumask *aff; + + c->adapter = adapter; + c->netdev = adapter->netdev; + c->chl_idx = idx; + c->num_tc = adapter->nic_param.num_tc; + c->stats = &stats->channel_stats[idx].ch; + + /*1rq per channel, and may have multi sqs per channel*/ + c->qp.rq_num = 1; + c->qp.sq_num = c->num_tc; + + if (xdev->caps.msix_enable) { + ret = xsc_vector2eqn(xdev, c->chl_idx, &eqn, &irqn); + if (ret) + goto err; + aff = irq_get_affinity_mask(irqn); + c->aff_mask = aff; + c->cpu = cpumask_first(aff); + } + + if ((c->qp.sq_num > XSC_MAX_NUM_TC) || + (c->qp.rq_num > XSC_MAX_NUM_TC)) { + ret = -EINVAL; + goto err; + } + + for (i = 0; i < c->qp.rq_num; i++) { + ret = xsc_eth_open_cq(c, &c->qp.rq[i].cq, &chl_param->rqcq_param); + if (ret) { + j = i - 1; + goto err_open_rq_cq; + } + } + + for (i = 0; i < c->qp.sq_num; i++) { + ret = xsc_eth_open_cq(c, &c->qp.sq[i].cq, &chl_param->sqcq_param); + if (ret) { + j = i - 1; + goto err_open_sq_cq; + } + } + +#ifndef XSC_RSS_SUPPORT + for (i = 0; i < c->qp.rq_num; i++) { + ret = xsc_eth_open_qp_rq(c, &c->qp.rq[i], &chl_param->rq_param, i); + if (ret) { + j = i - 1; + goto err_open_rq; + } + } +#endif + + for (i = 0; i < c->qp.sq_num; i++) { + ret = xsc_eth_open_qp_sq(c, &c->qp.sq[i], &chl_param->sq_param, i); + if (ret) { + j = i - 1; + goto err_open_sq; + } + } + + netif_napi_add(netdev, &c->napi, xsc_eth_napi_poll, XSC_CQ_POLL_BUDGET); + + xsc_core_dbg(adapter->xdev, "%s ch%d ok\n", __func__, idx); + return 0; + +err_open_sq: + for (; j >= 0; j--) + xsc_eth_close_qp_sq(c, &c->qp.sq[j]); + j = (c->qp.rq_num - 1); +#ifndef XSC_RSS_SUPPORT +err_open_rq: + for (; j >= 0; j--) + xsc_eth_close_qp_rq(c, &c->qp.rq[j]); + j = (c->qp.sq_num - 1); +#endif +err_open_sq_cq: + for (; j >= 0; j--) + xsc_eth_close_cq(c, &c->qp.sq[j].cq); + j = (c->qp.rq_num - 1); +err_open_rq_cq: + for (; j >= 0; j--) + xsc_eth_close_cq(c, &c->qp.rq[j].cq); +err: + xsc_core_warn(adapter->xdev, + "failed to open channel: ch%d, sq_num=%d, rq_num=%d, err=%d\n", + idx, c->qp.sq_num, c->qp.rq_num, ret); + return ret; +} + +static void xsc_build_rq_frags_info(struct xsc_queue_attr *attr, + struct xsc_rq_frags_info *frags_info) +{ + int i; + int ds_num = attr->ele_size/XSC_RECV_WQE_DS; + + for (i = 0; i < ds_num; i++) { + frags_info->arr[i].frag_size = XSC_RX_FRAG_SZ; + frags_info->arr[i].frag_stride = + roundup_pow_of_two(frags_info->arr[i].frag_size); + } + + frags_info->num_frags = ds_num; + frags_info->log_num_frags = order_base_2(frags_info->num_frags); + frags_info->wqe_bulk = 1 + (frags_info->num_frags % 2); + frags_info->wqe_bulk = max_t(u8, frags_info->wqe_bulk, 8); +} + +static void xsc_eth_build_channel_param(struct xsc_adapter *adapter, + struct xsc_channel_param *chl_param) +{ + xsc_eth_build_queue_param(adapter, &chl_param->rqcq_param.cq_attr, + XSC_QUEUE_TYPE_RQCQ); + xsc_eth_build_queue_param(adapter, &chl_param->sqcq_param.cq_attr, + XSC_QUEUE_TYPE_SQCQ); + xsc_eth_build_queue_param(adapter, &chl_param->rq_param.rq_attr, + XSC_QUEUE_TYPE_RQ); + xsc_build_rq_frags_info(&chl_param->rq_param.rq_attr, + &chl_param->rq_param.frags_info); + xsc_eth_build_queue_param(adapter, &chl_param->sq_param.sq_attr, + XSC_QUEUE_TYPE_SQ); +} + +int xsc_eth_open_channels(struct xsc_adapter *adapter) +{ + int ret = 0; + int i; + struct xsc_channel_param *chl_param; + struct xsc_eth_channels *chls = &adapter->channels; + struct xsc_core_device *xdev = adapter->xdev; + bool free_rq = false; + + chls->num_chl = adapter->nic_param.num_channels; + chls->c = kcalloc_node(chls->num_chl, sizeof(struct xsc_channel), + GFP_KERNEL, xdev->priv.numa_node); + if (!chls->c) { + ret = -ENOMEM; + goto err; + } + + chl_param = kvzalloc(sizeof(struct xsc_channel_param), GFP_KERNEL); + if (!chl_param) { + ret = -ENOMEM; + goto err_free_ch; + } + + xsc_eth_build_channel_param(adapter, chl_param); + + for (i = 0; i < chls->num_chl; i++) { + ret = xsc_eth_open_channel(adapter, i, &chls->c[i], chl_param); + if (ret) + goto err_open_channel; +#ifndef XSC_RSS_SUPPORT + free_rq = true; +#endif + } + +#ifdef XSC_RSS_SUPPORT + ret = xsc_eth_open_rss_qp_rqs(adapter, &chl_param->rq_param, chls, chls->num_chl); + if (ret) + goto err_open_channel; + free_rq = true; +#endif + + for (i = 0; i < chls->num_chl; i++) + napi_enable(&chls->c[i].napi); + + ret = xsc_eth_modify_qps(adapter, chls); + if (ret) + goto err_modify_qps; + + kvfree(chl_param); + xsc_core_info(adapter->xdev, "%s ok, ch_num=%d\n", __func__, chls->num_chl); + return 0; + +err_modify_qps: + i = chls->num_chl; +err_open_channel: + for (--i; i >= 0; i--) + xsc_eth_close_channel(&chls->c[i], free_rq); + + kvfree(chl_param); +err_free_ch: + kfree(chls->c); +err: + chls->num_chl = 0; + xsc_core_warn(adapter->xdev, "failed to open %d channels, err=%d\n", + chls->num_chl, ret); + return ret; +} + +static void xsc_eth_activate_txqsq(struct xsc_channel *c) +{ + int tc = c->num_tc; + struct xsc_sq *psq; + + for (tc = 0; tc < c->num_tc; tc++) { + psq = &c->qp.sq[tc]; + psq->txq = netdev_get_tx_queue(psq->channel->netdev, psq->txq_ix); + set_bit(XSC_ETH_SQ_STATE_ENABLED, &psq->state); + netdev_tx_reset_queue(psq->txq); + netif_tx_start_queue(psq->txq); + } +} + +static void xsc_eth_deactivate_txqsq(struct xsc_channel *c) +{ + int tc = c->num_tc; + struct xsc_sq *psq; + + for (tc = 0; tc < c->num_tc; tc++) { + psq = &c->qp.sq[tc]; + clear_bit(XSC_ETH_SQ_STATE_ENABLED, &psq->state); + } +} + +static void xsc_activate_rq(struct xsc_channel *c) +{ + int i; + + for (i = 0; i < c->qp.rq_num; i++) + set_bit(XSC_ETH_RQ_STATE_ENABLED, &c->qp.rq[i].state); +} + +static void xsc_deactivate_rq(struct xsc_channel *c) +{ + int i; + + for (i = 0; i < c->qp.rq_num; i++) + clear_bit(XSC_ETH_RQ_STATE_ENABLED, &c->qp.rq[i].state); +} + +void xsc_eth_activate_channel(struct xsc_channel *c) +{ + xsc_eth_activate_txqsq(c); + xsc_activate_rq(c); +} + +void xsc_eth_deactivate_channel(struct xsc_channel *c) +{ + xsc_deactivate_rq(c); + xsc_eth_deactivate_txqsq(c); +} + +static void xsc_eth_activate_channels(struct xsc_eth_channels *chs) +{ + int i; + + for (i = 0; i < chs->num_chl; i++) + xsc_eth_activate_channel(&chs->c[i]); +} + +static void xsc_eth_deactivate_channels(struct xsc_eth_channels *chs) +{ + int i; + + for (i = 0; i < chs->num_chl; i++) + xsc_eth_deactivate_channel(&chs->c[i]); + + /* Sync with all NAPIs to wait until they stop using queues. */ + synchronize_net(); + + for (i = 0; i < chs->num_chl; i++) + /* last doorbell out */ + napi_disable(&chs->c[i].napi); +} + +static void xsc_eth_build_tx2sq_maps(struct xsc_adapter *adapter) +{ + struct xsc_channel *c; + struct xsc_sq *psq; + int i, tc; + + for (i = 0; i < adapter->channels.num_chl; i++) { + c = &adapter->channels.c[i]; + for (tc = 0; tc < c->num_tc; tc++) { + psq = &c->qp.sq[tc]; + adapter->txq2sq[psq->txq_ix] = psq; + } + } +} + +void xsc_eth_activate_priv_channels(struct xsc_adapter *adapter) +{ + int num_txqs; + struct net_device *netdev = adapter->netdev; + + num_txqs = adapter->channels.num_chl * adapter->nic_param.num_tc; + netif_set_real_num_tx_queues(netdev, num_txqs); + netif_set_real_num_rx_queues(netdev, adapter->channels.num_chl); + + xsc_eth_build_tx2sq_maps(adapter); + xsc_eth_activate_channels(&adapter->channels); + netif_tx_start_all_queues(adapter->netdev); +} + +void xsc_eth_deactivate_priv_channels(struct xsc_adapter *adapter) +{ + netif_tx_disable(adapter->netdev); + xsc_eth_deactivate_channels(&adapter->channels); +} + +static int xsc_eth_sw_init(struct xsc_adapter *adapter) +{ + int ret; + + ret = xsc_eth_open_channels(adapter); + if (ret) + return ret; + + xsc_eth_activate_priv_channels(adapter); + + return 0; +} + +static void xsc_eth_close_channel(struct xsc_channel *c, bool free_rq) +{ + int i; + + for (i = 0; i < c->qp.rq_num; i++) { + if (free_rq) + xsc_eth_close_qp_rq(c, &c->qp.rq[i]); + xsc_eth_close_cq(c, &c->qp.rq[i].cq); + memset(&c->qp.rq[i], 0, sizeof(struct xsc_rq)); + } + + for (i = 0; i < c->qp.sq_num; i++) { + xsc_eth_close_qp_sq(c, &c->qp.sq[i]); + xsc_eth_close_cq(c, &c->qp.sq[i].cq); + } + + netif_napi_del(&c->napi); +} + +static void xsc_eth_close_channels(struct xsc_adapter *adapter) +{ + int i; + struct xsc_channel *c = NULL; + + for (i = 0; i < adapter->channels.num_chl; i++) { + c = &adapter->channels.c[i]; + xsc_eth_close_channel(c, true); + } + + kfree(adapter->channels.c); + adapter->channels.num_chl = 0; +} + +static void xsc_eth_set_port_status(struct xsc_core_device *xdev, + enum xsc_port_status status) +{ + +} + +#ifdef NEED_WAIT_RDMA_READY +static void xsc_wait_rdma_ready(struct xsc_core_device *xdev) +{ + int count = 10; + int i = 0; + + do { + if (xdev->rdma_ready == true) + break; + + msleep(100); + } while (i++ < count); + + xsc_core_dbg(xdev, "waiting for %d ms, rdma is %s\n", i * 100, + xdev->rdma_ready ? "ready" : "not ready"); +} +#endif + +int xsc_eth_set_led_status(int id, struct xsc_adapter *adapter) +{ + int err; + + struct xsc_event_set_led_status_mbox_in in; + struct xsc_event_set_led_status_mbox_out out; + + /*query linkstatus cmd*/ + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_SET_LED_STATUS); + in.port_id = id; + + err = xsc_cmd_exec(adapter->xdev, &in, sizeof(in), &out, sizeof(out)); + if (err || out.status) { + xsc_core_err(adapter->xdev, "%s set led to %d status res %d err %d\n", + __func__, id, out.status, err); + return -1; + } + + return XSCALE_RET_SUCCESS; +} + +#ifdef NEED_AGILEX_TRAINING +int xsc_eth_get_linkinfo(struct xsc_event_linkstatus_resp *plinkinfo, struct xsc_adapter *adapter) +{ + int err; + + struct xsc_event_query_linkstatus_mbox_in in; + struct xsc_event_query_linkstatus_mbox_out out; + + /*query linkstatus cmd*/ + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_PHYPORT_STATE); + + err = xsc_cmd_exec(adapter->xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return XSCALE_RET_ERROR; + + /*0:down, 1:up*/ + xsc_core_dbg(adapter->xdev, "status = %d, speed mode = %d\n", + out.ctx.linkstatus, out.ctx.linkspeed); + + plinkinfo->linkspeed = out.ctx.linkspeed; + plinkinfo->linkstatus = out.ctx.linkstatus; + + return XSCALE_RET_SUCCESS; +} + +bool xsc_eth_get_phyport_state(struct xsc_adapter *adapter) +{ + struct xsc_event_linkstatus_resp linkinfo; + + if (xsc_eth_get_linkinfo(&linkinfo, adapter)) { + xsc_core_err(adapter->xdev, "%s fail to get linkinfo\n", __func__); + return XSCALE_ETH_PHYPORT_DOWN; + } + + return linkinfo.linkstatus; +} + +#if defined(MSIX_SUPPORT) +int xsc_eth_change_linkstatus(struct xsc_adapter *adapter) +{ + struct xsc_event_linkstatus_resp linkinfo; + + if (xsc_eth_get_linkinfo(&linkinfo, adapter)) { + xsc_core_err(adapter->xdev, "%s fail to get linkinfo\n", __func__); + return XSCALE_RET_ERROR; + } + + /*save get_linkstatus*/ + if ((linkinfo.linkstatus == XSCALE_ETH_PHYPORT_UP) && !netif_carrier_ok(adapter->netdev)) { + netdev_info(adapter->netdev, "Link up\n"); + netif_carrier_on(adapter->netdev); + } else if ((linkinfo.linkstatus != XSCALE_ETH_PHYPORT_UP) && + netif_carrier_ok(adapter->netdev)) { + netdev_info(adapter->netdev, "Link down\n"); + netif_carrier_off(adapter->netdev); + } + + return XSCALE_RET_SUCCESS; +} + +static void xsc_eth_event_work(struct work_struct *work) +{ + int err; + struct xsc_event_query_type_mbox_in in; + struct xsc_event_query_type_mbox_out out; + struct xsc_adapter *adapter = container_of(work, struct xsc_adapter, event_work); + + if (adapter->status != XSCALE_ETH_DRIVER_OK) + return; + + /*query cmd_type cmd*/ + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_EVENT_TYPE); + + err = xsc_cmd_exec(adapter->xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) { + xsc_core_err(adapter->xdev, "failed to xsc_event_work, err=%d\n", err); + goto failed; + } + + switch (out.ctx.resp_cmd_type) { + case XSC_CMD_EVENT_RESP_CHANGE_LINK: + err = xsc_eth_change_linkstatus(adapter); + if (err) { + xsc_core_err(adapter->xdev, "failed to change linkstatus, err=%d\n", err); + goto failed; + } + + xsc_core_dbg(adapter->xdev, "event cmdtype=%04x\n", out.ctx.resp_cmd_type); + break; + default: + xsc_core_info(adapter->xdev, "unknown event cmdtype=%04x\n", + out.ctx.resp_cmd_type); + break; + } + +failed: + return; +} + +void xsc_eth_event_handler(void *arg) +{ + struct xsc_adapter *adapter = (struct xsc_adapter *)arg; + + queue_work(adapter->workq, &adapter->event_work); +} +#endif +#endif + +int xsc_eth_enable_nic_hca(struct xsc_adapter *adapter) +{ + struct xsc_core_device *xdev = adapter->xdev; + struct net_device *netdev = adapter->netdev; + struct xsc_cmd_enable_nic_hca_mbox_in in = {}; + struct xsc_cmd_enable_nic_hca_mbox_out out = {}; + u16 caps = 0; + u16 caps_mask = 0; + int err; + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_ENABLE_NIC_HCA); + in.nic.info.pf = xdev->pf; + in.nic.info.pcie = xdev->pcie; + in.nic.info.mac_port = xdev->mac_port; + in.nic.info.pcie_port = xdev->pcie_port; + in.nic.info.pf_id = xdev->pf_id; + in.nic.info.vf_id = cpu_to_be16(xdev->vf_id); + in.nic.info.glb_func_id = cpu_to_be16(xdev->glb_func_id); + in.nic.info.logic_port = cpu_to_be16(xdev->logic_port); + in.nic.info.pf_logic_port = cpu_to_be16(xdev->pf_logic_port); + in.nic.info.mac_logic_port = cpu_to_be16(xdev->mac_logic_port); + in.nic.info.gsi_qpn = cpu_to_be16(xdev->gsi_qpn); + + if (!xsc_core_is_pf(xdev) && !xsc_vf_pp_init_enable(xdev)) + caps_mask = caps |= (BIT(XSC_TBM_CAP_IPAT_BYPASS) | + BIT(XSC_TBM_CAP_PCT_BYPASS)) | (BIT(XSC_TBM_CAP_BC_BYPASS)); + +#ifdef XSC_RSS_SUPPORT + in.rss.rss_en = 1; + in.rss.rqn_base = cpu_to_be16(adapter->channels.rqn_base - + xdev->caps.raweth_rss_qp_id_base); + in.rss.rqn_num = cpu_to_be16(adapter->channels.num_chl); + in.rss.hash_tmpl = cpu_to_be32(adapter->rss_params.rss_hash_tmpl); + in.rss.hfunc = hash_func_type(adapter->rss_params.hfunc); +#else + in.rss.rss_en = 0; + if (adapter->channels.c) + in.rss.rqn_base = cpu_to_be16(adapter->channels.c[0].qp.rq[0].rqn - + xdev->caps.raweth_rss_qp_id_base); +#endif + caps_mask |= BIT(XSC_TBM_CAP_RSS); + + if (netdev->features & NETIF_F_RXCSUM) + caps |= BIT(XSC_TBM_CAP_HASH_PPH); + caps_mask |= BIT(XSC_TBM_CAP_HASH_PPH); + + memcpy(in.nic.mac_addr, netdev->dev_addr, ETH_ALEN); + + in.nic.caps = cpu_to_be16(caps); + in.nic.caps_mask = cpu_to_be16(caps_mask); + + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err || out.hdr.status) { + xsc_core_warn(xdev, "failed!! err=%d, status=%d\n", err, out.hdr.status); + return -ENOEXEC; + } + + xdev->bomt_idx = be16_to_cpu(out.res.bomt_idx); + xsc_core_info(xdev, "rss_qp_base=%d bomt_idx=%d\n", in.rss.rqn_base, xdev->bomt_idx); + + return 0; +} + +int xsc_eth_disable_nic_hca(struct xsc_adapter *adapter) +{ + struct xsc_core_device *xdev = adapter->xdev; + struct xsc_cmd_disable_nic_hca_mbox_in in = {}; + struct xsc_cmd_disable_nic_hca_mbox_out out = {}; + u16 caps = 0; + int err; + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DISABLE_NIC_HCA); + in.nic.info.pf = xdev->pf; + in.nic.info.pcie = xdev->pcie; + in.nic.info.mac_port = xdev->mac_port; + in.nic.info.pcie_port = xdev->pcie_port; + in.nic.info.pf_id = xdev->pf_id; + in.nic.info.vf_id = cpu_to_be16(xdev->vf_id); + in.nic.info.glb_func_id = cpu_to_be16(xdev->glb_func_id); + in.nic.info.logic_port = cpu_to_be16(xdev->logic_port); + in.nic.info.pf_logic_port = cpu_to_be16(xdev->pf_logic_port); + in.nic.info.mac_logic_port = cpu_to_be16(xdev->mac_logic_port); + + in.bc.bomt_idx = cpu_to_be16(xdev->bomt_idx); + + if (!xsc_core_is_pf(xdev) && !xsc_vf_pp_init_enable(xdev)) + caps |= BIT(XSC_TBM_CAP_IPAT_BYPASS) | + BIT(XSC_TBM_CAP_PCT_BYPASS) | (BIT(XSC_TBM_CAP_BC_BYPASS)); + in.nic.caps = cpu_to_be16(caps); + + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err || out.hdr.status) { + xsc_core_warn(xdev, "failed!! err=%d, status=%d\n", err, out.hdr.status); + return -ENOEXEC; + } + + return 0; +} + +void xsc_eth_rss_params_change(struct xsc_adapter *adapter, u32 change, void *modify) +{ + struct xsc_core_device *xdev = adapter->xdev; + struct xsc_rss_params *rss = &adapter->rss_params; + struct xsc_eth_params *params = &adapter->nic_param; + struct xsc_cmd_modify_nic_hca_mbox_in *in = + (struct xsc_cmd_modify_nic_hca_mbox_in *)modify; + u32 hash_field = 0; + int key_len; + u8 rss_caps_mask = 0; + + if (change & BIT(XSC_RSS_RXQ_DROP)) { + in->rss.rqn_base = cpu_to_be16(adapter->channels.rqn_base - + xdev->caps.raweth_rss_qp_id_base); + in->rss.rqn_num = 0; + rss_caps_mask |= BIT(XSC_RSS_RXQ_DROP); + goto rss_caps; + } + + if (change & BIT(XSC_RSS_RXQ_UPDATE)) { + in->rss.rqn_base = cpu_to_be16(adapter->channels.rqn_base - + xdev->caps.raweth_rss_qp_id_base); + in->rss.rqn_num = cpu_to_be16(params->num_channels); + rss_caps_mask |= BIT(XSC_RSS_RXQ_UPDATE); + } + + if (change & BIT(XSC_RSS_HASH_KEY_UPDATE)) { + key_len = min(sizeof(in->rss.hash_key), sizeof(rss->toeplitz_hash_key)); + memcpy(&in->rss.hash_key, rss->toeplitz_hash_key, key_len); + rss_caps_mask |= BIT(XSC_RSS_HASH_KEY_UPDATE); + } + + if (change & BIT(XSC_RSS_HASH_TEMP_UPDATE)) { + hash_field = rss->rx_hash_fields[XSC_TT_IPV4_TCP] | + rss->rx_hash_fields[XSC_TT_IPV6_TCP]; + in->rss.hash_tmpl = cpu_to_be32(hash_field); + rss_caps_mask |= BIT(XSC_RSS_HASH_TEMP_UPDATE); + } + + if (change & BIT(XSC_RSS_HASH_FUNC_UPDATE)) { + in->rss.hfunc = hash_func_type(rss->hfunc); + rss_caps_mask |= BIT(XSC_RSS_HASH_FUNC_UPDATE); + } + +rss_caps: + if (rss_caps_mask) { + in->rss.caps_mask = rss_caps_mask; + in->rss.rss_en = 1; + in->nic.caps = in->nic.caps_mask = cpu_to_be16(BIT(XSC_TBM_CAP_RSS)); + } +} + +int xsc_eth_modify_nic_hca(struct xsc_adapter *adapter, u32 flags) +{ + struct xsc_core_device *xdev = adapter->xdev; + struct xsc_cmd_modify_nic_hca_mbox_in in = {}; + struct xsc_cmd_modify_nic_hca_mbox_out out = {}; + int err = 0; + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_MODIFY_NIC_HCA); + in.nic.info.pf = xdev->pf; + in.nic.info.pcie = xdev->pcie; + in.nic.info.mac_port = xdev->mac_port; + in.nic.info.pcie_port = xdev->pcie_port; + in.nic.info.pf_id = xdev->pf_id; + in.nic.info.vf_id = cpu_to_be16(xdev->vf_id); + in.nic.info.glb_func_id = cpu_to_be16(xdev->glb_func_id); + in.nic.info.logic_port = cpu_to_be16(xdev->logic_port); + in.nic.info.pf_logic_port = cpu_to_be16(xdev->pf_logic_port); + in.nic.info.mac_logic_port = cpu_to_be16(xdev->mac_logic_port); + + xsc_eth_rss_params_change(adapter, flags, &in); + + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err || out.hdr.status) { + xsc_core_warn(xdev, "failed!! err=%d, status=%d\n", err, out.hdr.status); + return -ENOEXEC; + } + + return 0; +} + +int xsc_eth_open(struct net_device *netdev) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + int ret = XSCALE_RET_SUCCESS; + + xsc_core_info(adapter->xdev, "%s entry\n", __func__); + + mutex_lock(&adapter->state_lock); + if (adapter->status == XSCALE_ETH_DRIVER_OK) { + xsc_core_warn(adapter->xdev, "unnormal ndo_open when status=%d\n", + adapter->status); + goto ret; + } + + spin_lock_init(&adapter->lock); + + ret = xsc_eth_sw_init(adapter); + if (ret) + goto ret; + + ret = xsc_eth_reset(adapter->xdev); + if (ret) + goto ret; + + ret = xsc_eth_enable_nic_hca(adapter); + if (ret) + goto ret; + +#ifdef NEED_WAIT_RDMA_READY + xsc_wait_rdma_ready(adapter->xdev); +#endif + +#ifdef NEED_CREATE_RX_THREAD + ret = xsc_eth_rx_thread_create(adapter); + if (ret) { + xsc_core_warn(adapter->xdev, "xsc_eth_rx_thread_create failed, err=%d\n", + ret); + goto ret; + } +#endif + +#ifdef NEED_AGILEX_TRAINING +#if defined(MSIX_SUPPORT) + if (xsc_core_is_pf(adapter->xdev)) { + /*INIT_WORK*/ + INIT_WORK(&adapter->event_work, xsc_eth_event_work); + adapter->xdev->event_handler = xsc_eth_event_handler; + + if (xsc_eth_get_phyport_state(adapter)) { + netdev_info(netdev, "Link up\n"); + netif_carrier_on(adapter->netdev); + } else + netdev_info(netdev, "Link down\n"); + } +#else /*no msix*/ + if (xsc_core_is_pf(adapter->xdev)) { + timer_setup(&adapter->link_timer, xsc_eth_update_carrier_timer, 0); + mod_timer(&adapter->link_timer, jiffies + 2*HZ); + } +#endif + if (!xsc_core_is_pf(adapter->xdev)) + netif_carrier_on(netdev); +#else + netif_carrier_on(netdev); +#endif + + adapter->status = XSCALE_ETH_DRIVER_OK; + +ret: + mutex_unlock(&adapter->state_lock); + xsc_core_info(adapter->xdev, "%s: return %s, ret=%d\n", __func__, + ret?"fail":"ok", ret); + if (ret) + return XSCALE_RET_ERROR; + else + return XSCALE_RET_SUCCESS; +} + +int xsc_eth_close(struct net_device *netdev) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + int ret = 0; + + mutex_lock(&adapter->state_lock); + + if (!netif_device_present(netdev)) { + ret = -ENODEV; + goto ret; + } + + if (adapter->status != XSCALE_ETH_DRIVER_OK) + goto ret; + + adapter->status = XSCALE_ETH_DRIVER_CLOSE; + +#ifdef NEED_CREATE_RX_THREAD + if (adapter->task) + kthread_stop(adapter->task); +#endif + + xsc_eth_set_port_status(adapter->xdev, XSC_PORT_DOWN); + netif_carrier_off(adapter->netdev); + xsc_eth_deactivate_priv_channels(adapter); + + xsc_eth_close_channels(adapter); + + ret = xsc_eth_disable_nic_hca(adapter); + if (ret) + xsc_core_warn(adapter->xdev, "failed to disable nic hca, err=%d\n", ret); + +ret: + mutex_unlock(&adapter->state_lock); + xsc_core_info(adapter->xdev, "%s: return %s, ret=%d\n", __func__, + ret?"fail":"ok", ret); + + return ret; +} + +static void xsc_eth_set_rx_mode(struct net_device *netdev) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + + queue_work(adapter->workq, &adapter->set_rx_mode_work); +} + +void xsc_eth_set_mac_hw(unsigned char *dev_addr, unsigned char addr_len) +{ + /****TBD****/ +} + +static int xsc_eth_set_mac(struct net_device *netdev, void *addr) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + struct sockaddr *saddr = addr; + + if (!is_valid_ether_addr(saddr->sa_data)) + return -EADDRNOTAVAIL; + + memcpy(netdev->dev_addr, saddr->sa_data, netdev->addr_len); + xsc_eth_set_mac_hw(netdev->dev_addr, netdev->addr_len); + + queue_work(adapter->workq, &adapter->set_rx_mode_work); + + return 0; +} + +static void xsc_netdev_set_tcs(struct xsc_adapter *priv, u16 nch, u8 ntc) +{ + int tc; + + netdev_reset_tc(priv->netdev); + + if (ntc == 1) + return; + + netdev_set_num_tc(priv->netdev, ntc); + + /* Map netdev TCs to offset 0 + * We have our own UP to TXQ mapping for QoS + */ + for (tc = 0; tc < ntc; tc++) + netdev_set_tc_queue(priv->netdev, tc, nch, 0); +} + +static int xsc_update_netdev_queues(struct xsc_adapter *priv) +{ + struct net_device *netdev = priv->netdev; + int num_txqs, num_rxqs, nch, ntc; + int old_num_txqs, old_ntc; + int err; + bool disabling; + + old_num_txqs = netdev->real_num_tx_queues; + old_ntc = netdev->num_tc ? : 1; + + nch = priv->nic_param.num_channels; + ntc = priv->nic_param.num_tc; + num_txqs = nch * ntc; + num_rxqs = nch;// * priv->profile->rq_groups; + + disabling = num_txqs < netdev->real_num_tx_queues; + + xsc_netdev_set_tcs(priv, nch, ntc); + + err = netif_set_real_num_tx_queues(netdev, num_txqs); + if (err) { + netdev_warn(netdev, "netif_set_real_num_tx_queues failed, txqs=%d->%d, tc=%d->%d, err=%d\n", + old_num_txqs, num_txqs, old_ntc, ntc, err); + goto err_tcs; + } + + err = netif_set_real_num_rx_queues(netdev, num_rxqs); + if (err) { + netdev_warn(netdev, "netif_set_real_num_rx_queues failed, rxqs=%d, err=%d\n", + num_rxqs, err); + goto err_txqs; + } + + if (disabling) + synchronize_net(); + + return 0; + +err_txqs: + /* netif_set_real_num_rx_queues could fail only when nch increased. Only + * one of nch and ntc is changed in this function. That means, the call + * to netif_set_real_num_tx_queues below should not fail, because it + * decreases the number of TX queues. + */ + WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs)); + +err_tcs: + xsc_netdev_set_tcs(priv, old_num_txqs / old_ntc, old_ntc); + return err; +} + +static void xsc_set_default_xps_cpumasks(struct xsc_adapter *priv, + struct xsc_eth_params *params) +{ + struct xsc_core_device *xdev = priv->xdev; + int num_comp_vectors, irq; + + num_comp_vectors = priv->nic_param.comp_vectors; + cpumask_clear(xdev->xps_cpumask); + + for (irq = 0; irq < num_comp_vectors; irq++) { + mask_cpu_by_node(xdev->priv.numa_node, xdev->xps_cpumask); + netif_set_xps_queue(priv->netdev, xdev->xps_cpumask, irq); + } +} + +void xsc_build_default_indir_rqt(u32 *indirection_rqt, int len, + int num_channels) +{ + int i; + + for (i = 0; i < len; i++) + indirection_rqt[i] = i % num_channels; +} + +int xsc_eth_num_channels_changed(struct xsc_adapter *priv) +{ + struct net_device *netdev = priv->netdev; + u16 count = priv->nic_param.num_channels; + int err; + + err = xsc_update_netdev_queues(priv); + if (err) + goto err; + + if (!netif_is_rxfh_configured(priv->netdev)) + xsc_build_default_indir_rqt(priv->rss_params.indirection_rqt, + XSC_INDIR_RQT_SIZE, count); + + return 0; + +err: + netdev_err(netdev, "%s: failed to change rss rxq number %d, err=%d\n", + __func__, count, err); + return err; +} + +int xsc_safe_switch_channels(struct xsc_adapter *adapter, + xsc_eth_fp_preactivate preactivate, + xsc_eth_fp_postactivate postactivate) +{ + struct net_device *netdev = adapter->netdev; + int carrier_ok; + int ret = 0; + + adapter->status = XSCALE_ETH_DRIVER_CLOSE; + + carrier_ok = netif_carrier_ok(netdev); + netif_carrier_off(netdev); +#ifdef NEED_CREATE_RX_THREAD + if (adapter->task) + kthread_stop(adapter->task); +#endif + ret = xsc_eth_modify_nic_hca(adapter, BIT(XSC_RSS_RXQ_DROP)); + if (ret) + goto close_channels; + + xsc_eth_deactivate_priv_channels(adapter); + xsc_eth_close_channels(adapter); + + if (preactivate) { + ret = preactivate(adapter); + if (ret) + goto out; + } + + ret = xsc_eth_open_channels(adapter); + if (ret) + goto close_channels; + + if (postactivate) { + ret = postactivate(adapter); + if (ret) + goto close_channels; + } + + xsc_eth_activate_priv_channels(adapter); + ret = xsc_eth_modify_nic_hca(adapter, BIT(XSC_RSS_RXQ_UPDATE)); + if (ret) + goto close_channels; + +#ifdef NEED_CREATE_RX_THREAD + ret = xsc_eth_rx_thread_create(adapter); + if (ret) + goto close_channels; +#endif + + adapter->status = XSCALE_ETH_DRIVER_OK; + + goto out; + +close_channels: + xsc_eth_deactivate_priv_channels(adapter); + xsc_eth_close_channels(adapter); + +out: + if (carrier_ok) + netif_carrier_on(netdev); + xsc_core_dbg(adapter->xdev, "channels=%d, mtu=%d, err=%d\n", + adapter->nic_param.num_channels, + adapter->nic_param.mtu, ret); + return ret; +} + +int xsc_eth_nic_mtu_changed(struct xsc_adapter *priv) +{ + u32 new_mtu = priv->nic_param.mtu; + u32 frags_size = XSC_SW2HW_FRAG_SIZE(new_mtu); + int ret = 0; + + ret = xsc_eth_set_hw_mtu(priv->xdev, XSC_SW2HW_MTU(new_mtu)); + if (ret) + return ret; + + frags_size = roundup(frags_size, XSC_RX_FRAG_SZ); + if (frags_size != priv->nic_param.rq_frags_size) + priv->nic_param.rq_frags_size = frags_size; + + return 0; +} + +static int xsc_eth_change_mtu(struct net_device *netdev, int new_mtu) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + int old_mtu = netdev->mtu; + int ret = 0; + + if (new_mtu > netdev->max_mtu || new_mtu < netdev->min_mtu) { + netdev_err(netdev, "%s: Bad MTU (%d), valid range is: [%d..%d]\n", + __func__, new_mtu, netdev->min_mtu, netdev->max_mtu); + return -EINVAL; + } + + mutex_lock(&adapter->state_lock); + adapter->nic_param.mtu = new_mtu; + if (adapter->status != XSCALE_ETH_DRIVER_OK) { + ret = xsc_eth_nic_mtu_changed(adapter); + if (ret) + adapter->nic_param.mtu = old_mtu; + else + netdev->mtu = adapter->nic_param.mtu; + goto out; + } + + ret = xsc_safe_switch_channels(adapter, xsc_eth_nic_mtu_changed, NULL); + if (ret) + goto out; + + netdev->mtu = adapter->nic_param.mtu; + +out: + mutex_unlock(&adapter->state_lock); + xsc_core_info(adapter->xdev, "%s: mtu: %d->%d, expected_mtu=%d, err=%d\n", + __func__, old_mtu, netdev->mtu, new_mtu, ret); + return ret; +} + +static void xsc_get_stats(struct net_device *netdev, struct rtnl_link_stats64 *stats) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + + xsc_fold_sw_stats64(adapter, stats); +} + +int xsc_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + struct xsc_core_sriov *sriov = &adapter->xdev->priv.sriov; + struct xsc_core_device *vf_xdev; + + netdev_info(netdev, "%s: vf_idx=%d, sriov=%p\n", __func__, vf, sriov); + if (vf >= sriov->num_vfs) + return -EINVAL; + + vf_xdev = sriov->vfs[vf].dev; + netdev_info(netdev, "%s: vf_idx=%d, mac[5]=0x%02x\n", __func__, vf, mac[5]); + return xsc_eth_set_mac(vf_xdev->netdev, mac); +} + +int set_feature_rxcsum(struct net_device *netdev, bool enable) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + struct xsc_core_device *xdev = adapter->xdev; + struct xsc_cmd_modify_nic_hca_mbox_in in = {}; + struct xsc_cmd_modify_nic_hca_mbox_out out = {}; + int err; + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_MODIFY_NIC_HCA); + in.nic.info.pcie = xdev->pcie; + in.nic.info.pf = xsc_core_is_pf(xdev) ? 1 : 0; + in.nic.info.pf_id = xdev->pf_id; + in.nic.info.vf_id = cpu_to_be16(xdev->vf_id); + in.nic.caps_mask = cpu_to_be16(BIT(XSC_TBM_CAP_HASH_PPH)); + in.nic.caps = cpu_to_be16(enable << XSC_TBM_CAP_HASH_PPH); + + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err || out.hdr.status) { + netdev_err(netdev, "failed to change rxcsum=%d, err=%d, status=%d\n", + enable, err, out.hdr.status); + return -ENOEXEC; + } + + return 0; +} + +static int xsc_handle_feature(struct net_device *netdev, + netdev_features_t *features, + netdev_features_t wanted_features, + netdev_features_t feature, + xsc_feature_handler feature_handler) +{ + netdev_features_t changes = wanted_features ^ netdev->features; + bool enable = !!(wanted_features & feature); + int err; + + if (!(changes & feature)) + return 0; + + err = feature_handler(netdev, enable); + if (err) { + netdev_err(netdev, "%s feature %pNF failed, err %d\n", + enable ? "Enable" : "Disable", &feature, err); + return err; + } + + XSC_SET_FEATURE(features, feature, enable); + return 0; +} + +int xsc_eth_set_features(struct net_device *netdev, netdev_features_t features) +{ + netdev_features_t oper_features = netdev->features; + int err = 0; + +#define XSC_HANDLE_FEATURE(feature, handler) \ + xsc_handle_feature(netdev, &oper_features, features, feature, handler) + + err |= XSC_HANDLE_FEATURE(NETIF_F_RXCSUM, set_feature_rxcsum); + if (err) { + netdev->features = oper_features; + return -EINVAL; + } + + return 0; +} + +static const struct net_device_ops xsc_netdev_ops = { + + .ndo_open = xsc_eth_open, + .ndo_stop = xsc_eth_close, + .ndo_start_xmit = xsc_eth_xmit_start, + + .ndo_set_rx_mode = xsc_eth_set_rx_mode, + .ndo_validate_addr = NULL, + .ndo_set_mac_address = xsc_eth_set_mac, + .ndo_change_mtu = xsc_eth_change_mtu, + .ndo_tx_timeout = NULL, + .ndo_set_tx_maxrate = NULL, + .ndo_vlan_rx_add_vid = NULL, + .ndo_vlan_rx_kill_vid = NULL, + .ndo_do_ioctl = NULL, + .ndo_set_vf_mac = xsc_set_vf_mac, + .ndo_set_vf_vlan = NULL, + .ndo_set_vf_rate = NULL, + .ndo_set_vf_spoofchk = NULL, + .ndo_set_vf_rss_query_en = NULL, + .ndo_set_vf_trust = NULL, + .ndo_get_vf_config = NULL, + .ndo_get_stats64 = xsc_get_stats, + .ndo_setup_tc = NULL, + + .ndo_set_features = xsc_eth_set_features, + .ndo_fix_features = NULL, + .ndo_fdb_add = NULL, + .ndo_bridge_setlink = NULL, + .ndo_bridge_getlink = NULL, + .ndo_dfwd_add_station = NULL, + .ndo_dfwd_del_station = NULL, + .ndo_udp_tunnel_add = NULL, + .ndo_udp_tunnel_del = NULL, + .ndo_features_check = NULL, + .ndo_bpf = NULL, + .ndo_xdp_xmit = NULL, +}; + +static int xsc_eth_check_required_cap(struct xsc_core_device *xdev) +{ + int err = -1; + + /*get cap from hw*/ + + err = 0; + return err; +} + +static int xsc_get_max_num_channels(struct xsc_core_device *xdev) +{ +#ifdef NEED_CREATE_RX_THREAD + return 8; +#else + return min_t(int, xdev->dev_res->eq_table.num_comp_vectors, + XSC_ETH_MAX_NUM_CHANNELS); +#endif +} + +static void xsc_eth_update_carrier_work(struct work_struct *work) +{ + //TBD +} + +static void xsc_eth_set_rx_mode_work(struct work_struct *work) +{ +} + +static int xsc_eth_netdev_init(struct xsc_adapter *adapter) +{ + unsigned int node, tc, nch; + + tc = adapter->nic_param.num_tc; + nch = adapter->nic_param.max_num_ch; + node = dev_to_node(adapter->dev); + adapter->txq2sq = kcalloc_node(nch * tc, + sizeof(*adapter->txq2sq), GFP_KERNEL, node); + if (!adapter->txq2sq) + goto err_out; + + mutex_init(&adapter->state_lock); + + xsc_set_default_xps_cpumasks(adapter, &adapter->nic_param); + /*INIT_WORK*/ + INIT_WORK(&adapter->update_carrier_work, xsc_eth_update_carrier_work); + INIT_WORK(&adapter->set_rx_mode_work, xsc_eth_set_rx_mode_work); + adapter->workq = create_singlethread_workqueue("xsc_eth"); + if (!adapter->workq) + goto err_free_priv; + + netif_carrier_off(adapter->netdev); + +// dev_net_set(netdev, devlink_net(priv_to_devlink(dev))); + return 0; + +err_free_priv: + kfree(adapter->txq2sq); +err_out: + return -ENOMEM; +} + +static const struct xsc_tirc_config tirc_default_config[XSC_NUM_INDIR_TIRS] = { + [XSC_TT_IPV4] = { + .l3_prot_type = XSC_L3_PROT_TYPE_IPV4, + .l4_prot_type = 0, + .rx_hash_fields = XSC_HASH_IP, + }, + [XSC_TT_IPV4_TCP] = { + .l3_prot_type = XSC_L3_PROT_TYPE_IPV4, + .l4_prot_type = XSC_L4_PROT_TYPE_TCP, + .rx_hash_fields = XSC_HASH_IP_PORTS, + }, + [XSC_TT_IPV4_UDP] = { + .l3_prot_type = XSC_L3_PROT_TYPE_IPV4, + .l4_prot_type = XSC_L4_PROT_TYPE_UDP, + .rx_hash_fields = XSC_HASH_IP_PORTS, + }, + [XSC_TT_IPV6] = { + .l3_prot_type = XSC_L3_PROT_TYPE_IPV6, + .l4_prot_type = 0, + .rx_hash_fields = XSC_HASH_IP6, + }, + [XSC_TT_IPV6_TCP] = { + .l3_prot_type = XSC_L3_PROT_TYPE_IPV6, + .l4_prot_type = XSC_L4_PROT_TYPE_TCP, + .rx_hash_fields = XSC_HASH_IP6_PORTS, + }, + [XSC_TT_IPV6_UDP] = { + .l3_prot_type = XSC_L3_PROT_TYPE_IPV6, + .l4_prot_type = XSC_L4_PROT_TYPE_UDP, + .rx_hash_fields = XSC_HASH_IP6_PORTS, + }, +}; + +struct xsc_tirc_config xsc_tirc_get_default_config(enum xsc_traffic_types tt) +{ + return tirc_default_config[tt]; +} + +void xsc_build_rss_params(struct xsc_rss_params *rss_params, u16 num_channels) +{ + enum xsc_traffic_types tt; + + rss_params->hfunc = ETH_RSS_HASH_TOP; + netdev_rss_key_fill(rss_params->toeplitz_hash_key, + sizeof(rss_params->toeplitz_hash_key)); + + xsc_build_default_indir_rqt(rss_params->indirection_rqt, + XSC_INDIR_RQT_SIZE, num_channels); + + for (tt = 0; tt < XSC_NUM_INDIR_TIRS; tt++) { + rss_params->rx_hash_fields[tt] = + tirc_default_config[tt].rx_hash_fields; + } + rss_params->rss_hash_tmpl = XSC_HASH_IP_PORTS | XSC_HASH_IP6_PORTS; +} + +void xsc_eth_build_nic_params(struct xsc_adapter *adapter, u32 ch_num, u32 tc_num) +{ + struct xsc_core_device *xdev = adapter->xdev; + + adapter->nic_param.mtu = SW_DEFAULT_MTU; + adapter->nic_param.num_tc = tc_num; + + adapter->nic_param.comp_vectors = xdev->dev_res->eq_table.num_comp_vectors; + adapter->nic_param.max_num_ch = ch_num; + adapter->nic_param.num_channels = ch_num; + + adapter->nic_param.rq_max_size = BIT(xdev->caps.log_max_qp_depth); + adapter->nic_param.sq_max_size = BIT(xdev->caps.log_max_qp_depth); + adapter->nic_param.rq_frags_size = XSC_RX_FRAG_SZ; + + xsc_build_rss_params(&adapter->rss_params, adapter->nic_param.num_channels); + xsc_core_info(xdev, "%s: mtu=%d, num_ch=%d(max=%d), num_tc=%d, frags_size=%d\n", + __func__, adapter->nic_param.mtu, + adapter->nic_param.num_channels, + adapter->nic_param.max_num_ch, + adapter->nic_param.num_tc, + adapter->nic_param.rq_frags_size); +} + +void xsc_eth_build_nic_netdev(struct xsc_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + struct xsc_core_device *xdev = adapter->xdev; + + /* Set up network device as normal. */ + netdev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE; + netdev->netdev_ops = &xsc_netdev_ops; + +#ifdef CONFIG_XSC_CORE_EN_DCB + netdev->dcbnl_ops = &xsc_dcbnl_ops; +#endif + eth_set_ethtool_ops(netdev); + + netdev->min_mtu = SW_MIN_MTU; + netdev->max_mtu = SW_MAX_MTU; + /*mtu - macheaderlen - ipheaderlen should be aligned in 8B*/ + netdev->mtu = SW_DEFAULT_MTU; + + netdev->vlan_features |= NETIF_F_SG; + netdev->vlan_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;//NETIF_F_HW_CSUM; + netdev->vlan_features |= NETIF_F_GRO; + netdev->vlan_features |= NETIF_F_TSO;//NETIF_F_TSO_ECN + netdev->vlan_features |= NETIF_F_TSO6; + //todo: enable rx csum + netdev->vlan_features |= NETIF_F_RXCSUM; + netdev->vlan_features |= NETIF_F_RXHASH; + netdev->vlan_features |= NETIF_F_GSO_PARTIAL; + + netdev->hw_features = netdev->vlan_features; +// netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; +// netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; +// netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; +// netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; + +// netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; +// netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; + + if (xsc_vxlan_allowed(xdev) || xsc_geneve_tx_allowed(xdev) || + xsc_any_tunnel_proto_supported(xdev)) { + netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; + netdev->hw_enc_features |= NETIF_F_TSO; //NETIF_F_TSO_ECN + netdev->hw_enc_features |= NETIF_F_TSO6; + netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; + } + + netdev->features |= netdev->hw_features; + netdev->features |= NETIF_F_HIGHDMA; +// netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; +// netdev->hw_features |= NETIF_F_RXCSUM; +} + +static int xsc_eth_nic_init(struct xsc_adapter *adapter, + void *rep_priv, u32 ch_num, u32 tc_num) +{ + int err = -1; + + xsc_eth_build_nic_params(adapter, ch_num, tc_num); + + err = xsc_eth_netdev_init(adapter); + if (err) + return err; + + adapter->workq = create_singlethread_workqueue("xsc_eth"); + if (!adapter->workq) + return err; + + xsc_eth_build_nic_netdev(adapter); + + return 0; +} + +static void xsc_eth_nic_cleanup(struct xsc_adapter *adapter) +{ + destroy_workqueue(adapter->workq); + kfree(adapter->txq2sq); +} + +/* create xdev resource,pd/domain/mkey */ +int xsc_eth_create_xdev_resources(struct xsc_core_device *xdev) +{ + return 0; +} + +static int xsc_eth_init_nic_tx(struct xsc_adapter *adapter) +{ + /*create tis table*/ +#ifdef CONFIG_XSC_CORE_EN_DCB + xsc_dcbnl_initialize(adapter); +#endif + + return 0; +} + +static int xsc_eth_cleanup_nic_tx(struct xsc_adapter *adapter) +{ + return 0; +} + +/* init tx: create hw resource, set register according to spec */ +int xsc_eth_init_nic_rx(struct xsc_adapter *adapter) +{ + /* create rqt and tir table + * tir table:base on traffic type like ip4_tcp/ipv6_tcp/ + * each rqt table for a traffic type + */ + + return 0; +} + +static int xsc_eth_cleanup_nic_rx(struct xsc_adapter *adapter) +{ + return 0; +} + +static void xsc_eth_l2_addr_init(struct xsc_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + char mac[6] = {0}; + int ret = 0; + + ret = xsc_eth_get_mac(adapter->xdev, adapter->xdev->mac_port, mac); + if (ret) { + xsc_core_warn(adapter->xdev, "get mac failed %d, generate random mac...", ret); + eth_random_addr(mac); + } + memcpy(netdev->dev_addr, mac, 6); + + if (!is_valid_ether_addr(netdev->perm_addr)) + memcpy(netdev->perm_addr, netdev->dev_addr, netdev->addr_len); +} + +static int xsc_eth_nic_enable(struct xsc_adapter *adapter) +{ + struct xsc_core_device *xdev = adapter->xdev; + + xsc_lag_add(xdev, adapter->netdev); + + xsc_eth_l2_addr_init(adapter); + xsc_eth_set_hw_mtu(xdev, XSC_SW2HW_MTU(adapter->nic_param.mtu)); + +#ifdef CONFIG_XSC_CORE_EN_DCB + xsc_dcbnl_init_app(adapter); +#endif + queue_work(adapter->workq, &adapter->set_rx_mode_work); + + xsc_eth_set_port_status(xdev, XSC_PORT_UP); + + rtnl_lock(); + netif_device_attach(adapter->netdev); + rtnl_unlock(); + + return 0; +} + +static void xsc_eth_nic_disable(struct xsc_adapter *adapter) +{ + rtnl_lock(); + if (netif_running(adapter->netdev)) + xsc_eth_close(adapter->netdev); + netif_device_detach(adapter->netdev); + rtnl_unlock(); + + queue_work(adapter->workq, &adapter->set_rx_mode_work); + + xsc_lag_remove(adapter->xdev); +} + +/* call init tx/rx, enable function about nic init */ +static int xsc_attach_netdev(struct xsc_adapter *adapter) +{ + int err = -1; + + err = xsc_eth_init_nic_tx(adapter); + if (err) + return err; + + err = xsc_eth_init_nic_rx(adapter); + if (err) + return err; + + err = xsc_eth_nic_enable(adapter); + if (err) + return err; + + xsc_core_info(adapter->xdev, "%s:ok\r\n", __func__); + return 0; +} + +static void xsc_detach_netdev(struct xsc_adapter *adapter) +{ + xsc_eth_nic_disable(adapter); + + flush_workqueue(adapter->workq); + + xsc_eth_cleanup_nic_rx(adapter); + xsc_eth_cleanup_nic_tx(adapter); + adapter->status = XSCALE_ETH_DRIVER_DETACH; +} + +static int xsc_eth_attach(struct xsc_core_device *xdev, struct xsc_adapter *adapter) +{ + int err = -1; + + if (netif_device_present(adapter->netdev)) + return 0; + + err = xsc_eth_create_xdev_resources(xdev); + if (err) + return err; + + err = xsc_attach_netdev(adapter); + if (err) + return err; + + xsc_core_info(adapter->xdev, "%s:ok\r\n", __func__); + return 0; +} + +static void xsc_eth_detach(struct xsc_core_device *xdev, struct xsc_adapter *adapter) +{ + if (!netif_device_present(adapter->netdev)) + return; + + xsc_detach_netdev(adapter); +} + +static void *xsc_eth_add(struct xsc_core_device *xdev) +{ + int err = -1; + int num_chl, num_tc; + struct net_device *netdev; + struct xsc_adapter *adapter = NULL; + void *rep_priv = NULL; + + xsc_core_info(xdev, "%s enter\n", __func__); + + err = xsc_eth_check_required_cap(xdev); + if (err) + return NULL; + + num_chl = xsc_get_max_num_channels(xdev); + num_tc = XSC_TX_NUM_TC; + + /* Allocate ourselves a network device with room for our info */ + netdev = alloc_etherdev_mqs(sizeof(struct xsc_adapter), + num_chl * num_tc, num_chl); + if (unlikely(!netdev)) { + xsc_core_warn(xdev, "alloc_etherdev_mqs failed, txq=%d, rxq=%d\n", + (num_chl * num_tc), num_chl); + return NULL; + } + + /* Set up our device-specific information */ + netdev->dev.parent = &xdev->pdev->dev; + adapter = netdev_priv(netdev); + adapter->netdev = netdev; + adapter->pdev = xdev->pdev; + adapter->dev = &adapter->pdev->dev; + adapter->xdev = (void *)xdev; + xdev->eth_priv = adapter; + + err = xsc_eth_nic_init(adapter, rep_priv, num_chl, num_tc); + if (err) { + xsc_core_warn(xdev, "xsc_nic_init failed, num_ch=%d, num_tc=%d, err=%d\n", + num_chl, num_tc, err); + goto err_free_netdev; + } + xsc_core_info(xdev, "xsc_nic_init ok: ch=%d, tc=%d\n", num_chl, num_tc); + + err = xsc_eth_attach(xdev, adapter); + if (err) { + xsc_core_warn(xdev, "xsc_eth_attach failed, err=%d\n", err); + goto err_cleanup_netdev; + } + + adapter->stats = kvzalloc(sizeof(struct xsc_stats), GFP_KERNEL); + if (unlikely(!adapter->stats)) + goto err_detach; + + err = register_netdev(netdev); + if (err) { + xsc_core_warn(xdev, "register_netdev failed, err=%d\n", err); + goto err_reg_netdev; + } + + xdev->netdev = (void *)netdev; + adapter->status = XSCALE_ETH_DRIVER_INIT; + + xsc_core_info(xdev, "%s success\n", __func__); + return adapter; + +err_reg_netdev: + kfree(adapter->stats); +err_detach: + xsc_eth_detach(xdev, adapter); +err_cleanup_netdev: + xsc_eth_nic_cleanup(adapter); +err_free_netdev: + free_netdev(netdev); + + return NULL; +} + +static void xsc_eth_remove(struct xsc_core_device *xdev, void *context) +{ + struct xsc_adapter *adapter = NULL; + + if (!xdev) + return; + + adapter = xdev->eth_priv; + if (!adapter) { + xsc_core_warn(xdev, "%s: adapter is null\n", __func__); + return; + } + + xsc_core_info(xdev, "%s: adapter=%p status=%d\n", + __func__, adapter, adapter->status); + + unregister_netdev(adapter->netdev); + + kfree(adapter->stats); + + xsc_eth_detach(xdev, adapter); + xsc_eth_nic_cleanup(adapter); + + free_netdev(adapter->netdev); + + xdev->netdev = NULL; + xdev->eth_priv = NULL; + xsc_core_info(xdev, "%s: ok\n", __func__); +} + +static struct xsc_interface xsc_interface = { + .add = xsc_eth_add, + .remove = xsc_eth_remove, + .event = NULL, + .protocol = XSC_INTERFACE_PROTOCOL_ETH, +}; + +static __init int xsc_net_driver_init(void) +{ + int ret; + + ret = xsc_register_interface(&xsc_interface); + if (ret != 0) { + pr_err("failed to register interface\n"); + goto out; + } + + ret = xsc_eth_ctrl_init(); + if (ret != 0) { + pr_err("failed to register port control node\n"); + xsc_unregister_interface(&xsc_interface); + goto out; + } + + return 0; +out: + return -1; +} + +static __exit void xsc_net_driver_exit(void) +{ + xsc_eth_ctrl_fini(); + xsc_unregister_interface(&xsc_interface); +} + +module_init(xsc_net_driver_init); +module_exit(xsc_net_driver_exit); + +MODULE_DESCRIPTION("Yunsilicon XSC Ethernet driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0.0"); diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/ut_main.c b/drivers/net/ethernet/yunsilicon/xsc/net/ut_main.c new file mode 100644 index 0000000000000000000000000000000000000000..e35501cf9a24af69ea076fde67f5262abaf9b20e --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/ut_main.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "xsc_eth.h" +#include "xsc_accel.h" +#include +#include +#include "xsc_eth_txrx.h" +#include "xsc_eth_stats.h" +#include "xsc_eth_debug.h" + +#ifdef NEED_CREATE_RX_THREAD + +extern void xsc_cq_notify_hw(struct xsc_cq *cq); + +DEFINE_PER_CPU(bool, txcqe_get); +EXPORT_PER_CPU_SYMBOL(txcqe_get); + +u32 xsc_eth_process_napi(struct xsc_adapter *adapter) +{ + int work_done = 0; + bool err = false; + int budget = 1; + int i, chl; + int errtx = false; + struct xsc_channel *c; + struct xsc_rq *prq; + struct xsc_ch_stats *ch_stats; + + if (adapter->status == XSCALE_ETH_DRIVER_OK) { + for (chl = 0; chl < adapter->channels.num_chl; chl++) { + c = &adapter->channels.c[chl]; + prq = &c->qp.rq[0]; + ch_stats = c->stats; + ch_stats->poll++; + + for (i = 0; i < c->num_tc; i++) { + errtx |= xsc_poll_tx_cq(&c->qp.sq[i].cq, budget); + ETH_DEBUG_LOG("errtx=%u.\r\n", errtx); + if (likely(__this_cpu_read(txcqe_get))) { + xsc_cq_notify_hw(&c->qp.sq[i].cq); + __this_cpu_write(txcqe_get, false); + } + } + + work_done = xsc_poll_rx_cq(&prq->cq, budget); + + ETH_DEBUG_LOG("work_done=%d.\r\n", work_done); + + if (work_done != 0) { + xsc_cq_notify_hw(&prq->cq); + err |= prq->post_wqes(prq); + + ETH_DEBUG_LOG("err=%u.\r\n", err); + } else { + ETH_DEBUG_LOG("no-load.\r\n"); + } + + ch_stats->arm++; + } + } + + return XSCALE_RET_SUCCESS; +} + +int xsc_eth_rx_thread(void *arg) +{ + u32 uiRet = XSCALE_RET_SUCCESS; + struct xsc_adapter *adapter = (struct xsc_adapter *)arg; + + while (kthread_should_stop() == 0) { + if (need_resched()) + schedule(); + uiRet = xsc_eth_process_napi(adapter); + if (uiRet != XSCALE_RET_SUCCESS) + ETH_DEBUG_LOG("unexpected branch.\r\n"); + + ETH_DEBUG_LOG("adapter=%p\r\n", adapter); + } + ETH_DEBUG_LOG("do_exit.\r\n"); + + return XSCALE_RET_SUCCESS; +} + +u32 g_thread_count; +u32 xsc_eth_rx_thread_create(struct xsc_adapter *adapter) +{ + struct task_struct *task = NULL; + + task = kthread_create(xsc_eth_rx_thread, (void *)adapter, + "xsc_rx%i", g_thread_count); + if (!task) + return XSCALE_RET_ERROR; + + ETH_DEBUG_LOG("thread_count=%d\r\n", g_thread_count); + + kthread_bind(task, g_thread_count); + wake_up_process(task); + adapter->task = task; + + g_thread_count++; + + return XSCALE_RET_SUCCESS; +} +#endif + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_accel.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_accel.h new file mode 100644 index 0000000000000000000000000000000000000000..f628761512b5dfefe9a4b9b05e7a996f19da902c --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_accel.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ACCEL_H +#define XSC_ACCEL_H + +#include +#include +#include + +static inline void xsc_udp_gso_handle_tx_skb(struct sk_buff *skb) +{ + int payload_len = skb_shinfo(skb)->gso_size + sizeof(struct udphdr); + + udp_hdr(skb)->len = htons(payload_len); +} + +static inline struct sk_buff *xsc_accel_handle_tx(struct sk_buff *skb) +{ + /*no not consider tls and ipsec*/ + if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + xsc_udp_gso_handle_tx_skb(skb); + + return skb; +} + +static inline bool xsc_vxlan_allowed(struct xsc_core_device *dev) +{ + return false; +} + +static inline bool xsc_geneve_tx_allowed(struct xsc_core_device *dev) +{ + return false; +} + +static inline bool xsc_any_tunnel_proto_supported(struct xsc_core_device *dev) +{ + return false; +} + +#endif /* XSC_ACCEL_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_dcbnl.c b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_dcbnl.c new file mode 100644 index 0000000000000000000000000000000000000000..69b1a654eaaf3f40ca2d2cdf07eae48f7c57a05b --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_dcbnl.c @@ -0,0 +1,1284 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "xsc_eth.h" +#include "xsc_eth_debug.h" + +#define XSC_100MB (100000) +#define XSC_1GB (1000000) + +#define XSC_CEE_STATE_UP 1 +#define XSC_CEE_STATE_DOWN 0 + +/* Max supported cable length is 1000 meters */ +#define XSC_MAX_CABLE_LENGTH 1000 + +enum { + XSC_VENDOR_TC_GROUP_NUM = 7, + XSC_LOWEST_PRIO_GROUP = 0, +}; + +#define XSC_DCBX_STUB 1 + +#ifdef CONFIG_XSC_CORE_EN_DCB +static int xsc_set_trust_state(struct xsc_adapter *priv, u8 trust_state); +static int xsc_set_dscp2prio(struct xsc_adapter *priv, u8 dscp, u8 prio); + +static int xsc_max_tc(struct xsc_core_device *dev) +{ + u8 num_tc = dev->caps.max_tc ? : 8; + + return num_tc - 1; +} + +static int xsc_dcbnl_set_dcbx_mode(struct xsc_adapter *priv, + enum xsc_dcbx_oper_mode mode) +{ + return 1; +} + +static int xsc_dcbnl_switch_to_host_mode(struct xsc_adapter *priv) +{ + struct xsc_dcbx *dcbx = &priv->dcbx; + + dcbx->mode = XSC_DCBX_PARAM_VER_OPER_HOST; + return 0; +} + +static int xsc_dcbnl_ieee_getets(struct net_device *netdev, + struct ieee_ets *ets) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_core_device *xdev = priv->xdev; + u8 tc_group[IEEE_8021QAZ_MAX_TCS]; + bool is_tc_group_6_exist = false; + bool is_zero_bw_ets_tc = false; + int err = 0; + int i; + + if (!xdev->caps.ets) + return -EOPNOTSUPP; + + ets->ets_cap = xsc_max_tc(priv->xdev) + 1; + for (i = 0; i < ets->ets_cap; i++) { +#ifndef XSC_DCBX_STUB + err = xsc_query_port_prio_tc(xdev, i, &ets->prio_tc[i]); + if (err) + return err; + + err = xsc_query_port_tc_group(xdev, i, &tc_group[i]); + if (err) + return err; + + err = xsc_query_port_tc_bw_alloc(xdev, i, &ets->tc_tx_bw[i]); + if (err) + return err; +#else + ets->prio_tc[i] = i; + tc_group[i] = XSC_VENDOR_TC_GROUP_NUM; + ets->tc_tx_bw[i] = XSC_MAX_BW_ALLOC; +#endif + if (ets->tc_tx_bw[i] < XSC_MAX_BW_ALLOC && + tc_group[i] == (XSC_LOWEST_PRIO_GROUP + 1)) + is_zero_bw_ets_tc = true; + + if (tc_group[i] == (XSC_VENDOR_TC_GROUP_NUM - 1)) + is_tc_group_6_exist = true; + } + + /* Report 0% ets tc if exits*/ + if (is_zero_bw_ets_tc) { + for (i = 0; i < ets->ets_cap; i++) + if (tc_group[i] == XSC_LOWEST_PRIO_GROUP) + ets->tc_tx_bw[i] = 0; + } + + /* Update tc_tsa based on fw setting*/ + for (i = 0; i < ets->ets_cap; i++) { + if (ets->tc_tx_bw[i] < XSC_MAX_BW_ALLOC) + priv->dcbx.tc_tsa[i] = IEEE_8021QAZ_TSA_ETS; + else if (tc_group[i] == XSC_VENDOR_TC_GROUP_NUM && + !is_tc_group_6_exist) + priv->dcbx.tc_tsa[i] = IEEE_8021QAZ_TSA_VENDOR; + xsc_eth_dbg(HW, priv, "%s: tc%d, group=%d, bw=%d\n", + __func__, i, tc_group[i], ets->tc_tx_bw[i]); + } + memcpy(ets->tc_tsa, priv->dcbx.tc_tsa, sizeof(ets->tc_tsa)); + + return err; +} + +static void xsc_build_tc_group(struct ieee_ets *ets, u8 *tc_group, int max_tc) +{ + bool any_tc_mapped_to_ets = false; + bool ets_zero_bw = false; + int strict_group; + int i; + + for (i = 0; i <= max_tc; i++) { + if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_ETS) { + any_tc_mapped_to_ets = true; + if (!ets->tc_tx_bw[i]) + ets_zero_bw = true; + } + } + + /* strict group has higher priority than ets group */ + strict_group = XSC_LOWEST_PRIO_GROUP; + if (any_tc_mapped_to_ets) + strict_group++; + if (ets_zero_bw) + strict_group++; + + for (i = 0; i <= max_tc; i++) { + switch (ets->tc_tsa[i]) { + case IEEE_8021QAZ_TSA_VENDOR: + tc_group[i] = XSC_VENDOR_TC_GROUP_NUM; + break; + case IEEE_8021QAZ_TSA_STRICT: + tc_group[i] = strict_group++; + break; + case IEEE_8021QAZ_TSA_ETS: + tc_group[i] = XSC_LOWEST_PRIO_GROUP; + if (ets->tc_tx_bw[i] && ets_zero_bw) + tc_group[i] = XSC_LOWEST_PRIO_GROUP + 1; + break; + } + } +} + +static void xsc_build_tc_tx_bw(struct ieee_ets *ets, u8 *tc_tx_bw, + u8 *tc_group, int max_tc) +{ + int bw_for_ets_zero_bw_tc = 0; + int last_ets_zero_bw_tc = -1; + int num_ets_zero_bw = 0; + int i; + + for (i = 0; i <= max_tc; i++) { + if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_ETS && + !ets->tc_tx_bw[i]) { + num_ets_zero_bw++; + last_ets_zero_bw_tc = i; + } + } + + if (num_ets_zero_bw) + bw_for_ets_zero_bw_tc = XSC_MAX_BW_ALLOC / num_ets_zero_bw; + + for (i = 0; i <= max_tc; i++) { + switch (ets->tc_tsa[i]) { + case IEEE_8021QAZ_TSA_VENDOR: + tc_tx_bw[i] = XSC_MAX_BW_ALLOC; + break; + case IEEE_8021QAZ_TSA_STRICT: + tc_tx_bw[i] = XSC_MAX_BW_ALLOC; + break; + case IEEE_8021QAZ_TSA_ETS: + tc_tx_bw[i] = ets->tc_tx_bw[i] ? + ets->tc_tx_bw[i] : + bw_for_ets_zero_bw_tc; + break; + } + } + + /* Make sure the total bw for ets zero bw group is 100% */ + if (last_ets_zero_bw_tc != -1) + tc_tx_bw[last_ets_zero_bw_tc] += + XSC_MAX_BW_ALLOC % num_ets_zero_bw; +} + +/* If there are ETS BW 0, + * Set ETS group # to 1 for all ETS non zero BW tcs. Their sum must be 100%. + * Set group #0 to all the ETS BW 0 tcs and + * equally splits the 100% BW between them + * Report both group #0 and #1 as ETS type. + * All the tcs in group #0 will be reported with 0% BW. + */ +int xsc_dcbnl_ieee_setets_core(struct xsc_adapter *priv, struct ieee_ets *ets) +{ + struct xsc_core_device *xdev = priv->xdev; + u8 tc_tx_bw[IEEE_8021QAZ_MAX_TCS]; + u8 tc_group[IEEE_8021QAZ_MAX_TCS]; + int max_tc = xsc_max_tc(xdev); + int err = 0; + + memset(tc_tx_bw, 0, IEEE_8021QAZ_MAX_TCS); + memset(tc_group, 0, IEEE_8021QAZ_MAX_TCS); + xsc_build_tc_group(ets, tc_group, max_tc); + xsc_build_tc_tx_bw(ets, tc_tx_bw, tc_group, max_tc); +#ifndef XSC_DCBX_STUB + err = xsc_set_port_prio_tc(xdev, ets->prio_tc); + if (err) + return err; + + err = xsc_set_port_tc_group(xdev, tc_group); + if (err) + return err; + + err = xsc_set_port_tc_bw_alloc(xdev, tc_tx_bw); + + if (err) + return err; +#endif + memcpy(priv->dcbx.tc_tsa, ets->tc_tsa, sizeof(ets->tc_tsa)); + + return err; +} + +static int xsc_dbcnl_validate_ets(struct net_device *netdev, + struct ieee_ets *ets, + bool zero_sum_allowed) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + bool have_ets_tc = false; + int bw_sum = 0; + int i; + + xsc_eth_dbg(HW, priv, "%s\n", __func__); + /* Validate Priority */ + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { + if (ets->prio_tc[i] >= XSC_MAX_PRIORITY) { + netdev_err(netdev, + "Failed to validate ETS: priority value greater than max(%d)\n", + XSC_MAX_PRIORITY); + return -EINVAL; + } + } + + /* Validate Bandwidth Sum */ + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { + if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_ETS) { + have_ets_tc = true; + bw_sum += ets->tc_tx_bw[i]; + } + } + + if (have_ets_tc && bw_sum != 100) { + if (bw_sum || (!bw_sum && !zero_sum_allowed)) + netdev_err(netdev, + "Failed to validate ETS: BW sum is illegal\n"); + return -EINVAL; + } + return 0; +} + +static int xsc_dcbnl_ieee_setets(struct net_device *dev, + struct ieee_ets *ets) +{ + struct xsc_adapter *priv = netdev_priv(dev); + int err; + + if (!priv->xdev->caps.ets) + return -EOPNOTSUPP; + + err = xsc_dbcnl_validate_ets(dev, ets, false); + if (err) + return err; + + err = xsc_dcbnl_ieee_setets_core(priv, ets); + if (err) + return err; + + return 0; +} + +static int xsc_dcbnl_ieee_getpfc(struct net_device *dev, + struct ieee_pfc *pfc) +{ + struct xsc_adapter *priv = netdev_priv(dev); + struct xsc_core_device *xdev = priv->xdev; +// struct xsc_pport_stats *pstats = &priv->stats.pport; +// struct xsc_cee_config cee_cfg = priv->dcbx.cee_cfg; + int i; + + pfc->pfc_cap = xsc_max_tc(xdev) + 1; +#ifndef XSC_DCBX_STUB + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { + pfc->requests[i] = PPORT_PER_PRIO_GET(pstats, i, tx_pause); + pfc->indications[i] = PPORT_PER_PRIO_GET(pstats, i, rx_pause); + } + + if (xdev->caps.port_buf) + pfc->delay = priv->dcbx.cable_len; + return xsc_query_port_pfc(xdev, &pfc->pfc_en, NULL); +#else + pfc->pfc_en = 0; + +// for (i = 0; i < CEE_DCBX_MAX_PRIO; i++) +// pfc->pfc_en |= cee_cfg.pfc_setting[i] << i; + + for (i = 0; i < pfc->pfc_cap; i++) + pfc->pfc_en |= 1 << i; + + xsc_eth_dbg(HW, priv, "%s: pfc_en=0x%x\n", __func__, pfc->pfc_en); + + return 0; +#endif +} + +static int xsc_dcbnl_ieee_setpfc(struct net_device *dev, + struct ieee_pfc *pfc) +{ + struct xsc_adapter *priv = netdev_priv(dev); + u32 changed = 0; + u8 curr_pfc_en = 0; + int ret = 0; +#ifndef XSC_DCBX_STUB + struct xsc_core_device *xdev = priv->xdev; + u32 old_cable_len = priv->dcbx.cable_len; + struct ieee_pfc pfc_new; +#else + struct xsc_cee_config *cee_cfg = &priv->dcbx.cee_cfg; + int i; +#endif + + /* pfc_en */ +#ifndef XSC_DCBX_STUB + xsc_query_port_pfc(xdev, &curr_pfc_en, NULL); + if (pfc->pfc_en != curr_pfc_en) { + ret = xsc_set_port_pfc(xdev, pfc->pfc_en, pfc->pfc_en); + if (ret) + return ret; + xsc_toggle_port_link(xdev); + changed |= XSC_PORT_BUFFER_PFC; + } +#else + for (i = 0; i < CEE_DCBX_MAX_PRIO; i++) + curr_pfc_en |= cee_cfg->pfc_setting[i] << i; + + if (pfc->pfc_en != curr_pfc_en) { + changed |= XSC_PORT_BUFFER_PFC; + for (i = 0; i < CEE_DCBX_MAX_PRIO; i++) { + if (pfc->pfc_en & (1<pfc_setting[i] = 1; + else + cee_cfg->pfc_setting[i] = 0; + } + } +#endif + xsc_eth_dbg(HW, priv, "%s: new_pfc_en=0x%x, cur_pfc_en=0x%x\n", + __func__, pfc->pfc_en, curr_pfc_en); + + if (pfc->delay && + pfc->delay < XSC_MAX_CABLE_LENGTH && + pfc->delay != priv->dcbx.cable_len) { + priv->dcbx.cable_len = pfc->delay; + changed |= XSC_PORT_BUFFER_CABLE_LEN; + } + +#ifndef XSC_DCBX_STUB + if (xdev->caps.port_buf) { + pfc_new.pfc_en = (changed & XSC_PORT_BUFFER_PFC) ? + pfc->pfc_en : curr_pfc_en; + if (priv->dcbx.manual_buffer) + ret = xsc_port_manual_buffer_config(priv, changed, + dev->mtu, &pfc_new, + NULL, NULL); + if (ret && (changed & XSC_PORT_BUFFER_CABLE_LEN)) + priv->dcbx.cable_len = old_cable_len; + } +#endif + + if (!ret) + xsc_eth_dbg(HW, priv, + "%s: PFC per priority bit mask: 0x%x\n", + __func__, pfc->pfc_en); + + return ret; +} + +static u8 xsc_dcbnl_getdcbx(struct net_device *dev) +{ + struct xsc_adapter *priv = netdev_priv(dev); + + xsc_eth_dbg(HW, priv, "%s: dcbx->cap=0x%x\n", __func__, priv->dcbx.cap); + return priv->dcbx.cap; +} + +static u8 xsc_dcbnl_setdcbx(struct net_device *dev, u8 mode) +{ + struct xsc_adapter *priv = netdev_priv(dev); + struct xsc_dcbx *dcbx = &priv->dcbx; + + xsc_eth_dbg(HW, priv, "%s: mode=%d\n", __func__, mode); + if (mode & DCB_CAP_DCBX_LLD_MANAGED) + return 1; + + if ((!mode) && priv->xdev->caps.dcbx) { + if (dcbx->mode == XSC_DCBX_PARAM_VER_OPER_AUTO) + return 0; + + /* set dcbx to fw controlled */ + if (!xsc_dcbnl_set_dcbx_mode(priv, XSC_DCBX_PARAM_VER_OPER_AUTO)) { + dcbx->mode = XSC_DCBX_PARAM_VER_OPER_AUTO; + dcbx->cap &= ~DCB_CAP_DCBX_HOST; + return 0; + } + + return 1; + } + + if (!(mode & DCB_CAP_DCBX_HOST)) + return 1; + + if (xsc_dcbnl_switch_to_host_mode(netdev_priv(dev))) + return 1; + + dcbx->cap = mode; + + return 0; +} + +static int xsc_dcbnl_ieee_setapp(struct net_device *dev, struct dcb_app *app) +{ + struct xsc_adapter *priv = netdev_priv(dev); + struct dcb_app temp; + bool is_new; + int err; + + xsc_eth_dbg(HW, priv, "%s\n", __func__); + if (!priv->xdev->caps.dscp) + return -EOPNOTSUPP; + + if ((app->selector != IEEE_8021QAZ_APP_SEL_DSCP) || + (app->protocol >= XSC_MAX_DSCP)) + return -EINVAL; + + /* Save the old entry info */ + temp.selector = IEEE_8021QAZ_APP_SEL_DSCP; + temp.protocol = app->protocol; + temp.priority = priv->dcbx_dp.dscp2prio[app->protocol]; + + /* Check if need to switch to dscp trust state */ + if (!priv->dcbx.dscp_app_cnt) { + err = xsc_set_trust_state(priv, XSC_QPTS_TRUST_DSCP); + if (err) + return err; + } + + /* Skip the fw command if new and old mapping are the same */ + if (app->priority != priv->dcbx_dp.dscp2prio[app->protocol]) { + err = xsc_set_dscp2prio(priv, app->protocol, app->priority); + if (err) + goto fw_err; + } + + /* Delete the old entry if exists */ + is_new = false; + err = dcb_ieee_delapp(dev, &temp); + if (err) + is_new = true; + + /* Add new entry and update counter */ + err = dcb_ieee_setapp(dev, app); + if (err) + return err; + + if (is_new) + priv->dcbx.dscp_app_cnt++; + + return err; + +fw_err: + xsc_set_trust_state(priv, XSC_QPTS_TRUST_PCP); + return err; +} + +static int xsc_dcbnl_ieee_delapp(struct net_device *dev, struct dcb_app *app) +{ + struct xsc_adapter *priv = netdev_priv(dev); + int err; + + xsc_eth_dbg(HW, priv, "%s\n", __func__); + if (!priv->xdev->caps.dscp) + return -EOPNOTSUPP; + + if ((app->selector != IEEE_8021QAZ_APP_SEL_DSCP) || + (app->protocol >= XSC_MAX_DSCP)) + return -EINVAL; + + /* Skip if no dscp app entry */ + if (!priv->dcbx.dscp_app_cnt) + return -ENOENT; + + /* Check if the entry matches fw setting */ + if (app->priority != priv->dcbx_dp.dscp2prio[app->protocol]) + return -ENOENT; + + /* Delete the app entry */ + err = dcb_ieee_delapp(dev, app); + if (err) + return err; + + /* Reset the priority mapping back to zero */ + err = xsc_set_dscp2prio(priv, app->protocol, 0); + if (err) + goto fw_err; + + priv->dcbx.dscp_app_cnt--; + + /* Check if need to switch to pcp trust state */ + if (!priv->dcbx.dscp_app_cnt) + err = xsc_set_trust_state(priv, XSC_QPTS_TRUST_PCP); + + return err; + +fw_err: + xsc_set_trust_state(priv, XSC_QPTS_TRUST_PCP); + return err; +} + +static int xsc_dcbnl_ieee_getmaxrate(struct net_device *netdev, + struct ieee_maxrate *maxrate) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_core_device *xdev = priv->xdev; + u8 max_bw_value[IEEE_8021QAZ_MAX_TCS]; + u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS]; + int i; +#ifndef XSC_DCBX_STUB + int err; + + err = xsc_query_port_ets_rate_limit(xdev, max_bw_value, max_bw_unit); + if (err) + return err; +#else + for (i = 0; i <= xsc_max_tc(xdev); i++) { + max_bw_unit[i] = XSC_GBPS_UNIT; + max_bw_value[i] = 25; + } +#endif + + memset(maxrate->tc_maxrate, 0, sizeof(maxrate->tc_maxrate)); + + for (i = 0; i <= xsc_max_tc(xdev); i++) { + switch (max_bw_unit[i]) { + case XSC_100_MBPS_UNIT: + maxrate->tc_maxrate[i] = max_bw_value[i] * XSC_100MB; + break; + case XSC_GBPS_UNIT: + maxrate->tc_maxrate[i] = max_bw_value[i] * XSC_1GB; + break; + case XSC_BW_NO_LIMIT: + break; + default: + WARN(true, "non-supported BW unit"); + break; + } + } + + xsc_eth_dbg(HW, priv, "%s\n", __func__); + return 0; +} + +static int xsc_dcbnl_ieee_setmaxrate(struct net_device *netdev, + struct ieee_maxrate *maxrate) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_core_device *xdev = priv->xdev; + u8 max_bw_value[IEEE_8021QAZ_MAX_TCS]; + u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS]; + __u64 upper_limit_mbps = roundup(255 * XSC_100MB, XSC_1GB); + int i; + + xsc_eth_dbg(HW, priv, "%s\n", __func__); + memset(max_bw_value, 0, sizeof(max_bw_value)); + memset(max_bw_unit, 0, sizeof(max_bw_unit)); + + for (i = 0; i <= xsc_max_tc(xdev); i++) { + if (!maxrate->tc_maxrate[i]) { + max_bw_unit[i] = XSC_BW_NO_LIMIT; + continue; + } + if (maxrate->tc_maxrate[i] < upper_limit_mbps) { + max_bw_value[i] = div_u64(maxrate->tc_maxrate[i], + XSC_100MB); + max_bw_value[i] = max_bw_value[i] ? max_bw_value[i] : 1; + max_bw_unit[i] = XSC_100_MBPS_UNIT; + } else { + max_bw_value[i] = div_u64(maxrate->tc_maxrate[i], + XSC_1GB); + max_bw_unit[i] = XSC_GBPS_UNIT; + } + } + + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) + netdev_dbg(netdev, "%s: tc_%d <=> max_bw %d Gbps\n", + __func__, i, max_bw_value[i]); +#ifndef XSC_DCBX_STUB + return xsc_modify_port_ets_rate_limit(xdev, max_bw_value, max_bw_unit); +#else + return 0; +#endif +} + +static u8 xsc_dcbnl_setall(struct net_device *netdev) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_cee_config *cee_cfg = &priv->dcbx.cee_cfg; + struct xsc_core_device *xdev = priv->xdev; + struct ieee_ets ets; + struct ieee_pfc pfc; + int err = -EOPNOTSUPP; + int i; + + if (!xdev->caps.ets) + goto out; + + memset(&ets, 0, sizeof(ets)); + memset(&pfc, 0, sizeof(pfc)); + + ets.ets_cap = IEEE_8021QAZ_MAX_TCS; + for (i = 0; i < CEE_DCBX_MAX_PGS; i++) { + ets.tc_tx_bw[i] = cee_cfg->pg_bw_pct[i]; + ets.tc_rx_bw[i] = cee_cfg->pg_bw_pct[i]; + ets.tc_tsa[i] = IEEE_8021QAZ_TSA_ETS; + ets.prio_tc[i] = cee_cfg->prio_to_pg_map[i]; + } + +#ifndef XSC_DCBX_STUB + err = xsc_dbcnl_validate_ets(netdev, &ets, true); + if (err) + goto out; +#endif + + err = xsc_dcbnl_ieee_setets_core(priv, &ets); + if (err) { + netdev_err(netdev, + "%s, Failed to set ETS: %d\n", __func__, err); + goto out; + } + + /* Set PFC */ + pfc.pfc_cap = xsc_max_tc(xdev) + 1; + if (!cee_cfg->pfc_enable) + pfc.pfc_en = 0; + else + for (i = 0; i < CEE_DCBX_MAX_PRIO; i++) + pfc.pfc_en |= cee_cfg->pfc_setting[i] << i; + + err = xsc_dcbnl_ieee_setpfc(netdev, &pfc); + if (err) { + netdev_err(netdev, + "%s, Failed to set PFC: %d\n", __func__, err); + goto out; + } +out: + return err ? XSC_DCB_NO_CHG : XSC_DCB_CHG_RESET; +} + +static u8 xsc_dcbnl_getstate(struct net_device *netdev) +{ + return XSC_CEE_STATE_UP; +} + +static void xsc_dcbnl_getpermhwaddr(struct net_device *netdev, + u8 *perm_addr) +{ +#ifndef XSC_DCBX_STUB + struct xsc_adapter *priv = netdev_priv(netdev); +#endif + + if (!perm_addr) + return; + + memset(perm_addr, 0xff, MAX_ADDR_LEN); +#ifndef XSC_DCBX_STUB + xsc_query_nic_vport_mac_address(priv->xdev, 0, perm_addr); +#endif +} + +static void xsc_dcbnl_setpgtccfgtx(struct net_device *netdev, + int priority, u8 prio_type, + u8 pgid, u8 bw_pct, u8 up_map) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_cee_config *cee_cfg = &priv->dcbx.cee_cfg; + + xsc_eth_dbg(HW, priv, "%s: prio=%d, type=%d, pgid=%d, bw_pct=%d, up_map=%d\n", + __func__, priority, prio_type, pgid, + bw_pct, up_map); + if (priority >= CEE_DCBX_MAX_PRIO) { + netdev_err(netdev, + "%s, priority is out of range\n", __func__); + return; + } + + if (pgid >= CEE_DCBX_MAX_PGS) { + netdev_err(netdev, + "%s, priority group is out of range\n", __func__); + return; + } + + cee_cfg->prio_to_pg_map[priority] = pgid; +} + +static void xsc_dcbnl_setpgbwgcfgtx(struct net_device *netdev, + int pgid, u8 bw_pct) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_cee_config *cee_cfg = &priv->dcbx.cee_cfg; + + xsc_eth_dbg(HW, priv, "%s: pgid=%d, bw_pct=%d\n", + __func__, pgid, bw_pct); + if (pgid >= CEE_DCBX_MAX_PGS) { + netdev_err(netdev, + "%s, priority group is out of range\n", __func__); + return; + } + + cee_cfg->pg_bw_pct[pgid] = bw_pct; +} + +static void xsc_dcbnl_getpgtccfgtx(struct net_device *netdev, + int priority, u8 *prio_type, + u8 *pgid, u8 *bw_pct, u8 *up_map) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_core_device *xdev = priv->xdev; + + if (!xdev->caps.ets) { + netdev_err(netdev, "%s, ets is not supported\n", __func__); + return; + } + + if (priority >= CEE_DCBX_MAX_PRIO) { + netdev_err(netdev, + "%s, priority is out of range\n", __func__); + return; + } + + *prio_type = 0; + *bw_pct = 0; + *up_map = 0; +#ifndef XSC_DCBX_STUB + if (xsc_query_port_prio_tc(xdev, priority, pgid)) +#endif + *pgid = 0; + + xsc_eth_dbg(HW, priv, "%s: prio=%d, pgid=%d, bw_pct=%d\n", + __func__, priority, *pgid, *bw_pct); +} + +static void xsc_dcbnl_getpgbwgcfgtx(struct net_device *netdev, + int pgid, u8 *bw_pct) +{ + struct ieee_ets ets; + struct xsc_adapter *priv = netdev_priv(netdev); + + if (pgid >= CEE_DCBX_MAX_PGS) { + netdev_err(netdev, + "%s, priority group is out of range\n", __func__); + return; + } + + xsc_dcbnl_ieee_getets(netdev, &ets); + *bw_pct = ets.tc_tx_bw[pgid]; + xsc_eth_dbg(HW, priv, "%s: pgid=%d, bw_pct=%d\n", + __func__, pgid, *bw_pct); +} + +static void xsc_dcbnl_setpfccfg(struct net_device *netdev, + int priority, u8 setting) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_cee_config *cee_cfg = &priv->dcbx.cee_cfg; + + xsc_eth_dbg(HW, priv, "%s: prio=%d, setting=%d\n", + __func__, priority, setting); + if (priority >= CEE_DCBX_MAX_PRIO) { + netdev_err(netdev, + "%s, priority is out of range\n", __func__); + return; + } + + if (setting > 1) + return; + + cee_cfg->pfc_setting[priority] = setting; +} + +static int +xsc_dcbnl_get_priority_pfc(struct net_device *netdev, + int priority, u8 *setting) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct ieee_pfc pfc; + int err; + + err = xsc_dcbnl_ieee_getpfc(netdev, &pfc); + + if (err) + *setting = 0; + else + *setting = (pfc.pfc_en >> priority) & 0x01; + + xsc_eth_dbg(HW, priv, "%s: prio=%d, setting=%d\n", + __func__, priority, *setting); + return err; +} + +static void xsc_dcbnl_getpfccfg(struct net_device *netdev, + int priority, u8 *setting) +{ + if (priority >= CEE_DCBX_MAX_PRIO) { + netdev_err(netdev, + "%s, priority is out of range\n", __func__); + return; + } + + if (!setting) + return; + + xsc_dcbnl_get_priority_pfc(netdev, priority, setting); +} + +static u8 xsc_dcbnl_getcap(struct net_device *netdev, + int capid, u8 *cap) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_core_device *xdev = priv->xdev; + u8 rval = 0; + + switch (capid) { + case DCB_CAP_ATTR_PG: + *cap = true; + break; + case DCB_CAP_ATTR_PFC: + *cap = true; + break; + case DCB_CAP_ATTR_UP2TC: + *cap = false; + break; + case DCB_CAP_ATTR_PG_TCS: + *cap = 1 << xsc_max_tc(xdev); + break; + case DCB_CAP_ATTR_PFC_TCS: + *cap = 1 << xsc_max_tc(xdev); + break; + case DCB_CAP_ATTR_GSP: + *cap = false; + break; + case DCB_CAP_ATTR_BCN: + *cap = false; + break; + case DCB_CAP_ATTR_DCBX: + *cap = priv->dcbx.cap | + DCB_CAP_DCBX_VER_CEE | + DCB_CAP_DCBX_VER_IEEE; + break; + default: + *cap = 0; + rval = 1; + break; + } + + xsc_eth_dbg(HW, priv, "%s: capid=%d, cap=%d, ret=%d\n", + __func__, capid, *cap, rval); + return rval; +} + +static int xsc_dcbnl_getnumtcs(struct net_device *netdev, + int tcs_id, u8 *num) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_core_device *xdev = priv->xdev; + + switch (tcs_id) { + case DCB_NUMTCS_ATTR_PG: + case DCB_NUMTCS_ATTR_PFC: + *num = xsc_max_tc(xdev) + 1; + break; + default: + return -EINVAL; + } + + xsc_eth_dbg(HW, priv, "%s: tcs_id=%d, tc_num=%d\n", + __func__, tcs_id, *num); + return 0; +} + +static u8 xsc_dcbnl_getpfcstate(struct net_device *netdev) +{ + struct ieee_pfc pfc; + + if (xsc_dcbnl_ieee_getpfc(netdev, &pfc)) + return XSC_CEE_STATE_DOWN; + + return pfc.pfc_en ? XSC_CEE_STATE_UP : XSC_CEE_STATE_DOWN; +} + +static void xsc_dcbnl_setpfcstate(struct net_device *netdev, u8 state) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_cee_config *cee_cfg = &priv->dcbx.cee_cfg; + + if ((state != XSC_CEE_STATE_UP) && (state != XSC_CEE_STATE_DOWN)) + return; + + cee_cfg->pfc_enable = state; +} + +static int xsc_dcbnl_getbuffer(struct net_device *dev, + struct dcbnl_buffer *dcb_buffer) +{ + struct xsc_adapter *priv = netdev_priv(dev); + struct xsc_core_device *xdev = priv->xdev; + int err = 0; +#ifndef XSC_DCBX_STUB + int i; + struct xsc_port_buffer port_buffer; + u8 buffer[XSC_MAX_PRIORITY]; +#endif + + if (!xdev->caps.port_buf) + return -EOPNOTSUPP; + +#ifndef XSC_DCBX_STUB + err = xsc_port_query_priority2buffer(xdev, buffer); + if (err) + return err; + + for (i = 0; i < XSC_MAX_PRIORITY; i++) + dcb_buffer->prio2buffer[i] = buffer[i]; + + err = xsc_port_query_buffer(priv, &port_buffer); + if (err) + return err; + + for (i = 0; i < XSC_MAX_BUFFER; i++) + dcb_buffer->buffer_size[i] = port_buffer.buffer[i].size; + dcb_buffer->total_size = port_buffer.port_buffer_size; +#endif + return err; +} + +static int xsc_dcbnl_setbuffer(struct net_device *dev, + struct dcbnl_buffer *dcb_buffer) +{ + struct xsc_adapter *priv = netdev_priv(dev); + struct xsc_core_device *xdev = priv->xdev; + int i; + int err = 0; +#ifndef XSC_DCBX_STUB + struct xsc_port_buffer port_buffer; + u8 old_prio2buffer[XSC_MAX_PRIORITY]; + u32 *buffer_size = NULL; + u8 *prio2buffer = NULL; + u32 changed = 0; +#endif + + if (!xdev->caps.port_buf) + return -EOPNOTSUPP; + + for (i = 0; i < DCBX_MAX_BUFFERS; i++) + netdev_dbg(dev, "buffer[%d]=%d\n", i, dcb_buffer->buffer_size[i]); + + for (i = 0; i < XSC_MAX_PRIORITY; i++) + netdev_dbg(dev, "priority %d buffer%d\n", i, dcb_buffer->prio2buffer[i]); + +#ifndef XSC_DCBX_STUB + err = xsc_port_query_priority2buffer(xdev, old_prio2buffer); + if (err) + return err; + + for (i = 0; i < XSC_MAX_PRIORITY; i++) { + if (dcb_buffer->prio2buffer[i] != old_prio2buffer[i]) { + changed |= XSC_PORT_BUFFER_PRIO2BUFFER; + prio2buffer = dcb_buffer->prio2buffer; + break; + } + } + + err = xsc_port_query_buffer(priv, &port_buffer); + if (err) + return err; + + for (i = 0; i < XSC_MAX_BUFFER; i++) { + if (port_buffer.buffer[i].size != dcb_buffer->buffer_size[i]) { + changed |= XSC_PORT_BUFFER_SIZE; + buffer_size = dcb_buffer->buffer_size; + break; + } + } + + if (!changed) + return 0; + + priv->dcbx.manual_buffer = true; + err = xsc_port_manual_buffer_config(priv, changed, dev->mtu, NULL, + buffer_size, prio2buffer); +#endif + return err; +} + +const struct dcbnl_rtnl_ops xsc_dcbnl_ops = { + .ieee_getets = xsc_dcbnl_ieee_getets, + .ieee_setets = xsc_dcbnl_ieee_setets, + .ieee_getmaxrate = xsc_dcbnl_ieee_getmaxrate, + .ieee_setmaxrate = xsc_dcbnl_ieee_setmaxrate, + .ieee_getpfc = xsc_dcbnl_ieee_getpfc, + .ieee_setpfc = xsc_dcbnl_ieee_setpfc, + .ieee_setapp = xsc_dcbnl_ieee_setapp, + .ieee_delapp = xsc_dcbnl_ieee_delapp, + .getdcbx = xsc_dcbnl_getdcbx, + .setdcbx = xsc_dcbnl_setdcbx, + .dcbnl_getbuffer = xsc_dcbnl_getbuffer, + .dcbnl_setbuffer = xsc_dcbnl_setbuffer, + +/* CEE interfaces */ + .setall = xsc_dcbnl_setall, + .getstate = xsc_dcbnl_getstate, + .getpermhwaddr = xsc_dcbnl_getpermhwaddr, + + .setpgtccfgtx = xsc_dcbnl_setpgtccfgtx, + .setpgbwgcfgtx = xsc_dcbnl_setpgbwgcfgtx, + .getpgtccfgtx = xsc_dcbnl_getpgtccfgtx, + .getpgbwgcfgtx = xsc_dcbnl_getpgbwgcfgtx, + + .setpfccfg = xsc_dcbnl_setpfccfg, + .getpfccfg = xsc_dcbnl_getpfccfg, + .getcap = xsc_dcbnl_getcap, + .getnumtcs = xsc_dcbnl_getnumtcs, + .getpfcstate = xsc_dcbnl_getpfcstate, + .setpfcstate = xsc_dcbnl_setpfcstate, +}; + +static void xsc_dcbnl_query_dcbx_mode(struct xsc_adapter *priv, + enum xsc_dcbx_oper_mode *mode) +{ + *mode = XSC_DCBX_PARAM_VER_OPER_HOST; + + /* From driver's point of view, we only care if the mode + * is host (HOST) or non-host (AUTO) + */ + if (*mode != XSC_DCBX_PARAM_VER_OPER_HOST) + *mode = XSC_DCBX_PARAM_VER_OPER_AUTO; +} + +static void xsc_ets_init(struct xsc_adapter *priv) +{ +#ifdef XSC_DCBX_STUB + struct ieee_ets ets; + int err; + int i; + + if (!priv->xdev->caps.ets) + return; + memset(&ets, 0, sizeof(ets)); + ets.ets_cap = xsc_max_tc(priv->xdev) + 1; + for (i = 0; i < ets.ets_cap; i++) { + ets.tc_tx_bw[i] = XSC_MAX_BW_ALLOC; + ets.tc_tsa[i] = IEEE_8021QAZ_TSA_VENDOR; + ets.prio_tc[i] = i; + } + + err = xsc_dcbnl_ieee_setets_core(priv, &ets); + if (err) + netdev_err(priv->netdev, + "%s, Failed to init ETS: %d\n", __func__, err); +#endif +} + +enum { + INIT, + DELETE, +}; + +static void xsc_dcbnl_dscp_app(struct xsc_adapter *priv, int action) +{ + struct dcb_app temp; + int i; + + xsc_eth_dbg(HW, priv, "%s: action=%d\n", __func__, action); + if (!priv->xdev->caps.dscp) + return; + + /* No SEL_DSCP entry in non DSCP state */ + if (priv->dcbx_dp.trust_state != XSC_QPTS_TRUST_DSCP) + return; + + temp.selector = IEEE_8021QAZ_APP_SEL_DSCP; + for (i = 0; i < XSC_MAX_DSCP; i++) { + temp.protocol = i; + temp.priority = priv->dcbx_dp.dscp2prio[i]; + if (action == INIT) + dcb_ieee_setapp(priv->netdev, &temp); + else + dcb_ieee_delapp(priv->netdev, &temp); + } + + priv->dcbx.dscp_app_cnt = (action == INIT) ? XSC_MAX_DSCP : 0; +} + +void xsc_dcbnl_init_app(struct xsc_adapter *priv) +{ + xsc_dcbnl_dscp_app(priv, INIT); +} + +void xsc_dcbnl_delete_app(struct xsc_adapter *priv) +{ + xsc_dcbnl_dscp_app(priv, DELETE); +} + +static void xsc_trust_update_tx_min_inline_mode(struct xsc_adapter *priv) +{ + struct xsc_eth_params *params = &priv->nic_param; + +#ifndef XSC_DCBX_STUB + xsc_query_min_inline(priv->xdev, ¶ms->tx_min_inline_mode); +#endif + if (priv->dcbx_dp.trust_state == XSC_QPTS_TRUST_DSCP && + params->tx_min_inline_mode == XSC_INLINE_MODE_L2) + params->tx_min_inline_mode = XSC_INLINE_MODE_IP; +} + +static void xsc_trust_update_sq_inline_mode(struct xsc_adapter *priv) +{ + + int old_mode = priv->nic_param.tx_min_inline_mode; + + mutex_lock(&priv->state_lock); + + xsc_trust_update_tx_min_inline_mode(priv); + + /* Skip if tx_min_inline is the same */ + if (old_mode == priv->nic_param.tx_min_inline_mode) + goto out; +#ifndef XSC_DCBX_STUB + xsc_safe_switch_channels(priv, NULL, NULL); +#endif + +out: + mutex_unlock(&priv->state_lock); +} + +static int xsc_set_trust_state(struct xsc_adapter *priv, u8 trust_state) +{ + int err = 0; + +#ifndef XSC_DCBX_STUB + err = xsc_cmd_set_trust_state(priv->xdev, trust_state); + if (err) + return err; +#endif + priv->dcbx_dp.trust_state = trust_state; + xsc_trust_update_sq_inline_mode(priv); + + /* In DSCP trust state, we need 8 send queues per channel */ +#ifndef XSC_DCBX_STUB + struct tc_mqprio_qopt mqprio = {.num_tc = XSC_MAX_NUM_TC}; + + if (priv->dcbx_dp.trust_state == XSC_QPTS_TRUST_DSCP) + xsc_setup_tc_mqprio(priv->netdev, &mqprio); +#endif + + return err; +} + +static int xsc_set_dscp2prio(struct xsc_adapter *priv, u8 dscp, u8 prio) +{ + int err = 0; + + xsc_eth_dbg(HW, priv, "%s: dscp=%d, prio=%d\n", + __func__, dscp, prio); +#ifndef XSC_DCBX_STUB + err = xsc_cmd_set_dscp2prio(priv->xdev, dscp, prio); + if (err) + return err; +#endif + priv->dcbx_dp.dscp2prio[dscp] = prio; + return err; +} + +static int xsc_trust_initialize(struct xsc_adapter *priv) +{ + struct xsc_core_device *xdev = priv->xdev; +#ifndef XSC_DCBX_STUB + int err; +#endif + + priv->dcbx_dp.trust_state = XSC_QPTS_TRUST_PCP; + + if (!xdev->caps.dscp) + return 0; + +#ifndef XSC_DCBX_STUB + err = xsc_query_trust_state(priv->xdev, &priv->dcbx_dp.trust_state); + if (err) + return err; +#endif + + xsc_trust_update_tx_min_inline_mode(priv); + if (priv->dcbx_dp.trust_state == XSC_QPTS_TRUST_DSCP) + priv->nic_param.num_tc = XSC_MAX_NUM_TC; +#ifndef XSC_DCBX_STUB + err = xsc_query_dscp2prio(priv->xdev, priv->dcbx_dp.dscp2prio); + if (err) + return err; +#endif + + return 0; +} + +#define XSC_BUFFER_CELL_SHIFT 7 + +static u16 xsc_query_port_buffers_cell_size(struct xsc_adapter *priv) +{ + return (1 << XSC_BUFFER_CELL_SHIFT); +} + +static void xsc_cee_init(struct xsc_adapter *priv) +{ + struct xsc_cee_config *cee_cfg = &priv->dcbx.cee_cfg; + struct xsc_core_device *xdev = priv->xdev; + int i, max_tc; + + memset(cee_cfg, 0, sizeof(*cee_cfg)); + + cee_cfg->pfc_enable = 1; + + max_tc = xsc_max_tc(xdev) + 1; + for (i = 0; i < max_tc; i++) + cee_cfg->pfc_setting[i] = 1; + + for (i = 0; i < CEE_DCBX_MAX_PGS; i++) + cee_cfg->prio_to_pg_map[i] = i % max_tc; +} + +void xsc_dcbnl_initialize(struct xsc_adapter *priv) +{ + struct xsc_dcbx *dcbx = &priv->dcbx; + + xsc_eth_dbg(HW, priv, "%s\n", __func__); + xsc_trust_initialize(priv); + + if (!priv->xdev->caps.qos) + return; + + if (priv->xdev->caps.dcbx) + xsc_dcbnl_query_dcbx_mode(priv, &dcbx->mode); + + priv->dcbx.cap = DCB_CAP_DCBX_VER_CEE | + DCB_CAP_DCBX_VER_IEEE; + if (priv->dcbx.mode == XSC_DCBX_PARAM_VER_OPER_HOST) + priv->dcbx.cap |= DCB_CAP_DCBX_HOST; + + priv->dcbx.port_buff_cell_sz = xsc_query_port_buffers_cell_size(priv); + priv->dcbx.manual_buffer = false; + priv->dcbx.cable_len = XSC_DEFAULT_CABLE_LEN; + + xsc_cee_init(priv); + xsc_ets_init(priv); +} +#endif + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth.h new file mode 100644 index 0000000000000000000000000000000000000000..e00849b46b705f02d4d34f7891fbc8c4511aa92d --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth.h @@ -0,0 +1,227 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ETH_H +#define XSC_ETH_H + +#include +#include "xsc_eth_common.h" +#include "xsc_eth_stats.h" +#include +#include + +#define XSC_INVALID_LKEY 0x100 + +#define XSCALE_ETH_PHYPORT_DOWN 0 +#define XSCALE_ETH_PHYPORT_UP 1 + +#define CONFIG_XSC_CORE_EN_DCB 1 +#define XSC_PAGE_CACHE 1 + +//#define XSC_UDP_FRAG_HW_CSUM 1 +//#define UDP_CHECK_0 1 +//#define UDP_CSUM_DEBUG 1 + +#ifndef COSIM +#define NEED_AGILEX_TRAINING +#endif +#define XSCALE_DRIVER_NAME "xsc_eth" +#define XSCALE_RET_SUCCESS 0 +#define XSCALE_RET_ERROR 1 + +enum { + XSCALE_ETH_DRIVER_INIT, + XSCALE_ETH_DRIVER_OK, + XSCALE_ETH_DRIVER_CLOSE, + XSCALE_ETH_DRIVER_DETACH, +}; + +#define XSCALE_ETH_QP_NUM_MAX 1 +#define XSCALE_RX_THREAD_MAX 128 + +enum { + XSC_BW_NO_LIMIT = 0, + XSC_100_MBPS_UNIT = 3, + XSC_GBPS_UNIT = 4, +}; + +struct xsc_cee_config { + /* bw pct for priority group */ + u8 pg_bw_pct[CEE_DCBX_MAX_PGS]; + u8 prio_to_pg_map[CEE_DCBX_MAX_PRIO]; + bool pfc_setting[CEE_DCBX_MAX_PRIO]; + bool pfc_enable; +}; + +enum { + XSC_DCB_CHG_RESET, + XSC_DCB_NO_CHG, + XSC_DCB_CHG_NO_RESET, +}; + +enum xsc_qpts_trust_state { + XSC_QPTS_TRUST_PCP = 1, + XSC_QPTS_TRUST_DSCP = 2, +}; + +enum xsc_dcbx_oper_mode { + XSC_DCBX_PARAM_VER_OPER_HOST = 0x0, + XSC_DCBX_PARAM_VER_OPER_AUTO = 0x3, +}; + +enum { + XSC_PORT_BUFFER_CABLE_LEN = BIT(0), + XSC_PORT_BUFFER_PFC = BIT(1), + XSC_PORT_BUFFER_PRIO2BUFFER = BIT(2), + XSC_PORT_BUFFER_SIZE = BIT(3), +}; + +struct xsc_dcbx { + enum xsc_dcbx_oper_mode mode; + struct xsc_cee_config cee_cfg; /* pending configuration */ + u8 dscp_app_cnt; + + /* The only setting that cannot be read from FW */ + u8 tc_tsa[IEEE_8021QAZ_MAX_TCS]; + u8 cap; + + /* Buffer configuration */ + bool manual_buffer; + u32 cable_len; + u32 xoff; + u16 port_buff_cell_sz; +}; + +struct xsc_bufferx_reg { + u8 lossy; + u8 epsb; + u32 size; + u32 xoff; + u32 xon; +}; + +struct xsc_port_buffer { + u32 port_buffer_size; + u32 spare_buffer_size; + struct xsc_bufferx_reg buffer[XSC_MAX_BUFFER]; +}; + +struct xsc_dcbx_dp { + u8 dscp2prio[XSC_MAX_DSCP]; + u8 trust_state; +}; + +struct xsc_rss_params { + u32 indirection_rqt[XSC_INDIR_RQT_SIZE]; + u32 rx_hash_fields[XSC_NUM_INDIR_TIRS]; + u8 toeplitz_hash_key[52]; + u8 hfunc; + u32 rss_hash_tmpl; +}; + +struct xsc_adapter { + struct net_device *netdev; + struct pci_dev *pdev; + struct device *dev; + struct xsc_core_device *xdev; + + struct xsc_eth_params nic_param; + struct xsc_rss_params rss_params; + + struct workqueue_struct *workq; + struct work_struct update_carrier_work; + struct work_struct set_rx_mode_work; + struct work_struct event_work; + +#ifdef NEED_AGILEX_TRAINING + struct timer_list link_timer; +#endif + + struct xsc_eth_channels channels; + struct xsc_sq **txq2sq; + + u32 status; + spinlock_t lock; + + struct mutex state_lock; /* Protects Interface state */ + struct xsc_stats *stats; + + struct xsc_dcbx dcbx; + struct xsc_dcbx_dp dcbx_dp; + + u32 msglevel; + + struct task_struct *task; +}; + +struct xsc_rx_buffer { + struct sk_buff *skb; + dma_addr_t dma; + u32 len; + struct page *page; +#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) + u32 page_offset; +#else + u16 page_offset; +#endif + u16 pagecnt_bias; +}; + +struct xsc_tx_buffer { + struct sk_buff *skb; + unsigned long *h_skb_data; + dma_addr_t dma; + u32 len; + struct page *page; +#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) + u32 page_offset; +#else + u16 page_offset; +#endif + u16 pagecnt_bias; +}; + +struct xsc_tx_wqe { + struct xsc_send_wqe_ctrl_seg ctrl; + struct xsc_wqe_data_seg data[0]; +}; + +typedef int (*xsc_eth_fp_preactivate)(struct xsc_adapter *priv); +typedef int (*xsc_eth_fp_postactivate)(struct xsc_adapter *priv); + +int xsc_safe_switch_channels(struct xsc_adapter *adapter, + xsc_eth_fp_preactivate preactivate, + xsc_eth_fp_postactivate postactivate); +int xsc_eth_num_channels_changed(struct xsc_adapter *priv); +int xsc_eth_modify_nic_hca(struct xsc_adapter *adapter, u32 change); +bool xsc_eth_get_phyport_state(struct xsc_adapter *adapter); +#ifdef NEED_AGILEX_TRAINING +int xsc_eth_get_linkinfo(struct xsc_event_linkstatus_resp *plinkinfo, struct xsc_adapter *adapter); +#endif +int xsc_eth_set_led_status(int id, struct xsc_adapter *adapter); + +/* Use this function to get max num channels after netdev was created */ +static inline int xsc_get_netdev_max_channels(struct xsc_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + + return min_t(unsigned int, netdev->num_rx_queues, + netdev->num_tx_queues); +} + +static inline int xsc_get_netdev_max_tc(struct xsc_adapter *adapter) +{ + return adapter->nic_param.num_tc; +} + +#ifdef CONFIG_XSC_CORE_EN_DCB +extern const struct dcbnl_rtnl_ops xsc_dcbnl_ops; +int xsc_dcbnl_ieee_setets_core(struct xsc_adapter *priv, struct ieee_ets *ets); +void xsc_dcbnl_initialize(struct xsc_adapter *priv); +void xsc_dcbnl_init_app(struct xsc_adapter *priv); +void xsc_dcbnl_delete_app(struct xsc_adapter *priv); +#endif +#endif /* XSC_ETH_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_common.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_common.h new file mode 100644 index 0000000000000000000000000000000000000000..9f6d88639d71d80a655dd0c5cb23a1b16a808afe --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_common.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ETH_COMMON_H +#define XSC_ETH_COMMON_H + +#include "xsc_queue.h" +#include "xsc_eth_compat.h" +#include + +#define SW_MIN_MTU 64 +#define SW_DEFAULT_MTU 1500 +#define SW_MAX_MTU 9600 + +#define XSC_ETH_HW_MTU_SEND 9800 /*need to obtain from hardware*/ +#define XSC_ETH_HW_MTU_RECV 9800 /*need to obtain from hardware*/ +#define XSC_SW2HW_MTU(mtu) (mtu + 14 + 4) +#define XSC_SW2HW_FRAG_SIZE(mtu) (mtu + 14 + 4 + XSC_PPH_HEAD_LEN) + +#define XSC_RX_MAX_HEAD (256) +#define XSC_RX_HEADROOM NET_SKB_PAD + +#define XSC_QPN_SQN_STUB 1025 +#define XSC_QPN_RQN_STUB 1024 + +#define XSC_LOG_INDIR_RQT_SIZE 0x8 + +#define XSC_INDIR_RQT_SIZE BIT(XSC_LOG_INDIR_RQT_SIZE) +#ifdef XSC_RSS_SUPPORT +#define XSC_ETH_MIN_NUM_CHANNELS 2 +#else +#define XSC_ETH_MIN_NUM_CHANNELS 1 +#endif +#define XSC_ETH_MAX_NUM_CHANNELS XSC_INDIR_RQT_SIZE + +#define XSC_TX_NUM_TC 1 +#define XSC_MAX_NUM_TC 8 +#define XSC_ETH_MAX_TC_TOTAL (XSC_ETH_MAX_NUM_CHANNELS * XSC_MAX_NUM_TC) +#define XSC_ETH_MAX_QP_NUM_PER_CH (XSC_MAX_NUM_TC + 1) + +#define XSC_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) +#define XSC_MIN_SKB_FRAG_SZ (XSC_SKB_FRAG_SZ(XSC_RX_HEADROOM)) +#define XSC_LOG_MAX_RX_WQE_BULK \ + (ilog2(PAGE_SIZE / roundup_pow_of_two(XSC_MIN_SKB_FRAG_SZ))) + +#define XSC_MIN_LOG_RQ_SZ (1 + XSC_LOG_MAX_RX_WQE_BULK) +#define XSC_DEF_LOG_RQ_SZ 0xa +#define XSC_MAX_LOG_RQ_SZ 0xd + +#define XSC_MIN_LOG_SQ_SZ 0x6 +#define XSC_DEF_LOG_SQ_SZ 0xa +#define XSC_MAX_LOG_SQ_SZ 0xd + +#define XSC_SQ_ELE_NUM_DEF BIT(XSC_DEF_LOG_SQ_SZ) +#define XSC_RQ_ELE_NUM_DEF BIT(XSC_DEF_LOG_RQ_SZ) + +#define XSC_RQCQ_ELE_NUM XSC_RQ_ELE_NUM_DEF //number of rqcq entry +#define XSC_SQCQ_ELE_NUM XSC_SQ_ELE_NUM_DEF //number of sqcq entry +#define XSC_RQ_ELE_NUM XSC_RQ_ELE_NUM_DEF //ds number of a wqebb +#define XSC_SQ_ELE_NUM XSC_SQ_ELE_NUM_DEF //DS number +#define XSC_EQ_ELE_NUM XSC_SQ_ELE_NUM_DEF //number of eq entry??? + +#define XSC_RQCQ_ELE_SZ 32 //size of a rqcq entry +#define XSC_SQCQ_ELE_SZ 32 //size of a sqcq entry +#define XSC_RQ_ELE_SZ XSC_RECV_WQE_BB +#define XSC_SQ_ELE_SZ XSC_SEND_WQE_BB +#define XSC_EQ_ELE_SZ 8 //size of a eq entry + +#define XSC_CQ_POLL_BUDGET 64 + +#define XSC_MAX_BW_ALLOC 100 /* Max percentage of BW allocation */ +#define XSC_MAX_PRIORITY 8 +#define XSC_MAX_DSCP 64 +#define XSC_MAX_BUFFER 8 +#define XSC_DEFAULT_CABLE_LEN 7 /* 7 meters */ + +enum xsc_port_status { + XSC_PORT_UP = 1, + XSC_PORT_DOWN = 2, +}; + +/*all attributes of queue, MAYBE no use for some special queue*/ + +enum xsc_queue_type { + XSC_QUEUE_TYPE_EQ = 0, + XSC_QUEUE_TYPE_RQCQ, + XSC_QUEUE_TYPE_SQCQ, + XSC_QUEUE_TYPE_RQ, + XSC_QUEUE_TYPE_SQ, + XSC_QUEUE_TYPE_MAX, +}; + +struct xsc_queue_attr { + u8 q_type; + u32 ele_num; + u32 ele_size; + u8 ele_log_size; + u8 q_log_size; +}; + +/*MUST set value before create queue*/ +struct xsc_eth_eq_attr { + struct xsc_queue_attr xsc_eq_attr; +}; + +struct xsc_eth_cq_attr { + struct xsc_queue_attr xsc_cq_attr; +}; + +struct xsc_eth_rq_attr { + struct xsc_queue_attr xsc_rq_attr; +}; + +struct xsc_eth_sq_attr { + struct xsc_queue_attr xsc_sq_attr; +}; + +struct xsc_eth_qp_attr { + struct xsc_queue_attr xsc_qp_attr; +}; + +struct xsc_eth_rx_wqe_cyc { + struct xsc_wqe_data_seg data[0]; +}; + +struct xsc_eq_param { +// struct xsc_eq_cmd_param eqc; + struct xsc_queue_attr eq_attr; +}; + +struct xsc_cq_param { + struct xsc_wq_param wq; + struct cq_cmd { + u8 abc[16]; + } cqc; + struct xsc_queue_attr cq_attr; +}; + +struct xsc_rq_param { +// struct xsc_rq_cmd_param rqc; + struct xsc_queue_attr rq_attr; + struct xsc_rq_frags_info frags_info; + +}; + +struct xsc_sq_param { +// struct xsc_rq_cmd_param sqc; + struct xsc_queue_attr sq_attr; +}; + +struct xsc_qp_param { +// struct xsc_qp_cmd_param qpc; + struct xsc_queue_attr qp_attr; +}; + +struct xsc_channel_param { + struct xsc_cq_param rqcq_param; + struct xsc_cq_param sqcq_param; + struct xsc_rq_param rq_param; + struct xsc_sq_param sq_param; + struct xsc_qp_param qp_param; +}; + +struct xsc_eth_qp { + u16 rq_num; + u16 sq_num; + struct xsc_rq rq[XSC_MAX_NUM_TC]; /*may be use one only*/ + struct xsc_sq sq[XSC_MAX_NUM_TC]; /*reserved to tc*/ +}; + +enum channel_flags { + XSC_CHANNEL_NAPI_SCHED = 1, +}; + +struct xsc_channel { + /* data path */ + struct xsc_eth_qp qp; + struct napi_struct napi; + u8 num_tc; + int chl_idx; + + /*relationship*/ + struct xsc_adapter *adapter; + struct net_device *netdev; + int cpu; + unsigned long flags; + + /* data path - accessed per napi poll */ + struct cpumask *aff_mask; + struct irq_desc *irq_desc; + struct xsc_ch_stats *stats; +}; + +enum xsc_eth_priv_flag { + XSC_PFLAG_RX_NO_CSUM_COMPLETE, + XSC_PFLAG_SNIFFER, + XSC_PFLAG_DROPLESS_RQ, + XSC_NUM_PFLAGS, /* Keep last */ +}; + +#define XSC_SET_PFLAG(params, pflag, enable) \ + do { \ + if (enable) \ + (params)->pflags |= BIT(pflag); \ + else \ + (params)->pflags &= ~(BIT(pflag)); \ + } while (0) + +#define XSC_GET_PFLAG(params, pflag) (!!((params)->pflags & (BIT(pflag)))) + +struct xsc_eth_params { + u16 num_channels; + u16 max_num_ch; + u8 num_tc; + u32 mtu; + u32 hard_mtu; + u32 comp_vectors; + u32 sq_size; + u32 sq_max_size; + u8 rq_wq_type; + u32 rq_size; + u32 rq_max_size; + u32 rq_frags_size; + + u16 num_rl_txqs; + bool rx_cqe_compress_def; +// struct net_dim_cq_moder rx_cq_moderation; +// struct net_dim_cq_moder tx_cq_moderation; + bool tunneled_offload_en; + bool lro_en; + u8 tx_min_inline_mode; + bool vlan_strip_disable; + bool scatter_fcs_en; + bool rx_dim_enabled; + bool tx_dim_enabled; + u32 lro_timeout; + u32 pflags; +}; + +struct xsc_eth_channels { + struct xsc_channel *c; + unsigned int num_chl; + u32 rqn_base; +}; + +struct xsc_eth_redirect_rqt_param { + bool is_rss; + union { + u32 rqn; /* Direct RQN (Non-RSS) */ + struct { + u8 hfunc; + struct xsc_eth_channels *channels; + } rss; /* RSS data */ + }; +}; + +union xsc_send_doorbell { + struct{ + int32_t next_pid : 16; + u32 qp_num : 15; + }; + u32 send_data; +}; + +union xsc_recv_doorbell { + struct{ + int32_t next_pid : 13; + u32 qp_num : 15; + }; + u32 recv_data; +}; + +#endif /* XSC_ETH_COMMON_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_compat.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_compat.h new file mode 100644 index 0000000000000000000000000000000000000000..b55953fe62e8cf64a5defe955814ed4b14d897a7 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_compat.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ETH_COMPAT_H +#define XSC_ETH_COMPAT_H + +#define xsc_netdev_xmit_more(skb) netdev_xmit_more() + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ctrl.c b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ctrl.c new file mode 100644 index 0000000000000000000000000000000000000000..1c8ff3b2de296a3cdb437b199d36386cc527063d --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ctrl.c @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define XSC_ETH_CTRL_NAME "eth_ctrl" + +static void encode_rlimit_set(void *data, u32 mac_port) +{ + struct xsc_rate_limit_set *req = (struct xsc_rate_limit_set *) data; + + req->rate_cir = __cpu_to_be32(req->rate_cir); + req->limit_id = __cpu_to_be32(req->limit_id); +} + +static void decode_rlimit_get(void *data) +{ + struct xsc_rate_limit_get *resp = (struct xsc_rate_limit_get *) data; + int i; + + for (i = 0; i <= QOS_PRIO_MAX; i++) + resp->rate_cir[i] = __be32_to_cpu(resp->rate_cir[i]); + + resp->max_limit_id = __be32_to_cpu(resp->max_limit_id); +} + +static int _eth_ctrl_ioctl_qos(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr, u16 expect_req_size, + u16 expect_resp_size, void (*encode)(void *, u32), void (*decode)(void *)) +{ + struct xsc_qos_mbox_in *in; + struct xsc_qos_mbox_out *out; + u16 user_size; + int err; + + user_size = expect_req_size > expect_resp_size ? expect_req_size : expect_resp_size; + if (hdr->attr.length != user_size) + return -EINVAL; + + in = kvzalloc(sizeof(struct xsc_qos_mbox_in) + expect_req_size, GFP_KERNEL); + if (!in) + goto err_in; + out = kvzalloc(sizeof(struct xsc_qos_mbox_out) + expect_resp_size, GFP_KERNEL); + if (!out) + goto err_out; + + err = copy_from_user(&in->data, user_hdr->attr.data, expect_req_size); + if (err) + goto err; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + in->req_prfx.mac_port = xdev->mac_port; + + if (encode) + encode((void *)in->data, xdev->mac_port); + + err = xsc_cmd_exec( + xdev, in, sizeof(*in) + expect_req_size, out, sizeof(*out) + expect_resp_size); + + hdr->attr.error = __be32_to_cpu(out->hdr.status); + if (decode) + decode((void *)out->data); + + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + goto err; + if (copy_to_user((void *)user_hdr->attr.data, &out->data, expect_resp_size)) + goto err; + + kvfree(in); + kvfree(out); + return 0; + +err: + kvfree(out); +err_out: + kvfree(in); +err_in: + return -EFAULT; +} + +static int _eth_ctrl_ioctl_hwconfig(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr, u16 expect_req_size, + u16 expect_resp_size, void (*encode)(void *, u32), void (*decode)(void *)) +{ + struct xsc_hwc_mbox_in *in; + struct xsc_hwc_mbox_out *out; + u16 user_size; + int err; + + user_size = expect_req_size > expect_resp_size ? expect_req_size : expect_resp_size; + if (hdr->attr.length != user_size) + return -EINVAL; + + in = kvzalloc(sizeof(struct xsc_hwc_mbox_in) + expect_req_size, GFP_KERNEL); + if (!in) + goto err_in; + out = kvzalloc(sizeof(struct xsc_hwc_mbox_out) + expect_resp_size, GFP_KERNEL); + if (!out) + goto err_out; + + err = copy_from_user(&in->data, user_hdr->attr.data, expect_req_size); + if (err) + goto err; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + if (encode) + encode((void *)in->data, xdev->mac_port); + + err = xsc_cmd_exec( + xdev, in, sizeof(*in) + expect_req_size, out, sizeof(*out) + expect_resp_size); + + hdr->attr.error = __be32_to_cpu(out->hdr.status); + if (decode) + decode((void *)out->data); + + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + goto err; + if (copy_to_user((void *)user_hdr->attr.data, &out->data, expect_resp_size)) + goto err; + + kvfree(in); + kvfree(out); + return 0; + +err: + kvfree(out); +err_out: + kvfree(in); +err_in: + return -EFAULT; +} + +static long _eth_ctrl_ioctl_cmdq(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr) +{ + struct xsc_ioctl_hdr hdr; + int err; + void *in; + void *out; + + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + if (err) + return -EFAULT; + + /* check valid */ + if (hdr.check_filed != XSC_IOCTL_CHECK_FILED) + return -EINVAL; + + /* check ioctl cmd */ + switch (hdr.attr.opcode) { + case XSC_CMD_OP_IOCTL_SET_DSCP_PMT: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_dscp_pmt_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_DSCP_PMT: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, 0, sizeof(struct xsc_dscp_pmt_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_TRUST_MODE: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_trust_mode_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_TRUST_MODE: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, 0, sizeof(struct xsc_trust_mode_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_PCP_PMT: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_pcp_pmt_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_PCP_PMT: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, 0, sizeof(struct xsc_pcp_pmt_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_DEFAULT_PRI: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_default_pri_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_DEFAULT_PRI: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, 0, sizeof(struct xsc_default_pri_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_PFC: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_pfc_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_PFC: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, 0, sizeof(struct xsc_pfc_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_RATE_LIMIT: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_rate_limit_set), 0, + encode_rlimit_set, NULL); + case XSC_CMD_OP_IOCTL_GET_RATE_LIMIT: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_rate_limit_get), + sizeof(struct xsc_rate_limit_get), NULL, decode_rlimit_get); + case XSC_CMD_OP_IOCTL_SET_SP: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_sp_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_SP: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, 0, sizeof(struct xsc_sp_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_WEIGHT: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, sizeof(struct xsc_weight_set), 0, + NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_WEIGHT: + return _eth_ctrl_ioctl_qos( + xdev, user_hdr, &hdr, 0, sizeof(struct xsc_weight_get), + NULL, NULL); + case XSC_CMD_OP_IOCTL_SET_HWC: + return _eth_ctrl_ioctl_hwconfig(xdev, user_hdr, &hdr, + sizeof(struct hwc_set_t), 0, NULL, NULL); + case XSC_CMD_OP_IOCTL_GET_HWC: + return _eth_ctrl_ioctl_hwconfig(xdev, user_hdr, &hdr, + sizeof(struct hwc_get_t), sizeof(struct hwc_get_t), NULL, NULL); + default: + return -EINVAL; + } + + in = kvzalloc(hdr.attr.length, GFP_KERNEL); + if (!in) + return -ENOMEM; + out = kvzalloc(hdr.attr.length, GFP_KERNEL); + if (!out) { + kfree(in); + return -ENOMEM; + } + + err = copy_from_user(in, user_hdr->attr.data, hdr.attr.length); + if (err) { + err = -EFAULT; + goto err_exit; + } + + xsc_cmd_exec(xdev, in, hdr.attr.length, out, hdr.attr.length); + + if (copy_to_user((void *)user_hdr, &hdr, sizeof(hdr))) + err = -EFAULT; + if (copy_to_user((void *)user_hdr->attr.data, out, hdr.attr.length)) + err = -EFAULT; +err_exit: + kfree(in); + kfree(out); + return err; +} + +static void _eth_ctrl_reg_cb(struct xsc_core_device *xdev, unsigned int cmd, + struct xsc_ioctl_hdr __user *user_hdr, void *data) +{ + int err; + + switch (cmd) { + case XSC_IOCTL_CMDQ: + err = _eth_ctrl_ioctl_cmdq(xdev, user_hdr); + break; + default: + err = -EFAULT; + break; + } +} + +static void _eth_ctrl_reg_fini(void) +{ + xsc_port_ctrl_cb_dereg(XSC_ETH_CTRL_NAME); +} + +static int _eth_ctrl_reg_init(void) +{ + int ret; + + ret = xsc_port_ctrl_cb_reg(XSC_ETH_CTRL_NAME, _eth_ctrl_reg_cb, NULL); + if (ret != 0) + pr_err("failed to register port control node for %s\n", XSC_ETH_CTRL_NAME); + + return ret; +} + +void xsc_eth_ctrl_fini(void) +{ + _eth_ctrl_reg_fini(); +} + +int xsc_eth_ctrl_init(void) +{ + return _eth_ctrl_reg_init(); +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ctrl.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ctrl.h new file mode 100644 index 0000000000000000000000000000000000000000..f91f93009af2262dda6ee488cdb8338d140c3271 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ctrl.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ETH_CTRL_H +#define XSC_ETH_CTRL_H + +void xsc_eth_ctrl_fini(void); +int xsc_eth_ctrl_init(void); + +#endif /* XSC_RXTX_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_debug.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_debug.h new file mode 100644 index 0000000000000000000000000000000000000000..ca8053093dfdef1ad2a2659385daf45133f48967 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_debug.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ETH_DEBUG_H +#define XSC_ETH_DEBUG_H + +#include +#include +#include +#include "xsc_eth.h" + +static bool debug; +#define FUN_LINE_FMT "%s %d " +#define ETH_DEBUG_LOG(fmt, ...) \ + do { \ + if (debug) \ + pr_info(FUN_LINE_FMT fmt, __func__, __LINE__, ##__VA_ARGS__); \ + } while (0) + +#define XSC_MSG_LEVEL (NETIF_MSG_LINK) // | NETIF_MSG_HW) + +#define xsc_eth_dbg(mlevel, priv, format, ...) \ +do { \ + if (NETIF_MSG_##mlevel & (priv)->msglevel) \ + netdev_warn(priv->netdev, format, \ + ##__VA_ARGS__); \ +} while (0) + +#define WQE_CSEG_DUMP(seg_name, seg) \ + do { \ + ETH_DEBUG_LOG("dump %s:\n", seg_name); \ + ETH_DEBUG_LOG("cseg->has_pph: %d\n", seg->has_pph); \ + ETH_DEBUG_LOG("cseg->so_type: %d\n", seg->so_type); \ + ETH_DEBUG_LOG("cseg->so_hdr_len: %d\n", seg->so_hdr_len); \ + ETH_DEBUG_LOG("cseg->so_data_size: %d\n", seg->so_data_size); \ + ETH_DEBUG_LOG("cseg->msg_opcode: %d\n", seg->msg_opcode); \ + ETH_DEBUG_LOG("cseg->wqe_id: %d\n", seg->wqe_id); \ + ETH_DEBUG_LOG("cseg->ds_data_num: %d\n", seg->ds_data_num); \ + ETH_DEBUG_LOG("cseg->msg_len: %d\n", seg->msg_len); \ + } while (0) + +#define WQE_DSEG_DUMP(seg_name, seg) \ + do { \ + ETH_DEBUG_LOG("dump %s:\n", seg_name); \ + ETH_DEBUG_LOG("dseg->va: %#llx\n", seg->va); \ + ETH_DEBUG_LOG("dseg->in_line: %d\n", seg->in_line); \ + ETH_DEBUG_LOG("dseg->mkey: %d\n", seg->mkey); \ + ETH_DEBUG_LOG("dseg->seg_len: %d\n", seg->seg_len); \ + } while (0) + +static inline void skbdata_debug_dump(struct sk_buff *skb, u16 headlen, int direct) +{ + if (!debug) + return; + + netdev_info(skb->dev, "pkt[%s]: skb_len=%d, head_len=%d\n", + (direct?"tx":"rx"), skb->len, headlen); + + if (skb) { + char *buf = skb->data; + int i, j; + int pos; + + for (i = 0; i < headlen; i++) { + if (i % 16 == 0) + pr_info("%#4.4x ", i); + pr_info("%2.2x ", ((unsigned char *)buf)[i]); + } + + pr_info("\n"); + + pos = headlen; + for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; + int fsz = skb_frag_size(frag); + + buf = (char *)(page_address(frag->bv_page) + frag->bv_offset); + for (i = 0; i < fsz; i++) { + if (i % 16 == 0) + pr_info("%#4.4x ", i); + pr_info("%2.2x ", ((unsigned char *)buf)[i]); + } + + pos += frag->bv_len; + } + pr_info("\n"); + } +} + +#define ETH_SQ_STATE(sq) \ + do { \ + if (test_bit(__QUEUE_STATE_STACK_XOFF, &sq->txq->state)) \ + ETH_DEBUG_LOG("sq is __QUEUE_STATE_STACK_XOFF\n"); \ + else if (test_bit(__QUEUE_STATE_DRV_XOFF, &sq->txq->state)) \ + ETH_DEBUG_LOG("sq is __QUEUE_STATE_DRV_XOFF\n"); \ + else \ + ETH_DEBUG_LOG("sq is %ld\n", sq->txq->state); \ + } while (0) + +static inline void xsc_pkt_pph_dump(char *data, int len) +{ + int i; + + if (!debug) + return; + + for (i = 0; i < len; i++) { + if (i % 16 == 0) + pr_info("%#4.4x ", i); + pr_info("%2.2x ", ((unsigned char *)data)[i]); + } +} + +#endif /* XSC_ETH_DEBUG_H */ + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ethtool.c b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ethtool.c new file mode 100644 index 0000000000000000000000000000000000000000..eb8cc44d26b47617cd3da24729aae8f1cd5e720d --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ethtool.c @@ -0,0 +1,974 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include "xsc_eth_stats.h" +#include "xsc_eth_debug.h" +#include "xsc_eth_ethtool.h" +#include "xsc_eth.h" +#include +#include +#include "../pci/fw/xsc_tbm.h" + +typedef int (*xsc_pflag_handler)(struct net_device *dev, bool enable); + +struct pflag_desc { + char name[ETH_GSTRING_LEN]; + xsc_pflag_handler handler; +}; + +const static char *fpga_type_name[] = {"S", "L"}; +const static char *hps_ddr_name[] = {"1", "2", "4", "unknown"}; +const static char *onchip_ft_name[] = {"N", "O" }; +const static char *rdma_icrc_name[] = {"N", "C" }; +const static char *ma_xbar_name[] = {"N", "X" }; +const static char *anlt_fec_name[] = {"N", "A" }; +const static char *pp_tbl_dma_name[] = {"N", "D" }; +const static char *pct_exp_name[] = {"N", "E" }; + +enum { + XSC_ST_LINK_STATE, + XSC_ST_LINK_SPEED, + XSC_ST_HEALTH_INFO, +#ifdef CONFIG_INET + XSC_ST_LOOPBACK, +#endif + XSC_ST_NUM, +}; + +const char xsc_self_tests[XSC_ST_NUM][ETH_GSTRING_LEN] = { + "Link Test", + "Speed Test", + "Health Test", +#ifdef CONFIG_INET + "Loopback Test", +#endif +}; + +static int xsc_test_loopback(struct xsc_adapter *adapter) +{ + + if (adapter->status != XSCALE_ETH_DRIVER_OK) { + netdev_err(adapter->netdev, + "\tCan't perform loopback test while device is down\n"); + return -ENODEV; + } + return 0; +} + +static int xsc_test_health_info(struct xsc_adapter *adapter) +{ + struct xsc_core_health *health = &adapter->xdev->priv.health; + + return health->sick ? 1 : 0; +} + +static int xsc_test_link_state(struct xsc_adapter *adapter) +{ + u8 port_state; + + if (!netif_carrier_ok(adapter->netdev)) + return 1; + + port_state = xsc_eth_get_phyport_state(adapter); + return port_state == 0 ? 1 : 0; +} + +static int xsc_test_link_speed(struct xsc_adapter *adapter) +{ +#ifdef NEED_AGILEX_TRAINING + + struct xsc_event_linkstatus_resp linkinfo; + + if (xsc_eth_get_linkinfo(&linkinfo, adapter)) { + xsc_core_err(adapter->xdev, "%s fail to get linkinfo\n", __func__); + return 1; + } +#endif + return 0; +} + +static int set_pflag_rx_no_csum_complete(struct net_device *dev, + bool enable) +{ + struct xsc_adapter *priv = netdev_priv(dev); + + XSC_SET_PFLAG(&priv->nic_param, + XSC_PFLAG_RX_NO_CSUM_COMPLETE, enable); + + return 0; +} + +static int set_pflag_sniffer(struct net_device *dev, bool enable) +{ + struct xsc_adapter *priv = netdev_priv(dev); + + XSC_SET_PFLAG(&priv->nic_param, XSC_PFLAG_SNIFFER, enable); + + return 0; +} + +static int set_pflag_dropless_rq(struct net_device *dev, + bool enable) +{ + struct xsc_adapter *priv = netdev_priv(dev); + + XSC_SET_PFLAG(&priv->nic_param, XSC_PFLAG_DROPLESS_RQ, enable); + + return 0; +} + +static const struct pflag_desc xsc_priv_flags[XSC_NUM_PFLAGS] = { + { "rx_no_csum_complete", set_pflag_rx_no_csum_complete }, + { "sniffer", set_pflag_sniffer }, + { "dropless_rq", set_pflag_dropless_rq}, +}; + +int xsc_priv_flags_num(void) +{ + return ARRAY_SIZE(xsc_priv_flags); +} + +const char *xsc_priv_flags_name(int flag) +{ + return xsc_priv_flags[flag].name; +} + +static int xsc_handle_pflag(struct net_device *dev, + u32 wanted_flags, + enum xsc_eth_priv_flag flag) +{ + struct xsc_adapter *priv = netdev_priv(dev); + bool enable = !!(wanted_flags & BIT(flag)); + u32 changes = wanted_flags ^ priv->nic_param.pflags; + int err; + + if (!(changes & BIT(flag))) + return 0; + + err = xsc_priv_flags[flag].handler(dev, enable); + if (err) + netdev_err(dev, "%s private flag '%s' failed err %d\n", + enable ? "Enable" : "Disable", + xsc_priv_flags[flag].name, err); + + return err; +} + +int xsc_set_priv_flags(struct net_device *dev, u32 pflags) +{ + struct xsc_adapter *priv = netdev_priv(dev); + enum xsc_eth_priv_flag pflag; + int err; + + mutex_lock(&priv->state_lock); + + for (pflag = 0; pflag < XSC_NUM_PFLAGS; pflag++) { + err = xsc_handle_pflag(dev, pflags, pflag); + if (err) + break; + } + + mutex_unlock(&priv->state_lock); + + /* Need to fix some features.. */ + netdev_update_features(dev); + + return err; +} + +static int xsc_get_module_info(struct net_device *netdev, + struct ethtool_modinfo *modinfo) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_core_device *xdev = priv->xdev; + int size_read = 0; + u8 data[4] = {0}; + + size_read = xsc_query_module_eeprom(xdev, 0, 2, data); + if (size_read < 2) + return -EIO; + + /* data[0] = identifier byte */ + switch (data[0]) { + case XSC_MODULE_ID_QSFP: + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_MAX_LEN; + break; + case XSC_MODULE_ID_QSFP_PLUS: + case XSC_MODULE_ID_QSFP28: + /* data[1] = revision id */ + if (data[0] == XSC_MODULE_ID_QSFP28 || data[1] >= 0x3) { + modinfo->type = ETH_MODULE_SFF_8636; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_MAX_LEN; + } else { + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_MAX_LEN; + } + break; + case XSC_MODULE_ID_SFP: + modinfo->type = ETH_MODULE_SFF_8472; + modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; + break; + default: + netdev_err(priv->netdev, "%s: cable type not recognized:0x%x\n", + __func__, data[0]); + return -EINVAL; + } + + return 0; +} + +static int xsc_get_module_eeprom(struct net_device *netdev, + struct ethtool_eeprom *ee, + u8 *data) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_core_device *xdev = priv->xdev; + int offset = ee->offset; + int size_read; + int i = 0; + + if (!ee->len) + return -EINVAL; + + memset(data, 0, ee->len); + + while (i < ee->len) { + size_read = xsc_query_module_eeprom(xdev, offset, ee->len - i, + data + i); + + if (!size_read) + /* Done reading */ + return 0; + + if (size_read < 0) { + netdev_err(priv->netdev, "%s: xsc_query_eeprom failed:0x%x\n", + __func__, size_read); + return 0; + } + + i += size_read; + offset += size_read; + } + + return 0; +} + +u32 xsc_get_priv_flags(struct net_device *dev) +{ + struct xsc_adapter *priv = netdev_priv(dev); + + return priv->nic_param.pflags; +} + +static void xsc_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *info) +{ + struct xsc_adapter *adapter = netdev_priv(dev); + struct xsc_feature_flag *ff = (struct xsc_feature_flag *)&adapter->xdev->feature_flag; + + snprintf(info->driver, sizeof(info->driver), "%s(cmdq-%d)", XSCALE_DRIVER_NAME, + adapter->xdev->cmdq_ver); + + if (HOTFIX_NUM == 0) + snprintf(info->version, sizeof(info->version), "%d.%d.%d.%d", + BRANCH_VERSION, MAJOR_VERSION, MINOR_VERSION, BUILD_VERSION); + else + snprintf(info->version, sizeof(info->version), "%d.%d.%d.%d.H%d", + BRANCH_VERSION, MAJOR_VERSION, MINOR_VERSION, BUILD_VERSION, HOTFIX_NUM); + + if (adapter->xdev->hotfix_num >= 0x27) + snprintf(info->fw_version, + sizeof(info->fw_version), + "%x.%x.%x.%s%s%s%s%s%s%s%s", + adapter->xdev->chip_ver_h, + adapter->xdev->hotfix_num, + adapter->xdev->chip_ver_l, + fpga_type_name[ff->fpga_type], + hps_ddr_name[ff->hps_ddr], + onchip_ft_name[ff->onchip_ft], + rdma_icrc_name[ff->rdma_icrc], + ma_xbar_name[ff->ma_xbar], + anlt_fec_name[ff->anlt_fec], + pp_tbl_dma_name[ff->pp_tbl_dma], + pct_exp_name[ff->pct_exp]); + else + snprintf(info->fw_version, + sizeof(info->fw_version), + "%x.%x.%x.%s%s%s%s%s%s", + adapter->xdev->chip_ver_h, + adapter->xdev->hotfix_num, + adapter->xdev->chip_ver_l, + fpga_type_name[ff->fpga_type], + hps_ddr_name[ff->hps_ddr], + onchip_ft_name[ff->onchip_ft], + rdma_icrc_name[ff->rdma_icrc], + ma_xbar_name[ff->ma_xbar], + anlt_fec_name[ff->anlt_fec]); + strlcpy(info->bus_info, pci_name(adapter->pdev), sizeof(info->bus_info)); +} + +static void xsc_fill_stats_strings(struct xsc_adapter *adapter, u8 *data) +{ + int i, idx = 0; + + for (i = 0; i < xsc_num_stats_grps; i++) + idx = xsc_stats_grps[i].fill_strings(adapter, data, idx); +} + +static int xsc_self_test_num(struct xsc_adapter *adapter) +{ + return ARRAY_SIZE(xsc_self_tests); +} + +static void xsc_ethtool_get_strings(struct xsc_adapter *adapter, u32 stringset, u8 *data) +{ + int i; + + switch (stringset) { + case ETH_SS_STATS: + xsc_fill_stats_strings(adapter, data); + break; + + case ETH_SS_TEST: + for (i = 0; i < xsc_self_test_num(adapter); i++) + strcpy(data + i * ETH_GSTRING_LEN, + xsc_self_tests[i]); + break; + + case ETH_SS_PRIV_FLAGS: + for (i = 0; i < XSC_NUM_PFLAGS; i++) + strcpy(data + i * ETH_GSTRING_LEN, xsc_priv_flags[i].name); + break; + + default: + ETH_DEBUG_LOG("wrong stringset\n"); + break; + } +} + +static void xsc_get_strings(struct net_device *dev, u32 stringset, u8 *data) +{ + struct xsc_adapter *adapter = netdev_priv(dev); + + xsc_ethtool_get_strings(adapter, stringset, data); +} + +static int xsc_ethtool_get_sset_count(struct xsc_adapter *adapter, int sset) +{ + int i, num_stats = 0; + + switch (sset) { + case ETH_SS_STATS: + for (i = 0; i < xsc_num_stats_grps; i++) + num_stats += xsc_stats_grps[i].get_num_stats(adapter); + return num_stats; + case ETH_SS_PRIV_FLAGS: + return XSC_NUM_PFLAGS; + case ETH_SS_TEST: + return xsc_self_test_num(adapter); + default: + return -EOPNOTSUPP; + } +} + +static int xsc_get_sset_count(struct net_device *dev, int sset) +{ + struct xsc_adapter *adapter = netdev_priv(dev); + + return xsc_ethtool_get_sset_count(adapter, sset); +} + +static int (*xsc_st_func[XSC_ST_NUM])(struct xsc_adapter *) = { + xsc_test_link_state, + xsc_test_link_speed, + xsc_test_health_info, +#ifdef CONFIG_INET + xsc_test_loopback, +#endif +}; + +static void xsc_self_test(struct net_device *ndev, struct ethtool_test *etest, + u64 *buf) +{ + struct xsc_adapter *priv = netdev_priv(ndev); + int i; + + memset(buf, 0, sizeof(u64) * XSC_ST_NUM); + + mutex_lock(&priv->state_lock); + netdev_info(ndev, "Self test begin..\n"); + + for (i = 0; i < XSC_ST_NUM; i++) { + netdev_info(ndev, "\t[%d] %s start..\n", + i, xsc_self_tests[i]); + buf[i] = xsc_st_func[i](priv); + netdev_info(ndev, "\t[%d] %s end: result(%lld)\n", + i, xsc_self_tests[i], buf[i]); + } + + mutex_unlock(&priv->state_lock); + + for (i = 0; i < XSC_ST_NUM; i++) { + if (buf[i]) { + etest->flags |= ETH_TEST_FL_FAILED; + break; + } + } + netdev_info(ndev, "Self test out: status flags(0x%x)\n", + etest->flags); + +} + +static void xsc_update_stats(struct xsc_adapter *adapter) +{ + int i; + + for (i = xsc_num_stats_grps - 1; i >= 0; i--) + if (xsc_stats_grps[i].update_stats) + xsc_stats_grps[i].update_stats(adapter); +} + +static void xsc_ethtool_get_ethtool_stats(struct xsc_adapter *adapter, + struct ethtool_stats *stats, u64 *data) +{ + int i, idx = 0; + + mutex_lock(&adapter->state_lock); + xsc_update_stats(adapter); + mutex_unlock(&adapter->state_lock); + + for (i = 0; i < xsc_num_stats_grps; i++) + idx = xsc_stats_grps[i].fill_stats(adapter, data, idx); +} + +static void xsc_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct xsc_adapter *adapter = netdev_priv(dev); + + xsc_ethtool_get_ethtool_stats(adapter, stats, data); +} + +static u32 xsc_get_msglevel(struct net_device *dev) +{ + return ((struct xsc_adapter *)netdev_priv(dev))->msglevel; +} + +static void xsc_set_msglevel(struct net_device *dev, u32 val) +{ + ((struct xsc_adapter *)netdev_priv(dev))->msglevel = val; +} + +static void xsc_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) +{ + struct xsc_adapter *priv = netdev_priv(dev); + + param->rx_max_pending = priv->nic_param.rq_max_size; + param->rx_pending = priv->nic_param.rq_size; + param->tx_max_pending = priv->nic_param.sq_max_size; + param->tx_pending = priv->nic_param.sq_size; +} + +static int xsc_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) +{ + struct xsc_adapter *priv = netdev_priv(dev); + u32 old_rq_size, old_sq_size; + int err = 0; + + if (param->rx_jumbo_pending) { + netdev_info(priv->netdev, "%s: rx_jumbo_pending not supported\n", + __func__); + return -EINVAL; + } + if (param->rx_mini_pending) { + netdev_info(priv->netdev, "%s: rx_mini_pending not supported\n", + __func__); + return -EINVAL; + } + + if (param->rx_pending < BIT(XSC_MIN_LOG_RQ_SZ)) { + netdev_info(priv->netdev, "%s: rx_pending (%d) < min (%ld)\n", + __func__, param->rx_pending, BIT(XSC_MIN_LOG_RQ_SZ)); + return -EINVAL; + } + if (param->rx_pending > priv->nic_param.rq_max_size) { + netdev_info(priv->netdev, "%s: rx_pending (%d) > max (%d)\n", + __func__, param->rx_pending, priv->nic_param.rq_max_size); + return -EINVAL; + } + + if (param->tx_pending < BIT(XSC_MIN_LOG_SQ_SZ)) { + netdev_info(priv->netdev, "%s: tx_pending (%d) < min (%ld)\n", + __func__, param->tx_pending, BIT(XSC_MIN_LOG_SQ_SZ)); + return -EINVAL; + } + if (param->tx_pending > priv->nic_param.sq_max_size) { + netdev_info(priv->netdev, "%s: tx_pending (%d) > max (%d)\n", + __func__, param->tx_pending, priv->nic_param.sq_max_size); + return -EINVAL; + } + + if (param->rx_pending == priv->nic_param.rq_size && + param->tx_pending == priv->nic_param.sq_size) + return 0; + + mutex_lock(&priv->state_lock); + + if (priv->status != XSCALE_ETH_DRIVER_OK) + goto unlock; + + old_rq_size = priv->nic_param.rq_size; + old_sq_size = priv->nic_param.sq_size; + priv->nic_param.rq_size = param->rx_pending; + priv->nic_param.sq_size = param->tx_pending; + + netdev_info(priv->netdev, "%s: tx_pending(%d->%d), rx_pending(%d->%d)\n", + __func__, old_sq_size, param->tx_pending, + old_rq_size, priv->nic_param.rq_size); + err = xsc_safe_switch_channels(priv, NULL, NULL); + if (err) { + priv->nic_param.rq_size = old_rq_size; + priv->nic_param.sq_size = old_sq_size; + netdev_err(priv->netdev, "%s: set ringparams failed, err=%d\n", + __func__, err); + } + +unlock: + mutex_unlock(&priv->state_lock); + + return err; +} + +static void xsc_get_channels(struct net_device *dev, + struct ethtool_channels *ch) +{ + struct xsc_adapter *priv = netdev_priv(dev); + + mutex_lock(&priv->state_lock); + + ch->max_combined = priv->nic_param.max_num_ch; + ch->combined_count = priv->nic_param.num_channels; + + mutex_unlock(&priv->state_lock); +} + +static int xsc_set_channels(struct net_device *dev, + struct ethtool_channels *ch) +{ + struct xsc_adapter *priv = netdev_priv(dev); + struct xsc_eth_params *params = &priv->nic_param; + unsigned int ch_max = params->max_num_ch; + unsigned int ch_num_old = params->num_channels; + unsigned int count = ch->combined_count; + int err = 0; + + if (!count) { + netdev_info(priv->netdev, "%s: combined_count=0 not supported\n", + __func__); + return -EINVAL; + } + + if (ch->rx_count || ch->tx_count) { + netdev_info(priv->netdev, "%s: separate rx/tx count not supported\n", + __func__); + return -EINVAL; + } + + if (count > ch_max) { + netdev_info(priv->netdev, "%s: count (%d) > max (%d)\n", + __func__, count, ch_max); + return -EINVAL; + } + + if (ch_num_old == count) + return 0; + + mutex_lock(&priv->state_lock); + + params->num_channels = count; + + if (priv->status != XSCALE_ETH_DRIVER_OK) { + err = xsc_eth_num_channels_changed(priv); + if (err) + params->num_channels = ch_num_old; + goto out; + } + + /* Switch to new channels, set new parameters and close old ones */ + err = xsc_safe_switch_channels(priv, NULL, xsc_eth_num_channels_changed); + +out: + mutex_unlock(&priv->state_lock); + netdev_info(priv->netdev, "set combined_cnt=%d, err=%d\n", count, err); + + return err; +} + +static int flow_type_to_traffic_type(u32 flow_type) +{ + switch (flow_type) { + case IPV4_FLOW: + return XSC_TT_IPV4; + case TCP_V4_FLOW: + return XSC_TT_IPV4_TCP; + case UDP_V4_FLOW: + return XSC_TT_IPV4_TCP; + case IPV6_FLOW: + return XSC_TT_IPV6; + case TCP_V6_FLOW: + return XSC_TT_IPV6_TCP; + case UDP_V6_FLOW: + return XSC_TT_IPV6_TCP; + case AH_V4_FLOW: + return XSC_TT_IPV4_IPSEC_AH; + case AH_V6_FLOW: + return XSC_TT_IPV6_IPSEC_AH; + case ESP_V4_FLOW: + return XSC_TT_IPV4_IPSEC_ESP; + case ESP_V6_FLOW: + return XSC_TT_IPV6_IPSEC_ESP; + default: + return -EINVAL; + } +} + +static int xsc_get_rss_hash_opt(struct xsc_adapter *priv, + struct ethtool_rxnfc *nfc) +{ + u32 hash_field = 0; + int tt; + + tt = flow_type_to_traffic_type(nfc->flow_type); + if (tt < 0) + return -EINVAL; + + hash_field = priv->rss_params.rx_hash_fields[tt]; + nfc->data = 0; + + if (hash_field & XSC_HASH_FIELD_SEL_PROTO) + nfc->data |= RXH_L3_PROTO; + if (tt == XSC_TT_IPV4_TCP) { + if (hash_field & XSC_HASH_FIELD_SEL_SRC_IP) + nfc->data |= RXH_IP_SRC; + if (hash_field & XSC_HASH_FIELD_SEL_DST_IP) + nfc->data |= RXH_IP_DST; + if (hash_field & XSC_HASH_FIELD_SEL_SPORT) + nfc->data |= RXH_L4_B_0_1; + if (hash_field & XSC_HASH_FIELD_SEL_DPORT) + nfc->data |= RXH_L4_B_2_3; + } else if (tt == XSC_TT_IPV6_TCP) { + if (hash_field & XSC_HASH_FIELD_SEL_SRC_IPV6) + nfc->data |= RXH_IP_SRC; + if (hash_field & XSC_HASH_FIELD_SEL_DST_IPV6) + nfc->data |= RXH_IP_DST; + if (hash_field & XSC_HASH_FIELD_SEL_SPORT_V6) + nfc->data |= RXH_L4_B_0_1; + if (hash_field & XSC_HASH_FIELD_SEL_DPORT_V6) + nfc->data |= RXH_L4_B_2_3; + } + + return 0; +} + +static int xsc_set_rss_hash_opt(struct xsc_adapter *priv, + struct ethtool_rxnfc *nfc) +{ + u32 rx_hash_field = XSC_HASH_FIELD_SEL_PROTO; + u32 change = 0; + int ret = 0; + int tt; + + tt = flow_type_to_traffic_type(nfc->flow_type); + if (tt < 0) + return -EINVAL; + + /* RSS does not support anything other than hashing to queues + * on src IP, dest IP, TCP/UDP src port and TCP/UDP dest + * port. + */ + if (nfc->flow_type != TCP_V4_FLOW && + nfc->flow_type != TCP_V6_FLOW && + nfc->flow_type != UDP_V4_FLOW && + nfc->flow_type != UDP_V6_FLOW) + return -EOPNOTSUPP; + + if (nfc->data & ~(RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3)) + return -EOPNOTSUPP; + + if (nfc->flow_type == TCP_V4_FLOW) { + if (nfc->data & RXH_IP_SRC) + rx_hash_field |= XSC_HASH_FIELD_SEL_SRC_IP; + if (nfc->data & RXH_IP_DST) + rx_hash_field |= XSC_HASH_FIELD_SEL_DST_IP; + if (nfc->data & RXH_L4_B_0_1) + rx_hash_field |= XSC_HASH_FIELD_SEL_SPORT; + if (nfc->data & RXH_L4_B_2_3) + rx_hash_field |= XSC_HASH_FIELD_SEL_DPORT; + } else if (nfc->flow_type == TCP_V6_FLOW) { + if (nfc->data & RXH_IP_SRC) + rx_hash_field |= XSC_HASH_FIELD_SEL_SRC_IPV6; + if (nfc->data & RXH_IP_DST) + rx_hash_field |= XSC_HASH_FIELD_SEL_DST_IPV6; + if (nfc->data & RXH_L4_B_0_1) + rx_hash_field |= XSC_HASH_FIELD_SEL_SPORT_V6; + if (nfc->data & RXH_L4_B_2_3) + rx_hash_field |= XSC_HASH_FIELD_SEL_DPORT_V6; + } else { + return 0; + } + + mutex_lock(&priv->state_lock); + if (rx_hash_field != priv->rss_params.rx_hash_fields[tt]) { + change |= BIT(XSC_RSS_HASH_TEMP_UPDATE); + priv->rss_params.rx_hash_fields[tt] = rx_hash_field; + } + + xsc_core_info(priv->xdev, + "%s: flow_type=%d, change=0x%x, hash_tmpl=0x%x\n", + __func__, nfc->flow_type, change, rx_hash_field); + if (change) + ret = xsc_eth_modify_nic_hca(priv, change); + + mutex_unlock(&priv->state_lock); + return ret; +} + +int xsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs) +{ + struct xsc_adapter *priv = netdev_priv(dev); + struct xsc_eth_params *params = &priv->nic_param; + int err = 0; + + if (info->cmd == ETHTOOL_GRXRINGS) { + info->data = params->num_channels; + return 0; + } + + switch (info->cmd) { + case ETHTOOL_GRXFH: + err = xsc_get_rss_hash_opt(priv, info); + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +int xsc_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) +{ + struct xsc_adapter *priv = netdev_priv(dev); + int err = 0; + + switch (cmd->cmd) { + case ETHTOOL_SRXFH: + err = xsc_set_rss_hash_opt(priv, cmd); + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static u32 xsc_get_rxfh_key_size(struct net_device *dev) +{ + struct xsc_adapter *priv = netdev_priv(dev); + + return sizeof(priv->rss_params.toeplitz_hash_key); +} + +static u32 xsc_get_rxfh_indir_size(struct net_device *netdev) +{ + return XSC_INDIR_RQT_SIZE; +} + +int xsc_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, + u8 *hfunc) +{ + struct xsc_adapter *priv = netdev_priv(netdev); + struct xsc_rss_params *rss = &priv->rss_params; + + if (indir) + memcpy(indir, rss->indirection_rqt, + sizeof(rss->indirection_rqt)); + + if (key) + memcpy(key, rss->toeplitz_hash_key, + sizeof(rss->toeplitz_hash_key)); + + if (hfunc) + *hfunc = rss->hfunc; + + return 0; +} + +int xsc_set_rxfh(struct net_device *dev, const u32 *indir, + const u8 *key, const u8 hfunc) +{ + struct xsc_adapter *priv = netdev_priv(dev); + struct xsc_rss_params *rss = &priv->rss_params; + u32 refresh = 0; + int err = 0; + + if ((hfunc != ETH_RSS_HASH_NO_CHANGE) && + (hfunc != ETH_RSS_HASH_XOR) && + (hfunc != ETH_RSS_HASH_TOP)) + return -EINVAL; + + mutex_lock(&priv->state_lock); + + if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != rss->hfunc) { + rss->hfunc = hfunc; + refresh |= BIT(XSC_RSS_HASH_FUNC_UPDATE); + } + + if (key) { + memcpy(rss->toeplitz_hash_key, key, + sizeof(rss->toeplitz_hash_key)); + if (rss->hfunc == ETH_RSS_HASH_TOP) + refresh |= BIT(XSC_RSS_HASH_KEY_UPDATE); + } + + if (refresh > 0 && (priv->status == XSCALE_ETH_DRIVER_OK)) + err = xsc_eth_modify_nic_hca(priv, refresh); + + mutex_unlock(&priv->state_lock); + + return err; +} + +static int xsc_get_link_ksettings(struct net_device *netdev, + struct ethtool_link_ksettings *cmd) +{ + struct xsc_adapter *adapter = netdev_priv(netdev); + struct xsc_feature_flag *ff = (struct xsc_feature_flag *)&adapter->xdev->feature_flag; + struct xsc_event_linkstatus_resp linkinfo; + /*this stub is add for 802.3ad bond test, function has been tested*/ + u32 supported, advertising; + + if (xsc_eth_get_linkinfo(&linkinfo, adapter)) { + xsc_core_err(adapter->xdev, "%s fail to get linkinfo\n", __func__); + return -EINVAL; + } + + cmd->base.port = PORT_FIBRE; + cmd->base.duplex = DUPLEX_FULL; + cmd->base.autoneg = ff->anlt_fec ? AUTONEG_ENABLE : AUTONEG_DISABLE; + + if (linkinfo.linkspeed == XSC_CMD_RESP_LINKSPEED_MODE_25G) { + cmd->base.speed = SPEED_25000; + + supported = (SUPPORTED_FIBRE | SUPPORTED_Autoneg); + advertising = (ADVERTISED_FIBRE | ADVERTISED_Autoneg); + + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, + supported); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, + advertising); + + ethtool_link_ksettings_add_link_mode(cmd, supported, 25000baseCR_Full); + ethtool_link_ksettings_add_link_mode(cmd, advertising, 25000baseCR_Full); + + ethtool_link_ksettings_add_link_mode(cmd, supported, 25000baseKR_Full); + ethtool_link_ksettings_add_link_mode(cmd, advertising, 25000baseKR_Full); + + ethtool_link_ksettings_add_link_mode(cmd, supported, 25000baseSR_Full); + ethtool_link_ksettings_add_link_mode(cmd, advertising, 25000baseSR_Full); + } else if (linkinfo.linkspeed == XSC_CMD_RESP_LINKSPEED_MODE_100G) { + cmd->base.speed = SPEED_100000; + + supported = (SUPPORTED_FIBRE | SUPPORTED_Autoneg); + advertising = (ADVERTISED_FIBRE | ADVERTISED_Autoneg); + + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, + supported); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, + advertising); + + ethtool_link_ksettings_add_link_mode(cmd, supported, 100000baseKR4_Full); + ethtool_link_ksettings_add_link_mode(cmd, advertising, 100000baseKR4_Full); + + ethtool_link_ksettings_add_link_mode(cmd, supported, 100000baseSR4_Full); + ethtool_link_ksettings_add_link_mode(cmd, advertising, 100000baseSR4_Full); + + ethtool_link_ksettings_add_link_mode(cmd, supported, 100000baseCR4_Full); + ethtool_link_ksettings_add_link_mode(cmd, advertising, 100000baseCR4_Full); + + ethtool_link_ksettings_add_link_mode(cmd, supported, 100000baseLR4_ER4_Full); + ethtool_link_ksettings_add_link_mode(cmd, advertising, 100000baseLR4_ER4_Full); + } + + return 0; +} + +static int xsc_set_phys_id(struct net_device *dev, enum ethtool_phys_id_state state) +{ + struct xsc_adapter *adapter = netdev_priv(dev); + struct xsc_core_device *xdev = adapter->xdev; + int ret = 0; + + switch (state) { + case ETHTOOL_ID_ACTIVE: + xsc_eth_set_led_status(xdev->pf_id, adapter); + break; + case ETHTOOL_ID_INACTIVE: + xsc_eth_set_led_status(LED_ACT_ON_HW, adapter); + break; + default: + return -EOPNOTSUPP; + } + + return ret; + +} + +static const struct ethtool_ops xsc_ethtool_ops = { + .get_drvinfo = xsc_get_drvinfo, + .get_link = ethtool_op_get_link, + .get_strings = xsc_get_strings, + .get_sset_count = xsc_get_sset_count, + .get_ethtool_stats = xsc_get_ethtool_stats, + .get_ringparam = xsc_get_ringparam, + .set_ringparam = xsc_set_ringparam, + .set_channels = xsc_set_channels, + .get_channels = xsc_get_channels, + .get_ts_info = NULL, + .get_link_ksettings = xsc_get_link_ksettings, + .set_link_ksettings = NULL, + .get_rxfh_key_size = xsc_get_rxfh_key_size, + .get_rxfh_indir_size = xsc_get_rxfh_indir_size, + .get_rxfh = xsc_get_rxfh, + .set_rxfh = xsc_set_rxfh, + .get_rxnfc = xsc_get_rxnfc, + .set_rxnfc = xsc_set_rxnfc, + .get_module_info = xsc_get_module_info, + .get_module_eeprom = xsc_get_module_eeprom, + .get_priv_flags = xsc_get_priv_flags, + .set_priv_flags = xsc_set_priv_flags, + .get_msglevel = xsc_get_msglevel, + .set_msglevel = xsc_set_msglevel, + .self_test = xsc_self_test, + .set_phys_id = xsc_set_phys_id, +}; + +void eth_set_ethtool_ops(struct net_device *dev) +{ + dev->ethtool_ops = &xsc_ethtool_ops; +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ethtool.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ethtool.h new file mode 100644 index 0000000000000000000000000000000000000000..4f7dd4c456ca3b4655a93819f327ea4903cd501b --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_ethtool.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_ETH_ETHTOOL_H +#define XSC_ETH_ETHTOOL_H + +void eth_set_ethtool_ops(struct net_device *dev); + +/* EEPROM Standards for plug in modules */ +#ifndef ETH_MODULE_SFF_8436_MAX_LEN +#define ETH_MODULE_SFF_8636_MAX_LEN 640 +#define ETH_MODULE_SFF_8436_MAX_LEN 640 +#endif + +#define LED_ACT_ON_HW 0xff + +#endif /* XSC_ETH_ETHTOOL_H */ + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_rx.c b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_rx.c new file mode 100644 index 0000000000000000000000000000000000000000..e77b416492ad9d5ac55a94672640fcff376de484 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_rx.c @@ -0,0 +1,939 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include "xsc_eth.h" +#include "xsc_eth_txrx.h" +#include "xsc_eth_common.h" +#include "xsc_eth_stats.h" +#include +#include "../pci/fw/xsc_tbm.h" + +#define PAGE_REF_ELEV (U16_MAX) +/* Upper bound on number of packets that share a single page */ +#define PAGE_REF_THRSD (PAGE_SIZE / 64) + +static inline void xsc_rq_notify_hw(struct xsc_rq *rq) +{ + struct xsc_core_device *xdev = rq->cq.xdev; + struct xsc_wq_cyc *wq = &rq->wqe.wq; + union xsc_recv_doorbell doorbell_value; + u64 rqwqe_id = wq->wqe_ctr << (ilog2(xdev->caps.recv_ds_num)); + + ETH_DEBUG_LOG("rq%d_db_val=0x%x, recv_ds=%d\n", + rq->rqn, doorbell_value.recv_data, + xdev->caps.recv_ds_num); + /*reverse wqe index to ds index*/ + doorbell_value.next_pid = rqwqe_id; + doorbell_value.qp_num = rq->rqn; + + /*keep order*/ + wmb(); + writel(doorbell_value.recv_data, REG_ADDR(xdev, xdev->regs.rx_db)); +} + +static inline void xsc_skb_set_hash(struct xsc_adapter *adapter, + struct xsc_cqe64 *cqe, + struct sk_buff *skb) +{ + struct xsc_rss_params *rss = &adapter->rss_params; + u32 hash_field; + bool l3_hash = false; + bool l4_hash = false; + int ht = 0; + + if (adapter->netdev->features & NETIF_F_RXHASH) { + if (skb->protocol == htons(ETH_P_IP)) { + hash_field = rss->rx_hash_fields[XSC_TT_IPV4_TCP]; + if (hash_field & XSC_HASH_FIELD_SEL_SRC_IP || + hash_field & XSC_HASH_FIELD_SEL_DST_IP) + l3_hash = true; + + if (hash_field & XSC_HASH_FIELD_SEL_SPORT || + hash_field & XSC_HASH_FIELD_SEL_DPORT) + l4_hash = true; + } else if (skb->protocol == htons(ETH_P_IPV6)) { + hash_field = rss->rx_hash_fields[XSC_TT_IPV6_TCP]; + if (hash_field & XSC_HASH_FIELD_SEL_SRC_IPV6 || + hash_field & XSC_HASH_FIELD_SEL_DST_IPV6) + l3_hash = true; + + if (hash_field & XSC_HASH_FIELD_SEL_SPORT_V6 || + hash_field & XSC_HASH_FIELD_SEL_DPORT_V6) + l4_hash = true; + } + + if (l3_hash && l4_hash) + ht = PKT_HASH_TYPE_L4; + else if (l3_hash) + ht = PKT_HASH_TYPE_L3; + if (ht) + skb_set_hash(skb, be32_to_cpu(cqe->vni), ht); + } +} + +static inline unsigned short from32to16(unsigned int x) +{ + /* add up 16-bit and 16-bit for 16+c bit */ + x = (x & 0xffff) + (x >> 16); + /* add up carry.. */ + x = (x & 0xffff) + (x >> 16); + return x; +} + +static inline unsigned int xsc_do_csum(const unsigned char *buff, + int len, unsigned int sum, bool plus) +{ + unsigned int result = 0; + u32 result0, result1, result2, result3, result4; + + if (len >= 4) { + const unsigned char *end = buff + ((unsigned int)len & ~3); + unsigned int carry = 0; + + do { + unsigned int w = *(unsigned int *) buff; + + buff += 4; + result += carry; + result += w; + carry = (w > result); + } while (buff < end); + result += carry; + result0 = result; + result = from32to16(result); + } + result1 = result; + if (len & 2) { + result += *(unsigned short *) buff; + buff += 2; + result2 = result; + result = from32to16(result); + } + result3 = result; + if (len & 1) +#ifdef __LITTLE_ENDIAN + result += *buff; +#else + result += (*buff << 8); +#endif + + if (plus) { + result = from32to16(sum) + from32to16(result); + result4 = result; + result = from32to16(result); + } else { + result = from32to16(sum) - from32to16(result); + } + + ETH_DEBUG_LOG("%s: len=%d, csum=0x%x -> %x -> %x -> %x -> %x\n", __func__, + len, result0, result1, result2, result3, result4); + return result; +} + +static inline bool handle_udp_frag_csum(struct sk_buff *skb, struct epp_pph *pph) +{ +#ifdef XSC_UDP_FRAG_HW_CSUM + struct iphdr *iph; + char *head = (char *)pph; + u8 l3_proto = PPH_OUTER_IP_TYPE(head); + u8 l4_proto = PPH_OUTER_TP_TYPE(head); + u8 iph_off = PPH_OUTER_IP_OFST(head); + u16 iph_len = PPH_OUTER_IP_LEN(pph); +#ifndef UDP_CHECK_0 + u16 csum_off = (u16)PPH_CSUM_OFST(head); + u8 payload_off = PPH_PAYLOAD_OFST(head); + u32 check = PPH_CSUM_VAL(head); +#endif + + if (l4_proto != L4_PROTO_UDP && l4_proto != L4_PROTO_NONE) + return false; + + if (l3_proto == L3_PROTO_IP) { + iph = (struct iphdr *)(skb->data + iph_off); + if (!ip_is_fragment(iph)) + return false; + +#ifdef UDP_CHECK_0 + if (l4_proto == L4_PROTO_UDP) { + struct udphdr *uh = (struct udphdr *)(skb->data + iph_off + iph_len); + + uh->check = 0; + } + return false; +#else + if (csum_off == 0) + csum_off = 256; + csum_off -= XSC_PPH_HEAD_LEN; + + if (csum_off < payload_off) { + head = (char *)(skb->data + csum_off); + check = xsc_do_csum(head, (payload_off - csum_off), check, false); + } else if (csum_off > payload_off) { + head = (char *)(skb->data + payload_off); + check = xsc_do_csum(head, (csum_off - payload_off), check, true); + } + skb->csum = csum_unfold((__force __sum16)check); + +#ifdef UDP_CSUM_DEBUG + int dump_len = 256; + + ETH_DEBUG_LOG( + "ip_id=%d frag_off=0x%x l4_proto=%d l3_proto=%d iph_off=%d ip_len=%d\n", + ntohs(iph->id), ntohs(iph->frag_off), + l4_proto, l3_proto, iph_off, iph_len); + ETH_DEBUG_LOG( + "skb_len=%d csum_off=%d payload_off=%d check=0x%llx -> 0x%x -> 0x%x\n", + skb->len, csum_off, payload_off, + PPH_CSUM_VAL(pph), check, skb->csum); + + if (skb->len < (256 - 64)) + dump_len = skb->len + 64; + xsc_pkt_pph_dump((char *)pph, dump_len); +#endif +#endif + return true; + } +#endif + return false; +} + +static inline void xsc_handle_csum(struct xsc_cqe64 *cqe, struct xsc_rq *rq, + struct sk_buff *skb, struct xsc_wqe_frag_info *wi) +{ + struct xsc_rq_stats *stats = rq->stats; + struct xsc_channel *c = rq->cq.channel; + struct net_device *netdev = c->adapter->netdev; + struct xsc_dma_info *dma_info = wi->di; + int offset_from = wi->offset; + struct epp_pph *hw_pph = page_address(dma_info->page) + offset_from; + + if (unlikely((netdev->features & NETIF_F_RXCSUM) == 0)) + goto csum_none; + + if (unlikely(XSC_GET_EPP2SOC_PPH_ERROR_BITMAP(hw_pph) & PACKET_UNKNOWN)) + goto csum_none; + + if (handle_udp_frag_csum(skb, hw_pph)) { + skb->ip_summed = CHECKSUM_COMPLETE; + stats->csum_succ++; + goto out; + } + + if (XSC_GET_EPP2SOC_PPH_EXT_TUNNEL_TYPE(hw_pph) && + (!(cqe->csum_err & OUTER_AND_INNER))) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->csum_level = 1; + skb->encapsulation = 1; + + stats->csum_unnecessary++; + } else if (XSC_GET_EPP2SOC_PPH_EXT_TUNNEL_TYPE(hw_pph) && + (!(cqe->csum_err & OUTER_BIT) && + (cqe->csum_err & INNER_BIT))) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->csum_level = 0; + skb->encapsulation = 1; + + stats->csum_unnecessary++; + } else if (!XSC_GET_EPP2SOC_PPH_EXT_TUNNEL_TYPE(hw_pph) && + (!(cqe->csum_err & OUTER_BIT))) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + + stats->csum_unnecessary++; + } else { + stats->csum_err++; + } + + goto out; + +csum_none: + skb->csum = 0; + skb->ip_summed = CHECKSUM_NONE; + stats->csum_none++; +out: + return; +} + +static inline void xsc_build_rx_skb(struct xsc_cqe64 *cqe, + u32 cqe_bcnt, + struct xsc_rq *rq, + struct sk_buff *skb, + struct xsc_wqe_frag_info *wi) +{ + struct xsc_channel *c = rq->cq.channel; + struct net_device *netdev = c->netdev; + struct xsc_adapter *adapter = c->adapter; + + skb->mac_len = ETH_HLEN; + + skb_record_rx_queue(skb, rq->ix); + xsc_handle_csum(cqe, rq, skb, wi); + + skb->protocol = eth_type_trans(skb, netdev); + xsc_skb_set_hash(adapter, cqe, skb); +} + +static inline void xsc_complete_rx_cqe(struct xsc_rq *rq, + struct xsc_cqe64 *cqe, + u32 cqe_bcnt, + struct sk_buff *skb, + struct xsc_wqe_frag_info *wi) +{ + struct xsc_rq_stats *stats = rq->stats; + + stats->packets++; + stats->bytes += cqe_bcnt; + xsc_build_rx_skb(cqe, cqe_bcnt, rq, skb, wi); +} + +static inline void xsc_add_skb_frag(struct xsc_rq *rq, + struct sk_buff *skb, + struct xsc_dma_info *di, + u32 frag_offset, u32 len, + unsigned int truesize) +{ + struct xsc_channel *c = rq->cq.channel; + struct device *dev = c->adapter->dev; + + dma_sync_single_for_cpu(dev, di->addr + frag_offset, len, DMA_FROM_DEVICE); + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + di->page, frag_offset, len, truesize); + + di->refcnt_bias--; +} + +static inline void xsc_copy_skb_header(struct device *dev, + struct sk_buff *skb, + struct xsc_dma_info *dma_info, + int offset_from, u32 headlen) +{ + void *from = page_address(dma_info->page) + offset_from; + /* Aligning len to sizeof(long) optimizes memcpy performance */ + unsigned int len = ALIGN(headlen, sizeof(long)); + + dma_sync_single_for_cpu(dev, dma_info->addr + offset_from, len, + DMA_FROM_DEVICE); + skb_copy_to_linear_data(skb, from, len); +} + +struct sk_buff *xsc_skb_from_cqe_nonlinear(struct xsc_rq *rq, + struct xsc_wqe_frag_info *wi, + u32 cqe_bcnt, u8 has_pph) +{ + struct xsc_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; + struct xsc_wqe_frag_info *head_wi = wi; + u16 headlen = min_t(u32, XSC_RX_MAX_HEAD, cqe_bcnt); + u16 frag_headlen = headlen; + u16 byte_cnt = cqe_bcnt - headlen; + struct sk_buff *skb; + struct xsc_channel *c = rq->cq.channel; + struct device *dev = c->adapter->dev; + struct net_device *netdev = c->adapter->netdev; + u8 fragcnt = 0; + u16 head_offset = head_wi->offset; + +#ifndef NEED_CREATE_RX_THREAD + skb = napi_alloc_skb(rq->cq.napi, ALIGN(XSC_RX_MAX_HEAD, sizeof(long))); +#else + skb = netdev_alloc_skb(netdev, ALIGN(XSC_RX_MAX_HEAD, sizeof(long))); +#endif + if (unlikely(!skb)) { + rq->stats->buff_alloc_err++; + return NULL; + } + + prefetchw(skb->data); + + if (likely(has_pph)) { + head_offset += XSC_PPH_HEAD_LEN; + headlen -= XSC_PPH_HEAD_LEN; + } + + while (byte_cnt) { + /*figure out whether the first fragment can be a page ?*/ + u16 frag_consumed_bytes = + min_t(u16, frag_info->frag_size - frag_headlen, byte_cnt); + + xsc_add_skb_frag(rq, skb, wi->di, wi->offset + frag_headlen, + frag_consumed_bytes, frag_info->frag_stride); + byte_cnt -= frag_consumed_bytes; + frag_headlen = 0; + ETH_DEBUG_LOG("consumed=%d, frag_size=%d, byte_cnt=%d, refcnt=%d, addr=0x%llx\n", + frag_consumed_bytes, frag_info->frag_size, byte_cnt, + wi->di->refcnt_bias, (u64)wi->di->addr); + + /*to protect extend wqe read, drop exceed bytes*/ + fragcnt++; + if (fragcnt == rq->wqe.info.num_frags) { + if (byte_cnt) { + rq->stats->oversize_pkts_sw_drop += byte_cnt; + netdev_warn(netdev, "large packet reach the maximum rev-wqe num.\n"); + netdev_warn(netdev, "%u bytes dropped: frag_num=%d, headlen=%d, cqe_cnt=%d, frag0_bytes=%d, frag_size=%d\n", + byte_cnt, fragcnt, headlen, cqe_bcnt, + frag_consumed_bytes, frag_info->frag_size); + } + break; + } + + frag_info++; + wi++; + } + + /* copy header */ + xsc_copy_skb_header(dev, skb, head_wi->di, head_offset, headlen); + + /* skb linear part was allocated with headlen and aligned to long */ + skb->tail += headlen; + skb->len += headlen; + + skbdata_debug_dump(skb, headlen, 0); + + return skb; +} + +static inline void page_ref_elev(struct xsc_dma_info *dma_info) +{ + page_ref_add(dma_info->page, PAGE_REF_ELEV); + dma_info->refcnt_bias += PAGE_REF_ELEV; +} + +static inline bool xsc_rx_cache_is_empty(struct xsc_page_cache *cache) +{ + return cache->head < 0; +} + +static inline bool xsc_rx_cache_page_busy(struct xsc_page_cache *cache, u32 i) +{ + struct xsc_dma_info *di = &cache->page_cache[i]; + + return (page_ref_count(di->page) - di->refcnt_bias) != 1; +} + +static inline void xsc_rx_cache_page_swap(struct xsc_page_cache *cache, + u32 a, u32 b) +{ + struct xsc_dma_info tmp; + + tmp = cache->page_cache[a]; + cache->page_cache[a] = cache->page_cache[b]; + cache->page_cache[b] = tmp; +} + +static inline void xsc_rx_cache_reduce_reset_watch(struct xsc_page_cache *cache) +{ + struct xsc_page_cache_reduce *reduce = &cache->reduce; + + reduce->next_ts = ilog2(cache->sz) == cache->log_min_sz ? + MAX_JIFFY_OFFSET : jiffies + reduce->grace_period; + reduce->success = 0; +} + +static inline bool xsc_page_is_reserved(struct page *page) +{ + return page_is_pfmemalloc(page) || page_to_nid(page) != numa_mem_id(); +} + +static inline bool xsc_rx_cache_check_reduce(struct xsc_rq *rq) +{ + struct xsc_page_cache *cache = &rq->page_cache; + + if (!cache->page_cache) + return false; + + if (unlikely(test_bit(XSC_ETH_RQ_STATE_CACHE_REDUCE_PENDING, &rq->state))) + return false; + + if (time_before(jiffies, cache->reduce.next_ts)) + return false; + + if (likely(!xsc_rx_cache_is_empty(cache)) && + xsc_rx_cache_page_busy(cache, cache->head)) + goto reset_watch; + + if (ilog2(cache->sz) == cache->log_min_sz) + goto reset_watch; + + /* would like to reduce */ + if (cache->reduce.success < XSC_PAGE_CACHE_REDUCE_SUCCESS_CNT) { + cache->reduce.success++; + return false; + } + + return true; + +reset_watch: + xsc_rx_cache_reduce_reset_watch(cache); + return false; + +} + +static inline void xsc_rx_cache_may_reduce(struct xsc_rq *rq) +{ + struct xsc_page_cache *cache = &rq->page_cache; + struct xsc_page_cache_reduce *reduce = &cache->reduce; + int max_new_head; + + if (!xsc_rx_cache_check_reduce(rq)) + return; + + /* do reduce */ + rq->stats->cache_rdc++; + cache->sz >>= 1; + max_new_head = (cache->sz >> 1) - 1; + if (cache->head > max_new_head) { + u32 npages = cache->head - max_new_head; + + cache->head = max_new_head; + if (cache->lrs >= cache->head) + cache->lrs = 0; + + memcpy(reduce->pending, &cache->page_cache[cache->head + 1], + npages * sizeof(*reduce->pending)); + reduce->npages = npages; + set_bit(XSC_ETH_RQ_STATE_CACHE_REDUCE_PENDING, &rq->state); + } + + xsc_rx_cache_reduce_reset_watch(cache); +} + +static inline bool xsc_rx_cache_extend(struct xsc_rq *rq) +{ + struct xsc_page_cache *cache = &rq->page_cache; + struct xsc_page_cache_reduce *reduce = &cache->reduce; + u8 log_limit_sz = cache->log_min_sz + XSC_PAGE_CACHE_LOG_MAX_RQ_MULT; + + if (ilog2(cache->sz) >= log_limit_sz) + return false; + + rq->stats->cache_ext++; + cache->sz <<= 1; + + xsc_rx_cache_reduce_reset_watch(cache); + schedule_delayed_work_on(smp_processor_id(), &reduce->reduce_work, + reduce->delay); + return true; +} + +static inline bool xsc_rx_cache_get(struct xsc_rq *rq, + struct xsc_dma_info *dma_info) +{ + struct xsc_page_cache *cache = &rq->page_cache; + struct xsc_rq_stats *stats = rq->stats; + struct xsc_core_device *xdev = rq->cq.xdev; + + if (unlikely(xsc_rx_cache_is_empty(cache))) + goto err_no_page; + + xsc_rx_cache_page_swap(cache, cache->head, cache->lrs); + cache->lrs++; + if (cache->lrs >= cache->head) + cache->lrs = 0; + if (xsc_rx_cache_page_busy(cache, cache->head)) + goto err_no_page; + + stats->cache_reuse++; + *dma_info = cache->page_cache[cache->head--]; + + dma_sync_single_for_device(&xdev->pdev->dev, dma_info->addr, + PAGE_SIZE, DMA_FROM_DEVICE); + if (unlikely(page_ref_count(dma_info->page) <= PAGE_REF_THRSD)) + page_ref_elev(dma_info); + + return true; + +err_no_page: + stats->cache_alloc++; + cache->reduce.success = 0; + + return false; +} + +static inline bool xsc_rx_cache_put(struct xsc_rq *rq, + struct xsc_dma_info *dma_info) +{ + struct xsc_page_cache *cache = &rq->page_cache; + struct xsc_rq_stats *stats = rq->stats; + + if (unlikely(cache->head == cache->sz - 1)) { + if (!xsc_rx_cache_extend(rq)) { + rq->stats->cache_full++; + return false; + } + } + + if (unlikely(xsc_page_is_reserved(dma_info->page))) { + stats->cache_waive++; + return false; + } + + cache->page_cache[++cache->head] = *dma_info; + return true; +} + +void xsc_page_dma_unmap(struct xsc_rq *rq, struct xsc_dma_info *dma_info) +{ + struct xsc_channel *c = rq->cq.channel; + struct device *dev = c->adapter->dev; + + dma_unmap_page(dev, dma_info->addr, XSC_RX_FRAG_SZ, rq->buff.map_dir); +} + +static inline void xsc_put_page(struct xsc_dma_info *dma_info) +{ + page_ref_sub(dma_info->page, dma_info->refcnt_bias); + ETH_DEBUG_LOG("free addr=0x%llx, refcnt=(%d, %d)\n", + dma_info->addr, page_ref_count(dma_info->page), dma_info->refcnt_bias); + put_page(dma_info->page); +} + +void xsc_page_release_dynamic(struct xsc_rq *rq, + struct xsc_dma_info *dma_info, bool recycle) +{ + if (likely(recycle)) { +#ifdef XSC_PAGE_CACHE + if (xsc_rx_cache_put(rq, dma_info)) + return; +#endif + + xsc_page_dma_unmap(rq, dma_info); + page_ref_sub(dma_info->page, dma_info->refcnt_bias); + page_pool_recycle_direct(rq->page_pool, dma_info->page); + } else { + xsc_page_dma_unmap(rq, dma_info); + + page_pool_release_page(rq->page_pool, dma_info->page); + + xsc_put_page(dma_info); + } +} + +static inline void xsc_put_rx_frag(struct xsc_rq *rq, + struct xsc_wqe_frag_info *frag, bool recycle) +{ + if (frag->last_in_page) + xsc_page_release_dynamic(rq, frag->di, recycle); +} + +static inline void xsc_free_rx_wqe(struct xsc_rq *rq, + struct xsc_wqe_frag_info *wi, bool recycle) +{ + int i; + + for (i = 0; i < rq->wqe.info.num_frags; i++, wi++) { + if (i < rq->frags_reuse_num || !recycle) + xsc_put_rx_frag(rq, wi, recycle); + } +} + +static inline void xsc_free_rx_wqe_reserve_ds(struct xsc_rq *rq, + struct xsc_wqe_frag_info *wi) +{ + int i; + + for (i = 0; i < rq->wqe.info.num_frags; i++, wi++) { + if (i >= rq->frags_reuse_num) { +#ifdef XSC_DEBUG + if (page_ref_count(wi->di->page) - wi->di->refcnt_bias > 1) + ETH_DEBUG_LOG("frag%d, last_in_page=%d, refcnt=%d, bias=%d\n", + i, wi->last_in_page, + page_ref_count(wi->di->page), wi->di->refcnt_bias); +#endif + xsc_put_rx_frag(rq, wi, false); + } + } +} + +static inline struct xsc_wqe_frag_info *get_frag(struct xsc_rq *rq, u16 ix) +{ + return &rq->wqe.frags[ix << rq->wqe.info.log_num_frags]; +} + +void xsc_eth_handle_rx_cqe(struct xsc_cqwq *cqwq, + struct xsc_rq *rq, struct xsc_cqe64 *cqe) +{ + struct xsc_wq_cyc *wq = &rq->wqe.wq; + struct xsc_channel *c = rq->cq.channel; + struct xsc_wqe_frag_info *wi; + struct sk_buff *skb; + u32 cqe_bcnt; + u16 ci; + u8 cqe_opcode = get_cqe_opcode(cqe); + + ci = xsc_wq_cyc_ctr2ix(wq, cqwq->cc); + wi = get_frag(rq, ci); + if (unlikely(cqe_opcode & BIT(7))) { + rq->stats->wqe_err++; + goto free_wqe; + } + + cqe_bcnt = le32_to_cpu(cqe->msg_len); + ETH_DEBUG_LOG("data_len=%d, cc=%d", cqe_bcnt, ci); + + /* Check packet size. */ + if (unlikely(cqe_bcnt > rq->hw_mtu)) { + if (!XSC_GET_PFLAG(&c->adapter->nic_param, XSC_PFLAG_DROPLESS_RQ)) { + rq->stats->oversize_pkts_sw_drop++; + goto free_wqe; + } + } + + skb = rq->wqe.skb_from_cqe(rq, wi, cqe_bcnt, cqe->has_pph); + if (!skb) + goto free_wqe; + + xsc_complete_rx_cqe(rq, cqe, cqe_bcnt, skb, wi); + +#ifdef NEED_CREATE_RX_THREAD + netif_rx_ni(skb); +#else + napi_gro_receive(rq->cq.napi, skb); +#endif + +free_wqe: + xsc_free_rx_wqe(rq, wi, true); + xsc_wq_cyc_pop(wq); +} + +static void xsc_dump_error_rqcqe(struct xsc_rq *rq, + struct xsc_cqe64 *cqe) +{ + struct xsc_channel *c = rq->cq.channel; + struct net_device *netdev = c->adapter->netdev; + u32 ci = xsc_cqwq_get_ci(&rq->cq.wq); + + net_err_ratelimited( + "Error cqe on dev=%s, cqn=%d, ci=%d, rqn=%d, qpn=%d, error_code=0x%x\n", + netdev->name, rq->cq.xcq.cqn, ci, + rq->rqn, cqe->qp_id, get_cqe_opcode(cqe)); + +#ifdef XSC_DEBUG + xsc_dump_err_cqe(rq->cq.xdev, cqe); +#endif +} + +int xsc_poll_rx_cq(struct xsc_cq *cq, int budget) +{ + struct xsc_rq *rq = container_of(cq, struct xsc_rq, cq); + struct xsc_cqwq *cqwq = &cq->wq; + struct xsc_cqe64 *cqe; + int work_done = 0; + struct xsc_ch_stats *ch_stats = cq->channel->stats; + + if (!test_bit(XSC_ETH_RQ_STATE_ENABLED, &rq->state)) + return 0; + + while ((work_done < budget) && (cqe = xsc_cqwq_get_cqe(cqwq))) { + if (unlikely(get_cqe_opcode(cqe) & BIT(7))) { + xsc_dump_error_rqcqe(rq, cqe); + rq->stats->cqe_err++; + break; + } + + rq->stats->cqes++; + + rq->handle_rx_cqe(cqwq, rq, cqe); + ++work_done; + + xsc_cqwq_pop(cqwq); + } + + if (!work_done) + return 0; + + xsc_cq_notify_hw(cq); + /* ensure cq space is freed before enabling more cqes */ + wmb(); + + rq->post_wqes(rq); + + ch_stats->poll += work_done; + if (work_done < budget) { + if (ch_stats->poll == 0) + ch_stats->poll_0++; + else if (ch_stats->poll <= 64) + ch_stats->poll_1_64++; + else if (ch_stats->poll < 512) + ch_stats->poll_65_511++; + else if (ch_stats->poll < 1024) + ch_stats->poll_512_1023++; + else if (ch_stats->poll >= 1024) + cq->channel->stats->poll_1024++; + } + + return work_done; +} + +static inline int xsc_page_alloc_mapped(struct xsc_rq *rq, + struct xsc_dma_info *dma_info) +{ + struct xsc_channel *c = rq->cq.channel; + struct device *dev = c->adapter->dev; + +#ifdef XSC_PAGE_CACHE + if (xsc_rx_cache_get(rq, dma_info)) + return 0; +#endif + + dma_info->page = page_pool_dev_alloc_pages(rq->page_pool); + if (unlikely(!dma_info->page)) + return -ENOMEM; + + dma_info->refcnt_bias = 0; + page_ref_elev(dma_info); + + dma_info->addr = dma_map_page(dev, dma_info->page, 0, + XSC_RX_FRAG_SZ, rq->buff.map_dir); + if (unlikely(dma_mapping_error(dev, dma_info->addr))) { + page_ref_sub(dma_info->page, dma_info->refcnt_bias); + page_pool_recycle_direct(rq->page_pool, dma_info->page); + dma_info->page = NULL; + return -ENOMEM; + } + + return 0; +} + +static inline int xsc_get_rx_frag(struct xsc_rq *rq, + struct xsc_wqe_frag_info *frag) +{ + int err = 0; + + if (!frag->offset) + /* On first frag (offset == 0), replenish page (dma_info actually). + * Other frags that point to the same dma_info (with a different + * offset) should just use the new one without replenishing again + * by themselves. + */ + err = xsc_page_alloc_mapped(rq, frag->di); + + return err; +} + +static int xsc_alloc_rx_wqe(struct xsc_rq *rq, struct xsc_eth_rx_wqe_cyc *wqe, u16 ix) +{ + struct xsc_wqe_frag_info *frag = get_frag(rq, ix); + u64 addr; + int i; + int err; + + for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) { + if (i >= rq->frags_reuse_num && wqe->data[i].va != 0) + continue; + + err = xsc_get_rx_frag(rq, frag); + if (unlikely(err)) + goto free_frags; + + addr = cpu_to_le64(frag->di->addr + frag->offset + rq->buff.headroom); + wqe->data[i].va = addr; + wqe->data[i].seg_len = cpu_to_le32(XSC_RX_FRAG_SZ); + ETH_DEBUG_LOG( + "rq%d_wqe%d->frag%d, off=%d, last_in_page=%d, refcnt=(%d, %d), addr=0x%llx\n", + rq->rqn, ix, i, frag->offset, frag->last_in_page, + page_ref_count(frag->di->page), frag->di->refcnt_bias, addr); + } + + return 0; + +free_frags: + while (--i >= 0) + xsc_put_rx_frag(rq, --frag, true); + + return err; +} + +void xsc_eth_dealloc_rx_wqe(struct xsc_rq *rq, u16 ix, bool resv_ds) +{ + struct xsc_wqe_frag_info *wi = get_frag(rq, ix); + + if (!resv_ds) + xsc_free_rx_wqe(rq, wi, false); + else + xsc_free_rx_wqe_reserve_ds(rq, wi); +} + +void xsc_eth_free_rx_wqes(struct xsc_rq *rq) +{ + u16 wqe_ix; + u16 wqe_ix_head, wqe_ix_tail; + struct xsc_wq_cyc *wq = &rq->wqe.wq; + + wqe_ix_tail = xsc_wq_cyc_get_tail(wq); + wqe_ix_head = xsc_wq_cyc_get_head(wq); + + for (wqe_ix = wqe_ix_head; wqe_ix < wqe_ix_tail; wqe_ix++) + xsc_eth_dealloc_rx_wqe(rq, wqe_ix, true); + + while (!xsc_wq_cyc_is_empty(wq)) { + wqe_ix = xsc_wq_cyc_get_tail(wq); + xsc_eth_dealloc_rx_wqe(rq, wqe_ix, false); + xsc_wq_cyc_pop(wq); + } +} + +static int xsc_alloc_rx_wqes(struct xsc_rq *rq, u16 ix, u8 wqe_bulk) +{ + struct xsc_wq_cyc *wq = &rq->wqe.wq; + struct xsc_eth_rx_wqe_cyc *wqe; + int err; + int i; + int idx; + + for (i = 0; i < wqe_bulk; i++) { + idx = xsc_wq_cyc_ctr2ix(wq, (ix + i)); + wqe = xsc_wq_cyc_get_wqe(wq, idx); + + err = xsc_alloc_rx_wqe(rq, wqe, idx); + if (unlikely(err)) { + rq->stats->buff_alloc_err++; + goto free_wqes; + } + } + + return 0; + +free_wqes: + while (--i >= 0) + xsc_eth_dealloc_rx_wqe(rq, ix + i, false); + + return err; +} + +bool xsc_eth_post_rx_wqes(struct xsc_rq *rq) +{ + struct xsc_wq_cyc *wq = &rq->wqe.wq; + int left = xsc_wq_cyc_missing(wq); + u8 wqe_bulk; + u16 head; + int alloc; + int err; + + wqe_bulk = rq->wqe.info.wqe_bulk; + if (xsc_wq_cyc_missing(wq) < wqe_bulk) + return false; + + do { + alloc = min_t(int, wqe_bulk, left); + head = xsc_wq_cyc_get_head(wq); + + err = xsc_alloc_rx_wqes(rq, head, alloc); + if (unlikely(err)) + break; + + xsc_wq_cyc_push_n(wq, alloc); + rq->stats->wqes += alloc; + left -= alloc; + } while (left > 0); + + dma_wmb(); + + /* ensure wqes are visible to device before updating doorbell record */ + xsc_rq_notify_hw(rq); + + xsc_rx_cache_may_reduce(rq); + + return !!err; +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_stats.c b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_stats.c new file mode 100644 index 0000000000000000000000000000000000000000..160a88de62ae0aeba9934234a3115ae819689098 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_stats.c @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include + +#include "xsc_eth_stats.h" +#include "xsc_eth.h" + +static const struct counter_desc sw_stats_desc[] = { + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_packets) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_bytes) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_packets) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_bytes) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_tso_packets) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_tso_bytes) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_tso_inner_packets) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_tso_inner_bytes) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_csum_unnecessary) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_csum_none) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_csum_err) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_csum_succ) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_csum_partial) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_csum_partial_inner) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_queue_stopped) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_queue_dropped) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_xmit_more) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_cqes) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_queue_wake) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_cqe_err) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, tx_oversize_pkts_sw_drop) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, txdone_skb_null) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, txdone_skb_refcnt_err) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cqes) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cqe_err) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_wqes) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_wqe_err) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_oversize_pkts_sw_drop) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_buff_alloc_err) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cache_reuse) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cache_full) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cache_empty) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cache_busy) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cache_alloc) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cache_waive) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cache_ext) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, rx_cache_rdc) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_events) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_poll) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_poll_0) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_poll_1_64) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_poll_65_511) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_poll_512_1023) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_poll_1024) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_arm) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_noarm) }, + { XSC_DECLARE_STAT(struct xsc_sw_stats, ch_aff_change) }, +}; + +#define NUM_SW_COUNTERS ARRAY_SIZE(sw_stats_desc) + +static int xsc_grp_sw_get_num_stats(struct xsc_adapter *adapter) +{ + return NUM_SW_COUNTERS; +} + +static int xsc_grp_sw_fill_strings(struct xsc_adapter *adapter, u8 *data, int idx) +{ + int i; + + for (i = 0; i < NUM_SW_COUNTERS; i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, sw_stats_desc[i].format); + return idx; +} + +static int xsc_grp_sw_fill_stats(struct xsc_adapter *adapter, u64 *data, int idx) +{ + int i; + + for (i = 0; i < NUM_SW_COUNTERS; i++) + data[idx++] = XSC_READ_CTR64_CPU(&adapter->stats->sw, sw_stats_desc, i); + return idx; +} + +void xsc_grp_sw_update_stats(struct xsc_adapter *adapter) +{ + struct xsc_sw_stats *s = &adapter->stats->sw; + int max_tc = xsc_get_netdev_max_tc(adapter); + int i; + + memset(s, 0, sizeof(*s)); + + for (i = 0; i < xsc_get_netdev_max_channels(adapter); i++) { + struct xsc_channel_stats *channel_stats = + &adapter->stats->channel_stats[i]; + + struct xsc_rq_stats *rq_stats = &channel_stats->rq; + struct xsc_ch_stats *ch_stats = &channel_stats->ch; + int j; + + s->rx_packets += rq_stats->packets; + s->rx_bytes += rq_stats->bytes; + s->rx_csum_unnecessary += rq_stats->csum_unnecessary; + s->rx_csum_none += rq_stats->csum_none; + s->rx_csum_err += rq_stats->csum_err; + s->rx_csum_succ += rq_stats->csum_succ; + s->rx_cqes += rq_stats->cqes; + s->rx_cqe_err += rq_stats->cqe_err; + s->rx_wqes += rq_stats->wqes; + s->rx_wqe_err += rq_stats->wqe_err; + s->rx_oversize_pkts_sw_drop += rq_stats->oversize_pkts_sw_drop; + s->rx_buff_alloc_err += rq_stats->buff_alloc_err; + s->rx_cache_reuse += rq_stats->cache_reuse; + s->rx_cache_full += rq_stats->cache_full; + s->rx_cache_empty += rq_stats->cache_empty; + s->rx_cache_busy += rq_stats->cache_busy; + s->rx_cache_alloc += rq_stats->cache_alloc; + s->rx_cache_waive += rq_stats->cache_waive; + s->rx_cache_ext += rq_stats->cache_ext; + s->rx_cache_rdc += rq_stats->cache_rdc; + + s->ch_events += ch_stats->events; + s->ch_poll += ch_stats->poll; + s->ch_poll_0 += ch_stats->poll_0; + s->ch_poll_1_64 += ch_stats->poll_1_64; + s->ch_poll_65_511 += ch_stats->poll_65_511; + s->ch_poll_512_1023 += ch_stats->poll_512_1023; + s->ch_poll_1024 += ch_stats->poll_1024; + s->ch_arm += ch_stats->arm; + s->ch_noarm += ch_stats->noarm; + s->ch_aff_change += ch_stats->aff_change; + + for (j = 0; j < max_tc; j++) { + struct xsc_sq_stats *sq_stats = &channel_stats->sq[j]; + + s->tx_packets += sq_stats->packets; + s->tx_bytes += sq_stats->bytes; + s->tx_tso_packets += sq_stats->tso_packets; + s->tx_tso_bytes += sq_stats->tso_bytes; + s->tx_tso_inner_packets += sq_stats->tso_inner_packets; + s->tx_tso_inner_bytes += sq_stats->tso_inner_bytes; + s->tx_csum_partial += sq_stats->csum_partial; + s->tx_csum_partial_inner += sq_stats->csum_partial_inner; + s->tx_csum_none += sq_stats->csum_none; + s->tx_queue_stopped += sq_stats->stopped; + s->tx_queue_dropped += sq_stats->dropped; + s->tx_xmit_more += sq_stats->xmit_more; + s->tx_cqes += sq_stats->cqes; + s->tx_queue_wake += sq_stats->wake; + s->tx_cqe_err += sq_stats->cqe_err; + s->tx_oversize_pkts_sw_drop += sq_stats->oversize_pkts_sw_drop; + s->txdone_skb_null += sq_stats->txdone_skb_null; + s->txdone_skb_refcnt_err += sq_stats->txdone_skb_refcnt_err; + s->skb_linear += sq_stats->skb_linear; + } + } +} + +static const struct counter_desc rq_stats_desc[] = { + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, packets) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, bytes) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, csum_unnecessary) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, csum_none) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, csum_err) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, csum_succ) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cqes) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, wqe_err) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, oversize_pkts_sw_drop) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, buff_alloc_err) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cache_reuse) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cache_full) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cache_empty) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cache_busy) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cache_alloc) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cache_waive) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cache_ext) }, + { XSC_DECLARE_RX_STAT(struct xsc_rq_stats, cache_rdc) }, +}; + +static const struct counter_desc sq_stats_desc[] = { + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, packets) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, bytes) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, tso_packets) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, tso_bytes) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, tso_inner_packets) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, tso_inner_bytes) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, csum_partial) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, csum_partial_inner) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, csum_none) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, stopped) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, dropped) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, xmit_more) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, cqes) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, wake) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, cqe_err) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, oversize_pkts_sw_drop) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, txdone_skb_null) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, txdone_skb_refcnt_err) }, + { XSC_DECLARE_TX_STAT(struct xsc_sq_stats, skb_linear) }, +}; + +static const struct counter_desc ch_stats_desc[] = { + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, events) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, poll) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, poll_0) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, poll_1_64) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, poll_65_511) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, poll_512_1023) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, poll_1024) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, arm) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, noarm) }, + { XSC_DECLARE_CH_STAT(struct xsc_ch_stats, aff_change) }, +}; + +#define NUM_RQ_STATS ARRAY_SIZE(rq_stats_desc) +#define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc) +#define NUM_CH_STATS ARRAY_SIZE(ch_stats_desc) + +static int xsc_grp_channels_get_num_stats(struct xsc_adapter *adapter) +{ + int max_nch = xsc_get_netdev_max_channels(adapter); + int max_tc = xsc_get_netdev_max_tc(adapter); + + return (NUM_RQ_STATS * max_nch) + + (NUM_CH_STATS * max_nch) + + (NUM_SQ_STATS * max_nch * max_tc); +} + +static int xsc_grp_channels_fill_strings(struct xsc_adapter *adapter, u8 *data, + int idx) +{ + int max_nch = xsc_get_netdev_max_channels(adapter); + int max_tc = xsc_get_netdev_max_tc(adapter); + int i, j, tc; + + for (i = 0; i < max_nch; i++) + for (j = 0; j < NUM_CH_STATS; j++) + sprintf(data + (idx++) * ETH_GSTRING_LEN, + ch_stats_desc[j].format, i); + + for (i = 0; i < max_nch; i++) { + for (j = 0; j < NUM_RQ_STATS; j++) + sprintf(data + (idx++) * ETH_GSTRING_LEN, + rq_stats_desc[j].format, i); + } + + for (tc = 0; tc < max_tc; tc++) + for (i = 0; i < max_nch; i++) + for (j = 0; j < NUM_SQ_STATS; j++) + sprintf(data + (idx++) * ETH_GSTRING_LEN, + sq_stats_desc[j].format, + i + tc * max_nch); + + return idx; +} + +static int xsc_grp_channels_fill_stats(struct xsc_adapter *adapter, u64 *data, + int idx) +{ + int max_nch = xsc_get_netdev_max_channels(adapter); + int max_tc = xsc_get_netdev_max_tc(adapter); + int i, j, tc; + struct xsc_stats *stats = adapter->stats; + + for (i = 0; i < max_nch; i++) + for (j = 0; j < NUM_CH_STATS; j++) + data[idx++] = + XSC_READ_CTR64_CPU(&stats->channel_stats[i].ch, + ch_stats_desc, j); + + for (i = 0; i < max_nch; i++) { + for (j = 0; j < NUM_RQ_STATS; j++) + data[idx++] = + XSC_READ_CTR64_CPU(&stats->channel_stats[i].rq, + rq_stats_desc, j); + } + + for (tc = 0; tc < max_tc; tc++) + for (i = 0; i < max_nch; i++) + for (j = 0; j < NUM_SQ_STATS; j++) + data[idx++] = + XSC_READ_CTR64_CPU(&stats->channel_stats[i].sq[tc], + sq_stats_desc, j); + + return idx; +} + +static const struct counter_desc hw_prio_stats_desc[] = { + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_bytes, 0), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_bytes, 0), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_pkts, 0), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_pkts, 0), + + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_bytes, 1), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_bytes, 1), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_pkts, 1), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_pkts, 1), + + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_bytes, 2), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_bytes, 2), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_pkts, 2), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_pkts, 2), + + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_bytes, 3), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_bytes, 3), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_pkts, 3), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_pkts, 3), + + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_bytes, 4), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_bytes, 4), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_pkts, 4), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_pkts, 4), + + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_bytes, 5), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_bytes, 5), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_pkts, 5), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_pkts, 5), + + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_bytes, 6), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_bytes, 6), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_pkts, 6), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_pkts, 6), + + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_bytes, 7), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_bytes, 7), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, tx_pkts, 7), + XSC_DECLARE_HW_PRIO_STAT(struct xsc_prio_stats, rx_pkts, 7), +}; + +static const struct counter_desc hw_stats_desc[] = { + /*by mac port*/ + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_tx_pkts) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_tx_bytes) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_rx_pkts) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_rx_bytes) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, np_cnp_sent) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rp_cnp_handled) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, np_ecn_marked_roce_packets) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rp_cnp_ignored) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, tx_pause) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_pause) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_fcs_errors) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rx_discards) }, + + /*by global*/ + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_loopback_pkts) }, + { XSC_DECLARE_STAT(struct xsc_hw_stats, rdma_loopback_bytes) }, +}; + +static int xsc_hw_get_num_stats(struct xsc_adapter *adapter) +{ + return ARRAY_SIZE(hw_prio_stats_desc) + ARRAY_SIZE(hw_stats_desc); +} + +static int xsc_hw_fill_strings(struct xsc_adapter *adapter, u8 *data, int idx) +{ + int i; + struct xsc_core_device *xdev; + + xdev = adapter->xdev; + + for (i = 0; i < ARRAY_SIZE(hw_prio_stats_desc); i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, hw_prio_stats_desc[i].format); + + for (i = 0; i < ARRAY_SIZE(hw_stats_desc); i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, hw_stats_desc[i].format); + + return idx; +} + +static int xsc_hw_fill_stats(struct xsc_adapter *adapter, u64 *data, int idx) +{ + struct xsc_prio_stats_mbox_in in; + struct xsc_prio_stats_mbox_out out; + struct xsc_hw_stats_mbox_in hw_in; + struct xsc_hw_stats_mbox_out hw_out; + struct xsc_core_device *xdev; + int ret; + u32 i; + u64 val; + + xdev = adapter->xdev; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.hdr.opcode = __cpu_to_be16(XSC_CMD_OP_QUERY_PRIO_STATS); + in.pport = xdev->mac_port; + + ret = xsc_cmd_exec(adapter->xdev, (void *)&in, sizeof(struct xsc_prio_stats_mbox_in), + (void *)&out, sizeof(struct xsc_prio_stats_mbox_out)); + if ((ret == 0) && (out.hdr.status == 0)) { + for (i = 0; i < ARRAY_SIZE(hw_prio_stats_desc); i++) { + val = XSC_READ_CTR64_CPU(&out.prio_stats, hw_prio_stats_desc, i); + data[idx++] = __be64_to_cpu(val); + } + } + + memset(&hw_in, 0, sizeof(hw_in)); + memset(&hw_out, 0, sizeof(hw_out)); + hw_in.hdr.opcode = __cpu_to_be16(XSC_CMD_OP_QUERY_HW_STATS); + hw_in.mac_port = xdev->mac_port; + hw_in.is_lag = 0; + + ret = xsc_cmd_exec(adapter->xdev, (void *)&hw_in, sizeof(struct xsc_hw_stats_mbox_in), + (void *)&hw_out, sizeof(struct xsc_hw_stats_mbox_out)); + if ((ret == 0) && (hw_out.hdr.status == 0)) { + for (i = 0; i < ARRAY_SIZE(hw_stats_desc); i++) { + val = XSC_READ_CTR64_CPU(&hw_out.hw_stats, hw_stats_desc, i); + data[idx++] = __be64_to_cpu(val); + } + } + + return idx; +} + +/* The stats groups order is opposite to the update_stats() order calls */ +const struct xsc_stats_grp xsc_stats_grps[] = { + { + .get_num_stats = xsc_grp_sw_get_num_stats, + .fill_strings = xsc_grp_sw_fill_strings, + .fill_stats = xsc_grp_sw_fill_stats, + .update_stats = xsc_grp_sw_update_stats, + }, + + { + .get_num_stats = xsc_grp_channels_get_num_stats, + .fill_strings = xsc_grp_channels_fill_strings, + .fill_stats = xsc_grp_channels_fill_stats, + }, + + { + .get_num_stats = xsc_hw_get_num_stats, + .fill_strings = xsc_hw_fill_strings, + .fill_stats = xsc_hw_fill_stats, + }, +}; + +const int xsc_num_stats_grps = ARRAY_SIZE(xsc_stats_grps); + +void xsc_fold_sw_stats64(struct xsc_adapter *adapter, struct rtnl_link_stats64 *s) +{ + int i, j; + + for (i = 0; i < xsc_get_netdev_max_channels(adapter); i++) { + struct xsc_channel_stats *channel_stats = &adapter->stats->channel_stats[i]; + struct xsc_rq_stats *rq_stats = &channel_stats->rq; + + s->rx_packets += rq_stats->packets; + s->rx_bytes += rq_stats->bytes; + + for (j = 0; j < xsc_get_netdev_max_tc(adapter); j++) { + struct xsc_sq_stats *sq_stats = &channel_stats->sq[j]; + + s->tx_packets += sq_stats->packets; + s->tx_bytes += sq_stats->bytes; + s->tx_dropped += sq_stats->dropped; + } + } +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_stats.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_stats.h new file mode 100644 index 0000000000000000000000000000000000000000..eeacfbad2dbd8a6da3f24bf4053b03c96219e210 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_stats.h @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_EN_STATS_H +#define XSC_EN_STATS_H + +#include "xsc_eth_common.h" + +#define XSC_READ_CTR64_CPU(ptr, dsc, i) \ + (*(u64 *)((char *)ptr + dsc[i].offset)) + +#define ETH_GSTRING_LEN 32 + +#define XSC_DECLARE_STAT(type, fld) ""#fld, offsetof(type, fld) +#define XSC_DECLARE_RX_STAT(type, fld) "rx%d_"#fld, offsetof(type, fld) +#define XSC_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld) +#define XSC_DECLARE_CH_STAT(type, fld) "ch%d_"#fld, offsetof(type, fld) + +#define XSC_DECLARE_HW_PRIO_STAT_NAME(fld, prio) (#fld"_prio"#prio) +#define XSC_DECLARE_HW_PRIO_STAT_OFFSET(type, fld, prio) \ + (offsetof(type, fld) + (sizeof(type) * prio)) +#define XSC_DECLARE_HW_PRIO_STAT(type, fld, prio) \ + {XSC_DECLARE_HW_PRIO_STAT_NAME(fld, prio), \ + XSC_DECLARE_HW_PRIO_STAT_OFFSET(type, fld, prio)} + +struct xsc_rq_stats { + u64 packets; + u64 bytes; + u64 csum_unnecessary; + u64 csum_none; + u64 csum_err; + u64 csum_succ; + u64 cqes; + u64 cqe_err; + u64 wqes; + u64 wqe_err; + u64 oversize_pkts_sw_drop; + u64 buff_alloc_err; + u64 cache_reuse; + u64 cache_full; + u64 cache_empty; + u64 cache_busy; + u64 cache_alloc; + u64 cache_waive; + u64 cache_ext; + u64 cache_rdc; +}; + +struct xsc_sq_stats { + /* commonly accessed in data path */ + u64 packets; + u64 bytes; + u64 tso_packets; + u64 tso_bytes; + u64 tso_inner_packets; + u64 tso_inner_bytes; + u64 csum_partial; + u64 csum_partial_inner; + /* less likely accessed in data path */ + u64 csum_none; + u64 stopped; + u64 dropped; + u64 xmit_more; + /* dirtied @completion */ + u64 cqes; + u64 wake; + u64 cqe_err; + u64 oversize_pkts_sw_drop; + u64 txdone_skb_null; + u64 txdone_skb_refcnt_err; + u64 skb_linear; +}; + +struct xsc_ch_stats { + u64 events; + u64 poll; + u64 poll_0; + u64 poll_1_64; + u64 poll_65_511; + u64 poll_512_1023; + u64 poll_1024; + u64 arm; + u64 noarm; + u64 aff_change; +}; + +struct xsc_adapter; +struct xsc_stats_grp { + u16 update_stats_mask; + int (*get_num_stats)(struct xsc_adapter *adapter); + int (*fill_strings)(struct xsc_adapter *adapter, u8 *data, int idx); + int (*fill_stats)(struct xsc_adapter *adapter, u64 *data, int idx); + void (*update_stats)(struct xsc_adapter *adapter); +}; + +struct counter_desc { + + char format[ETH_GSTRING_LEN]; + size_t offset; /* Byte offset */ +}; + +struct xsc_sw_stats { + u64 rx_packets; + u64 rx_bytes; + u64 tx_packets; + u64 tx_bytes; + u64 tx_tso_packets; + u64 tx_tso_bytes; + u64 tx_tso_inner_packets; + u64 tx_tso_inner_bytes; + u64 rx_csum_unnecessary; + u64 rx_csum_none; + u64 rx_csum_err; + u64 rx_csum_succ; + u64 tx_csum_none; + u64 tx_csum_partial; + u64 tx_csum_partial_inner; + u64 tx_queue_stopped; + u64 tx_queue_dropped; + u64 tx_xmit_more; + u64 tx_cqes; + u64 tx_queue_wake; + u64 tx_cqe_err; + u64 tx_oversize_pkts_sw_drop; + u64 txdone_skb_null; + u64 txdone_skb_refcnt_err; + u64 skb_linear; + u64 rx_cqes; + u64 rx_cqe_err; + u64 rx_wqes; + u64 rx_wqe_err; + u64 rx_oversize_pkts_sw_drop; + u64 rx_buff_alloc_err; + u64 rx_cache_reuse; + u64 rx_cache_full; + u64 rx_cache_empty; + u64 rx_cache_busy; + u64 rx_cache_alloc; + u64 rx_cache_waive; + u64 rx_cache_ext; + u64 rx_cache_rdc; + u64 ch_events; + u64 ch_poll; + u64 ch_poll_0; + u64 ch_poll_1_64; + u64 ch_poll_65_511; + u64 ch_poll_512_1023; + u64 ch_poll_1024; + u64 ch_arm; + u64 ch_noarm; + u64 ch_aff_change; +}; + +struct xsc_channel_stats { + struct xsc_ch_stats ch; + struct xsc_sq_stats sq[XSC_MAX_NUM_TC]; + struct xsc_rq_stats rq; +}; + +struct xsc_stats { + struct xsc_sw_stats sw; + struct xsc_channel_stats channel_stats[XSC_ETH_MAX_NUM_CHANNELS]; +}; + +extern const struct xsc_stats_grp xsc_stats_grps[]; +extern const int xsc_num_stats_grps; + +void xsc_fold_sw_stats64(struct xsc_adapter *adapter, struct rtnl_link_stats64 *s); + +#endif /* XSC_EN_STATS_H */ + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_tx.c b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_tx.c new file mode 100644 index 0000000000000000000000000000000000000000..5ba433c44b5725e0d9176b6c22aae274e62b0910 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_tx.c @@ -0,0 +1,561 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include "xsc_eth_stats.h" +#include "xsc_eth_common.h" +#include +#include +#include "xsc_eth.h" +#include "xsc_eth_txrx.h" + +#define XSC_OPCODE_RAW 0x7 + +static inline void *xsc_sq_fetch_wqe(struct xsc_sq *sq, size_t size, u16 *pi) +{ + struct xsc_wq_cyc *wq = &sq->wq; + void *wqe; + + /*caution, sp->pc is default to be zero*/ + *pi = xsc_wq_cyc_ctr2ix(wq, sq->pc); + wqe = xsc_wq_cyc_get_wqe(wq, *pi); + memset(wqe, 0, size); + + return wqe; +} + +u16 xsc_tx_get_gso_ihs(struct xsc_sq *sq, struct sk_buff *skb) +{ + struct xsc_sq_stats *stats = sq->stats; + u16 ihs; + + if (skb->encapsulation) { + ihs = skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb); + stats->tso_inner_packets++; + stats->tso_inner_bytes += skb->len - ihs; + } else { + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + ihs = skb_transport_offset(skb) + sizeof(struct udphdr); + else + ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); + stats->tso_packets++; + stats->tso_bytes += skb->len - ihs; + } + + return ihs; +} + +void xsc_txwqe_build_cseg_csum(struct xsc_sq *sq, + struct sk_buff *skb, + struct xsc_send_wqe_ctrl_seg *cseg) +{ + if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { + if (skb->encapsulation) { + cseg->csum_en = XSC_ETH_WQE_INNER_AND_OUTER_CSUM; + sq->stats->csum_partial_inner++; + } else { + cseg->csum_en = XSC_ETH_WQE_OUTER_CSUM; + sq->stats->csum_partial++; + } + } else { + cseg->csum_en = XSC_ETH_WQE_NONE_CSUM; + sq->stats->csum_none++; + } +} + +static inline struct xsc_sq_dma *xsc_dma_get(struct xsc_sq *sq, u32 i) +{ + return &sq->db.dma_fifo[i & sq->dma_fifo_mask]; +} + +static inline void xsc_dma_push(struct xsc_sq *sq, dma_addr_t addr, u32 size, + enum xsc_dma_map_type map_type) +{ + struct xsc_sq_dma *dma = xsc_dma_get(sq, sq->dma_fifo_pc++); + + dma->addr = addr; + dma->size = size; + dma->type = map_type; + ETH_DEBUG_LOG("dma = %p, dma->addr = %#llx\n", dma, dma->addr); +} + +static inline void xsc_tx_dma_unmap(struct device *dev, struct xsc_sq_dma *dma) +{ + switch (dma->type) { + case XSC_DMA_MAP_SINGLE: + dma_unmap_single(dev, dma->addr, dma->size, DMA_TO_DEVICE); + break; + case XSC_DMA_MAP_PAGE: + dma_unmap_page(dev, dma->addr, dma->size, DMA_TO_DEVICE); + break; + default: + ETH_DEBUG_LOG("%s\n", "xsc_tx_dma_unmap unknown DMA type!\n"); + } +} + +static void xsc_dma_unmap_wqe_err(struct xsc_sq *sq, u8 num_dma) +{ + struct xsc_adapter *adapter = sq->channel->adapter; + struct device *dev = adapter->dev; + + int i; + + for (i = 0; i < num_dma; i++) { + struct xsc_sq_dma *last_pushed_dma = xsc_dma_get(sq, --sq->dma_fifo_pc); + + xsc_tx_dma_unmap(dev, last_pushed_dma); + } +} + +static void xsc_txwqe_build_csegs(struct xsc_sq *sq, struct sk_buff *skb, + u16 mss, u16 ihs, u16 headlen, + u8 opcode, u16 ds_cnt, u32 num_bytes, + struct xsc_send_wqe_ctrl_seg *cseg) +{ + struct xsc_core_device *xdev = sq->cq.xdev; + int send_wqe_ds_num_log = ilog2(xdev->caps.send_ds_num); + + xsc_txwqe_build_cseg_csum(sq, skb, cseg); + + if (mss != 0) { + cseg->has_pph = 0; + cseg->so_type = 1; + cseg->so_hdr_len = ihs; + cseg->so_data_size = cpu_to_le16(mss); + } + + cseg->msg_opcode = opcode; + cseg->wqe_id = cpu_to_le16(sq->pc << send_wqe_ds_num_log); + cseg->ds_data_num = ds_cnt - XSC_SEND_WQEBB_CTRL_NUM_DS; + cseg->msg_len = cpu_to_le32(num_bytes); + + cseg->ce = 1; + + WQE_CSEG_DUMP("cseg", cseg); + +} + +static int xsc_txwqe_build_dsegs(struct xsc_sq *sq, struct sk_buff *skb, + u16 ihs, u16 headlen, + struct xsc_wqe_data_seg *dseg) +{ + dma_addr_t dma_addr = 0; + u8 num_dma = 0; + int i; + struct xsc_adapter *adapter = sq->channel->adapter; + struct device *dev = adapter->dev; + + if (headlen) { + dma_addr = dma_map_single(dev, skb->data, headlen, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, dma_addr))) + goto dma_unmap_wqe_err; + + dseg->va = cpu_to_le64(dma_addr); + dseg->mkey = cpu_to_le32(sq->mkey_be); + dseg->seg_len = cpu_to_le32(headlen); + + WQE_DSEG_DUMP("dseg-headlen", dseg); + + xsc_dma_push(sq, dma_addr, headlen, XSC_DMA_MAP_SINGLE); + num_dma++; + dseg++; + } + + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + int fsz = skb_frag_size(frag); + + dma_addr = skb_frag_dma_map(dev, frag, 0, fsz, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, dma_addr))) + goto dma_unmap_wqe_err; + + dseg->va = cpu_to_le64(dma_addr); + dseg->mkey = cpu_to_le32(sq->mkey_be); + dseg->seg_len = cpu_to_le32(fsz); + + WQE_DSEG_DUMP("dseg-frag", dseg); + + xsc_dma_push(sq, dma_addr, fsz, XSC_DMA_MAP_PAGE); + num_dma++; + dseg++; + } + + return num_dma; + +dma_unmap_wqe_err: + xsc_dma_unmap_wqe_err(sq, num_dma); + return -ENOMEM; +} + +static inline bool xsc_wqc_has_room_for(struct xsc_wq_cyc *wq, + u16 cc, u16 pc, u16 n) +{ + return (xsc_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc); +} + +static inline void xsc_sq_notify_hw(struct xsc_wq_cyc *wq, u16 pc, + struct xsc_sq *sq) +{ + struct xsc_adapter *adapter = sq->channel->adapter; + struct xsc_core_device *xdev = adapter->xdev; + union xsc_send_doorbell doorbell_value; + int send_ds_num_log = ilog2(xdev->caps.send_ds_num); + + /*reverse wqe index to ds index*/ + doorbell_value.next_pid = pc << send_ds_num_log; + doorbell_value.qp_num = sq->sqn; + + /* keep order */ + wmb(); + ETH_DEBUG_LOG("pc = %d sqn = %d\n", pc, sq->sqn); + ETH_DEBUG_LOG("doorbell_value = %#x\n", doorbell_value.send_data); + writel(doorbell_value.send_data, REG_ADDR(xdev, xdev->regs.tx_db)); +} + +void xsc_txwqe_complete(struct xsc_sq *sq, struct sk_buff *skb, + u8 opcode, u16 ds_cnt, u8 num_wqebbs, u32 num_bytes, u8 num_dma, + struct xsc_tx_wqe_info *wi) +{ + struct xsc_wq_cyc *wq = &sq->wq; + + wi->num_bytes = num_bytes; + wi->num_dma = num_dma; + wi->num_wqebbs = num_wqebbs; + wi->skb = skb; + +#ifdef XSC_BQL_SUPPORT + ETH_SQ_STATE(sq); + netdev_tx_sent_queue(sq->txq, num_bytes); + ETH_SQ_STATE(sq); +#endif + + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + ETH_DEBUG_LOG("%s\n", "hw tstamp\n"); + } + + /*1*/ + sq->pc += wi->num_wqebbs; + ETH_DEBUG_LOG("%d\n", sq->pc); + + if (unlikely(!xsc_wqc_has_room_for(wq, sq->cc, sq->pc, sq->stop_room))) { + netif_tx_stop_queue(sq->txq); + sq->stats->stopped++; + ETH_DEBUG_LOG("%p %d %d %d\n", wq, sq->cc, sq->pc, sq->stop_room); + } + + ETH_DEBUG_LOG("%d %d\n", xsc_netdev_xmit_more(skb), netif_xmit_stopped(sq->txq)); + + if (!xsc_netdev_xmit_more(skb) || netif_xmit_stopped(sq->txq)) + xsc_sq_notify_hw(wq, sq->pc, sq); +} + +static void xsc_dump_error_sqcqe(struct xsc_sq *sq, + struct xsc_cqe64 *cqe) +{ + u32 ci = xsc_cqwq_get_ci(&sq->cq.wq); + struct net_device *netdev = sq->channel->netdev; + + net_err_ratelimited( + "Error cqe on dev %s, cqn 0x%x, ci 0x%x, sqn 0x%x, error_code 0x%x, qpid 0x%x\n", + netdev->name, sq->cq.xcq.cqn, ci, + sq->sqn, get_cqe_opcode(cqe), cqe->qp_id); + +#ifdef XSC_DEBUG + xsc_dump_err_cqe(sq->cq.xdev, cqe); +#endif +} + +void xsc_free_tx_wqe(struct device *dev, struct xsc_sq *sq) +{ + struct xsc_tx_wqe_info *wi; + struct sk_buff *skb; + u16 ci, npkts = 0; + u32 nbytes = 0; + int i; + + while (sq->cc != sq->pc) { + ci = xsc_wq_cyc_ctr2ix(&sq->wq, sq->cc); + wi = &sq->db.wqe_info[ci]; + skb = wi->skb; + + if (!skb) { /* nop */ + sq->cc++; + continue; + } + + for (i = 0; i < wi->num_dma; i++) { + struct xsc_sq_dma *dma = + xsc_dma_get(sq, sq->dma_fifo_cc++); + + xsc_tx_dma_unmap(dev, dma); + } + + dev_kfree_skb_any(skb); + npkts++; + nbytes += wi->num_bytes; + sq->cc += wi->num_wqebbs; + } + + netdev_tx_completed_queue(sq->txq, npkts, nbytes); +} + +#ifdef NEED_CREATE_RX_THREAD + DECLARE_PER_CPU(bool, txcqe_get); +#endif + +bool xsc_poll_tx_cq(struct xsc_cq *cq, int napi_budget) +{ + struct xsc_adapter *adapter; + struct device *dev; + struct xsc_sq_stats *stats; + struct xsc_sq *sq; + struct xsc_cqe64 *cqe; + u32 dma_fifo_cc; + u32 nbytes = 0; + u16 npkts = 0; + u16 sqcc; + int i = 0; + + sq = container_of(cq, struct xsc_sq, cq); + if (!test_bit(XSC_ETH_SQ_STATE_ENABLED, &sq->state)) + return false; + + adapter = sq->channel->adapter; + dev = adapter->dev; + + cqe = xsc_cqwq_get_cqe(&cq->wq); + if (!cqe) + goto out; + + stats = sq->stats; + + if (unlikely(get_cqe_opcode(cqe) & BIT(7))) { + xsc_dump_error_sqcqe(sq, cqe); + stats->cqe_err++; + return false; + } + +#ifdef NEED_CREATE_RX_THREAD + __this_cpu_write(txcqe_get, true); +#endif + + sqcc = sq->cc; + + /* avoid dirtying sq cache line every cqe */ + dma_fifo_cc = sq->dma_fifo_cc; + i = 0; + do { + struct xsc_tx_wqe_info *wi; + struct sk_buff *skb; + int j; + u16 ci; + + xsc_cqwq_pop(&cq->wq); + + ci = xsc_wq_cyc_ctr2ix(&sq->wq, sqcc); + wi = &sq->db.wqe_info[ci]; + skb = wi->skb; + + /*cqe may be overstanding in real test, not by nop in other*/ + if (unlikely(!skb)) { + stats->txdone_skb_null++; + continue; + } + + for (j = 0; j < wi->num_dma; j++) { + struct xsc_sq_dma *dma = xsc_dma_get(sq, dma_fifo_cc++); + + xsc_tx_dma_unmap(dev, dma); + } + + npkts++; + nbytes += wi->num_bytes; + sqcc += wi->num_wqebbs; + +#ifndef NEED_CREATE_RX_THREAD + napi_consume_skb(skb, napi_budget); +#else + if (refcount_read(&skb->users) < 1) + stats->txdone_skb_refcnt_err++; + + napi_consume_skb(skb, 0); +#endif + ETH_DEBUG_LOG("ci=%d, sqcc=%d, pkts=%d\n", ci, sqcc, npkts); + + } while ((++i <= napi_budget) && (cqe = xsc_cqwq_get_cqe(&cq->wq))); + + stats->cqes += i; + + xsc_cq_notify_hw(cq); + + /* ensure cq space is freed before enabling more cqes */ + wmb(); + + sq->dma_fifo_cc = dma_fifo_cc; + sq->cc = sqcc; + ETH_DEBUG_LOG("dma_fifo_cc=%d, sqcc=%d\n", dma_fifo_cc, sqcc); + +#ifdef XSC_BQL_SUPPORT + ETH_SQ_STATE(sq); + netdev_tx_completed_queue(sq->txq, npkts, nbytes); + ETH_SQ_STATE(sq); +#endif + + if (netif_tx_queue_stopped(sq->txq) && + xsc_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room)) { + netif_tx_wake_queue(sq->txq); + stats->wake++; + } + +out: + return (i == napi_budget); +} + +static u32 xsc_eth_xmit_frame(struct sk_buff *skb, + struct xsc_sq *sq, + struct xsc_tx_wqe *wqe, + u16 pi) +{ + struct xsc_send_wqe_ctrl_seg *cseg; + struct xsc_wqe_data_seg *dseg; + struct xsc_tx_wqe_info *wi; + struct xsc_sq_stats *stats = sq->stats; + struct xsc_core_device *xdev = sq->cq.xdev; + u16 ds_cnt; + u16 mss, ihs, headlen; + u8 opcode; + u32 num_bytes, num_dma; + u8 num_wqebbs; + +retry_send: + /* Calc ihs and ds cnt, no writes to wqe yet */ + /*ctrl-ds, it would be reduce in ds_data_num*/ + ds_cnt = XSC_SEND_WQEBB_CTRL_NUM_DS; + + /*in andes inline is bonding with gso*/ + if (skb_is_gso(skb)) { + opcode = XSC_OPCODE_RAW; + mss = skb_shinfo(skb)->gso_size; + ihs = xsc_tx_get_gso_ihs(sq, skb); + num_bytes = skb->len; + stats->packets += skb_shinfo(skb)->gso_segs; + } else { + opcode = XSC_OPCODE_RAW; + mss = 0; + ihs = 0; + num_bytes = skb->len; + stats->packets++; + } + + /*linear data in skb*/ + headlen = skb->len - skb->data_len; + ds_cnt += !!headlen; + ds_cnt += skb_shinfo(skb)->nr_frags; + ETH_DEBUG_LOG( + "skb_len=%d, data_len=%d, nr_frags=%d, mss=%d, ihs=%d, headlen=%d, ds_cnt=%d\n", + skb->len, skb->data_len, skb_shinfo(skb)->nr_frags, + mss, ihs, headlen, ds_cnt); + + /*to make the connection, only linear data is present*/ + skbdata_debug_dump(skb, headlen, 1); + + /* Check packet size. */ + if (unlikely(mss == 0 && num_bytes > sq->hw_mtu)) { + sq->stats->oversize_pkts_sw_drop++; + goto err_drop; + } + + num_wqebbs = DIV_ROUND_UP(ds_cnt, xdev->caps.send_ds_num); + /*if ds_cnt exceed one wqe, drop it*/ + if (num_wqebbs != 1) { + sq->stats->skb_linear++; + if (skb_linearize(skb)) + goto err_drop; + goto retry_send; + } + + /* fill wqe */ + wi = (struct xsc_tx_wqe_info *)&sq->db.wqe_info[pi]; + cseg = &wqe->ctrl; + dseg = &wqe->data[0]; + + xsc_txwqe_build_csegs(sq, skb, mss, ihs, headlen, + opcode, ds_cnt, num_bytes, cseg); + + /*inline header is also use dma to transport*/ + num_dma = xsc_txwqe_build_dsegs(sq, skb, ihs, headlen, dseg); + if (unlikely(num_dma < 0)) + goto err_drop; + + xsc_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes, + num_dma, wi); + + stats->bytes += num_bytes; + stats->xmit_more += xsc_netdev_xmit_more(skb); + return NETDEV_TX_OK; + +err_drop: + ETH_DEBUG_LOG("%s: drop skb, ds_cnt=%d, num_wqebbs=%d, num_dma=%d\n", + __func__, ds_cnt, num_wqebbs, num_dma); + stats->dropped++; + dev_kfree_skb_any(skb); + + return NETDEV_TX_OK; +} + +netdev_tx_t xsc_eth_xmit_start(struct sk_buff *skb, struct net_device *netdev) +{ + u32 ret; + u32 queue_id; + struct xsc_sq *sq; + struct xsc_tx_wqe *wqe; + u16 pi; + struct xsc_adapter *adapter = netdev_priv(netdev); + struct xsc_core_device *xdev = adapter->xdev; + + if (!skb) { + ETH_DEBUG_LOG("skb == NULL\n"); + return NETDEV_TX_OK; + } + + if (!adapter) { + ETH_DEBUG_LOG("adapter == NULL\n"); + return NETDEV_TX_BUSY; + } + + if (adapter->status != XSCALE_ETH_DRIVER_OK) { + ETH_DEBUG_LOG("adapter->status = %d\n", adapter->status); + return NETDEV_TX_BUSY; + } + + queue_id = skb_get_queue_mapping(skb); + ETH_DEBUG_LOG("queue_id = %d\n", queue_id); + assert(adapter->xdev, queue_id < XSC_ETH_MAX_TC_TOTAL); + + sq = adapter->txq2sq[queue_id]; + if (!sq) { + ETH_DEBUG_LOG("sq = NULL\n"); + return NETDEV_TX_BUSY; + } + ETH_DEBUG_LOG("sqn = %d\n", sq->sqn); + + wqe = xsc_sq_fetch_wqe(sq, xdev->caps.send_ds_num * XSC_SEND_WQE_DS, &pi); + ETH_DEBUG_LOG("wqe = %p pi = %d\n", wqe, pi); + assert(adapter->xdev, wqe); + +#ifndef ANDES_DRIVER + skb = xsc_accel_handle_tx(skb); +#endif + + ret = xsc_eth_xmit_frame(skb, sq, wqe, pi); + + ETH_DEBUG_LOG("ret = %d\n", ret); + + return ret; +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_txrx.c b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_txrx.c new file mode 100644 index 0000000000000000000000000000000000000000..1454ebbca89be2aa379f4bc4f778b105b61d1cdc --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_txrx.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include "xsc_eth_common.h" +#include "xsc_eth_stats.h" +#include "xsc_eth_txrx.h" + +void xsc_cq_notify_hw_rearm(struct xsc_cq *cq) +{ + union xsc_cq_doorbell db; + + ETH_DEBUG_LOG("cc = %d cqn = %d\n", cq->wq.cc, cq->xcq.cqn); + + db.val = 0; + db.cq_next_cid = cpu_to_le32(cq->wq.cc); + db.cq_id = cpu_to_le32(cq->xcq.cqn); + db.arm = 0; + + /* keep order */ + wmb(); + writel(db.val, REG_ADDR(cq->xdev, cq->xdev->regs.complete_db)); + if (cq->channel && cq->channel->stats) + cq->channel->stats->arm++; +} + +void xsc_cq_notify_hw(struct xsc_cq *cq) +{ + struct xsc_core_device *xdev = cq->xdev; + union xsc_cq_doorbell db; + + ETH_DEBUG_LOG("cc = %d cqn = %d\n", cq->wq.cc, cq->xcq.cqn); + + dma_wmb(); + + db.val = 0; + db.cq_next_cid = cpu_to_le32(cq->wq.cc); + db.cq_id = cpu_to_le32(cq->xcq.cqn); + + /* ensure doorbell record is visible to device before ringing the doorbell*/ +// wmb(); + writel(db.val, REG_ADDR(xdev, xdev->regs.complete_reg)); + if (cq->channel && cq->channel->stats) + cq->channel->stats->noarm++; +} + +static inline bool xsc_channel_no_affinity_change(struct xsc_channel *c) +{ + int current_cpu = smp_processor_id(); + + return cpumask_test_cpu(current_cpu, c->aff_mask); +} + +int xsc_eth_napi_poll(struct napi_struct *napi, int budget) +{ + struct xsc_channel *c = container_of(napi, struct xsc_channel, napi); + struct xsc_rq *rq = &c->qp.rq[0]; + bool busy = false; + int work_done = 0; + int i; + + rcu_read_lock(); + + clear_bit(XSC_CHANNEL_NAPI_SCHED, &c->flags); + + for (i = 0; i < c->num_tc; i++) + busy |= xsc_poll_tx_cq(&c->qp.sq[i].cq, budget); + + /* budget=0 means: don't poll rx rings */ + if (likely(budget)) { + work_done = xsc_poll_rx_cq(&rq->cq, budget); + busy |= work_done == budget; + } + + if (work_done < budget) { + if (unlikely(!napi_complete_done(napi, work_done))) + goto out; + + for (i = 0; i < c->num_tc; i++) + xsc_cq_notify_hw_rearm(&c->qp.sq[i].cq); + + xsc_cq_notify_hw_rearm(&rq->cq); + } + +out: + rcu_read_unlock(); + return work_done; +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_txrx.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_txrx.h new file mode 100644 index 0000000000000000000000000000000000000000..8c5df22db76546455d6948340b48d1da7e518a60 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_eth_txrx.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_RXTX_H +#define XSC_RXTX_H + +#include "xsc_eth.h" +#include +#include "xsc_eth_debug.h" + +enum { + XSC_ETH_WQE_NONE_CSUM, + XSC_ETH_WQE_INNER_CSUM, + XSC_ETH_WQE_OUTER_CSUM, + XSC_ETH_WQE_INNER_AND_OUTER_CSUM, +}; + +#define ANDES_DRIVER + +static inline u32 xsc_cqwq_get_size(struct xsc_cqwq *wq) +{ + return wq->fbc.sz_m1 + 1; +} + +static inline struct xsc_cqe64 *xsc_cqwq_get_wqe(struct xsc_cqwq *wq, u32 ix) +{ + struct xsc_cqe64 *cqe = xsc_frag_buf_get_wqe(&wq->fbc, ix); + + ETH_DEBUG_LOG("cqe = %p\n", cqe); + + return cqe; +} + +static inline struct xsc_cqe64 *xsc_cqwq_get_cqe(struct xsc_cqwq *wq) +{ + struct xsc_cqe64 *cqe; + u8 cqe_ownership_bit; + u8 sw_ownership_val; + u32 ci = xsc_cqwq_get_ci(wq); + + cqe = xsc_cqwq_get_wqe(wq, ci); + + cqe_ownership_bit = cqe->owner & XSC_CQE_OWNER_MASK; + sw_ownership_val = xsc_cqwq_get_wrap_cnt(wq) & 1; + ETH_DEBUG_LOG("ci=%d, cqe_owner=%d, sw_owner=%d\n", + ci, cqe_ownership_bit, sw_ownership_val); + + if (cqe_ownership_bit != sw_ownership_val) + return NULL; + + /* ensure cqe content is read after cqe ownership bit */ + dma_rmb(); + + return cqe; +} + +void xsc_free_tx_wqe(struct device *dev, struct xsc_sq *sq); +int xsc_eth_napi_poll(struct napi_struct *napi, int budget); +bool xsc_poll_tx_cq(struct xsc_cq *cq, int napi_budget); +int xsc_poll_rx_cq(struct xsc_cq *cq, int budget); +void xsc_eth_handle_rx_cqe(struct xsc_cqwq *cqwq, + struct xsc_rq *rq, struct xsc_cqe64 *cqe); +struct sk_buff *xsc_skb_from_cqe_nonlinear(struct xsc_rq *rq, + struct xsc_wqe_frag_info *wi, + u32 cqe_bcnt, u8 has_pph); +bool xsc_eth_post_rx_wqes(struct xsc_rq *rq); +void xsc_cq_notify_hw(struct xsc_cq *cq); +void xsc_cq_notify_hw_rearm(struct xsc_cq *cq); +void xsc_eth_free_rx_wqes(struct xsc_rq *rq); +netdev_tx_t xsc_eth_xmit_start(struct sk_buff *skb, struct net_device *netdev); + +void xsc_page_release_dynamic(struct xsc_rq *rq, + struct xsc_dma_info *dma_info, + bool recycle); + +#endif /* XSC_RXTX_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/net/xsc_queue.h b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_queue.h new file mode 100644 index 0000000000000000000000000000000000000000..e9a5ef745b4eb790b46f72c0460a99afd8e18159 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/net/xsc_queue.h @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_QUEUE_H +#define XSC_QUEUE_H + +#include + +#include "../pci/wq.h" + +enum { + XSC_SEND_WQE_DS = 16, + XSC_SEND_WQE_BB = 64, +}; + +enum { + XSC_RECV_WQE_DS = 16, + XSC_RECV_WQE_BB = 16, +}; + +#define XSC_SEND_WQEBB_NUM_DS (XSC_SEND_WQE_BB / XSC_SEND_WQE_DS) +#define XSC_LOG_SEND_WQEBB_NUM_DS ilog2(XSC_SEND_WQEBB_NUM_DS) + +#define XSC_RECV_WQEBB_NUM_DS (XSC_RECV_WQE_BB / XSC_RECV_WQE_DS) +#define XSC_LOG_RECV_WQEBB_NUM_DS ilog2(XSC_RECV_WQEBB_NUM_DS) + +#define XSC_SEND_WQEBB_CTRL_NUM_DS 1 + +enum { + XSC_ETH_RQ_STATE_ENABLED, +}; + +enum { + XSC_ETH_SQ_STATE_ENABLED, + XSC_ETH_RQ_STATE_CACHE_REDUCE_PENDING, +}; + +struct xsc_dma_info { + struct page *page; + dma_addr_t addr; + u32 refcnt_bias; +}; + +struct xsc_wqe_frag_info { + struct xsc_dma_info *di; + u32 offset; + bool last_in_page; +}; + +struct xsc_rq_frag_info { + int frag_size; + int frag_stride; +}; + +struct xsc_rq_frags_info { + struct xsc_rq_frag_info arr[XSC_MAX_RX_FRAGS]; + u8 num_frags; + u8 log_num_frags; + u8 wqe_bulk; +}; + +struct xsc_cq { + /* data path - accessed per cqe */ + struct xsc_cqwq wq; + + /* data path - accessed per napi poll */ + u16 event_ctr; + struct napi_struct *napi; + struct xsc_core_cq xcq; + struct xsc_channel *channel; + + /* control */ + struct xsc_core_device *xdev; + struct xsc_wq_ctrl wq_ctrl; +} ____cacheline_aligned_in_smp; + +#define XSC_PAGE_CACHE_LOG_MAX_RQ_MULT 4 +#define XSC_PAGE_CACHE_REDUCE_WORK_INTERVAL 200 /* msecs */ +#define XSC_PAGE_CACHE_REDUCE_GRACE_PERIOD 1000 /* msecs */ +#define XSC_PAGE_CACHE_REDUCE_SUCCESS_CNT 20 + +struct xsc_page_cache_reduce { + struct delayed_work reduce_work; + u32 success; + unsigned long next_ts; + unsigned long grace_period; + unsigned long delay; + struct xsc_dma_info *pending; + u32 npages; +}; + +struct xsc_page_cache { + struct xsc_dma_info *page_cache; + int head; + u32 sz; + u32 lrs; + u8 log_min_sz; + u8 log_max_sz; + struct xsc_page_cache_reduce reduce; +}; + +struct xsc_rq; +struct xsc_cqe64; +typedef void (*xsc_fp_handle_rx_cqe)(struct xsc_cqwq *cqwq, struct xsc_rq *rq, + struct xsc_cqe64 *cqe64); +typedef bool (*xsc_fp_post_rx_wqes)(struct xsc_rq *rq); +typedef void (*xsc_fp_dealloc_wqe)(struct xsc_rq *rq); +typedef struct sk_buff * (*xsc_fp_skb_from_cqe)(struct xsc_rq *rq, + struct xsc_wqe_frag_info *wi, u32 cqe_bcnt, u8 has_pph); + +struct xsc_rq { + struct xsc_core_qp cqp; + struct { + struct xsc_wq_cyc wq; + struct xsc_wqe_frag_info *frags; + struct xsc_dma_info *di; + struct xsc_rq_frags_info info; + xsc_fp_skb_from_cqe skb_from_cqe; + } wqe; + + struct { + u16 headroom; + u8 map_dir; /* dma map direction */ + } buff; + + struct page_pool *page_pool; + struct xsc_wq_ctrl wq_ctrl; + struct xsc_cq cq; + u32 rqn; + int ix; + + unsigned long state; + struct work_struct recover_work; + struct xsc_rq_stats *stats; + + u32 hw_mtu; + u8 frags_reuse_num; + + xsc_fp_handle_rx_cqe handle_rx_cqe; + xsc_fp_post_rx_wqes post_wqes; + xsc_fp_dealloc_wqe dealloc_wqes; + struct xsc_page_cache page_cache; +} ____cacheline_aligned_in_smp; + +struct xsc_tx_wqe_info { + struct sk_buff *skb; + u32 num_bytes; + u8 num_wqebbs; + u8 num_dma; +}; + +enum xsc_dma_map_type { + XSC_DMA_MAP_SINGLE, + XSC_DMA_MAP_PAGE +}; + +struct xsc_sq_dma { + dma_addr_t addr; + u32 size; + enum xsc_dma_map_type type; +}; + +struct xsc_sq { + struct xsc_core_qp cqp; + /* dirtied @completion */ + u16 cc; + u32 dma_fifo_cc; + /* dirtied @xmit */ + u16 pc ____cacheline_aligned_in_smp; + u32 dma_fifo_pc; + + struct xsc_cq cq; + + /* read only */ + struct xsc_wq_cyc wq; + u32 dma_fifo_mask; + struct xsc_sq_stats *stats; + struct { + struct xsc_sq_dma *dma_fifo; + struct xsc_tx_wqe_info *wqe_info; + } db; + void __iomem *uar_map; + struct netdev_queue *txq; + u32 sqn; + u16 stop_room; + + __be32 mkey_be; + unsigned long state; + unsigned int hw_mtu; + + /* control path */ + struct xsc_wq_ctrl wq_ctrl; + struct xsc_channel *channel; + int ch_ix; + int txq_ix; + struct work_struct recover_work; +} ____cacheline_aligned_in_smp; + +struct rdma_opcode_data { + u32 immdt_value; +} __packed __aligned(4); + +struct raw_opcode_data { + u16 has_pph : 1; + u16 so_type : 1; + u16 so_data_size : 14; + u8 rsv; + u8 so_hdr_len; +} __packed __aligned(4); + +struct rawtype_opcode_data { + u16 desc_id; + u16 is_last_wqe : 1; + u16 dst_qp_id : 15; +} __packed __aligned(4); + +struct xsc_wqe_ctrl_seg { + u8 msg_opcode; + u8 with_immdt : 1; + u8 csum_en : 2; + u8 ds_data_num : 5; + u16 wqe_id; + u32 msg_len; + union { + struct rdma_opcode_data _rdma_opcode_data; + struct raw_opcode_data _raw_opcode_data; + struct rawtype_opcode_data _rawtype_opcode_data; + } opcode_data; + u32 se : 1; + u32 ce : 1; + u32 rsv : 30; +}; + +static inline u8 get_cqe_opcode(struct xsc_cqe64 *cqe) +{ + return cqe->msg_opcode; +} + +static inline void xsc_dump_err_cqe(struct xsc_core_device *dev, + struct xsc_cqe64 *cqe) +{ + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, 16, 1, cqe, + sizeof(*cqe), false); +} + +#endif /* XSC_QUEUE_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/Kconfig b/drivers/net/ethernet/yunsilicon/xsc/pci/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..ce0ed40b204b0a30bd27ca157bddcb21a637d197 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. +# All rights reserved. +# Yunsilicon PCI configuration +# + +config YUNSILICON_XSC_PCI + tristate "Yunsilicon XSC PCI driver" + default n + select NET_DEVLINK + select PAGE_POOL + help + This driver is common for Yunsilicon XSC + ethernet and RDMA drivers. diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/Makefile b/drivers/net/ethernet/yunsilicon/xsc/pci/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..74258e8d41d77a5993b4476c91fc63cd82c05419 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/Makefile @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. +# All rights reserved. +# +# Makefile for the Yunsilicon xsc ethernet driver +# +ccflags-y := -I $(srctree)/drivers/net/ethernet/yunsilicon/xsc/ +ccflags-y += -Wno-implicit-fallthrough + +obj-$(CONFIG_YUNSILICON_XSC_PCI) += xsc_pci.o + +xsc_pci-y := main.o eq.o intf.o debugfs.o alloc.o wq.o cq.o qp.o \ + cmd2.o fw.o pagealloc.o port.o mr.o pd.o mad.o xsc_lag.o xsc_pci_ctrl.o\ + pci_irq.o vport.o sriov.o sriov_sysfs.o devlink.o eswitch.o xsc_port_ctrl.o \ + fw/cmd.o \ + fw/xsc_flow.o \ + fw/xsc_res.o \ + fw/osdep.o \ + fw/xsc_mem.o diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/alloc.c b/drivers/net/ethernet/yunsilicon/xsc/pci/alloc.c new file mode 100644 index 0000000000000000000000000000000000000000..093e6438672d9f70b1dbe8f65c7507b696c4b95d --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/alloc.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Handling for queue buffers -- we allocate a bunch of memory and + * register it in a memory region at HCA virtual address 0. If the + * requested size is > max_direct, we split the allocation into + * multiple pages, so we don't require too much contiguous memory. + */ + +int xsc_buf_alloc(struct xsc_core_device *xdev, int size, int max_direct, + struct xsc_buf *buf) +{ + dma_addr_t t; + + buf->size = size; + if (size <= max_direct) { + buf->nbufs = 1; + buf->npages = 1; + buf->page_shift = get_order(size) + PAGE_SHIFT; + buf->direct.buf = dma_alloc_coherent(&xdev->pdev->dev, + size, &t, GFP_KERNEL | __GFP_ZERO); + if (!buf->direct.buf) + return -ENOMEM; + + buf->direct.map = t; + + while (t & ((1 << buf->page_shift) - 1)) { + --buf->page_shift; + buf->npages *= 2; + } + } else { + int i; + + buf->direct.buf = NULL; + buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; + buf->npages = buf->nbufs; + buf->page_shift = PAGE_SHIFT; + buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), + GFP_KERNEL); + if (!buf->page_list) + return -ENOMEM; + + for (i = 0; i < buf->nbufs; i++) { + buf->page_list[i].buf = + dma_alloc_coherent(&xdev->pdev->dev, PAGE_SIZE, + &t, GFP_KERNEL | __GFP_ZERO); + if (!buf->page_list[i].buf) + goto err_free; + + buf->page_list[i].map = t; + } + + if (BITS_PER_LONG == 64) { + struct page **pages; + + pages = kmalloc_array(buf->nbufs, sizeof(*pages), GFP_KERNEL); + if (!pages) + goto err_free; + for (i = 0; i < buf->nbufs; i++) + pages[i] = virt_to_page(buf->page_list[i].buf); + buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); + kfree(pages); + if (!buf->direct.buf) + goto err_free; + } + } + + return 0; + +err_free: + xsc_buf_free(xdev, buf); + + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(xsc_buf_alloc); + +void xsc_buf_free(struct xsc_core_device *xdev, struct xsc_buf *buf) +{ + int i; + + if (buf->nbufs == 1) + dma_free_coherent(&xdev->pdev->dev, buf->size, buf->direct.buf, + buf->direct.map); + else { + if (BITS_PER_LONG == 64 && buf->direct.buf) + vunmap(buf->direct.buf); + + for (i = 0; i < buf->nbufs; i++) + if (buf->page_list[i].buf) + dma_free_coherent(&xdev->pdev->dev, PAGE_SIZE, + buf->page_list[i].buf, + buf->page_list[i].map); + kfree(buf->page_list); + } +} +EXPORT_SYMBOL_GPL(xsc_buf_free); + +void xsc_fill_page_array(struct xsc_buf *buf, __be64 *pas, int npages) +{ + u64 addr; + int i; + int shift = PAGE_SHIFT - PAGE_SHIFT_4K; + int mask = (1 << shift) - 1; + + for (i = 0; i < npages; i++) { + if (buf->nbufs == 1) + addr = buf->direct.map + (i << PAGE_SHIFT_4K); + else + addr = buf->page_list[i >> shift].map + ((i & mask) << PAGE_SHIFT_4K); + + pas[i] = cpu_to_be64(addr); + } +} +EXPORT_SYMBOL_GPL(xsc_fill_page_array); + +void xsc_fill_page_frag_array(struct xsc_frag_buf *buf, __be64 *pas, int npages) +{ + int i; + dma_addr_t addr; + int shift = PAGE_SHIFT - PAGE_SHIFT_4K; + int mask = (1 << shift) - 1; + + for (i = 0; i < npages; i++) { + addr = buf->frags[i >> shift].map + ((i & mask) << PAGE_SHIFT_4K); + pas[i] = cpu_to_be64(addr); + } +} +EXPORT_SYMBOL_GPL(xsc_fill_page_frag_array); + +static void *xsc_dma_zalloc_coherent_node(struct xsc_core_device *xdev, + size_t size, dma_addr_t *dma_handle, + int node) +{ + struct xsc_dev_resource *dev_res = xdev->dev_res; + struct device *device = &xdev->pdev->dev; + int original_node; + void *cpu_handle; + + /* WA for kernels that don't use numa_mem_id in alloc_pages_node */ + if (node == NUMA_NO_NODE) +#ifdef HAVE_NUMA_MEM_ID + node = numa_mem_id(); +#else + node = first_memory_node; +#endif + + mutex_lock(&dev_res->alloc_mutex); + original_node = dev_to_node(device); + set_dev_node(device, node); + cpu_handle = dma_alloc_coherent(device, size, dma_handle, + GFP_KERNEL); + set_dev_node(device, original_node); + mutex_unlock(&dev_res->alloc_mutex); + return cpu_handle; +} + +int xsc_frag_buf_alloc_node(struct xsc_core_device *xdev, int size, + struct xsc_frag_buf *buf, int node) +{ + int i; + + buf->size = size; + buf->npages = DIV_ROUND_UP(size, PAGE_SIZE); + buf->page_shift = PAGE_SHIFT; + buf->frags = kcalloc(buf->npages, sizeof(struct xsc_buf_list), + GFP_KERNEL); + if (!buf->frags) + goto err_out; + + for (i = 0; i < buf->npages; i++) { + struct xsc_buf_list *frag = &buf->frags[i]; + int frag_sz = min_t(int, size, PAGE_SIZE); + + frag->buf = xsc_dma_zalloc_coherent_node(xdev, frag_sz, + &frag->map, node); + if (!frag->buf) + goto err_free_buf; + if (frag->map & ((1 << buf->page_shift) - 1)) { + dma_free_coherent(&xdev->pdev->dev, frag_sz, + buf->frags[i].buf, buf->frags[i].map); + xsc_core_warn(xdev, "unexpected map alignment: %pad, page_shift=%d\n", + &frag->map, buf->page_shift); + goto err_free_buf; + } + size -= frag_sz; + } + + return 0; + +err_free_buf: + while (i--) + dma_free_coherent(&xdev->pdev->dev, PAGE_SIZE, buf->frags[i].buf, + buf->frags[i].map); + kfree(buf->frags); +err_out: + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(xsc_frag_buf_alloc_node); + +void xsc_frag_buf_free(struct xsc_core_device *xdev, struct xsc_frag_buf *buf) +{ + int size = buf->size; + int i; + + for (i = 0; i < buf->npages; i++) { + int frag_sz = min_t(int, size, PAGE_SIZE); + + dma_free_coherent(&xdev->pdev->dev, frag_sz, buf->frags[i].buf, + buf->frags[i].map); + size -= frag_sz; + } + kfree(buf->frags); +} +EXPORT_SYMBOL_GPL(xsc_frag_buf_free); + +static struct xsc_db_pgdir *xsc_alloc_db_pgdir(struct xsc_core_device *xdev, + int node) +{ + u32 db_per_page = PAGE_SIZE / cache_line_size(); + struct xsc_db_pgdir *pgdir; + + pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); + if (!pgdir) + return NULL; + + pgdir->bitmap = bitmap_zalloc(db_per_page, GFP_KERNEL); + if (!pgdir->bitmap) { + kfree(pgdir); + return NULL; + } + + bitmap_fill(pgdir->bitmap, db_per_page); + + pgdir->db_page = xsc_dma_zalloc_coherent_node(xdev, PAGE_SIZE, + &pgdir->db_dma, node); + if (!pgdir->db_page) { + bitmap_free(pgdir->bitmap); + kfree(pgdir); + return NULL; + } + + return pgdir; +} + +static int xsc_alloc_db_from_pgdir(struct xsc_db_pgdir *pgdir, + struct xsc_db *db) +{ + u32 db_per_page = PAGE_SIZE / cache_line_size(); + int offset; + int i; + + i = find_first_bit(pgdir->bitmap, db_per_page); + if (i >= db_per_page) + return -ENOMEM; + + __clear_bit(i, pgdir->bitmap); + + db->u.pgdir = pgdir; + db->index = i; + offset = db->index * cache_line_size(); + db->db = pgdir->db_page + offset / sizeof(*pgdir->db_page); + db->dma = pgdir->db_dma + offset; + + db->db[0] = 0; + db->db[1] = 0; + + return 0; +} + +int xsc_db_alloc_node(struct xsc_core_device *xdev, struct xsc_db *db, int node) +{ + struct xsc_db_pgdir *pgdir; + int ret = 0; + + mutex_lock(&xdev->dev_res->pgdir_mutex); + + list_for_each_entry(pgdir, &xdev->dev_res->pgdir_list, list) + if (!xsc_alloc_db_from_pgdir(pgdir, db)) + goto out; + + pgdir = xsc_alloc_db_pgdir(xdev, node); + if (!pgdir) { + ret = -ENOMEM; + goto out; + } + + list_add(&pgdir->list, &xdev->dev_res->pgdir_list); + + /* This should never fail -- we just allocated an empty page: */ + WARN_ON(xsc_alloc_db_from_pgdir(pgdir, db)); + +out: + mutex_unlock(&xdev->dev_res->pgdir_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(xsc_db_alloc_node); + +int xsc_db_alloc(struct xsc_core_device *xdev, struct xsc_db *db) +{ + return xsc_db_alloc_node(xdev, db, xdev->priv.numa_node); +} +EXPORT_SYMBOL_GPL(xsc_db_alloc); + +void xsc_db_free(struct xsc_core_device *xdev, struct xsc_db *db) +{ + u32 db_per_page = PAGE_SIZE / cache_line_size(); + + mutex_lock(&xdev->dev_res->pgdir_mutex); + + __set_bit(db->index, db->u.pgdir->bitmap); + + if (bitmap_full(db->u.pgdir->bitmap, db_per_page)) { + dma_free_coherent(&xdev->pdev->dev, PAGE_SIZE, + db->u.pgdir->db_page, db->u.pgdir->db_dma); + list_del(&db->u.pgdir->list); + bitmap_free(db->u.pgdir->bitmap); + kfree(db->u.pgdir); + } + + mutex_unlock(&xdev->dev_res->pgdir_mutex); +} +EXPORT_SYMBOL_GPL(xsc_db_free); + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/cmd2.c b/drivers/net/ethernet/yunsilicon/xsc/pci/cmd2.c new file mode 100644 index 0000000000000000000000000000000000000000..bb395daf7b36fd952f6a0244a3c9725121d7711f --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/cmd2.c @@ -0,0 +1,2022 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tmp_cmdq_defines.h" +#include + +enum { + CMD_IF_REV = 3, +}; + +enum { + CMD_MODE_POLLING, + CMD_MODE_EVENTS +}; + +enum { + NUM_LONG_LISTS = 2, + NUM_MED_LISTS = 64, + LONG_LIST_SIZE = (2ULL * 1024 * 1024 * 1024 / PAGE_SIZE) * 8 + 16 + + XSC_CMD_DATA_BLOCK_SIZE, + MED_LIST_SIZE = 16 + XSC_CMD_DATA_BLOCK_SIZE, +}; + +enum { + XSC_CMD_DELIVERY_STAT_OK = 0x0, + XSC_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1, + XSC_CMD_DELIVERY_STAT_TOK_ERR = 0x2, + XSC_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3, + XSC_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4, + XSC_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5, + XSC_CMD_DELIVERY_STAT_FW_ERR = 0x6, + XSC_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7, + XSC_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8, + XSC_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9, + XSC_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10, +}; + +enum { + XSC_CMD_STAT_OK = 0x0, + XSC_CMD_STAT_INT_ERR = 0x1, + XSC_CMD_STAT_BAD_OP_ERR = 0x2, + XSC_CMD_STAT_BAD_PARAM_ERR = 0x3, + XSC_CMD_STAT_BAD_SYS_STATE_ERR = 0x4, + XSC_CMD_STAT_BAD_RES_ERR = 0x5, + XSC_CMD_STAT_RES_BUSY = 0x6, + XSC_CMD_STAT_LIM_ERR = 0x8, + XSC_CMD_STAT_BAD_RES_STATE_ERR = 0x9, + XSC_CMD_STAT_IX_ERR = 0xa, + XSC_CMD_STAT_NO_RES_ERR = 0xf, + XSC_CMD_STAT_BAD_INP_LEN_ERR = 0x50, + XSC_CMD_STAT_BAD_OUTP_LEN_ERR = 0x51, + XSC_CMD_STAT_BAD_QP_STATE_ERR = 0x10, + XSC_CMD_STAT_BAD_PKT_ERR = 0x30, + XSC_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR = 0x40, +}; + +static struct xsc_cmd_work_ent *alloc_cmd(struct xsc_cmd *cmd, + struct xsc_cmd_msg *in, + struct xsc_rsp_msg *out, + xsc_cmd_cbk_t cbk, + void *context, int page_queue) +{ + gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL; + struct xsc_cmd_work_ent *ent; + + ent = kzalloc(sizeof(*ent), alloc_flags); + if (!ent) + return ERR_PTR(-ENOMEM); + + ent->in = in; + ent->out = out; + ent->callback = cbk; + ent->context = context; + ent->cmd = cmd; + ent->page_queue = page_queue; + + return ent; +} + +static u8 alloc_token(struct xsc_cmd *cmd) +{ + u8 token; + + spin_lock(&cmd->token_lock); + token = cmd->token++ % 255 + 1; + spin_unlock(&cmd->token_lock); + + return token; +} + +static int alloc_ent(struct xsc_cmd *cmd) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&cmd->alloc_lock, flags); + ret = find_first_bit(&cmd->bitmask, cmd->max_reg_cmds); + if (ret < cmd->max_reg_cmds) + clear_bit(ret, &cmd->bitmask); + spin_unlock_irqrestore(&cmd->alloc_lock, flags); + + return ret < cmd->max_reg_cmds ? ret : -ENOMEM; +} + +static void free_ent(struct xsc_cmd *cmd, int idx) +{ + unsigned long flags; + + spin_lock_irqsave(&cmd->alloc_lock, flags); + set_bit(idx, &cmd->bitmask); + spin_unlock_irqrestore(&cmd->alloc_lock, flags); +} + +static struct xsc_cmd_layout *get_inst(struct xsc_cmd *cmd, int idx) +{ + return cmd->cmd_buf + (idx << cmd->log_stride); +} + +static struct xsc_rsp_layout *get_cq_inst(struct xsc_cmd *cmd, int idx) +{ + return cmd->cq_buf + (idx << cmd->log_stride); +} + +static u8 xor8_buf(void *buf, int len) +{ + u8 *ptr = buf; + u8 sum = 0; + int i; + + for (i = 0; i < len; i++) + sum ^= ptr[i]; + + return sum; +} + +static int verify_block_sig(struct xsc_cmd_prot_block *block) +{ + if (xor8_buf(block->rsvd0, sizeof(*block) - sizeof(block->data) - 1) != 0xff) + return -EINVAL; + + if (xor8_buf(block, sizeof(*block)) != 0xff) + return -EINVAL; + + return 0; +} + +static void calc_block_sig(struct xsc_cmd_prot_block *block, u8 token) +{ + block->token = token; + block->ctrl_sig = ~xor8_buf(block->rsvd0, sizeof(*block) - sizeof(block->data) - 2); + block->sig = ~xor8_buf(block, sizeof(*block) - 1); +} + +static void calc_chain_sig(struct xsc_cmd_mailbox *head, u8 token) +{ + struct xsc_cmd_mailbox *next = head; + + while (next) { + calc_block_sig(next->buf, token); + next = next->next; + } +} + +static void set_signature(struct xsc_cmd_work_ent *ent) +{ + ent->lay->sig = ~xor8_buf(ent->lay, sizeof(*ent->lay)); + calc_chain_sig(ent->in->next, ent->token); + calc_chain_sig(ent->out->next, ent->token); +} + +static void free_cmd(struct xsc_cmd_work_ent *ent) +{ + kfree(ent); +} + +static int verify_signature(struct xsc_cmd_work_ent *ent) +{ + struct xsc_cmd_mailbox *next = ent->out->next; + int err; + u8 sig; + + sig = xor8_buf(ent->rsp_lay, sizeof(*ent->rsp_lay)); + if (sig != 0xff) + return -EINVAL; + + while (next) { + err = verify_block_sig(next->buf); + if (err) + return err; + + next = next->next; + } + + return 0; +} + +#ifdef COSIM +static void dump_buf(void *buf, int size, int data_only, int offset) +{ + __be32 *p = buf; + int i; + + for (i = 0; i < size; i += 16) { + pr_debug("%03x: %08x %08x %08x %08x\n", offset, be32_to_cpu(p[0]), + be32_to_cpu(p[1]), be32_to_cpu(p[2]), + be32_to_cpu(p[3])); + p += 4; + offset += 16; + } + if (!data_only) + pr_debug("\n"); +} +#endif + +const char *xsc_command_str(int command) +{ + switch (command) { + case XSC_CMD_OP_QUERY_HCA_CAP: + return "QUERY_HCA_CAP"; + + case XSC_CMD_OP_SET_HCA_CAP: + return "SET_HCA_CAP"; + + case XSC_CMD_OP_QUERY_ADAPTER: + return "QUERY_ADAPTER"; + + case XSC_CMD_OP_INIT_HCA: + return "INIT_HCA"; + + case XSC_CMD_OP_TEARDOWN_HCA: + return "TEARDOWN_HCA"; + + case XSC_CMD_OP_QUERY_PAGES: + return "QUERY_PAGES"; + + case XSC_CMD_OP_MANAGE_PAGES: + return "MANAGE_PAGES"; + + case XSC_CMD_OP_CREATE_MKEY: + return "CREATE_MKEY"; + + case XSC_CMD_OP_QUERY_MKEY: + return "QUERY_MKEY"; + + case XSC_CMD_OP_DESTROY_MKEY: + return "DESTROY_MKEY"; + + case XSC_CMD_OP_QUERY_SPECIAL_CONTEXTS: + return "QUERY_SPECIAL_CONTEXTS"; + + case XSC_CMD_OP_CREATE_EQ: + return "CREATE_EQ"; + + case XSC_CMD_OP_DESTROY_EQ: + return "DESTROY_EQ"; + + case XSC_CMD_OP_QUERY_EQ: + return "QUERY_EQ"; + + case XSC_CMD_OP_CREATE_CQ: + return "CREATE_CQ"; + + case XSC_CMD_OP_DESTROY_CQ: + return "DESTROY_CQ"; + + case XSC_CMD_OP_QUERY_CQ: + return "QUERY_CQ"; + + case XSC_CMD_OP_MODIFY_CQ: + return "MODIFY_CQ"; + + case XSC_CMD_OP_CREATE_QP: + return "CREATE_QP"; + + case XSC_CMD_OP_DESTROY_QP: + return "DESTROY_QP"; + + case XSC_CMD_OP_RST2INIT_QP: + return "RST2INIT_QP"; + + case XSC_CMD_OP_INIT2RTR_QP: + return "INIT2RTR_QP"; + + case XSC_CMD_OP_RTR2RTS_QP: + return "RTR2RTS_QP"; + + case XSC_CMD_OP_RTS2RTS_QP: + return "RTS2RTS_QP"; + + case XSC_CMD_OP_SQERR2RTS_QP: + return "SQERR2RTS_QP"; + + case XSC_CMD_OP_2ERR_QP: + return "2ERR_QP"; + + case XSC_CMD_OP_RTS2SQD_QP: + return "RTS2SQD_QP"; + + case XSC_CMD_OP_SQD2RTS_QP: + return "SQD2RTS_QP"; + + case XSC_CMD_OP_2RST_QP: + return "2RST_QP"; + + case XSC_CMD_OP_QUERY_QP: + return "QUERY_QP"; + + case XSC_CMD_OP_CONF_SQP: + return "CONF_SQP"; + + case XSC_CMD_OP_MAD_IFC: + return "MAD_IFC"; + + case XSC_CMD_OP_INIT2INIT_QP: + return "INIT2INIT_QP"; + + case XSC_CMD_OP_SUSPEND_QP: + return "SUSPEND_QP"; + + case XSC_CMD_OP_UNSUSPEND_QP: + return "UNSUSPEND_QP"; + + case XSC_CMD_OP_SQD2SQD_QP: + return "SQD2SQD_QP"; + + case XSC_CMD_OP_ALLOC_QP_COUNTER_SET: + return "ALLOC_QP_COUNTER_SET"; + + case XSC_CMD_OP_DEALLOC_QP_COUNTER_SET: + return "DEALLOC_QP_COUNTER_SET"; + + case XSC_CMD_OP_QUERY_QP_COUNTER_SET: + return "QUERY_QP_COUNTER_SET"; + + case XSC_CMD_OP_CREATE_PSV: + return "CREATE_PSV"; + + case XSC_CMD_OP_DESTROY_PSV: + return "DESTROY_PSV"; + + case XSC_CMD_OP_QUERY_PSV: + return "QUERY_PSV"; + + case XSC_CMD_OP_QUERY_SIG_RULE_TABLE: + return "QUERY_SIG_RULE_TABLE"; + + case XSC_CMD_OP_QUERY_BLOCK_SIZE_TABLE: + return "QUERY_BLOCK_SIZE_TABLE"; + + case XSC_CMD_OP_CREATE_SRQ: + return "CREATE_SRQ"; + + case XSC_CMD_OP_DESTROY_SRQ: + return "DESTROY_SRQ"; + + case XSC_CMD_OP_QUERY_SRQ: + return "QUERY_SRQ"; + + case XSC_CMD_OP_ARM_RQ: + return "ARM_RQ"; + + case XSC_CMD_OP_RESIZE_SRQ: + return "RESIZE_SRQ"; + + case XSC_CMD_OP_ALLOC_PD: + return "ALLOC_PD"; + + case XSC_CMD_OP_DEALLOC_PD: + return "DEALLOC_PD"; + + case XSC_CMD_OP_ALLOC_UAR: + return "ALLOC_UAR"; + + case XSC_CMD_OP_DEALLOC_UAR: + return "DEALLOC_UAR"; + + case XSC_CMD_OP_ATTACH_TO_MCG: + return "ATTACH_TO_MCG"; + + case XSC_CMD_OP_DETACH_FROM_MCG: + return "DETACH_FROM_MCG"; + + case XSC_CMD_OP_ALLOC_XRCD: + return "ALLOC_XRCD"; + + case XSC_CMD_OP_DEALLOC_XRCD: + return "DEALLOC_XRCD"; + + case XSC_CMD_OP_ACCESS_REG: + return "ACCESS_REG"; + + default: return "unknown command opcode"; + } +} + +#ifdef COSIM +static void dump_command(struct xsc_core_device *xdev, struct xsc_cmd_mailbox *next, + struct xsc_cmd_work_ent *ent, int input, int len) +{ + u16 op = be16_to_cpu(((struct xsc_inbox_hdr *)(ent->lay->in))->opcode); + int data_only; + int offset = 0; + int dump_len; + + data_only = !!(xsc_debug_mask & (1 << XSC_CMD_DATA)); + + if (data_only) + xsc_core_dbg_mask(xdev, 1 << XSC_CMD_DATA, + "dump command data %s(0x%x) %s\n", + xsc_command_str(op), op, + input ? "INPUT" : "OUTPUT"); + else + xsc_core_dbg(xdev, "dump command %s(0x%x) %s\n", + xsc_command_str(op), op, + input ? "INPUT" : "OUTPUT"); + + if (data_only) { + if (input) { + dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset); + offset += sizeof(ent->lay->in); + } else { + dump_buf(ent->rsp_lay->out, sizeof(ent->rsp_lay->out), 1, offset); + offset += sizeof(ent->rsp_lay->out); + } + } else { + if (input) { + dump_buf(ent->lay, sizeof(*ent->lay), 0, offset); + offset += sizeof(*ent->lay); + } else { + dump_buf(ent->rsp_lay, sizeof(*ent->rsp_lay), 0, offset); + offset += sizeof(*ent->rsp_lay); + } + } + + while (next && offset < len) { + if (data_only) { + dump_len = min_t(int, XSC_CMD_DATA_BLOCK_SIZE, len - offset); + dump_buf(next->buf, dump_len, 1, offset); + offset += XSC_CMD_DATA_BLOCK_SIZE; + } else { + xsc_core_dbg(xdev, "command block:\n"); + dump_buf(next->buf, sizeof(struct xsc_cmd_prot_block), 0, offset); + offset += sizeof(struct xsc_cmd_prot_block); + } + next = next->next; + } + + if (data_only) + pr_debug("\n"); +} +#endif + +static void cmd_work_handler(struct work_struct *work) +{ + struct xsc_cmd_work_ent *ent = container_of(work, struct xsc_cmd_work_ent, work); + struct xsc_cmd *cmd = ent->cmd; + struct xsc_core_device *xdev = container_of(cmd, struct xsc_core_device, cmd); + struct xsc_cmd_layout *lay; + struct semaphore *sem; + unsigned long flags; + + sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem; + down(sem); + if (!ent->page_queue) { + ent->idx = alloc_ent(cmd); + if (ent->idx < 0) { + xsc_core_err(xdev, "failed to allocate command entry\n"); + up(sem); + return; + } + } else { + ent->idx = cmd->max_reg_cmds; + } + + ent->token = alloc_token(cmd); + cmd->ent_arr[ent->idx] = ent; + + spin_lock_irqsave(&cmd->doorbell_lock, flags); + lay = get_inst(cmd, cmd->cmd_pid); + ent->lay = lay; + memset(lay, 0, sizeof(*lay)); + memcpy(lay->in, ent->in->first.data, sizeof(lay->in)); + if (ent->in->next) + lay->in_ptr = cpu_to_be64(ent->in->next->dma); + lay->inlen = cpu_to_be32(ent->in->len); + if (ent->out->next) + lay->out_ptr = cpu_to_be64(ent->out->next->dma); + lay->outlen = cpu_to_be32(ent->out->len); + lay->type = XSC_PCI_CMD_XPORT; + lay->token = ent->token; + lay->idx = ent->idx; + if (!cmd->checksum_disabled) + set_signature(ent); + else + lay->sig = 0xff; +#ifdef COSIM + dump_command(xdev, ent->in->next, ent, 1, ent->in->len); +#endif + + ktime_get_ts64(&ent->ts1); + + /* ring doorbell after the descriptor is valid */ + wmb(); + + cmd->cmd_pid = (cmd->cmd_pid + 1) % (1<log_sz); + writel(cmd->cmd_pid, REG_ADDR(xdev, cmd->reg.req_pid_addr)); + mmiowb(); + spin_unlock_irqrestore(&cmd->doorbell_lock, flags); + +#ifdef XSC_DEBUG + xsc_core_dbg(xdev, "write 0x%x to command doorbell, idx %u\n", cmd->cmd_pid, ent->idx); +#endif +} + +static const char *deliv_status_to_str(u8 status) +{ + switch (status) { + case XSC_CMD_DELIVERY_STAT_OK: + return "no errors"; + case XSC_CMD_DELIVERY_STAT_SIGNAT_ERR: + return "signature error"; + case XSC_CMD_DELIVERY_STAT_TOK_ERR: + return "token error"; + case XSC_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: + return "bad block number"; + case XSC_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: + return "output pointer not aligned to block size"; + case XSC_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: + return "input pointer not aligned to block size"; + case XSC_CMD_DELIVERY_STAT_FW_ERR: + return "firmware internal error"; + case XSC_CMD_DELIVERY_STAT_IN_LENGTH_ERR: + return "command input length error"; + case XSC_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: + return "command output length error"; + case XSC_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: + return "reserved fields not cleared"; + case XSC_CMD_DELIVERY_STAT_CMD_DESCR_ERR: + return "bad command descriptor type"; + default: + return "unknown status code"; + } +} + +static u16 msg_to_opcode(struct xsc_cmd_msg *in) +{ + struct xsc_inbox_hdr *hdr = (struct xsc_inbox_hdr *)(in->first.data); + + return be16_to_cpu(hdr->opcode); +} + +static int wait_func(struct xsc_core_device *xdev, struct xsc_cmd_work_ent *ent) +{ + unsigned long timeout = msecs_to_jiffies(XSC_CMD_TIMEOUT_MSEC); + int err; + + if (!wait_for_completion_timeout(&ent->done, timeout)) + err = -ETIMEDOUT; + else + err = ent->ret; + + if (err == -ETIMEDOUT) { + xsc_core_warn(xdev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", + xsc_command_str(msg_to_opcode(ent->in)), + msg_to_opcode(ent->in)); + } else if (err) { + xsc_core_dbg(xdev, "err %d, delivery status %s(%d)\n", err, + deliv_status_to_str(ent->status), ent->status); + } + + return err; +} + +/* Notes: + * 1. Callback functions may not sleep + * 2. page queue commands do not support asynchrous completion + */ +static int xsc_cmd_invoke(struct xsc_core_device *xdev, struct xsc_cmd_msg *in, + struct xsc_rsp_msg *out, xsc_cmd_cbk_t callback, + void *context, int page_queue, u8 *status) +{ + struct xsc_cmd *cmd = &xdev->cmd; + struct xsc_cmd_work_ent *ent; + ktime_t t1, t2, delta; + struct xsc_cmd_stats *stats; + int err = 0; + s64 ds; + u16 op; + struct semaphore *sem; + + if (callback && page_queue) + return -EINVAL; + + ent = alloc_cmd(cmd, in, out, callback, context, page_queue); + if (IS_ERR(ent)) + return PTR_ERR(ent); + + if (!callback) + init_completion(&ent->done); + + INIT_WORK(&ent->work, cmd_work_handler); + if (page_queue) { + cmd_work_handler(&ent->work); + } else if (!queue_work(cmd->wq, &ent->work)) { + xsc_core_warn(xdev, "failed to queue work\n"); + err = -ENOMEM; + goto out_free; + } + + if (!callback) { + err = wait_func(xdev, ent); + if (err == -ETIMEDOUT) + goto out; + t1 = timespec64_to_ktime(ent->ts1); + t2 = timespec64_to_ktime(ent->ts2); + delta = ktime_sub(t2, t1); + ds = ktime_to_ns(delta); + op = be16_to_cpu(((struct xsc_inbox_hdr *)in->first.data)->opcode); + if (op < ARRAY_SIZE(cmd->stats)) { + stats = &cmd->stats[op]; + spin_lock(&stats->lock); + stats->sum += ds; + ++stats->n; + spin_unlock(&stats->lock); + } + xsc_core_dbg_mask(xdev, 1 << XSC_CMD_TIME, + "fw exec time for %s is %lld nsec\n", + xsc_command_str(op), ds); + *status = ent->status; + free_cmd(ent); + } + + return err; + +out_free: + free_cmd(ent); +out: + sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem; + up(sem); + return err; +} + +static ssize_t dbg_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xsc_core_device *xdev = filp->private_data; + struct xsc_cmd_debug *dbg = &xdev->cmd.dbg; + char lbuf[3]; + int err; + + if (!dbg->in_msg || !dbg->out_msg) + return -ENOMEM; + + if (copy_from_user(lbuf, buf, sizeof(lbuf))) + return -EPERM; + + lbuf[sizeof(lbuf) - 1] = 0; + + if (strcmp(lbuf, "go")) + return -EINVAL; + + err = xsc_cmd_exec(xdev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen); + + return err ? err : count; +} + +static const struct file_operations fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dbg_write, +}; + +static int xsc_copy_to_cmd_msg(struct xsc_cmd_msg *to, void *from, int size) +{ + struct xsc_cmd_prot_block *block; + struct xsc_cmd_mailbox *next; + int copy; + + if (!to || !from) + return -ENOMEM; + + copy = min_t(int, size, sizeof(to->first.data)); + memcpy(to->first.data, from, copy); + size -= copy; + from += copy; + + next = to->next; + while (size) { + if (!next) { + /* this is a BUG */ + return -ENOMEM; + } + + copy = min_t(int, size, XSC_CMD_DATA_BLOCK_SIZE); + block = next->buf; + memcpy(block->data, from, copy); + block->owner_status = 0; + from += copy; + size -= copy; + next = next->next; + } + + return 0; +} + +static int xsc_copy_from_rsp_msg(void *to, struct xsc_rsp_msg *from, int size) +{ + struct xsc_cmd_prot_block *block; + struct xsc_cmd_mailbox *next; + int copy; + + if (!to || !from) + return -ENOMEM; + + copy = min_t(int, size, sizeof(from->first.data)); + memcpy(to, from->first.data, copy); + size -= copy; + to += copy; + + next = from->next; + while (size) { + if (!next) { + /* this is a BUG */ + return -ENOMEM; + } + + copy = min_t(int, size, XSC_CMD_DATA_BLOCK_SIZE); + block = next->buf; + if (block->owner_status != 1) { + //xsc_core_warn(xdev, "dam buf not ready\n"); + mdelay(10); + continue; + } + + memcpy(to, block->data, copy); + to += copy; + size -= copy; + next = next->next; + } + + return 0; +} + +static struct xsc_cmd_mailbox *alloc_cmd_box(struct xsc_core_device *xdev, + gfp_t flags) +{ + struct xsc_cmd_mailbox *mailbox; + + mailbox = kmalloc(sizeof(*mailbox), flags); + if (!mailbox) + return ERR_PTR(-ENOMEM); + + mailbox->buf = pci_pool_alloc(xdev->cmd.pool, flags, + &mailbox->dma); + if (!mailbox->buf) { + xsc_core_dbg(xdev, "failed allocation\n"); + kfree(mailbox); + return ERR_PTR(-ENOMEM); + } + memset(mailbox->buf, 0, sizeof(struct xsc_cmd_prot_block)); + mailbox->next = NULL; + + return mailbox; +} + +static void free_cmd_box(struct xsc_core_device *xdev, + struct xsc_cmd_mailbox *mailbox) +{ + pci_pool_free(xdev->cmd.pool, mailbox->buf, mailbox->dma); + kfree(mailbox); +} + +static struct xsc_cmd_msg *xsc_alloc_cmd_msg(struct xsc_core_device *xdev, + gfp_t flags, int size) +{ + struct xsc_cmd_mailbox *tmp, *head = NULL; + struct xsc_cmd_prot_block *block; + struct xsc_cmd_msg *msg; + int blen; + int err; + int n; + int i; + + msg = kzalloc(sizeof(*msg), GFP_KERNEL); + if (!msg) + return ERR_PTR(-ENOMEM); + + blen = size - min_t(int, sizeof(msg->first.data), size); + n = (blen + XSC_CMD_DATA_BLOCK_SIZE - 1) / XSC_CMD_DATA_BLOCK_SIZE; + + for (i = 0; i < n; i++) { + tmp = alloc_cmd_box(xdev, flags); + if (IS_ERR(tmp)) { + xsc_core_warn(xdev, "failed allocating block\n"); + err = PTR_ERR(tmp); + goto err_alloc; + } + + block = tmp->buf; + tmp->next = head; + block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0); + block->block_num = cpu_to_be32(n - i - 1); + head = tmp; + } + msg->next = head; + msg->len = size; + return msg; + +err_alloc: + while (head) { + tmp = head->next; + free_cmd_box(xdev, head); + head = tmp; + } + kfree(msg); + + return ERR_PTR(err); +} + +static void xsc_free_cmd_msg(struct xsc_core_device *xdev, + struct xsc_cmd_msg *msg) +{ + struct xsc_cmd_mailbox *head = msg->next; + struct xsc_cmd_mailbox *next; + + while (head) { + next = head->next; + free_cmd_box(xdev, head); + head = next; + } + kfree(msg); +} + +static struct xsc_rsp_msg *xsc_alloc_rsp_msg(struct xsc_core_device *xdev, + gfp_t flags, int size) +{ + struct xsc_cmd_mailbox *tmp, *head = NULL; + struct xsc_cmd_prot_block *block; + struct xsc_rsp_msg *msg; + int blen; + int err; + int n; + int i; + + msg = kzalloc(sizeof(*msg), GFP_KERNEL); + if (!msg) + return ERR_PTR(-ENOMEM); + + blen = size - min_t(int, sizeof(msg->first.data), size); + n = (blen + XSC_CMD_DATA_BLOCK_SIZE - 1) / XSC_CMD_DATA_BLOCK_SIZE; + + for (i = 0; i < n; i++) { + tmp = alloc_cmd_box(xdev, flags); + if (IS_ERR(tmp)) { + xsc_core_warn(xdev, "failed allocating block\n"); + err = PTR_ERR(tmp); + goto err_alloc; + } + + block = tmp->buf; + tmp->next = head; + block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0); + block->block_num = cpu_to_be32(n - i - 1); + head = tmp; + } + msg->next = head; + msg->len = size; + return msg; + +err_alloc: + while (head) { + tmp = head->next; + free_cmd_box(xdev, head); + head = tmp; + } + kfree(msg); + + return ERR_PTR(err); +} + +static void xsc_free_rsp_msg(struct xsc_core_device *xdev, + struct xsc_rsp_msg *msg) +{ + struct xsc_cmd_mailbox *head = msg->next; + struct xsc_cmd_mailbox *next; + + while (head) { + next = head->next; + free_cmd_box(xdev, head); + head = next; + } + kfree(msg); +} + +static ssize_t data_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xsc_core_device *xdev = filp->private_data; + struct xsc_cmd_debug *dbg = &xdev->cmd.dbg; + void *ptr; + int err; + + if (*pos != 0) + return -EINVAL; + + kfree(dbg->in_msg); + dbg->in_msg = NULL; + dbg->inlen = 0; + + ptr = kzalloc(count, GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + if (copy_from_user(ptr, buf, count)) { + err = -EPERM; + goto out; + } + dbg->in_msg = ptr; + dbg->inlen = count; + + *pos = count; + + return count; + +out: + kfree(ptr); + return err; +} + +static ssize_t data_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct xsc_core_device *xdev = filp->private_data; + struct xsc_cmd_debug *dbg = &xdev->cmd.dbg; + int copy; + + if (*pos) + return 0; + + if (!dbg->out_msg) + return -ENOMEM; + + copy = min_t(int, count, dbg->outlen); + if (copy_to_user(buf, dbg->out_msg, copy)) + return -EPERM; + + *pos += copy; + + return copy; +} + +static const struct file_operations dfops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = data_write, + .read = data_read, +}; + +static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct xsc_core_device *xdev = filp->private_data; + struct xsc_cmd_debug *dbg = &xdev->cmd.dbg; + char outlen[8]; + int err; + + if (*pos) + return 0; + + err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen); + if (err < 0) + return err; + + if (copy_to_user(buf, &outlen, err)) + return -EPERM; + + *pos += err; + + return err; +} + +static ssize_t outlen_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xsc_core_device *xdev = filp->private_data; + struct xsc_cmd_debug *dbg = &xdev->cmd.dbg; + char outlen_str[8]; + int outlen; + void *ptr; + int err; + + if (*pos != 0 || count > 6) + return -EINVAL; + + kfree(dbg->out_msg); + dbg->out_msg = NULL; + dbg->outlen = 0; + + if (copy_from_user(outlen_str, buf, count)) + return -EPERM; + + outlen_str[7] = 0; + + err = kstrtoint(outlen_str, 10, &outlen); + if (err < 0) + return err; + + ptr = kzalloc(outlen, GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + dbg->out_msg = ptr; + dbg->outlen = outlen; + + *pos = count; + + return count; +} + +static const struct file_operations olfops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = outlen_write, + .read = outlen_read, +}; + +static void set_wqname(struct xsc_core_device *xdev) +{ + struct xsc_cmd *cmd = &xdev->cmd; + + snprintf(cmd->wq_name, sizeof(cmd->wq_name), "xsc_cmd_%s", + dev_name(&xdev->pdev->dev)); +} + +static void clean_debug_files(struct xsc_core_device *xdev) +{ + struct xsc_cmd_debug *dbg = &xdev->cmd.dbg; + + if (!xsc_debugfs_root) + return; + + xsc_cmdif_debugfs_cleanup(xdev); + debugfs_remove_recursive(dbg->dbg_root); +} + +static int create_debugfs_files(struct xsc_core_device *xdev) +{ + struct xsc_cmd_debug *dbg = &xdev->cmd.dbg; + int err = -ENOMEM; + + if (!xsc_debugfs_root) + return 0; + + dbg->dbg_root = debugfs_create_dir("cmd", xdev->dev_res->dbg_root); + if (!dbg->dbg_root) + return err; + + dbg->dbg_in = debugfs_create_file("in", 0400, dbg->dbg_root, + xdev, &dfops); + if (!dbg->dbg_in) + goto err_dbg; + + dbg->dbg_out = debugfs_create_file("out", 0200, dbg->dbg_root, + xdev, &dfops); + if (!dbg->dbg_out) + goto err_dbg; + + dbg->dbg_outlen = debugfs_create_file("out_len", 0600, dbg->dbg_root, + xdev, &olfops); + if (!dbg->dbg_outlen) + goto err_dbg; + + debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status); + + dbg->dbg_run = debugfs_create_file("run", 0200, dbg->dbg_root, xdev, &fops); + if (!dbg->dbg_run) + goto err_dbg; + + xsc_cmdif_debugfs_init(xdev); + + return 0; + +err_dbg: + clean_debug_files(xdev); + return err; +} + +void xsc_cmd_use_events(struct xsc_core_device *xdev) +{ + struct xsc_cmd *cmd = &xdev->cmd; + int i; + + for (i = 0; i < cmd->max_reg_cmds; i++) + down(&cmd->sem); + + down(&cmd->pages_sem); + + flush_workqueue(cmd->wq); + + cmd->mode = CMD_MODE_EVENTS; + + while (cmd->cmd_pid != cmd->cq_cid) + msleep(20); + kthread_stop(cmd->cq_task); + cmd->cq_task = NULL; + + up(&cmd->pages_sem); + for (i = 0; i < cmd->max_reg_cmds; i++) + up(&cmd->sem); +} + +static int cmd_cq_polling(void *data); +void xsc_cmd_use_polling(struct xsc_core_device *xdev) +{ + struct xsc_cmd *cmd = &xdev->cmd; + int i; + + for (i = 0; i < cmd->max_reg_cmds; i++) + down(&cmd->sem); + + down(&cmd->pages_sem); + + flush_workqueue(cmd->wq); + cmd->mode = CMD_MODE_POLLING; + cmd->cq_task = kthread_create(cmd_cq_polling, (void *)xdev, "xsc_cmd_cq_polling"); + if (cmd->cq_task) + wake_up_process(cmd->cq_task); + + up(&cmd->pages_sem); + for (i = 0; i < cmd->max_reg_cmds; i++) + up(&cmd->sem); +} + +static int status_to_err(u8 status) +{ + return status ? -1 : 0; /* TBD more meaningful codes */ +} + +static struct xsc_cmd_msg *alloc_msg(struct xsc_core_device *xdev, int in_size) +{ + struct xsc_cmd_msg *msg = ERR_PTR(-ENOMEM); + struct xsc_cmd *cmd = &xdev->cmd; + struct cache_ent *ent = NULL; + + if (in_size > MED_LIST_SIZE && in_size <= LONG_LIST_SIZE) + ent = &cmd->cache.large; + else if (in_size > 16 && in_size <= MED_LIST_SIZE) + ent = &cmd->cache.med; + + if (ent) { + spin_lock(&ent->lock); + if (!list_empty(&ent->head)) { + msg = list_entry(ent->head.next, typeof(*msg), list); + /* For cached lists, we must explicitly state what is + * the real size + */ + msg->len = in_size; + list_del(&msg->list); + } + spin_unlock(&ent->lock); + } + + if (IS_ERR(msg)) + msg = xsc_alloc_cmd_msg(xdev, GFP_KERNEL, in_size); + + return msg; +} + +static void free_msg(struct xsc_core_device *xdev, struct xsc_cmd_msg *msg) +{ + if (msg->cache) { + spin_lock(&msg->cache->lock); + list_add_tail(&msg->list, &msg->cache->head); + spin_unlock(&msg->cache->lock); + } else { + xsc_free_cmd_msg(xdev, msg); + } +} + +static int is_manage_pages(struct xsc_inbox_hdr *in) +{ + return be16_to_cpu(in->opcode) == XSC_CMD_OP_MANAGE_PAGES; +} + +static int dummy_work(struct xsc_core_device *xdev, struct xsc_cmd_msg *in, + struct xsc_rsp_msg *out, u16 dummy_cnt, u16 dummy_start_pid) +{ + struct xsc_cmd *cmd = &xdev->cmd; + struct xsc_cmd_work_ent **dummy_ent_arr; + struct xsc_cmd_layout *lay; + struct semaphore *sem; + int err = 0; + u16 i; + u16 free_cnt = 0; + u16 temp_pid = dummy_start_pid; + + sem = &cmd->sem; + + dummy_ent_arr = kcalloc(dummy_cnt, sizeof(struct xsc_cmd_work_ent *), GFP_KERNEL); + if (!dummy_ent_arr) { + err = -ENOMEM; + goto alloc_ent_arr_err; + } + + for (i = 0; i < dummy_cnt; i++) { + dummy_ent_arr[i] = alloc_cmd(cmd, in, out, NULL, NULL, 0); + if (IS_ERR(dummy_ent_arr[i])) { + xsc_core_err(xdev, "failed to alloc cmd buffer\n"); + err = -ENOMEM; + free_cnt = i; + goto alloc_ent_err; + } + + down(sem); + + dummy_ent_arr[i]->idx = alloc_ent(cmd); + if (dummy_ent_arr[i]->idx < 0) { + xsc_core_err(xdev, "failed to allocate command entry\n"); + err = -1; + free_cnt = i; + goto get_cmd_ent_idx_err; + } + dummy_ent_arr[i]->token = alloc_token(cmd); + cmd->ent_arr[dummy_ent_arr[i]->idx] = dummy_ent_arr[i]; + init_completion(&dummy_ent_arr[i]->done); + + lay = get_inst(cmd, temp_pid); + dummy_ent_arr[i]->lay = lay; + memset(lay, 0, sizeof(*lay)); + memcpy(lay->in, dummy_ent_arr[i]->in->first.data, + sizeof(dummy_ent_arr[i]->in)); + lay->inlen = cpu_to_be32(dummy_ent_arr[i]->in->len); + lay->outlen = cpu_to_be32(dummy_ent_arr[i]->out->len); + lay->type = XSC_PCI_CMD_XPORT; + lay->token = dummy_ent_arr[i]->token; + lay->idx = dummy_ent_arr[i]->idx; + if (!cmd->checksum_disabled) + set_signature(dummy_ent_arr[i]); + else + lay->sig = 0xff; + temp_pid = (temp_pid + 1) % (1<log_sz); + } + + /* ring doorbell after the descriptor is valid */ + wmb(); + writel(cmd->cmd_pid, REG_ADDR(xdev, cmd->reg.req_pid_addr)); + if (readl(REG_ADDR(xdev, cmd->reg.interrupt_stat_addr)) != 0) + writel(0xF, REG_ADDR(xdev, cmd->reg.interrupt_stat_addr)); + + mmiowb(); + xsc_core_dbg(xdev, "write 0x%x to command doorbell, idx %u ~ %u\n", cmd->cmd_pid, + dummy_ent_arr[0]->idx, dummy_ent_arr[dummy_cnt-1]->idx); + + if (wait_for_completion_timeout(&dummy_ent_arr[dummy_cnt - 1]->done, + msecs_to_jiffies(3000)) == 0) { + xsc_core_err(xdev, "dummy_cmd %d ent timeout, cmdq fail\n", dummy_cnt - 1); + err = -ETIMEDOUT; + } else { + xsc_core_dbg(xdev, "%d ent done\n", dummy_cnt); + } + + for (i = 0; i < dummy_cnt; i++) + free_cmd(dummy_ent_arr[i]); + + kfree(dummy_ent_arr); + return err; + +get_cmd_ent_idx_err: + free_cmd(dummy_ent_arr[free_cnt]); + up(sem); +alloc_ent_err: + for (i = 0; i < free_cnt; i++) { + free_ent(cmd, dummy_ent_arr[i]->idx); + up(sem); + free_cmd(dummy_ent_arr[i]); + } + kfree(dummy_ent_arr); +alloc_ent_arr_err: + return err; +} + +static int xsc_dummy_cmd_exec(struct xsc_core_device *xdev, void *in, int in_size, void *out, + int out_size, u16 dmmy_cnt, u16 dummy_start) +{ + struct xsc_cmd_msg *inb; + struct xsc_rsp_msg *outb; + int err; + + inb = alloc_msg(xdev, in_size); + if (IS_ERR(inb)) { + err = PTR_ERR(inb); + return err; + } + + err = xsc_copy_to_cmd_msg(inb, in, in_size); + if (err) { + xsc_core_warn(xdev, "err %d\n", err); + goto out_in; + } + + outb = xsc_alloc_rsp_msg(xdev, GFP_KERNEL, out_size); + if (IS_ERR(outb)) { + err = PTR_ERR(outb); + goto out_in; + } + + err = dummy_work(xdev, inb, outb, dmmy_cnt, dummy_start); + + if (err) + goto out_out; + + err = xsc_copy_from_rsp_msg(out, outb, out_size); + +out_out: + xsc_free_rsp_msg(xdev, outb); + +out_in: + free_msg(xdev, inb); + return err; +} + +static int xsc_send_dummy_cmd(struct xsc_core_device *xdev, u16 gap, u16 dummy_start) +{ + struct xsc_cmd_dummy_mbox_out *out; + struct xsc_cmd_dummy_mbox_in in; + int err; + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) { + err = -ENOMEM; + goto no_mem_out; + } + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DUMMY); + in.hdr.opmod = cpu_to_be16(0x1); + + err = xsc_dummy_cmd_exec(xdev, &in, sizeof(in), out, sizeof(*out), gap, dummy_start); + if (err) + goto out_out; + + if (out->hdr.status) { + err = xsc_cmd_status_to_err(&out->hdr); + goto out_out; + } + +out_out: + kfree(out); +no_mem_out: + return err; +} + +static int request_pid_cid_mismatch_restore(struct xsc_core_device *xdev) +{ + struct xsc_cmd *cmd = &xdev->cmd; + u16 req_pid, req_cid; + u16 gap; + + int err; + + req_pid = readl(REG_ADDR(xdev, cmd->reg.req_pid_addr)); + req_cid = readl(REG_ADDR(xdev, cmd->reg.req_cid_addr)); + if (req_pid >= (1<log_sz) || req_cid >= (1<log_sz)) { + xsc_core_err(xdev, "req_pid %d, req_cid %d, out of normal range!!! max value is %d\n", + req_pid, req_cid, (1<log_sz)); + return -1; + } + + if (req_pid == req_cid) + return 0; + + gap = (req_pid > req_cid) ? (req_pid - req_cid) : ((1<log_sz) + req_pid - req_cid); + xsc_core_info(xdev, "Cmdq req_pid %d, req_cid %d, send %d dummy cmds\n", + req_pid, req_cid, gap); + + err = xsc_send_dummy_cmd(xdev, gap, req_cid); + if (err) { + xsc_core_err(xdev, "Send dummy cmd failed\n"); + goto send_dummy_fail; + } + +send_dummy_fail: + return err; +} + +int _xsc_cmd_exec(struct xsc_core_device *xdev, void *in, int in_size, void *out, + int out_size) +{ + struct xsc_cmd_msg *inb; + struct xsc_rsp_msg *outb; + int pages_queue; + int err; + u8 status = 0; + + pages_queue = is_manage_pages(in); + + inb = alloc_msg(xdev, in_size); + if (IS_ERR(inb)) { + err = PTR_ERR(inb); + return err; + } + + err = xsc_copy_to_cmd_msg(inb, in, in_size); + if (err) { + xsc_core_warn(xdev, "err %d\n", err); + goto out_in; + } + + outb = xsc_alloc_rsp_msg(xdev, GFP_KERNEL, out_size); + if (IS_ERR(outb)) { + err = PTR_ERR(outb); + goto out_in; + } + + err = xsc_cmd_invoke(xdev, inb, outb, NULL, NULL, pages_queue, &status); + if (err) + goto out_out; + + if (status) { + xsc_core_err(xdev, "err %d, status %d\n", err, status); + err = status_to_err(status); + goto out_out; + } + + err = xsc_copy_from_rsp_msg(out, outb, out_size); + +out_out: + xsc_free_rsp_msg(xdev, outb); + +out_in: + free_msg(xdev, inb); + return err; +} +EXPORT_SYMBOL(_xsc_cmd_exec); + +static void destroy_msg_cache(struct xsc_core_device *xdev) +{ + struct xsc_cmd *cmd = &xdev->cmd; + struct xsc_cmd_msg *msg; + struct xsc_cmd_msg *n; + + list_for_each_entry_safe(msg, n, &cmd->cache.large.head, list) { + list_del(&msg->list); + xsc_free_cmd_msg(xdev, msg); + } + + list_for_each_entry_safe(msg, n, &cmd->cache.med.head, list) { + list_del(&msg->list); + xsc_free_cmd_msg(xdev, msg); + } +} + +static int create_msg_cache(struct xsc_core_device *xdev) +{ + struct xsc_cmd *cmd = &xdev->cmd; + struct xsc_cmd_msg *msg; + int err; + int i; + + spin_lock_init(&cmd->cache.large.lock); + INIT_LIST_HEAD(&cmd->cache.large.head); + spin_lock_init(&cmd->cache.med.lock); + INIT_LIST_HEAD(&cmd->cache.med.head); + + for (i = 0; i < NUM_LONG_LISTS; i++) { + msg = xsc_alloc_cmd_msg(xdev, GFP_KERNEL, LONG_LIST_SIZE); + if (IS_ERR(msg)) { + err = PTR_ERR(msg); + goto ex_err; + } + msg->cache = &cmd->cache.large; + list_add_tail(&msg->list, &cmd->cache.large.head); + } + + for (i = 0; i < NUM_MED_LISTS; i++) { + msg = xsc_alloc_cmd_msg(xdev, GFP_KERNEL, MED_LIST_SIZE); + if (IS_ERR(msg)) { + err = PTR_ERR(msg); + goto ex_err; + } + msg->cache = &cmd->cache.med; + list_add_tail(&msg->list, &cmd->cache.med.head); + } + + return 0; + +ex_err: + destroy_msg_cache(xdev); + return err; +} + +static void xsc_cmd_comp_handler(struct xsc_core_device *xdev, u8 idx, struct xsc_rsp_layout *rsp) +{ + struct xsc_cmd *cmd = &xdev->cmd; + struct xsc_cmd_work_ent *ent; + xsc_cmd_cbk_t callback; + void *context; + int err; + struct xsc_inbox_hdr *hdr; + + if (idx > cmd->max_reg_cmds || (cmd->bitmask & (1 << idx))) { + xsc_core_err(xdev, "idx[%d] exceed max cmds, or has no relative request.\n", + idx); + return; + } + ent = cmd->ent_arr[idx]; + ent->rsp_lay = rsp; + ktime_get_ts64(&ent->ts2); + + memcpy(ent->out->first.data, ent->rsp_lay->out, sizeof(ent->rsp_lay->out)); +#ifdef COSIM + dump_command(xdev, ent->out->next, ent, 0, ent->out->len); +#endif + if (!cmd->checksum_disabled) + ent->ret = verify_signature(ent); + else + ent->ret = 0; + ent->status = 0; +// ent->status = ((struct xsc_outbox_hdr *)ent->rsp_lay->out)->status; + + hdr = (struct xsc_inbox_hdr *)ent->in->first.data; +#ifdef XSC_DEBUG + xsc_core_dbg(xdev, "delivery status:%s(%d), rsp status=%d, opcode %#x, idx:%d,%d, ret=%d\n", + deliv_status_to_str(ent->status), ent->status, + ((struct xsc_outbox_hdr *)ent->rsp_lay->out)->status, + __be16_to_cpu(hdr->opcode), idx, ent->lay->idx, ent->ret); +#endif + free_ent(cmd, ent->idx); + if (ent->callback) { + callback = ent->callback; + context = ent->context; + err = ent->ret; + free_cmd(ent); + callback(err, context); + } else { + complete(&ent->done); + } + if (ent->page_queue) + up(&cmd->pages_sem); + else + up(&cmd->sem); +} + +static int cmd_cq_polling(void *data) +{ + struct xsc_core_device *xdev = data; + struct xsc_cmd *cmd = &xdev->cmd; + struct xsc_rsp_layout *rsp; + u32 cq_pid; + + while (!kthread_should_stop()) { + if (need_resched()) + schedule(); + cq_pid = readl(REG_ADDR(xdev, cmd->reg.rsp_pid_addr)); + if (cmd->cq_cid == cq_pid) { +#ifdef COSIM + mdelay(1000); +#endif + continue; + } + + //get cqe + rsp = get_cq_inst(cmd, cmd->cq_cid); + if (!cmd->ownerbit_learned) { + cmd->ownerbit_learned = 1; + cmd->owner_bit = rsp->owner_bit; + } + if (cmd->owner_bit != rsp->owner_bit) { + //hw update cq doorbell but buf may not ready + xsc_core_err(xdev, "hw update cq doorbell but buf not ready %u %u\n", + cmd->cq_cid, cq_pid); + continue; + } + + xsc_cmd_comp_handler(xdev, rsp->idx, rsp); + + cmd->cq_cid = (cmd->cq_cid + 1) % (1<log_sz); + + writel(cmd->cq_cid, REG_ADDR(xdev, cmd->reg.rsp_cid_addr)); + if (cmd->cq_cid == 0) + cmd->owner_bit = !cmd->owner_bit; + } + return 0; +} + +int xsc_cmd_err_handler(struct xsc_core_device *xdev) +{ + union interrupt_stat { + struct { + u32 hw_read_req_err:1; + u32 hw_write_req_err:1; + u32 req_pid_err:1; + u32 rsp_cid_err:1; + }; + u32 raw; + } stat; + int err = 0; + int retry = 0; + + stat.raw = readl(REG_ADDR(xdev, xdev->cmd.reg.interrupt_stat_addr)); + while (stat.raw != 0) { + err++; + if (stat.hw_read_req_err) { + retry = 1; + stat.hw_read_req_err = 0; + xsc_core_err(xdev, "hw report read req from host failed!\n"); + } else if (stat.hw_write_req_err) { + retry = 1; + stat.hw_write_req_err = 0; + xsc_core_err(xdev, "hw report write req to fw failed!\n"); + } else if (stat.req_pid_err) { + stat.req_pid_err = 0; + xsc_core_err(xdev, "hw report unexpected req pid!\n"); + } else if (stat.rsp_cid_err) { + stat.rsp_cid_err = 0; + xsc_core_err(xdev, "hw report unexpected rsp cid!\n"); + } else { + stat.raw = 0; + xsc_core_err(xdev, "ignore unknown interrupt!\n"); + } + } + + if (retry) + writel(xdev->cmd.cmd_pid, REG_ADDR(xdev, xdev->cmd.reg.req_pid_addr)); + + if (err) + writel(0xf, REG_ADDR(xdev, xdev->cmd.reg.interrupt_stat_addr)); + + return err; +} + +void xsc_cmd_resp_handler(struct xsc_core_device *xdev) +{ + struct xsc_cmd *cmd = &xdev->cmd; + struct xsc_rsp_layout *rsp; + u32 cq_pid; + const int budget = 32; + int count = 0; + + while (count < budget) { + cq_pid = readl(REG_ADDR(xdev, cmd->reg.rsp_pid_addr)); + if (cq_pid == cmd->cq_cid) + return; + + rsp = get_cq_inst(cmd, cmd->cq_cid); + if (!cmd->ownerbit_learned) { + cmd->ownerbit_learned = 1; + cmd->owner_bit = rsp->owner_bit; + } + if (cmd->owner_bit != rsp->owner_bit) { + xsc_core_err(xdev, "hw update cq doorbell but buf not ready %u %u\n", + cmd->cq_cid, cq_pid); + return; + } + + xsc_cmd_comp_handler(xdev, rsp->idx, rsp); + + cmd->cq_cid = (cmd->cq_cid + 1) % (1 << cmd->log_sz); + writel(cmd->cq_cid, REG_ADDR(xdev, cmd->reg.rsp_cid_addr)); + if (cmd->cq_cid == 0) + cmd->owner_bit = !cmd->owner_bit; + + count++; + } +} + +static void xsc_cmd_handle_rsp_before_reload +(struct xsc_cmd *cmd, struct xsc_core_device *xdev) +{ + u32 rsp_pid, rsp_cid; + + rsp_pid = readl(REG_ADDR(xdev, cmd->reg.rsp_pid_addr)); + rsp_cid = readl(REG_ADDR(xdev, cmd->reg.rsp_cid_addr)); + if (rsp_pid == rsp_cid) + return; + + cmd->cq_cid = rsp_pid; + + writel(cmd->cq_cid, REG_ADDR(xdev, cmd->reg.rsp_cid_addr)); +} + +int xsc_cmd_init(struct xsc_core_device *xdev) +{ + int size = sizeof(struct xsc_cmd_prot_block); + int align = roundup_pow_of_two(size); + struct xsc_cmd *cmd = &xdev->cmd; + u32 cmd_h, cmd_l; + u32 err_stat; + int err; + int i; + +// cmd_if_rev = cmdif_rev(dev); +// if (cmd_if_rev != CMD_IF_REV) { +// dev_err(&dev->pdev->dev, +// "Driver cmdif rev(%d) differs from firmware's(%d)\n", +// CMD_IF_REV, cmd_if_rev); +// return -EINVAL; +// } + + //sriov need adapt for this process. + //now there is 544 cmdq resource, soc using from id 514 + if (xsc_core_is_pf(xdev)) { + cmd->reg.req_pid_addr = HIF_CMDQM_HOST_REQ_PID_MEM_ADDR; + cmd->reg.req_cid_addr = HIF_CMDQM_HOST_REQ_CID_MEM_ADDR; + cmd->reg.rsp_pid_addr = HIF_CMDQM_HOST_RSP_PID_MEM_ADDR; + cmd->reg.rsp_cid_addr = HIF_CMDQM_HOST_RSP_CID_MEM_ADDR; + cmd->reg.req_buf_h_addr = HIF_CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_ADDR; + cmd->reg.req_buf_l_addr = HIF_CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_ADDR; + cmd->reg.rsp_buf_h_addr = HIF_CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_ADDR; + cmd->reg.rsp_buf_l_addr = HIF_CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_ADDR; + cmd->reg.msix_vec_addr = HIF_CMDQM_VECTOR_ID_MEM_ADDR; + cmd->reg.element_sz_addr = HIF_CMDQM_Q_ELEMENT_SZ_REG_ADDR; + cmd->reg.q_depth_addr = HIF_CMDQM_HOST_Q_DEPTH_REG_ADDR; + cmd->reg.interrupt_stat_addr = HIF_CMDQM_HOST_VF_ERR_STS_MEM_ADDR; + } else { + cmd->reg.req_pid_addr = CMDQM_HOST_REQ_PID_MEM_ADDR; + cmd->reg.req_cid_addr = CMDQM_HOST_REQ_CID_MEM_ADDR; + cmd->reg.rsp_pid_addr = CMDQM_HOST_RSP_PID_MEM_ADDR; + cmd->reg.rsp_cid_addr = CMDQM_HOST_RSP_CID_MEM_ADDR; + cmd->reg.req_buf_h_addr = CMDQM_HOST_REQ_BUF_BASE_H_ADDR_MEM_ADDR; + cmd->reg.req_buf_l_addr = CMDQM_HOST_REQ_BUF_BASE_L_ADDR_MEM_ADDR; + cmd->reg.rsp_buf_h_addr = CMDQM_HOST_RSP_BUF_BASE_H_ADDR_MEM_ADDR; + cmd->reg.rsp_buf_l_addr = CMDQM_HOST_RSP_BUF_BASE_L_ADDR_MEM_ADDR; + cmd->reg.msix_vec_addr = CMDQM_VECTOR_ID_MEM_ADDR; + cmd->reg.element_sz_addr = CMDQM_Q_ELEMENT_SZ_REG_ADDR; + cmd->reg.q_depth_addr = CMDQM_HOST_Q_DEPTH_REG_ADDR; + cmd->reg.interrupt_stat_addr = CMDQM_HOST_VF_ERR_STS_MEM_ADDR; + } + + cmd->pool = pci_pool_create("xsc_cmd", xdev->pdev, size, align, 0); + if (!cmd->pool) + return -ENOMEM; + + cmd->cmd_buf = (void *)__get_free_pages(GFP_ATOMIC, 0); + if (!cmd->cmd_buf) { + err = -ENOMEM; + goto err_free_pool; + } + cmd->cq_buf = (void *)__get_free_pages(GFP_ATOMIC, 0); + if (!cmd->cq_buf) { + err = -ENOMEM; + goto err_free_cmd; + } + + cmd->dma = dma_map_single(&xdev->pdev->dev, cmd->cmd_buf, PAGE_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&xdev->pdev->dev, cmd->dma)) { + err = -ENOMEM; + goto err_free; + } + + cmd->cq_dma = dma_map_single(&xdev->pdev->dev, cmd->cq_buf, PAGE_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&xdev->pdev->dev, cmd->cq_dma)) { + err = -ENOMEM; + goto err_map_cmd; + } + + cmd->cmd_pid = readl(REG_ADDR(xdev, cmd->reg.req_pid_addr)); + cmd->cq_cid = readl(REG_ADDR(xdev, cmd->reg.rsp_cid_addr)); + cmd->ownerbit_learned = 0; + + xsc_cmd_handle_rsp_before_reload(cmd, xdev); + +#define ELEMENT_SIZE_LOG 6 //64B +#define Q_DEPTH_LOG 5 //32 + + cmd->log_sz = Q_DEPTH_LOG; + cmd->log_stride = readl(REG_ADDR(xdev, cmd->reg.element_sz_addr)); + writel(1<log_sz, REG_ADDR(xdev, cmd->reg.q_depth_addr)); + if (cmd->log_stride != ELEMENT_SIZE_LOG) { + dev_err(&xdev->pdev->dev, "firmware failed to init cmdq, log_stride=(%d, %d)\n", + cmd->log_stride, ELEMENT_SIZE_LOG); + err = -ENODEV; + goto err_map; + } + + if (1 << cmd->log_sz > XSC_MAX_COMMANDS) { + dev_err(&xdev->pdev->dev, "firmware reports too many outstanding commands %d\n", + 1 << cmd->log_sz); + err = -EINVAL; + goto err_map; + } + + if (cmd->log_sz + cmd->log_stride > PAGE_SHIFT) { + dev_err(&xdev->pdev->dev, "command queue size overflow\n"); + err = -EINVAL; + goto err_map; + } + + cmd->checksum_disabled = 1; + cmd->max_reg_cmds = (1 << cmd->log_sz) - 1; + cmd->bitmask = (1 << cmd->max_reg_cmds) - 1; + +// cmd->cmdif_rev = ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; +// if (cmd->cmdif_rev > CMD_IF_REV) { +// err = -ENOTSUPP; +// goto err_map; +// } + + spin_lock_init(&cmd->alloc_lock); + spin_lock_init(&cmd->token_lock); + spin_lock_init(&cmd->doorbell_lock); + for (i = 0; i < ARRAY_SIZE(cmd->stats); i++) + spin_lock_init(&cmd->stats[i].lock); + + sema_init(&cmd->sem, cmd->max_reg_cmds); + sema_init(&cmd->pages_sem, 1); + + cmd_h = (u32)((u64)(cmd->dma) >> 32); + cmd_l = (u32)(cmd->dma); + if (cmd_l & 0xfff) { + dev_err(&xdev->pdev->dev, "invalid command queue address\n"); + err = -ENOMEM; + goto err_map; + } + +// iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h); +// iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz); +// u32 *ptr = (u32 *)&cmd->dma; +// IA_WRITE(xdev, CMDQ_PA_REG_ADDR, ptr, sizeof(u64) / sizeof(u32)); + writel(cmd_h, REG_ADDR(xdev, cmd->reg.req_buf_h_addr)); + writel(cmd_l, REG_ADDR(xdev, cmd->reg.req_buf_l_addr)); + + cmd_h = (u32)((u64)(cmd->cq_dma) >> 32); + cmd_l = (u32)(cmd->cq_dma); + if (cmd_l & 0xfff) { + dev_err(&xdev->pdev->dev, "invalid command queue address\n"); + err = -ENOMEM; + goto err_map; + } + writel(cmd_h, REG_ADDR(xdev, cmd->reg.rsp_buf_h_addr)); + writel(cmd_l, REG_ADDR(xdev, cmd->reg.rsp_buf_l_addr)); + + /* Make sure firmware sees the complete address before we proceed */ + wmb(); + + xsc_core_dbg(xdev, "descriptor at dma 0x%llx 0x%llx\n", + (unsigned long long)(cmd->dma), (unsigned long long)(cmd->cq_dma)); + + cmd->mode = CMD_MODE_POLLING; + + err = create_msg_cache(xdev); + if (err) { + dev_err(&xdev->pdev->dev, "failed to create command cache\n"); + goto err_map; + } + + set_wqname(xdev); + cmd->wq = create_singlethread_workqueue(cmd->wq_name); + if (!cmd->wq) { + dev_err(&xdev->pdev->dev, "failed to create command workqueue\n"); + err = -ENOMEM; + goto err_cache; + } + + cmd->cq_task = kthread_create(cmd_cq_polling, (void *)xdev, "xsc_cmd_cq_polling"); + if (!cmd->cq_task) { + dev_err(&xdev->pdev->dev, "failed to create cq task\n"); + err = -ENOMEM; + goto err_wq; + } + wake_up_process(cmd->cq_task); + + err = create_debugfs_files(xdev); + if (err) { + err = -ENOMEM; + goto err_task; + } + + err = request_pid_cid_mismatch_restore(xdev); + if (err) { + dev_err(&xdev->pdev->dev, "request pid,cid wrong, restore failed\n"); + goto err_req_restore; + } + + // clear abnormal state to avoid the impact of previous error + err_stat = readl(REG_ADDR(xdev, xdev->cmd.reg.interrupt_stat_addr)); + if (err_stat) { + xsc_core_warn(xdev, "err_stat 0x%x when initializing, clear it\n", err_stat); + writel(0xf, REG_ADDR(xdev, xdev->cmd.reg.interrupt_stat_addr)); + } + + return 0; + +err_req_restore: +err_task: + kthread_stop(cmd->cq_task); + +err_wq: + destroy_workqueue(cmd->wq); + +err_cache: + destroy_msg_cache(xdev); + +err_map: + dma_unmap_single(&xdev->pdev->dev, cmd->cq_dma, PAGE_SIZE, + DMA_BIDIRECTIONAL); + +err_map_cmd: + dma_unmap_single(&xdev->pdev->dev, cmd->dma, PAGE_SIZE, + DMA_BIDIRECTIONAL); +err_free: + free_pages((unsigned long)cmd->cq_buf, 0); + +err_free_cmd: + free_pages((unsigned long)cmd->cmd_buf, 0); + +err_free_pool: + pci_pool_destroy(cmd->pool); + + return err; +} +EXPORT_SYMBOL(xsc_cmd_init); + +void xsc_cmd_cleanup(struct xsc_core_device *xdev) +{ + struct xsc_cmd *cmd = &xdev->cmd; + + clean_debug_files(xdev); + destroy_workqueue(cmd->wq); + if (cmd->cq_task) + kthread_stop(cmd->cq_task); + destroy_msg_cache(xdev); + dma_unmap_single(&xdev->pdev->dev, cmd->dma, PAGE_SIZE, + DMA_BIDIRECTIONAL); + free_pages((unsigned long)cmd->cq_buf, 0); + dma_unmap_single(&xdev->pdev->dev, cmd->cq_dma, PAGE_SIZE, + DMA_BIDIRECTIONAL); + free_pages((unsigned long)cmd->cmd_buf, 0); + pci_pool_destroy(cmd->pool); +} +EXPORT_SYMBOL(xsc_cmd_cleanup); + +static const char *cmd_status_str(u8 status) +{ + switch (status) { + case XSC_CMD_STAT_OK: + return "OK"; + case XSC_CMD_STAT_INT_ERR: + return "internal error"; + case XSC_CMD_STAT_BAD_OP_ERR: + return "bad operation"; + case XSC_CMD_STAT_BAD_PARAM_ERR: + return "bad parameter"; + case XSC_CMD_STAT_BAD_SYS_STATE_ERR: + return "bad system state"; + case XSC_CMD_STAT_BAD_RES_ERR: + return "bad resource"; + case XSC_CMD_STAT_RES_BUSY: + return "resource busy"; + case XSC_CMD_STAT_LIM_ERR: + return "limits exceeded"; + case XSC_CMD_STAT_BAD_RES_STATE_ERR: + return "bad resource state"; + case XSC_CMD_STAT_IX_ERR: + return "bad index"; + case XSC_CMD_STAT_NO_RES_ERR: + return "no resources"; + case XSC_CMD_STAT_BAD_INP_LEN_ERR: + return "bad input length"; + case XSC_CMD_STAT_BAD_OUTP_LEN_ERR: + return "bad output length"; + case XSC_CMD_STAT_BAD_QP_STATE_ERR: + return "bad QP state"; + case XSC_CMD_STAT_BAD_PKT_ERR: + return "bad packet (discarded)"; + case XSC_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: + return "bad size too many outstanding CQEs"; + default: + return "unknown status"; + } +} + +int xsc_cmd_status_to_err(struct xsc_outbox_hdr *hdr) +{ + if (!hdr->status) + return 0; + + pr_warn("command failed, status %s(0x%x), syndrome 0x%x\n", + cmd_status_str(hdr->status), hdr->status, + be32_to_cpu(hdr->syndrome)); + + switch (hdr->status) { + case XSC_CMD_STAT_OK: return 0; + case XSC_CMD_STAT_INT_ERR: return -EIO; + case XSC_CMD_STAT_BAD_OP_ERR: return -EINVAL; + case XSC_CMD_STAT_BAD_PARAM_ERR: return -EINVAL; + case XSC_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO; + case XSC_CMD_STAT_BAD_RES_ERR: return -EINVAL; + case XSC_CMD_STAT_RES_BUSY: return -EBUSY; + case XSC_CMD_STAT_LIM_ERR: return -EINVAL; + case XSC_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL; + case XSC_CMD_STAT_IX_ERR: return -EINVAL; + case XSC_CMD_STAT_NO_RES_ERR: return -EAGAIN; + case XSC_CMD_STAT_BAD_INP_LEN_ERR: return -EIO; + case XSC_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO; + case XSC_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL; + case XSC_CMD_STAT_BAD_PKT_ERR: return -EINVAL; + case XSC_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL; + default: return -EIO; + } +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/cq.c b/drivers/net/ethernet/yunsilicon/xsc/pci/cq.c new file mode 100644 index 0000000000000000000000000000000000000000..712524130295dac14e5f37ad02ad1bfa1f5040dc --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/cq.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include + +void xsc_cq_event(struct xsc_core_device *xdev, u32 cqn, int event_type) +{ + struct xsc_cq_table *table = &xdev->dev_res->cq_table; + struct xsc_core_cq *cq; + + spin_lock(&table->lock); + + cq = radix_tree_lookup(&table->tree, cqn); + if (cq) + atomic_inc(&cq->refcount); + + spin_unlock(&table->lock); + + if (!cq) { + xsc_core_warn(xdev, "Async event for bogus CQ 0x%x\n", cqn); + return; + } + + cq->event(cq, event_type); + + if (atomic_dec_and_test(&cq->refcount)) + complete(&cq->free); +} + +int xsc_core_create_cq(struct xsc_core_device *dev, struct xsc_core_cq *cq, + struct xsc_create_cq_mbox_in *in, int inlen) +{ + int err; + struct xsc_cq_table *table = &dev->dev_res->cq_table; + struct xsc_create_cq_mbox_out out; + struct xsc_destroy_cq_mbox_in din; + struct xsc_destroy_cq_mbox_out dout; + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_CREATE_CQ); + memset(&out, 0, sizeof(out)); + err = xsc_cmd_exec(dev, in, inlen, &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + return xsc_cmd_status_to_err(&out.hdr); + + cq->cqn = be32_to_cpu(out.cqn); + cq->cons_index = 0; + cq->arm_sn = 0; + cq->arm_db = dev->regs.complete_db; + cq->ci_db = dev->regs.complete_reg; + cq->dev = dev; + atomic_set(&cq->refcount, 1); + init_completion(&cq->free); + + spin_lock_irq(&table->lock); + err = radix_tree_insert(&table->tree, cq->cqn, cq); + spin_unlock_irq(&table->lock); + if (err) + goto err_cmd; + + cq->pid = current->pid; + err = xsc_debug_cq_add(dev, cq); + if (err) + xsc_core_dbg(dev, "failed adding CP 0x%x to debug file system\n", + cq->cqn); + + return 0; + +err_cmd: + memset(&din, 0, sizeof(din)); + memset(&dout, 0, sizeof(dout)); + din.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_CQ); + xsc_cmd_exec(dev, &din, sizeof(din), &dout, sizeof(dout)); + return err; +} +EXPORT_SYMBOL(xsc_core_create_cq); + +int xsc_core_destroy_cq(struct xsc_core_device *dev, struct xsc_core_cq *cq) +{ + struct xsc_cq_table *table = &dev->dev_res->cq_table; + struct xsc_destroy_cq_mbox_in in; + struct xsc_destroy_cq_mbox_out out; + struct xsc_core_cq *tmp; + int err; + + spin_lock_irq(&table->lock); + tmp = radix_tree_delete(&table->tree, cq->cqn); + spin_unlock_irq(&table->lock); + if (!tmp) { + xsc_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn); + return -EINVAL; + } + if (tmp != cq) { + xsc_core_warn(dev, "corruption on srqn 0x%x\n", cq->cqn); + return -EINVAL; + } + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_CQ); + in.cqn = cpu_to_be32(cq->cqn); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + return xsc_cmd_status_to_err(&out.hdr); + + //synchronize_irq(cq->irqn); + + xsc_debug_cq_remove(dev, cq); + if (atomic_dec_and_test(&cq->refcount)) + complete(&cq->free); + wait_for_completion(&cq->free); + + return 0; +} +EXPORT_SYMBOL(xsc_core_destroy_cq); + +int xsc_core_query_cq(struct xsc_core_device *dev, struct xsc_core_cq *cq, + struct xsc_query_cq_mbox_out *out) +{ + struct xsc_query_cq_mbox_in in; + int err; + + memset(&in, 0, sizeof(in)); + memset(out, 0, sizeof(*out)); + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_CQ); + in.cqn = cpu_to_be32(cq->cqn); + err = xsc_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); + if (err) + return err; + + if (out->hdr.status) + return xsc_cmd_status_to_err(&out->hdr); + + return err; +} +EXPORT_SYMBOL(xsc_core_query_cq); + +void xsc_init_cq_table(struct xsc_core_device *dev) +{ + struct xsc_cq_table *table = &dev->dev_res->cq_table; + + spin_lock_init(&table->lock); + INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); + xsc_cq_debugfs_init(dev); +} + +void xsc_cleanup_cq_table(struct xsc_core_device *dev) +{ + xsc_cq_debugfs_cleanup(dev); +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/debugfs.c b/drivers/net/ethernet/yunsilicon/xsc/pci/debugfs.c new file mode 100644 index 0000000000000000000000000000000000000000..5c31712f01ca7b5c821f04bd2cf7de59bf6bd5e1 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/debugfs.c @@ -0,0 +1,916 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "fw/xsc_tbm.h" + +enum { + QP_PID, + QP_STATE, + QP_XPORT, + QP_MTU, + QP_N_RECV, + QP_RECV_SZ, + QP_N_SEND, + QP_LOG_PG_SZ, + QP_RQPN, +}; + +static char *qp_fields[] = { + [QP_PID] = "pid", + [QP_STATE] = "state", + [QP_XPORT] = "transport", + [QP_MTU] = "mtu", + [QP_N_RECV] = "num_recv", + [QP_RECV_SZ] = "rcv_wqe_sz", + [QP_N_SEND] = "num_send", + [QP_LOG_PG_SZ] = "log2_page_sz", + [QP_RQPN] = "remote_qpn", +}; + +enum { + EQ_NUM_EQES, + EQ_INTR, + EQ_LOG_PG_SZ, +}; + +static char *eq_fields[] = { + [EQ_NUM_EQES] = "num_eqes", + [EQ_INTR] = "intr", + [EQ_LOG_PG_SZ] = "log_page_size", +}; + +enum { + CQ_PID, + CQ_NUM_CQES, + CQ_LOG_PG_SZ, +}; + +static char *cq_fields[] = { + [CQ_PID] = "pid", + [CQ_NUM_CQES] = "num_cqes", + [CQ_LOG_PG_SZ] = "log_page_size", +}; + +struct dentry *xsc_debugfs_root; +EXPORT_SYMBOL(xsc_debugfs_root); + +static ssize_t xsc_debugfs_reg_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + char *buf; + int len; + char xsc_debugfs_reg_buf[256] = ""; + + /* don't allow partial reads */ + if (*ppos != 0) + return 0; + + buf = kasprintf(GFP_KERNEL, "%s: %s\n", + "xsc debugfs", + xsc_debugfs_reg_buf); + if (!buf) + return -ENOMEM; + + if (count < strlen(buf)) { + kfree(buf); + return -ENOSPC; + } + + len = simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf)); + + kfree(buf); + + return len; +} + +static ssize_t xsc_debugfs_reg_write(struct file *filp, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct xsc_core_device *xdev = filp->private_data; + u64 reg; + int cnt, len; + int num; + int offset; + char xsc_debugfs_reg_buf[256] = ""; + + /* don't allow partial writes */ + if (*ppos != 0) + return 0; + + if (count >= sizeof(xsc_debugfs_reg_buf)) + return -ENOSPC; + + len = simple_write_to_buffer(xsc_debugfs_reg_buf, + sizeof(xsc_debugfs_reg_buf) - 1, + ppos, buffer, count); + if (len < 0) + return len; + + xsc_debugfs_reg_buf[len] = '\0'; + + if (strncmp(xsc_debugfs_reg_buf, "write", 5) == 0) { + cnt = sscanf(&xsc_debugfs_reg_buf[5], "%llx %n", + ®, &offset); + if (cnt == 1) { + int tmp; + int value; + int buf[8]; + int *ptr; + + offset += 5; + num = 0; + while (1) { + cnt = sscanf(&xsc_debugfs_reg_buf[offset], "%x %n", + &value, &tmp); + if (cnt < 2) + break; + xsc_core_info(xdev, "write: 0x%llx = 0x%x\n", + (reg + sizeof(int) * num), value); + offset += tmp; + buf[num++] = value; + if (num == 8) + break; + } + if (num > 1) { + ptr = &buf[0]; + IA_WRITE(xdev, reg, ptr, num); + } else if (num == 1) + REG_WR32(xdev, reg, buf[0]); + } else { + xsc_core_err(xdev, "write \n"); + } + } else if (strncmp(xsc_debugfs_reg_buf, "read", 4) == 0) { + cnt = sscanf(&xsc_debugfs_reg_buf[4], "%llx %d %n", ®, &num, &offset); + if (cnt == 2) { + int *buf; + int i; + int *ptr; + + buf = kcalloc(num, sizeof(int), GFP_KERNEL); + if (!buf) + return -ENOMEM; + ptr = buf; + IA_READ(xdev, reg, ptr, num); + xsc_core_info(xdev, "read: 0x%llx num:%d\n", reg, num); + for (i = 0; i < num; i++) + xsc_core_info(xdev, "read:0x%llx = %#x\n", + (reg + sizeof(int) * i), buf[i]); + } else if (cnt == 1) { + int value = REG_RD32(xdev, reg); + + xsc_core_info(xdev, "read: 0x%llx = %#x\n", reg, value); + } else { + xsc_core_err(xdev, "read \n"); + } + } else { + xsc_core_err(xdev, "Unknown command %s\n", xsc_debugfs_reg_buf); + xsc_core_err(xdev, "Available commands:\n"); + xsc_core_err(xdev, "read \n"); + xsc_core_err(xdev, "write \n"); + } + return count; +} + +static const struct file_operations xsc_debugfs_reg_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = xsc_debugfs_reg_read, + .write = xsc_debugfs_reg_write, +}; + +static ssize_t xsc_debugfs_vlan_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + char *buf; + int len; + char xsc_debugfs_vlan_buf[256] = ""; + + /* don't allow partial reads */ + if (*ppos != 0) + return 0; + + buf = kasprintf(GFP_KERNEL, "%s: %s\n", + "vlan debugfs", xsc_debugfs_vlan_buf); + if (!buf) + return -ENOMEM; + + if (count < strlen(buf)) { + kfree(buf); + return -ENOSPC; + } + + len = simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf)); + + kfree(buf); + return len; +} + +static ssize_t xsc_debugfs_vlan_write(struct file *filp, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct xsc_core_device *xdev = filp->private_data; + struct xsc_vlan_config config; + char proto[16]; + int off = 0; + int len, cnt; + char xsc_debugfs_vlan_buf[256] = ""; + + /* don't allow partial writes */ + if (*ppos != 0) + return 0; + + if (count >= sizeof(xsc_debugfs_vlan_buf)) + return -ENOSPC; + + len = simple_write_to_buffer(xsc_debugfs_vlan_buf, + sizeof(xsc_debugfs_vlan_buf) - 1, + ppos, buffer, count); + if (len < 0) + return len; + + xsc_debugfs_vlan_buf[len] = '\0'; + memset(&config, 0, sizeof(config)); + // + // trunk 100 0 4096 802.1q 0 0 + // tunnel 100 0 4096 802.1ad 1 0 + if (strncmp(xsc_debugfs_vlan_buf, "trunk", 5) == 0) { + off = 5; + config.mode = XSC_VLAN_MODE_TRUNK; + } else if (strncmp(xsc_debugfs_vlan_buf, "tunnel", 6) == 0) { + off = 6; + config.mode = XSC_VLAN_MODE_TUNNEL; + } else if (strncmp(xsc_debugfs_vlan_buf, "access", 6) == 0) { + off = 6; + config.mode = XSC_VLAN_MODE_ACCESS; + } else if (strncmp(xsc_debugfs_vlan_buf, "tagged", 6) == 0) { + off = 6; + config.mode = XSC_VLAN_MODE_NATIVE_TAGGED; + } else if (strncmp(xsc_debugfs_vlan_buf, "untagged", 8) == 0) { + off = 8; + config.mode = XSC_VLAN_MODE_NATIVE_UNTAGGED; + } else if (strncmp(xsc_debugfs_vlan_buf, "none", 4) == 0) { + off = 4; + config.mode = 0; + } else { + xsc_core_err(xdev, "invalid vlan mode: %s\n", xsc_debugfs_vlan_buf); + return 0; + } + + cnt = sscanf(&xsc_debugfs_vlan_buf[off], "%u %u %u %s %u %u", + &config.pvid, &config.vid_allow_base, + &config.vid_allow_num, proto, + &config.prio, &config.smac_filter_en); + if (cnt < 3) { + xsc_core_err(xdev, "error arguments: \n"); + return 0; + } + + if (strncmp(proto, "802.1q", 6) == 0) + config.proto = ETH_P_8021Q; + else if (strncmp(proto, "802.1ad", 7) == 0) + config.proto = ETH_P_8021AD; + else + config.proto = ETH_P_8021Q; + + if (config.prio > 7) { + xsc_core_err(xdev, "invalid vlan prio: %s\n", xsc_debugfs_vlan_buf); + return 0; + } + + xsc_core_info(xdev, "%s: vlan_mode=%d vid=%d vlan_allow=%d/%d proto=0x%x prio=%d smac_en=%d", + __func__, config.mode, config.pvid, config.vid_allow_base, + config.vid_allow_num, config.proto, config.prio, + config.smac_filter_en); + + return count; +} + +static const struct file_operations xsc_debugfs_vlan_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = xsc_debugfs_vlan_read, + .write = xsc_debugfs_vlan_write, +}; + +int xsc_vlan_debugfs_init(struct xsc_core_device *dev) +{ + struct dentry *pfile; + + if (dev->dev_res->dbg_root) { + pfile = debugfs_create_file("vlan", 0644, + dev->dev_res->dbg_root, dev, + &xsc_debugfs_vlan_fops); + if (!pfile) + xsc_core_err(dev, "failed to create vlan debugfs\n"); + else + xsc_core_info(dev, "create vlan debugfs ok\n"); + } + + return 0; +} + +int xsc_debugfs_init(struct xsc_core_device *dev) +{ + const char *name = pci_name(dev->pdev); + struct dentry *pfile; + + if (!xsc_debugfs_root) + return -ENOMEM; + + dev->dev_res->dbg_root = debugfs_create_dir(name, xsc_debugfs_root); + if (dev->dev_res->dbg_root) { + pfile = debugfs_create_file("reg_ops", 0600, + dev->dev_res->dbg_root, dev, + &xsc_debugfs_reg_fops); + if (!pfile) + xsc_core_err(dev, "failed to create debugfs ops for %s\n", + name); + } else { + xsc_core_err(dev, "failed to create debugfs dir for %s\n", name); + return -ENOMEM; + } + + xsc_vlan_debugfs_init(dev); + xsc_core_info(dev, "%s.dir_name=%s\r\n", __func__, name); + + return 0; +} + +void xsc_debugfs_fini(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return; + + debugfs_remove_recursive(dev->dev_res->dbg_root); +} + +void xsc_register_debugfs(void) +{ + xsc_debugfs_root = debugfs_create_dir("xsc_pci", NULL); +} + +void xsc_unregister_debugfs(void) +{ + debugfs_remove(xsc_debugfs_root); +} + +int xsc_qp_debugfs_init(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return 0; + + atomic_set(&dev->num_qps, 0); + + dev->dev_res->qp_debugfs = debugfs_create_dir("QPs", dev->dev_res->dbg_root); + if (!dev->dev_res->qp_debugfs) + return -ENOMEM; + + return 0; +} + +void xsc_qp_debugfs_cleanup(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return; + + debugfs_remove_recursive(dev->dev_res->qp_debugfs); +} + +int xsc_eq_debugfs_init(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return 0; + + dev->dev_res->eq_debugfs = debugfs_create_dir("EQs", dev->dev_res->dbg_root); + if (!dev->dev_res->eq_debugfs) + return -ENOMEM; + + return 0; +} + +void xsc_eq_debugfs_cleanup(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return; + + debugfs_remove_recursive(dev->dev_res->eq_debugfs); +} + +static ssize_t average_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct xsc_cmd_stats *stats; + u64 field = 0; + int ret; + int err; + char tbuf[22]; + + if (*pos) + return 0; + + stats = filp->private_data; + spin_lock(&stats->lock); + if (stats->n) + field = stats->sum / stats->n; + spin_unlock(&stats->lock); + ret = snprintf(tbuf, sizeof(tbuf), "%llu\n", field); + if (ret > 0) { + err = copy_to_user(buf, tbuf, ret); + if (err) + return err; + } + + *pos += ret; + return ret; +} + +static ssize_t average_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xsc_cmd_stats *stats; + + stats = filp->private_data; + spin_lock(&stats->lock); + stats->sum = 0; + stats->n = 0; + spin_unlock(&stats->lock); + + *pos += count; + + return count; +} + +static const struct file_operations stats_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = average_read, + .write = average_write, +}; + +int xsc_cmdif_debugfs_init(struct xsc_core_device *xdev) +{ + struct xsc_cmd_stats *stats; + struct xsc_cmd *cmd; + struct dentry **cmdif_debugfs; + const char *namep; + int err; + int i; + + if (!xsc_debugfs_root) + return 0; + + cmd = &xdev->cmd; + cmdif_debugfs = &xdev->dev_res->cmdif_debugfs; + *cmdif_debugfs = debugfs_create_dir("commands", xdev->dev_res->dbg_root); + if (!*cmdif_debugfs) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(cmd->stats); i++) { + stats = &cmd->stats[i]; + namep = xsc_command_str(i); + if (strcmp(namep, "unknown command opcode")) { + stats->root = debugfs_create_dir(namep, *cmdif_debugfs); + if (!stats->root) { + xsc_core_warn(xdev, "failed adding command %d\n", + i); + err = -ENOMEM; + goto out; + } + + stats->avg = debugfs_create_file("average", 0400, + stats->root, stats, + &stats_fops); + if (!stats->avg) { + xsc_core_warn(xdev, "failed creating debugfs file\n"); + err = -ENOMEM; + goto out; + } + + debugfs_create_u64("n", 0400, stats->root, &stats->n); + } + } + + return 0; +out: + debugfs_remove_recursive(xdev->dev_res->cmdif_debugfs); + return err; +} + +void xsc_cmdif_debugfs_cleanup(struct xsc_core_device *xdev) +{ + if (!xsc_debugfs_root) + return; + + debugfs_remove_recursive(xdev->dev_res->cmdif_debugfs); +} + +int xsc_cq_debugfs_init(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return 0; + + dev->dev_res->cq_debugfs = debugfs_create_dir("CQs", dev->dev_res->dbg_root); + if (!dev->dev_res->cq_debugfs) + return -ENOMEM; + + return 0; +} + +void xsc_cq_debugfs_cleanup(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return; + + debugfs_remove_recursive(dev->dev_res->cq_debugfs); +} + +int xsc_qptrace_debugfs_init(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return 0; + + dev->dev_res->qptrace_debugfs = + debugfs_create_dir("QPTrace", dev->dev_res->dbg_root); + if (!dev->dev_res->qptrace_debugfs) + return -ENOMEM; + + return 0; +} + +void xsc_qptrace_debugfs_cleanup(struct xsc_core_device *dev) +{ + if (!xsc_debugfs_root) + return; + + debugfs_remove_recursive(dev->dev_res->qptrace_debugfs); +} + +static u64 qp_read_field(struct xsc_core_device *dev, struct xsc_core_qp *qp, + int index) +{ + struct xsc_query_qp_mbox_out *out; + struct xsc_qp_context *ctx; + u64 param = 0; + int err; + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) + return param; + + err = xsc_core_qp_query(dev, qp, out, sizeof(*out)); + if (err) { + xsc_core_warn(dev, "failed to query qp\n"); + goto out; + } + + ctx = &out->ctx; + switch (index) { + case QP_PID: + param = qp->pid; + break; +// case QP_STATE: +// param = be32_to_cpu(ctx->flags) >> 28; +// break; +// case QP_XPORT: +// param = (be32_to_cpu(ctx->flags) >> 16) & 0xff; +// break; + case QP_MTU: + param = ctx->mtu_mode ? IB_MTU_1024 : IB_MTU_4096; + break; +// case QP_N_RECV: +// param = 1 << ((ctx->rq_size_stride >> 3) & 0xf); +// break; +// case QP_RECV_SZ: +// param = 1 << ((ctx->rq_size_stride & 7) + 4); +// break; +// case QP_N_SEND: +// no_sq = be16_to_cpu(ctx->sq_crq_size) >> 15; +// if (!no_sq) +// param = 1 << (be16_to_cpu(ctx->sq_crq_size) >> 11); +// else +// param = 0; +// break; +// case QP_LOG_PG_SZ: +// param = ((cpu_to_be32(ctx->log_pg_sz_remote_qpn) >> 24) & 0x1f); +// param += 12; +// break; + case QP_RQPN: + param = cpu_to_be32(ctx->remote_qpn) & 0xffffff; + break; + } + +out: + kfree(out); + return param; +} + +static u64 eq_read_field(struct xsc_core_device *dev, struct xsc_eq *eq, + int index) +{ + struct xsc_query_eq_mbox_out *out; + struct xsc_eq_context *ctx; + u64 param = 0; + int err; + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) + return param; + + ctx = &out->ctx; + + err = xsc_core_eq_query(dev, eq, out, sizeof(*out)); + if (err) { + xsc_core_warn(dev, "failed to query eq\n"); + goto out; + } + + switch (index) { + case EQ_NUM_EQES: +// param = 1 << ((be32_to_cpu(ctx->log_sz_usr_page) >> 24) & 0x1f); + break; + case EQ_INTR: +// param = ctx->intr; + break; + case EQ_LOG_PG_SZ: +// param = (ctx->log_page_size & 0x1f) + 12; + break; + } + +out: + kfree(out); + return param; +} + +static u64 cq_read_field(struct xsc_core_device *dev, struct xsc_core_cq *cq, + int index) +{ + struct xsc_query_cq_mbox_out *out; + struct xsc_cq_context *ctx; + u64 param = 0; + int err; + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) + return param; + + ctx = &out->ctx; + + err = xsc_core_query_cq(dev, cq, out); + if (err) { + xsc_core_warn(dev, "failed to query cq\n"); + goto out; + } + + switch (index) { + case CQ_PID: +// param = cq->pid; + break; + case CQ_NUM_CQES: +// param = 1 << ((be32_to_cpu(ctx->log_sz_usr_page) >> 24) & 0x1f); + break; + case CQ_LOG_PG_SZ: +// param = (ctx->log_pg_sz & 0x1f) + 12; + break; + } + +out: + kfree(out); + return param; +} + +static ssize_t dbg_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct xsc_field_desc *desc; + struct xsc_rsc_debug *d; + char tbuf[18]; + u64 field; + int ret; + int err; + + if (*pos) + return 0; + + desc = filp->private_data; + d = (void *)(desc - desc->i) - sizeof(*d); + switch (d->type) { + case XSC_DBG_RSC_QP: + field = qp_read_field(d->xdev, d->object, desc->i); + break; + + case XSC_DBG_RSC_EQ: + field = eq_read_field(d->xdev, d->object, desc->i); + break; + + case XSC_DBG_RSC_CQ: + field = cq_read_field(d->xdev, d->object, desc->i); + break; + + default: + xsc_core_warn(d->xdev, "invalid resource type %d\n", d->type); + return -EINVAL; + } + + ret = snprintf(tbuf, sizeof(tbuf), "0x%llx\n", field); + if (ret > 0) { + err = copy_to_user(buf, tbuf, ret); + if (err) + return err; + } + + *pos += ret; + return ret; +} + +static const struct file_operations fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = dbg_read, +}; + +static int add_res_tree(struct xsc_core_device *dev, enum dbg_rsc_type type, + struct dentry *root, struct xsc_rsc_debug **dbg, + int rsn, char **field, int nfile, void *data) +{ + struct xsc_rsc_debug *d; + char resn[32]; + int err; + int i; + + d = kzalloc(sizeof(*d) + nfile * sizeof(d->fields[0]), GFP_KERNEL); + if (!d) + return -ENOMEM; + + d->xdev = dev; + d->object = data; + d->type = type; + sprintf(resn, "0x%x", rsn); + d->root = debugfs_create_dir(resn, root); + if (!d->root) { + err = -ENOMEM; + goto out_free; + } + + for (i = 0; i < nfile; i++) { + d->fields[i].i = i; + d->fields[i].dent = debugfs_create_file(field[i], 0400, + d->root, &d->fields[i], + &fops); + if (!d->fields[i].dent) { + err = -ENOMEM; + goto out_rem; + } + } + *dbg = d; + + return 0; +out_rem: + debugfs_remove_recursive(d->root); + +out_free: + kfree(d); + return err; +} + +static void rem_res_tree(struct xsc_rsc_debug *d) +{ + debugfs_remove_recursive(d->root); + kfree(d); +} + +int xsc_debug_qp_add(struct xsc_core_device *dev, struct xsc_core_qp *qp) +{ + int err; + + if (!xsc_debugfs_root) + return 0; + + err = add_res_tree(dev, XSC_DBG_RSC_QP, dev->dev_res->qp_debugfs, + &qp->dbg, qp->qpn, qp_fields, + ARRAY_SIZE(qp_fields), qp); + if (err) + qp->dbg = NULL; + + return err; +} + +void xsc_debug_qp_remove(struct xsc_core_device *dev, struct xsc_core_qp *qp) +{ + if (!xsc_debugfs_root) + return; + + if (qp->dbg) + rem_res_tree(qp->dbg); +} + +static ssize_t trace_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) +{ + struct xsc_qp_trace *trace_info = filp->private_data; + int err; + int len; + + if (*pos) + return 0; + + len = sizeof(struct xsc_qp_trace); + err = copy_to_user(buf, trace_info, len); + if (err) + return err; + + *pos += len; + return len; +} + +static const struct file_operations fops_trace = { + .owner = THIS_MODULE, + .open = simple_open, + .read = trace_read, +}; + +int xsc_create_qptrace(struct xsc_core_device *dev, struct xsc_core_qp *qp) +{ + char name[16]; + + if (!xsc_debugfs_root) + return 0; + + snprintf(name, sizeof(name), "%d", qp->qpn); + + qp->trace = debugfs_create_file(name, 0400, dev->dev_res->qptrace_debugfs, + (void *)qp->trace_info, &fops_trace); + if (!qp->trace) + return -1; + + return 0; +} + +void xsc_remove_qptrace(struct xsc_core_device *dev, struct xsc_core_qp *qp) +{ + if (!xsc_debugfs_root) + return; + + debugfs_remove(qp->trace); +} + +int xsc_debug_eq_add(struct xsc_core_device *dev, struct xsc_eq *eq) +{ + int err; + + if (!xsc_debugfs_root) + return 0; + + err = add_res_tree(dev, XSC_DBG_RSC_EQ, dev->dev_res->eq_debugfs, + &eq->dbg, eq->eqn, eq_fields, + ARRAY_SIZE(eq_fields), eq); + if (err) + eq->dbg = NULL; + + return err; +} + +void xsc_debug_eq_remove(struct xsc_core_device *dev, struct xsc_eq *eq) +{ + if (!xsc_debugfs_root) + return; + + if (eq->dbg) + rem_res_tree(eq->dbg); +} + +int xsc_debug_cq_add(struct xsc_core_device *dev, struct xsc_core_cq *cq) +{ + int err; + + if (!xsc_debugfs_root) + return 0; + + err = add_res_tree(dev, XSC_DBG_RSC_CQ, dev->dev_res->cq_debugfs, + &cq->dbg, cq->cqn, cq_fields, + ARRAY_SIZE(cq_fields), cq); + if (err) + cq->dbg = NULL; + + return err; +} + +void xsc_debug_cq_remove(struct xsc_core_device *dev, struct xsc_core_cq *cq) +{ + if (!xsc_debugfs_root) + return; + + if (cq->dbg) + rem_res_tree(cq->dbg); +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/devlink.c b/drivers/net/ethernet/yunsilicon/xsc/pci/devlink.c new file mode 100644 index 0000000000000000000000000000000000000000..8493d61617b930c98b951d1d75dfcde1adffe067 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/devlink.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include "devlink.h" +#ifdef CONFIG_XSC_ESWITCH +#include "eswitch.h" +#endif + +static const struct devlink_ops xsc_devlink_ops = { +#ifdef CONFIG_XSC_ESWITCH + .eswitch_mode_set = xsc_devlink_eswitch_mode_set, + .eswitch_mode_get = xsc_devlink_eswitch_mode_get, +#endif +}; + +struct devlink *xsc_devlink_alloc(void) +{ + return devlink_alloc(&xsc_devlink_ops, sizeof(struct xsc_core_device)); +} + +void xsc_devlink_free(struct devlink *devlink) +{ + devlink_free(devlink); +} + +int xsc_devlink_register(struct devlink *devlink, struct device *dev) +{ + int err = 0; + + err = devlink_register(devlink, dev); + return err; +} + +void xsc_devlink_unregister(struct devlink *devlink) +{ + devlink_unregister(devlink); +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/devlink.h b/drivers/net/ethernet/yunsilicon/xsc/pci/devlink.h new file mode 100644 index 0000000000000000000000000000000000000000..0dbbadd87f85dd2d7e228761f96b21eb57596729 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/devlink.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_DEVLINK_H +#define XSC_DEVLINK_H + +#include + +struct devlink *xsc_devlink_alloc(void); +void xsc_devlink_free(struct devlink *devlink); +int xsc_devlink_register(struct devlink *devlink, struct device *dev); +void xsc_devlink_unregister(struct devlink *devlink); + +#endif /* XSC_DEVLINK_H */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/eq.c b/drivers/net/ethernet/yunsilicon/xsc/pci/eq.c new file mode 100644 index 0000000000000000000000000000000000000000..3a6d17fb7d9e9f25c13237d76bf34a75c6c5b348 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/eq.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include "fw/xsc_fw.h" +#include "wq.h" +#include + +enum { + XSC_EQE_SIZE = sizeof(struct xsc_eqe), + XSC_EQE_OWNER_INIT_VAL = 0x1, +}; + +enum { + XSC_NUM_SPARE_EQE = 0x80, + XSC_NUM_ASYNC_EQE = 0x100, +}; + +struct map_eq_in { + u64 mask; + u32 reserved; + u32 unmap_eqn; +}; + +struct cre_des_eq { + u8 reserved[15]; + u8 eqn; +}; + +static int xsc_cmd_destroy_eq(struct xsc_core_device *dev, u32 eqn) +{ + struct xsc_destroy_eq_mbox_in in; + struct xsc_destroy_eq_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_EQ); + in.eqn = cpu_to_be32(eqn); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); + if (!err) + goto ex; + + if (out.hdr.status) + err = xsc_cmd_status_to_err(&out.hdr); + +ex: + return err; +} + +static struct xsc_eqe *get_eqe(struct xsc_eq *eq, u32 entry) +{ + return xsc_buf_offset(&eq->buf, entry * XSC_EQE_SIZE); +} + +static struct xsc_eqe *next_eqe_sw(struct xsc_eq *eq) +{ + struct xsc_eqe *eqe = get_eqe(eq, eq->cons_index & (eq->nent - 1)); + + return ((eqe->owner & 1) ^ !!(eq->cons_index & eq->nent)) ? NULL : eqe; +} + +#ifdef XSC_DEBUG +static const char *eqe_type_str(u8 type) +{ + switch (type) { + case XSC_EVENT_TYPE_COMP: + return "XSC_EVENT_TYPE_COMP"; + default: + return "Unrecognized event"; + } +} +#endif +static void eq_update_ci(struct xsc_eq *eq, int arm) +{ + union xsc_eq_doorbell db; + + db.val = 0; + db.arm = !!arm; + db.eq_next_cid = eq->cons_index; + db.eq_id = eq->eqn; +#ifdef XSC_DEBUG + xsc_core_dbg(eq->dev, "ARM EQ %d ci 0x%x arm %d\n", + eq->eqn, eq->cons_index, arm); +#endif + writel(db.val, REG_ADDR(eq->dev, eq->doorbell)); + /* We still want ordering, just not swabbing, so add a barrier */ + mb(); +} + +void xsc_cq_completion(struct xsc_core_device *dev, u32 cqn) +{ + struct xsc_core_cq *cq; + struct xsc_cq_table *table = &dev->dev_res->cq_table; + + rcu_read_lock(); + cq = radix_tree_lookup(&table->tree, cqn); + if (likely(cq)) + atomic_inc(&cq->refcount); + rcu_read_unlock(); + + if (!cq) { + xsc_core_err(dev, "Completion event for bogus CQ 0x%x\n", cqn); + return; + } + + ++cq->arm_sn; + + if (!cq->comp) + xsc_core_err(dev, "cq->comp is NULL\n"); + else + cq->comp(cq); + + if (atomic_dec_and_test(&cq->refcount)) + complete(&cq->free); +} + +void xsc_eq_cq_event(struct xsc_core_device *dev, u32 cqn, int event_type) +{ + struct xsc_core_cq *cq; + struct xsc_cq_table *table = &dev->dev_res->cq_table; + + spin_lock(&table->lock); + cq = radix_tree_lookup(&table->tree, cqn); + if (likely(cq)) + atomic_inc(&cq->refcount); + spin_unlock(&table->lock); + + if (unlikely(!cq)) { + xsc_core_err(dev, "Async event for bogus CQ 0x%x\n", cqn); + return; + } + + cq->event(cq, event_type); + + if (atomic_dec_and_test(&cq->refcount)) + complete(&cq->free); + +} + +static int xsc_eq_int(struct xsc_core_device *dev, struct xsc_eq *eq) +{ + struct xsc_eqe *eqe; + int eqes_found = 0; + int set_ci = 0; + u32 cqn, qpn, queue_id; + + while ((eqe = next_eqe_sw(eq))) { + /* Make sure we read EQ entry contents after we've + * checked the ownership bit. + */ + rmb(); +#ifdef XSC_DEBUG + xsc_core_dbg(eq->dev, "eqn %d, eqe type %s cqn/qpn: %d\n", + eq->eqn, eqe_type_str(eqe->type), eqe->queue_id); +#endif + switch (eqe->type) { + case XSC_EVENT_TYPE_COMP: + case XSC_EVENT_TYPE_INTERNAL_ERROR: + /* eqe is changing */ + queue_id = eqe->queue_id; + cqn = queue_id; + xsc_cq_completion(dev, cqn); + break; + + case XSC_EVENT_TYPE_CQ_ERROR: + queue_id = eqe->queue_id; + cqn = queue_id; + xsc_eq_cq_event(dev, cqn, eqe->type); + break; + case XSC_EVENT_TYPE_WQ_CATAS_ERROR: + case XSC_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case XSC_EVENT_TYPE_WQ_ACCESS_ERROR: + queue_id = eqe->queue_id; + qpn = queue_id; + xsc_qp_event(dev, qpn, eqe->type); + break; + default: + xsc_core_warn(dev, "Unhandle event 0x%x on EQ 0x%x\n", eqe->type, eq->eqn); + break; + } + + ++eq->cons_index; + eqes_found = 1; + ++set_ci; + + /* The HCA will think the queue has overflowed if we + * don't tell it we've been processing events. We + * create our EQs with XSC_NUM_SPARE_EQE extra + * entries, so we must update our consumer index at + * least that often. + */ + if (unlikely(set_ci >= XSC_NUM_SPARE_EQE)) { + xsc_core_dbg(dev, "EQ%d eq_num=%d qpn=%d, db_noarm\n", + eq->eqn, set_ci, eqe->queue_id); + eq_update_ci(eq, 0); + set_ci = 0; + } + } + + eq_update_ci(eq, 1); +#ifdef XSC_DEBUG + xsc_core_dbg(dev, "EQ%d eq_num=%d qpn=%d, db_arm\n", + eq->eqn, set_ci, (eqe?eqe->queue_id:0)); +#endif + + return eqes_found; +} + +static irqreturn_t xsc_msix_handler(int irq, void *eq_ptr) +{ + struct xsc_eq *eq = eq_ptr; + struct xsc_core_device *dev = eq->dev; +#ifdef XSC_DEBUG + xsc_core_dbg(dev, "EQ %d hint irq: %d\n", eq->eqn, irq); +#endif + xsc_eq_int(dev, eq); + + /* MSI-X vectors always belong to us */ + return IRQ_HANDLED; +} + +static void init_eq_buf(struct xsc_eq *eq) +{ + struct xsc_eqe *eqe; + int i; + + for (i = 0; i < eq->nent; i++) { + eqe = get_eqe(eq, i); + eqe->owner = XSC_EQE_OWNER_INIT_VAL; + } +} + +int xsc_create_map_eq(struct xsc_core_device *dev, struct xsc_eq *eq, u8 vecidx, + int nent, const char *name) +{ + struct xsc_dev_resource *dev_res = dev->dev_res; + u16 msix_vec_offset = dev->msix_vec_base + vecidx; + struct xsc_create_eq_mbox_in *in; + struct xsc_create_eq_mbox_out out; + int err; + int inlen; + int hw_npages; + + eq->nent = roundup_pow_of_two(roundup(nent, XSC_NUM_SPARE_EQE)); + err = xsc_buf_alloc(dev, eq->nent * XSC_EQE_SIZE, PAGE_SIZE, + &eq->buf); + if (err) + return err; + + init_eq_buf(eq); + + hw_npages = DIV_ROUND_UP(eq->nent * XSC_EQE_SIZE, PAGE_SIZE_4K); + inlen = sizeof(*in) + sizeof(in->pas[0]) * hw_npages; + in = xsc_vzalloc(inlen); + if (!in) { + err = -ENOMEM; + goto err_buf; + } + memset(&out, 0, sizeof(out)); + + xsc_fill_page_array(&eq->buf, in->pas, hw_npages); + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_CREATE_EQ); + in->ctx.log_eq_sz = ilog2(eq->nent); + in->ctx.vecidx = cpu_to_be16(msix_vec_offset); + in->ctx.pa_num = cpu_to_be16(hw_npages); + in->ctx.glb_func_id = cpu_to_be16(dev->glb_func_id); + in->ctx.is_async_eq = (vecidx == XSC_EQ_VEC_ASYNC ? 1 : 0); + + err = xsc_cmd_exec(dev, in, inlen, &out, sizeof(out)); + if (err) + goto err_in; + + if (out.hdr.status) { + //err = xsc_cmd_status_to_err(&out.hdr); + err = -ENOSPC; + goto err_in; + } + + snprintf(dev_res->irq_info[vecidx].name, XSC_MAX_IRQ_NAME, "%s@pci:%s", + name, pci_name(dev->pdev)); + + eq->eqn = be32_to_cpu(out.eqn); + eq->irqn = pci_irq_vector(dev->pdev, vecidx); + eq->dev = dev; + eq->doorbell = dev->regs.event_db; + eq->index = vecidx; + xsc_core_dbg(dev, "msix%d request vector%d eq%d irq%d\n", + vecidx, msix_vec_offset, eq->eqn, eq->irqn); + + err = request_irq(eq->irqn, xsc_msix_handler, 0, + dev_res->irq_info[vecidx].name, eq); + if (err) + goto err_eq; + + /* EQs are created in ARMED state + */ + eq_update_ci(eq, 1); + xsc_vfree(in); + return 0; + +err_eq: + xsc_cmd_destroy_eq(dev, eq->eqn); + +err_in: + xsc_vfree(in); + +err_buf: + xsc_buf_free(dev, &eq->buf); + return err; +} +EXPORT_SYMBOL_GPL(xsc_create_map_eq); + +int xsc_destroy_unmap_eq(struct xsc_core_device *dev, struct xsc_eq *eq) +{ + int err; + + free_irq(eq->irqn, eq); + err = xsc_cmd_destroy_eq(dev, eq->eqn); + if (err) + xsc_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n", + eq->eqn); + xsc_buf_free(dev, &eq->buf); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_destroy_unmap_eq); + +int xsc_eq_init(struct xsc_core_device *dev) +{ + int err; + + spin_lock_init(&dev->dev_res->eq_table.lock); + + err = xsc_eq_debugfs_init(dev); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_eq_init); + +void xsc_eq_cleanup(struct xsc_core_device *dev) +{ + xsc_eq_debugfs_cleanup(dev); +} +EXPORT_SYMBOL_GPL(xsc_eq_cleanup); + +int xsc_start_eqs(struct xsc_core_device *dev) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + int err; + + err = xsc_create_map_eq(dev, &table->async_eq, XSC_EQ_VEC_ASYNC, + XSC_NUM_ASYNC_EQE, "xsc_async_eq"); + if (err) + xsc_core_warn(dev, "failed to create async EQ %d\n", err); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_start_eqs); + +void xsc_stop_eqs(struct xsc_core_device *dev) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + + xsc_destroy_unmap_eq(dev, &table->async_eq); +} + +int xsc_core_eq_query(struct xsc_core_device *dev, struct xsc_eq *eq, + struct xsc_query_eq_mbox_out *out, int outlen) +{ + struct xsc_query_eq_mbox_in in; + int err = 0; + + memset(&in, 0, sizeof(in)); + memset(out, 0, outlen); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_EQ); + in.eqn = eq->eqn; + + //err = xsc_cmd_exec(dev, &in, sizeof(in), out, outlen); + //if (err) + // return err; + + if (out->hdr.status) + err = xsc_cmd_status_to_err(&out->hdr); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_core_eq_query); diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/eswitch.c b/drivers/net/ethernet/yunsilicon/xsc/pci/eswitch.c new file mode 100644 index 0000000000000000000000000000000000000000..a9f37f7a81ee2274b82992d084341e6d59034b7c --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/eswitch.c @@ -0,0 +1,749 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include "fw/xsc_tbm.h" +#include "eswitch.h" +#include + +u8 xsc_eswitch_mode(struct xsc_eswitch *esw) +{ + return ESW_ALLOWED(esw) ? esw->mode : XSC_ESWITCH_NONE; +} +EXPORT_SYMBOL_GPL(xsc_eswitch_mode); + +static int xsc_eswitch_check(const struct xsc_core_device *dev) +{ + if (!ESW_ALLOWED(dev->priv.eswitch)) + return -EPERM; + if (!dev->priv.eswitch->num_vfs) + return -EOPNOTSUPP; + + return 0; +} + +struct xsc_vport *__must_check +xsc_eswitch_get_vport(struct xsc_eswitch *esw, u16 vport_num) +{ + u16 idx; + + if (!esw || !xsc_core_is_vport_manager(esw->dev)) + return ERR_PTR(-EPERM); + + idx = xsc_eswitch_vport_num_to_index(esw, vport_num); + + if (idx > esw->total_vports - 1) { + xsc_core_dbg(esw->dev, "vport out of range: num(0x%x), idx(0x%x)\n", + vport_num, idx); + return ERR_PTR(-EINVAL); + } + + return &esw->vports[idx]; +} + +static int eswitch_devlink_pf_support_check(const struct xsc_eswitch *esw) +{ + return 0; +} + +static int esw_mode_from_devlink(u16 mode, u16 *xsc_mode) +{ + switch (mode) { + case DEVLINK_ESWITCH_MODE_LEGACY: + *xsc_mode = XSC_ESWITCH_LEGACY; + break; + case DEVLINK_ESWITCH_MODE_SWITCHDEV: + *xsc_mode = XSC_ESWITCH_OFFLOADS; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int esw_mode_to_devlink(u16 xsc_mode, u16 *mode) +{ + switch (xsc_mode) { + case XSC_ESWITCH_LEGACY: + *mode = DEVLINK_ESWITCH_MODE_LEGACY; + break; + case XSC_ESWITCH_OFFLOADS: + *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; + break; + default: + return -EINVAL; + } + + return 0; +} + +int xsc_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode + , struct netlink_ext_ack *extack + ) +{ + struct xsc_core_device *dev = devlink_priv(devlink); + struct xsc_eswitch *esw = dev->priv.eswitch; + u16 cur_xsc_mode, xsc_mode = 0; + int err = 0; + + err = xsc_eswitch_check(dev); + if (err) + return err; + + if (esw_mode_from_devlink(mode, &xsc_mode)) + return -EINVAL; + + mutex_lock(&esw->mode_lock); + err = eswitch_devlink_pf_support_check(esw); + if (err) + goto done; + + cur_xsc_mode = esw->mode; + + if (cur_xsc_mode == xsc_mode) + goto done; + + if (xsc_mode == XSC_ESWITCH_OFFLOADS && cur_xsc_mode != XSC_ESWITCH_LEGACY) { + mutex_unlock(&esw->mode_lock); + return -EOPNOTSUPP; + } + + xsc_lag_disable(dev); + esw->mode = xsc_mode; + xsc_lag_enable(dev); + + if (esw->mode == XSC_ESWITCH_OFFLOADS) + xsc_set_vf_pp_status(dev, false); + +done: + mutex_unlock(&esw->mode_lock); + return err; +} + +int xsc_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) +{ + struct xsc_core_device *dev = devlink_priv(devlink); + struct xsc_eswitch *esw = dev->priv.eswitch; + int err = 0; + + err = xsc_eswitch_check(dev); + if (err) + return err; + + mutex_lock(&esw->mode_lock); + err = esw_mode_to_devlink(esw->mode, mode); + mutex_unlock(&esw->mode_lock); + + return err; +} + +static void esw_vport_change_handle_locked(struct xsc_vport *vport) +{ + struct xsc_core_device *dev = vport->dev; + //struct xsc_eswitch *esw = dev->priv.eswitch; + u8 mac[ETH_ALEN]; + + xsc_query_other_nic_vport_mac_address(dev, vport->vport, mac); +} + +static void esw_vport_change_handler(struct work_struct *work) +{ + struct xsc_vport *vport = + container_of(work, struct xsc_vport, vport_change_handler); + struct xsc_eswitch *esw = vport->dev->priv.eswitch; + + mutex_lock(&esw->state_lock); + esw_vport_change_handle_locked(vport); + mutex_unlock(&esw->state_lock); +} + +void xsc_eswitch_enable_vport(struct xsc_eswitch *esw, + struct xsc_vport *vport, + enum xsc_eswitch_vport_event enabled_events) +{ + u16 vport_num = vport->vport; + + mutex_lock(&esw->state_lock); + if (vport->enabled) + goto unlock_out; + + bitmap_zero(vport->req_vlan_bitmap, VLAN_N_VID); + bitmap_zero(vport->acl_vlan_8021q_bitmap, VLAN_N_VID); + bitmap_zero(vport->info.vlan_trunk_8021q_bitmap, VLAN_N_VID); + + /* Sync with current vport context */ + vport->enabled_events = enabled_events; + vport->enabled = true; + + /* Esw manager is trusted by default. Host PF (vport 0) is trusted as well + * in smartNIC as it's a vport group manager. + */ + //if (is_esw_manager_vport(esw, vport_num)) + // vport->info.trusted = true; + +// esw_vport_change_handle_locked(vport); + + esw->enabled_vports++; + xsc_core_dbg(esw->dev, "Enabled VPORT(%d)\n", vport_num); +unlock_out: + mutex_unlock(&esw->state_lock); +} + +void xsc_eswitch_disable_vport(struct xsc_eswitch *esw, + struct xsc_vport *vport) +{ + u16 vport_num = vport->vport; + + mutex_lock(&esw->state_lock); + if (!vport->enabled) + goto done; + + xsc_core_dbg(esw->dev, "Disabling vport(%d)\n", vport_num); + /* Mark this vport as disabled to discard new events */ + vport->enabled = false; + + /* Disable events from this vport */ +// arm_vport_context_events_cmd(esw->dev, vport->vport, 0); + /* We don't assume VFs will cleanup after themselves. + * Calling vport change handler while vport is disabled will cleanup + * the vport resources. + */ +// esw_vport_change_handle_locked(vport); + vport->enabled_events = 0; + esw->enabled_vports--; +done: + mutex_unlock(&esw->state_lock); +} + +void xsc_eswitch_enable_pf_vf_vports(struct xsc_eswitch *esw, + enum xsc_eswitch_vport_event enabled_events) +{ + struct xsc_vport *vport; + int i; + + vport = xsc_eswitch_get_vport(esw, XSC_VPORT_PF); + xsc_eswitch_enable_vport(esw, vport, enabled_events); + + xsc_esw_for_each_vf_vport(esw, i, vport, esw->num_vfs) + xsc_eswitch_enable_vport(esw, vport, enabled_events); +} + +#define XSC_LEGACY_SRIOV_VPORT_EVENTS (XSC_VPORT_UC_ADDR_CHANGE | \ + XSC_VPORT_MC_ADDR_CHANGE | \ + XSC_VPORT_PROMISC_CHANGE | \ + XSC_VPORT_VLAN_CHANGE) + +int esw_legacy_enable(struct xsc_eswitch *esw) +{ + struct xsc_vport *vport; + unsigned long i; + //int ret; + + xsc_esw_for_each_vf_vport(esw, i, vport, esw->num_vfs) { + vport->info.link_state = XSC_VPORT_ADMIN_STATE_AUTO; + } + xsc_eswitch_enable_pf_vf_vports(esw, XSC_LEGACY_SRIOV_VPORT_EVENTS); + return 0; +} + +int xsc_eswitch_enable_locked(struct xsc_eswitch *esw, int mode, int num_vfs) +{ + int err; + + //if (!ESW_ALLOWED(esw) || !esw->dev.caps.ft_support) { + // xsc_core_warn(esw->dev, "E-Switch FDB is not supported, aborting ...\n"); + // return -EOPNOTSUPP; + //} + + lockdep_assert_held(&esw->mode_lock); + + esw->num_vfs = num_vfs; + + if (esw->mode == XSC_ESWITCH_NONE) { + err = esw_legacy_enable(esw); + } else { + //err = esw_offloads_enable(esw); + err = -EOPNOTSUPP; + } + + if (err) + goto ret; + + esw->mode = mode; + + //xsc_eswitch_event_handlers_register(esw); + + xsc_core_info(esw->dev, "Enable: mode(%s), nvfs(%d), active vports(%d)\n", + mode == XSC_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", + num_vfs, esw->enabled_vports); + + return 0; + +ret: + return err; +} + +int xsc_eswitch_enable(struct xsc_eswitch *esw, int mode, int num_vfs) +{ + int ret; + + mutex_lock(&esw->mode_lock); + ret = xsc_eswitch_enable_locked(esw, mode, num_vfs); + mutex_unlock(&esw->mode_lock); + return ret; +} + +void xsc_eswitch_disable_locked(struct xsc_eswitch *esw, bool clear_vf) +{ + int old_mode; + + lockdep_assert_held(&esw->mode_lock); + + if (esw->mode == XSC_ESWITCH_NONE) + return; + + xsc_core_info(esw->dev, "Disable: mode(%s)\n", + esw->mode == XSC_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS"); + +// xsc_eswitch_event_handlers_unregister(esw); + + //if (esw->mode == XSC_ESWITCH_LEGACY) + // esw_legacy_disable(esw); + old_mode = esw->mode; + esw->mode = XSC_ESWITCH_NONE; + + //if (clear_vf) + // xsc_eswitch_clear_vf_vports_info(esw); + esw->num_vfs = 0; +} + +void xsc_eswitch_disable(struct xsc_eswitch *esw, bool clear_vf) +{ + if (!ESW_ALLOWED(esw)) + return; + + mutex_lock(&esw->mode_lock); + xsc_eswitch_disable_locked(esw, clear_vf); + mutex_unlock(&esw->mode_lock); +} + +int xsc_eswitch_init(struct xsc_core_device *dev) +{ + struct xsc_eswitch *esw; + struct xsc_vport *vport; + int i, total_vports, err; + + if (!XSC_VPORT_MANAGER(dev)) { + xsc_core_info(dev, "%s XSC_VPORT_MANAGER check fail\n", __func__); + return 0; + } + + total_vports = xsc_eswitch_get_total_vports(dev); + + xsc_core_info(dev, "Total vports %d\n", total_vports); + + esw = kzalloc(sizeof(*esw), GFP_KERNEL); + if (!esw) + return -ENOMEM; + + esw->dev = dev; + esw->manager_vport = xsc_eswitch_manager_vport(dev); + esw->first_host_vport = xsc_eswitch_first_host_vport_num(dev); + esw->work_queue = create_singlethread_workqueue("xsc_esw_wq"); + if (!esw->work_queue) { + err = -ENOMEM; + goto abort; + } + esw->vports = kcalloc(total_vports, sizeof(struct xsc_vport), + GFP_KERNEL); + if (!esw->vports) { + err = -ENOMEM; + goto abort; + } + esw->total_vports = total_vports; + + mutex_init(&esw->state_lock); + mutex_init(&esw->mode_lock); + + xsc_esw_for_all_vports(esw, i, vport) { + vport->vport = xsc_eswitch_index_to_vport_num(esw, i); + vport->info.link_state = XSC_VPORT_ADMIN_STATE_AUTO; + vport->info.vlan_proto = htons(ETH_P_8021Q); + vport->info.roce = true; + vport->dev = dev; + INIT_WORK(&vport->vport_change_handler, + esw_vport_change_handler); + } + esw->enabled_vports = 0; + esw->mode = XSC_ESWITCH_NONE; + + dev->priv.eswitch = esw; + return 0; + +abort: + if (esw->work_queue) + destroy_workqueue(esw->work_queue); + kfree(esw->vports); + kfree(esw); + return 0; +} + +void xsc_eswitch_cleanup(struct xsc_core_device *dev) +{ + if (!dev->priv.eswitch || !XSC_VPORT_MANAGER(dev)) + return; + + xsc_core_dbg(dev, "cleanup\n"); + + destroy_workqueue(dev->priv.eswitch->work_queue); + kfree(dev->priv.eswitch->vports); + kfree(dev->priv.eswitch); +} + +static void node_guid_gen_from_mac(u64 *node_guid, u8 mac[ETH_ALEN]) +{ + ((u8 *)node_guid)[7] = mac[0]; + ((u8 *)node_guid)[6] = mac[1]; + ((u8 *)node_guid)[5] = mac[2]; + ((u8 *)node_guid)[4] = 0xff; + ((u8 *)node_guid)[3] = 0xfe; + ((u8 *)node_guid)[2] = mac[3]; + ((u8 *)node_guid)[1] = mac[4]; + ((u8 *)node_guid)[0] = mac[5]; +} + +int xsc_eswitch_set_vport_mac(struct xsc_eswitch *esw, + u16 vport, u8 mac[ETH_ALEN]) +{ + struct xsc_vport *evport = xsc_eswitch_get_vport(esw, vport); + u64 node_guid; + int err = 0; + + if (IS_ERR(evport)) + return PTR_ERR(evport); + if (is_multicast_ether_addr(mac)) + return -EINVAL; + + mutex_lock(&esw->state_lock); + + if (evport->info.spoofchk && !is_valid_ether_addr(mac)) + xsc_core_warn(esw->dev, + "Set invalid MAC while spoofchk is on, vport(%d)\n", + vport); + + err = xsc_modify_other_nic_vport_mac_address(esw->dev, vport, mac); + if (err) { + xsc_core_warn(esw->dev, + "Failed to xsc_modify_nic_vport_mac vport(%d) err=(%d)\n", + vport, err); + goto unlock; + } + + node_guid_gen_from_mac(&node_guid, mac); + err = xsc_modify_other_nic_vport_node_guid(esw->dev, vport, node_guid); + if (err) + xsc_core_warn(esw->dev, + "Failed to set vport %d node guid, err = %d. RDMA_CM will not function properly for this VF.\n", + vport, err); + + ether_addr_copy(evport->info.mac, mac); + evport->info.node_guid = node_guid; + +#ifdef XSC_ESW_FDB_ENABLE + if (evport->enabled && esw->mode == XSC_ESWITCH_LEGACY) + err = esw_vport_ingress_config(esw, evport); +#endif + +unlock: + mutex_unlock(&esw->state_lock); + return err; +} + +int xsc_eswitch_get_vport_mac(struct xsc_eswitch *esw, + u16 vport, u8 *mac) +{ + struct xsc_vport *evport = xsc_eswitch_get_vport(esw, vport); + + if (IS_ERR(evport)) + return PTR_ERR(evport); + + mutex_lock(&esw->state_lock); + ether_addr_copy(mac, evport->info.mac); + mutex_unlock(&esw->state_lock); + return 0; +} + +int __xsc_eswitch_set_vport_vlan(struct xsc_eswitch *esw, int vport, u16 vlan, + u8 qos, __be16 proto, u8 set_flags) +{ + return 0; +} + +int xsc_eswitch_set_vport_vlan(struct xsc_eswitch *esw, int vport, + u16 vlan, u8 qos, __be16 vlan_proto) +{ + u8 set_flags = 0; + int err = 0; + + if (!ESW_ALLOWED(esw)) + return -EPERM; + + if (vlan || qos) + set_flags = SET_VLAN_STRIP | SET_VLAN_INSERT; + else + set_flags = CLR_VLAN_STRIP | CLR_VLAN_INSERT; + + mutex_lock(&esw->state_lock); + if (esw->mode != XSC_ESWITCH_LEGACY) { + if (!vlan) + goto unlock; /* compatibility with libvirt */ + + err = -EOPNOTSUPP; + goto unlock; + } + + err = __xsc_eswitch_set_vport_vlan(esw, vport, vlan, qos, vlan_proto, set_flags); + +unlock: + mutex_unlock(&esw->state_lock); + return err; +} + +int xsc_eswitch_set_vport_state(struct xsc_eswitch *esw, + u16 vport, int link_state) +{ + return 0; +} + +int xsc_eswitch_set_vport_spoofchk(struct xsc_eswitch *esw, + u16 vport, bool spoofchk) +{ + struct xsc_vport *evport = xsc_eswitch_get_vport(esw, vport); + bool pschk; + int err = 0; + + if (!ESW_ALLOWED(esw)) + return -EPERM; + if (IS_ERR(evport)) + return PTR_ERR(evport); + + mutex_lock(&esw->state_lock); + if (esw->mode != XSC_ESWITCH_LEGACY) { + err = -EOPNOTSUPP; + goto unlock; + } + pschk = evport->info.spoofchk; + evport->info.spoofchk = spoofchk; + if (pschk && !is_valid_ether_addr(evport->info.mac)) + xsc_core_warn(esw->dev, + "Spoofchk in set while MAC is invalid, vport(%d)\n", + evport->vport); + //if (evport->enabled && esw->mode == XSC_ESWITCH_LEGACY) + // err = __xsc_eswitch_set_spoofchk(esw, evport); + if (err) + evport->info.spoofchk = pschk; + +unlock: + mutex_unlock(&esw->state_lock); + return err; +} + +static int xsc_eswitch_update_vport_trunk(struct xsc_eswitch *esw, + struct xsc_vport *evport, + unsigned long *old_trunk) +{ + DECLARE_BITMAP(diff_vlan_bm, VLAN_N_VID); + int err = 0; + + bitmap_xor(diff_vlan_bm, old_trunk, + evport->info.vlan_trunk_8021q_bitmap, VLAN_N_VID); + if (!bitmap_weight(diff_vlan_bm, VLAN_N_VID)) + return err; + + if (err) { + bitmap_copy(evport->info.vlan_trunk_8021q_bitmap, old_trunk, + VLAN_N_VID); + //esw_update_acl_trunk_bitmap(esw, evport->vport); + //esw_acl_egress_lgcy_setup(esw, evport); + //esw_acl_ingress_lgcy_setup(esw, evport); + } + + return err; +} + +int xsc_eswitch_add_vport_trunk_range(struct xsc_eswitch *esw, + int vport, u16 start_vlan, u16 end_vlan) +{ + DECLARE_BITMAP(prev_vport_bitmap, VLAN_N_VID); + struct xsc_vport *evport = xsc_eswitch_get_vport(esw, vport); + int err = 0; + + if (!ESW_ALLOWED(esw)) + return -EPERM; + if (IS_ERR(evport)) + return PTR_ERR(evport); + + if (end_vlan > VLAN_N_VID || start_vlan > end_vlan) + return -EINVAL; + + mutex_lock(&esw->state_lock); + + if (evport->info.vlan || evport->info.qos) { + err = -EPERM; + xsc_core_warn(esw->dev, + "VGT+ is not allowed when operating in VST mode vport(%d)\n", + vport); + goto unlock; + } + + bitmap_copy(prev_vport_bitmap, evport->info.vlan_trunk_8021q_bitmap, + VLAN_N_VID); + bitmap_set(evport->info.vlan_trunk_8021q_bitmap, start_vlan, + end_vlan - start_vlan + 1); + err = xsc_eswitch_update_vport_trunk(esw, evport, prev_vport_bitmap); + +unlock: + mutex_unlock(&esw->state_lock); + + return err; +} + +int xsc_eswitch_del_vport_trunk_range(struct xsc_eswitch *esw, + int vport, u16 start_vlan, u16 end_vlan) +{ + DECLARE_BITMAP(prev_vport_bitmap, VLAN_N_VID); + struct xsc_vport *evport = xsc_eswitch_get_vport(esw, vport); + int err = 0; + + if (!ESW_ALLOWED(esw)) + return -EPERM; + if (IS_ERR(evport)) + return PTR_ERR(evport); + + if (end_vlan > VLAN_N_VID || start_vlan > end_vlan) + return -EINVAL; + + mutex_lock(&esw->state_lock); + bitmap_copy(prev_vport_bitmap, evport->info.vlan_trunk_8021q_bitmap, + VLAN_N_VID); + bitmap_clear(evport->info.vlan_trunk_8021q_bitmap, start_vlan, + end_vlan - start_vlan + 1); + err = xsc_eswitch_update_vport_trunk(esw, evport, prev_vport_bitmap); + mutex_unlock(&esw->state_lock); + + return err; +} + +int xsc_eswitch_set_vport_trust(struct xsc_eswitch *esw, + u16 vport_num, bool setting) +{ + struct xsc_vport *evport = xsc_eswitch_get_vport(esw, vport_num); + int err = 0; + + if (!ESW_ALLOWED(esw)) + return -EPERM; + if (IS_ERR(evport)) + return PTR_ERR(evport); + + mutex_lock(&esw->state_lock); + if (esw->mode != XSC_ESWITCH_LEGACY) { + err = -EOPNOTSUPP; + goto unlock; + } + evport->info.trusted = setting; +// if (evport->enabled) +// esw_vport_change_handle_locked(evport); + +unlock: + mutex_unlock(&esw->state_lock); + return err; +} + +int xsc_eswitch_set_vport_rate(struct xsc_eswitch *esw, u16 vport, + u32 max_rate, u32 min_rate) +{ + return 0; +} + +int xsc_eswitch_get_vport_config(struct xsc_eswitch *esw, + u16 vport, struct ifla_vf_info *ivi) +{ + struct xsc_vport *evport = xsc_eswitch_get_vport(esw, vport); + + if (IS_ERR(evport)) + return PTR_ERR(evport); + + memset(ivi, 0, sizeof(*ivi)); + ivi->vf = vport - 1; + + mutex_lock(&esw->state_lock); + ether_addr_copy(ivi->mac, evport->info.mac); + ivi->linkstate = evport->info.link_state; + ivi->vlan = evport->info.vlan; + ivi->qos = evport->info.qos; + ivi->vlan_proto = evport->info.vlan_proto; + ivi->spoofchk = evport->info.spoofchk; + ivi->trusted = evport->info.trusted; + ivi->min_tx_rate = evport->qos.min_rate; + ivi->max_tx_rate = evport->qos.max_rate; + mutex_unlock(&esw->state_lock); + + return 0; +} + +int xsc_eswitch_vport_update_group(struct xsc_eswitch *esw, int vport_num, + u32 group_id) +{ + return 0; +} + +int xsc_eswitch_set_vgroup_rate(struct xsc_eswitch *esw, int group_id, + u32 max_rate) +{ + return 0; +} + +int xsc_eswitch_set_vgroup_max_rate(struct xsc_eswitch *esw, int group_id, + u32 max_rate) +{ + return 0; +} + +int xsc_eswitch_set_vgroup_min_rate(struct xsc_eswitch *esw, int group_id, + u32 min_rate) +{ + return 0; +} + +int xsc_eswitch_modify_esw_vport_context(struct xsc_eswitch *esw, u16 vport, + bool other_vport, void *in, int inlen) +{ + return 0; +} + +int xsc_eswitch_query_esw_vport_context(struct xsc_eswitch *esw, u16 vport, + bool other_vport, void *out, int outlen) +{ + return 0; +} + +int xsc_eswitch_get_vport_stats(struct xsc_eswitch *esw, + u16 vport, struct ifla_vf_stats *vf_stats) +{ + return 0; +} + +int xsc_eswitch_query_vport_drop_stats(struct xsc_core_device *dev, + struct xsc_vport *vport, + struct xsc_vport_drop_stats *stats) +{ + return 0; +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/eswitch.h b/drivers/net/ethernet/yunsilicon/xsc/pci/eswitch.h new file mode 100644 index 0000000000000000000000000000000000000000..50442a0c758706e2d4b646e998c5e00f90a06fb3 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/eswitch.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef ESWITCH_H +#define ESWITCH_H + +#include +#include +#include +#include +#include +#include +#include +#include + +struct xsc_vport_drop_stats { + u64 rx_dropped; + u64 tx_dropped; +}; + +int xsc_eswitch_init(struct xsc_core_device *dev); +void xsc_eswitch_cleanup(struct xsc_core_device *dev); +int xsc_eswitch_enable_locked(struct xsc_eswitch *esw, int mode, int num_vfs); +int xsc_eswitch_enable(struct xsc_eswitch *esw, int mode, int num_vfs); +void xsc_eswitch_disable_locked(struct xsc_eswitch *esw, bool clear_vf); +void xsc_eswitch_disable(struct xsc_eswitch *esw, bool clear_vf); + +int xsc_devlink_eswitch_mode_set(struct devlink *devlink, u16 mod + , struct netlink_ext_ack *extack); +int xsc_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode); + +struct xsc_vport *__must_check +xsc_eswitch_get_vport(struct xsc_eswitch *esw, u16 vport_num); +int xsc_eswitch_set_vport_mac(struct xsc_eswitch *esw, + u16 vport, u8 mac[ETH_ALEN]); +int xsc_eswitch_get_vport_mac(struct xsc_eswitch *esw, + u16 vport, u8 *mac); +int xsc_eswitch_set_vport_vlan(struct xsc_eswitch *esw, int vport, + u16 vlan, u8 qos, __be16 vlan_proto); +int xsc_eswitch_set_vport_state(struct xsc_eswitch *esw, + u16 vport, int link_state); +int xsc_eswitch_set_vport_spoofchk(struct xsc_eswitch *esw, + u16 vport, bool spoofchk); +int xsc_eswitch_set_vport_trust(struct xsc_eswitch *esw, + u16 vport_num, bool setting); +int xsc_eswitch_set_vport_rate(struct xsc_eswitch *esw, u16 vport, + u32 max_rate, u32 min_rate); +int xsc_eswitch_vport_update_group(struct xsc_eswitch *esw, int vport_num, + u32 group_id); +int xsc_eswitch_set_vgroup_rate(struct xsc_eswitch *esw, int group_id, + u32 max_rate); +int xsc_eswitch_set_vgroup_max_rate(struct xsc_eswitch *esw, int group_id, + u32 max_rate); +int xsc_eswitch_set_vgroup_min_rate(struct xsc_eswitch *esw, int group_id, + u32 min_rate); +int xsc_eswitch_add_vport_trunk_range(struct xsc_eswitch *esw, + int vport, u16 start_vlan, u16 end_vlan); +int xsc_eswitch_del_vport_trunk_range(struct xsc_eswitch *esw, + int vport, u16 start_vlan, u16 end_vlan); +int xsc_eswitch_modify_esw_vport_context(struct xsc_eswitch *esw, u16 vport, + bool other_vport, + void *in, int inlen); +int xsc_eswitch_query_esw_vport_context(struct xsc_eswitch *esw, u16 vport, + bool other_vport, + void *out, int outlen); +int xsc_eswitch_get_vport_stats(struct xsc_eswitch *esw, + u16 vport, + struct ifla_vf_stats *vf_stats); +int xsc_eswitch_query_vport_drop_stats(struct xsc_core_device *dev, + struct xsc_vport *vport, + struct xsc_vport_drop_stats *stats); + +#define xsc_esw_for_all_vports(esw, i, vport) \ + for ((i) = XSC_VPORT_PF; \ + (vport) = &(esw)->vports[(i)], \ + (i) <= (esw)->total_vports; (i)++) + +#define xsc_esw_for_each_vf_vport(esw, i, vport, nvfs) \ + for ((i) = XSC_VPORT_FIRST_VF; \ + (vport) = &(esw)->vports[(i)], \ + (i) <= (nvfs); (i)++) + +static inline int xsc_eswitch_uplink_idx(struct xsc_eswitch *esw) +{ + /* Uplink always locate at the last element of the array.*/ + return esw->total_vports - 1; +} + +static inline int xsc_eswitch_ecpf_idx(struct xsc_eswitch *esw) +{ + return esw->total_vports - 2; +} + +static inline int xsc_eswitch_vport_num_to_index(struct xsc_eswitch *esw, + u16 vport_num) +{ + if (vport_num == XSC_VPORT_ECPF) { + if (!xsc_ecpf_vport_exists(esw->dev) && + !xsc_core_is_ecpf_esw_manager(esw->dev)) + xsc_core_warn(esw->dev, "ECPF vport doesn't exist!\n"); + return xsc_eswitch_ecpf_idx(esw); + } + + if (vport_num == XSC_VPORT_UPLINK) + return xsc_eswitch_uplink_idx(esw); + + /* PF and VF vports start from 0 to max_vfs */ + return vport_num; +} + +static inline u16 xsc_eswitch_index_to_vport_num(struct xsc_eswitch *esw, + int index) +{ + if (index == xsc_eswitch_uplink_idx(esw)) + return XSC_VPORT_UPLINK; + return index; +} + +static inline u16 xsc_eswitch_manager_vport(struct xsc_core_device *dev) +{ + return xsc_core_is_ecpf_esw_manager(dev) ? + XSC_VPORT_ECPF : XSC_VPORT_PF; +} + +static inline u16 xsc_eswitch_first_host_vport_num(struct xsc_core_device *dev) +{ + return xsc_core_is_ecpf_esw_manager(dev) ? + XSC_VPORT_PF : XSC_VPORT_FIRST_VF; +} + +#endif /* ESWITCH_H */ + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw.c b/drivers/net/ethernet/yunsilicon/xsc/pci/fw.c new file mode 100644 index 0000000000000000000000000000000000000000..34d7021d77ecfdacff4f28f5200a6ebebeaaa057 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include + +int xsc_cmd_query_hca_cap(struct xsc_core_device *dev, + struct xsc_caps *caps) +{ + struct xsc_cmd_query_hca_cap_mbox_out *out; + struct xsc_cmd_query_hca_cap_mbox_in in; + //struct xsc_query_special_ctxs_mbox_out ctx_out; + //struct xsc_query_special_ctxs_mbox_in ctx_in; + int err; + u16 t16; + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) + return -ENOMEM; + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_HCA_CAP); + in.hdr.opmod = cpu_to_be16(0x1); +#ifdef XSC_MSIX_BAR_EMUL + in.cpu_num = cpu_to_be16(num_online_cpus()); +#endif + err = xsc_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); + if (err) + goto out_out; + + if (out->hdr.status) { + err = xsc_cmd_status_to_err(&out->hdr); + goto out_out; + } + + dev->glb_func_id = be32_to_cpu(out->hca_cap.glb_func_id); + caps->raweth_qp_id_base = be16_to_cpu(out->hca_cap.raweth_qp_id_base); + caps->raweth_qp_id_end = be16_to_cpu(out->hca_cap.raweth_qp_id_end); + caps->raweth_rss_qp_id_base = be16_to_cpu(out->hca_cap.raweth_rss_qp_id_base); + caps->raw_tpe_qp_num = be16_to_cpu(out->hca_cap.raw_tpe_qp_num); + caps->max_cqes = 1 << out->hca_cap.log_max_cq_sz; + caps->max_wqes = 1 << out->hca_cap.log_max_qp_sz; + caps->max_sq_desc_sz = be16_to_cpu(out->hca_cap.max_desc_sz_sq); + caps->max_rq_desc_sz = be16_to_cpu(out->hca_cap.max_desc_sz_rq); + caps->flags = be64_to_cpu(out->hca_cap.flags); + caps->stat_rate_support = be16_to_cpu(out->hca_cap.stat_rate_support); + caps->log_max_msg = out->hca_cap.log_max_msg & 0x1f; + caps->num_ports = out->hca_cap.num_ports & 0xf; + caps->log_max_cq = out->hca_cap.log_max_cq & 0x1f; + caps->log_max_eq = out->hca_cap.log_max_eq & 0xf; + caps->log_max_msix = out->hca_cap.log_max_msix & 0xf; + caps->max_num_eqs = out->hca_cap.max_num_eqs & 0xff; + caps->mac_port = out->hca_cap.mac_port & 0xff; + if (caps->num_ports > XSC_MAX_FW_PORTS) { + xsc_core_err(dev, "device has %d ports while the driver supports max %d ports\n", + caps->num_ports, XSC_MAX_FW_PORTS); + err = -EINVAL; + goto out_out; + } + caps->send_ds_num = out->hca_cap.send_seg_num; + caps->send_wqe_shift = out->hca_cap.send_wqe_shift; + caps->recv_ds_num = out->hca_cap.recv_seg_num; + caps->recv_wqe_shift = out->hca_cap.recv_wqe_shift; + + caps->embedded_cpu = 0; + caps->ecpf_vport_exists = 0; +#ifdef CONFIG_XSC_ESWITCH + caps->eswitch_manager = 1; + caps->vport_group_manager = 1; +#endif + caps->log_max_current_uc_list = 0; + caps->log_max_current_mc_list = 0; + caps->log_max_vlan_list = 8; + caps->log_max_qp = out->hca_cap.log_max_qp & 0x1f; + caps->log_max_mkey = out->hca_cap.log_max_mkey & 0x3f; + caps->log_max_pd = out->hca_cap.log_max_pd & 0x1f; + caps->log_max_srq = out->hca_cap.log_max_srqs & 0x1f; + caps->local_ca_ack_delay = out->hca_cap.local_ca_ack_delay & 0x1f; + caps->log_max_mcg = out->hca_cap.log_max_mcg; + caps->log_max_mtt = out->hca_cap.log_max_mtt; + caps->log_max_tso = out->hca_cap.log_max_tso; + caps->hca_core_clock = be32_to_cpu(out->hca_cap.hca_core_clock); + caps->max_rwq_indirection_tables = + be32_to_cpu(out->hca_cap.max_rwq_indirection_tables); + caps->max_rwq_indirection_table_size = + be32_to_cpu(out->hca_cap.max_rwq_indirection_table_size); + caps->max_qp_mcg = be16_to_cpu(out->hca_cap.max_qp_mcg); + caps->max_ra_res_qp = 1 << (out->hca_cap.log_max_ra_res_qp & 0x3f); + caps->max_ra_req_qp = 1 << (out->hca_cap.log_max_ra_req_qp & 0x3f); + caps->max_srq_wqes = 1 << out->hca_cap.log_max_srq_sz; + caps->rx_pkt_len_max = be32_to_cpu(out->hca_cap.rx_pkt_len_max); + caps->max_vfs = be16_to_cpu(out->hca_cap.max_vfs); + caps->qp_rate_limit_min = be32_to_cpu(out->hca_cap.qp_rate_limit_min); + caps->qp_rate_limit_max = be32_to_cpu(out->hca_cap.qp_rate_limit_max); + +#ifdef MSIX_SUPPORT + caps->msix_enable = 1; +#else + caps->msix_enable = 0; +#endif +#ifdef XSC_MSIX_BAR_EMUL + caps->msix_base = be16_to_cpu(out->hca_cap.msix_base); + caps->msix_num = be16_to_cpu(out->hca_cap.msix_num); +#endif + + t16 = be16_to_cpu(out->hca_cap.bf_log_bf_reg_size); + if (t16 & 0x8000) { + caps->bf_reg_size = 1 << (t16 & 0x1f); + caps->bf_regs_per_page = XSC_BF_REGS_PER_PAGE; + } else { + caps->bf_reg_size = 0; + caps->bf_regs_per_page = 0; + } + caps->min_page_sz = ~(u32)((1 << PAGE_SHIFT) - 1); + + caps->dcbx = 1; + caps->qos = 1; + caps->ets = 1; + caps->dscp = 1; + caps->max_tc = PRI_NUM; + caps->log_max_qp_depth = out->hca_cap.log_max_qp_depth & 0xff; + + dev->chip_ver_h = be32_to_cpu(out->hca_cap.chip_ver_h); + dev->chip_ver_m = be32_to_cpu(out->hca_cap.chip_ver_m); + dev->chip_ver_l = be32_to_cpu(out->hca_cap.chip_ver_l); + dev->hotfix_num = be32_to_cpu(out->hca_cap.hotfix_num); + dev->feature_flag = be32_to_cpu(out->hca_cap.feature_flag); + + memcpy(dev->board_sn, out->hca_cap.board_sn, sizeof(out->hca_cap.board_sn)); + + if (xsc_core_is_pf(dev)) { + dev->regs.tx_db = be64_to_cpu(out->hca_cap.tx_db); + dev->regs.rx_db = be64_to_cpu(out->hca_cap.rx_db); + dev->regs.complete_db = be64_to_cpu(out->hca_cap.complete_db); + dev->regs.complete_reg = be64_to_cpu(out->hca_cap.complete_reg); + dev->regs.event_db = be64_to_cpu(out->hca_cap.event_db); + } + //memset(&ctx_in, 0, sizeof(ctx_in)); + //memset(&ctx_out, 0, sizeof(ctx_out)); + //ctx_in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_SPECIAL_CONTEXTS); + //err = xsc_cmd_exec(dev, &ctx_in, sizeof(ctx_in), + // &ctx_out, sizeof(ctx_out)); + //if (err) + // goto out_out; + + //if (ctx_out.hdr.status) + // err = xsc_cmd_status_to_err(&ctx_out.hdr); + + //caps->reserved_lkey = be32_to_cpu(ctx_out.reserved_lkey); + +out_out: + kfree(out); + + return err; +} + +int xsc_cmd_init_hca(struct xsc_core_device *dev) +{ + struct xsc_cmd_init_hca_mbox_in in; + struct xsc_cmd_init_hca_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_INIT_HCA); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + err = xsc_cmd_status_to_err(&out.hdr); + + return err; +} + +int xsc_cmd_teardown_hca(struct xsc_core_device *dev) +{ + struct xsc_cmd_teardown_hca_mbox_in in; + struct xsc_cmd_teardown_hca_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_TEARDOWN_HCA); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + err = xsc_cmd_status_to_err(&out.hdr); + + return err; +} + +int xsc_cmd_enable_hca(struct xsc_core_device *dev, u16 vf_idx) +{ + struct xsc_cmd_enable_hca_mbox_in in; + struct xsc_cmd_enable_hca_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_ENABLE_HCA); + in.pf = dev->pf; + in.pcie = dev->pcie; + in.pf_id = dev->pf_id; + in.vf_id = cpu_to_be16(vf_idx); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + err = xsc_cmd_status_to_err(&out.hdr); + + return err; +} + +int xsc_cmd_disable_hca(struct xsc_core_device *dev, u16 vf_idx) +{ + struct xsc_cmd_disable_hca_mbox_in in; + struct xsc_cmd_disable_hca_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DISABLE_HCA); + in.pf = dev->pf; + in.pcie = dev->pcie; + in.pf_id = dev->pf_id; + in.vf_id = cpu_to_be16(vf_idx); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + err = xsc_cmd_status_to_err(&out.hdr); + + return err; +} + +static u32 xsc_board_num; +static char xsc_board_sn[MAX_BOARD_NUM][XSC_BOARD_SN_LEN]; + +int xsc_get_board_id(struct xsc_core_device *dev) +{ + int i = 0; + + xsc_core_info(dev, "board_sn=%s, current_board_num=%d\n", + dev->board_sn, xsc_board_num); + if (strnlen(dev->board_sn, XSC_BOARD_SN_LEN) == 0) + return 0; + + for (i = 0; i < xsc_board_num; i++) { + if (!strncmp(xsc_board_sn[i], dev->board_sn, XSC_BOARD_SN_LEN)) { + dev->board_id = i; + return 0; + } + } + if (xsc_board_num < MAX_BOARD_NUM) { + memcpy(xsc_board_sn[xsc_board_num], dev->board_sn, XSC_BOARD_SN_LEN); + dev->board_id = xsc_board_num++; + } else { + return -ENOMEM; + } + + return 0; +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/bitops.h b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/bitops.h new file mode 100644 index 0000000000000000000000000000000000000000..0e6f193a6f8e95b47267a42d607d76e9f60ce7be --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/bitops.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef BITOPS_H +#define BITOPS_H + +#include +#include + +#define __round_mask(x, y) ((__typeof__(x))((y)-1)) +#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1) +#define round_down(x, y) ((x) & ~__round_mask(x, y)) + +unsigned long find_next_bit(const unsigned long *addr, unsigned long size, + unsigned long offset); + +#define find_first_bit(addr, size) find_next_bit((addr), (size), 0) + +#define clear_bit(bit, bitmap) __clear_bit(bit, bitmap) + +static inline void xsc_clear_bit(int bit, long *bitmap) +{ + clear_bit(bit, bitmap); +} + +static inline int xsc_test_bit(int bit, long *bitmap) +{ + return test_bit(bit, bitmap); +} + +static inline int xsc_test_and_set_bit(int bit, long *bitmap) +{ + return test_and_set_bit(bit, bitmap); +} + +static inline void xsc_set_bit(int bit, long *bitmap) +{ + set_bit(bit, bitmap); +} + +#endif + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/cmd.c b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/cmd.c new file mode 100644 index 0000000000000000000000000000000000000000..7a54bcf1f054017d885d1880452240b507bdbc70 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/cmd.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include + +#include "xsc_reg_struct.h" +#include "xsc_fw.h" +#include "xsc_flow.h" + +#include + +int xsc_alloc_iae_idx(struct xsc_core_device *dev, int *iae_idx) +{ + ACQUIRE_IA_LOCK(dev, *iae_idx); + return *iae_idx != -1 ? 0 : -1; +} + +void xsc_release_iae_idx(struct xsc_core_device *dev, int *iae_idx) +{ + RELEASE_IA_LOCK(dev, *iae_idx); + *iae_idx = -1; +} + +int xsc_get_iae_idx(struct xsc_core_device *dev) +{ + struct xsc_resources *res = get_xsc_res(dev); + + return res->iae_idx; +} + +static int xsc_cmd_exec_create_mkey(struct xsc_core_device *xdev, void *in, void *out) +{ + struct xsc_create_mkey_mbox_out *resp = out; + u32 mpt_idx = 0; + + if (alloc_mpt_entry(xdev, &mpt_idx)) + return -EINVAL; + + resp->mkey = cpu_to_be32(mpt_idx & 0xffffff); + resp->hdr.status = 0; + + return 0; +} + +int xsc_create_mkey(struct xsc_core_device *xdev, void *in, void *out) +{ + unsigned long flags; + struct xsc_resources *xres = get_xsc_res(xdev); + int ret = 0; + + xsc_acquire_lock(&xres->lock, &flags); + ret = xsc_cmd_exec_create_mkey(xdev, in, out); + xsc_release_lock(&xres->lock, flags); + return ret; +} + +static int xsc_cmd_exec_destroy_mkey(struct xsc_core_device *xdev, void *in, void *out) +{ + struct xsc_destroy_mkey_mbox_in *req = in; + struct xsc_destroy_mkey_mbox_out *resp = out; + u32 mkey = be32_to_cpu(req->mkey); + u32 mpt_idx = xsc_mkey_to_idx(mkey); + + dealloc_mpt_entry(xdev, &mpt_idx); + + resp->hdr.status = 0; + + return 0; +} + +int xsc_destroy_mkey(struct xsc_core_device *xdev, void *in, void *out) +{ + unsigned long flags; + struct xsc_resources *xres = get_xsc_res(xdev); + int ret = 0; + + xsc_acquire_lock(&xres->lock, &flags); + ret = xsc_cmd_exec_destroy_mkey(xdev, in, out); + xsc_release_lock(&xres->lock, flags); + return ret; +} + +static int xsc_cmd_exec_reg_mr(struct xsc_core_device *dev, void *in, void *out) +{ + struct xsc_register_mr_mbox_in *req = in; + struct xsc_register_mr_mbox_out *resp = out; + struct xsc_mpt_entry mpt_ent; + u32 mpt_idx = 0; + u32 mtt_base; + u64 va = be64_to_cpu(req->req.va_base); + u32 mem_size = be32_to_cpu(req->req.len); + u32 pdn = be32_to_cpu(req->req.pdn); + u32 key = be32_to_cpu(req->req.mkey); + int pages_num = be32_to_cpu(req->req.pa_num); + u32 *ptr; + u64 reg_addr; + int i; + int reg_stride; + + if (pages_num && alloc_mtt_entry(dev, pages_num, &mtt_base)) + return -EINVAL; + + mpt_idx = xsc_mkey_to_idx(key); + mpt_ent.va_l = va & 0xFFFFFFFF; + mpt_ent.va_h = va >> 32; + mpt_ent.mem_size = mem_size; + mpt_ent.pdn = pdn; + mpt_ent.key = key & 0xFF; + mpt_ent.mtt_base = mtt_base; + mpt_ent.acc = req->req.acc; + mpt_ent.page_mode = req->req.page_mode; + mpt_ent.mem_map_en = req->req.map_en; + mpt_ent.rsv = 0; + + get_xsc_res(dev)->mpt_entry[mpt_idx].va = va; + get_xsc_res(dev)->mpt_entry[mpt_idx].mtt_base = mtt_base; + get_xsc_res(dev)->mpt_entry[mpt_idx].page_num = pages_num; + + ptr = (u32 *)&mpt_ent; + reg_stride = REG_WIDTH_TO_STRIDE(MMC_MPT_TBL_MEM_WIDTH); + reg_addr = MMC_MPT_TBL_MEM_ADDR + + mpt_idx * roundup_pow_of_two(reg_stride); + + IA_WRITE_REG_MR(dev, reg_addr, ptr, sizeof(mpt_ent) / sizeof(u32), + xsc_get_iae_idx(dev)); + + xsc_core_dbg(dev, "reg mr, write mpt[%u]: va=%llx, mem_size=%u, pdn=%u\n", + mpt_idx, va, mpt_ent.mem_size, mpt_ent.pdn); + xsc_core_dbg(dev, "key=%u, mtt_base=%u, acc=%u, page_mode=%u, mem_map_en=%u\n", + mpt_ent.key, mpt_ent.mtt_base, mpt_ent.acc, mpt_ent.page_mode, mpt_ent.mem_map_en); + + for (i = 0; i < pages_num; i++) { + u64 pa = req->req.pas[i]; + + pa = be64_to_cpu(pa); + pa = pa >> PAGE_SHIFT_4K; + ptr = (u32 *)&pa; + reg_addr = MMC_MTT_TBL_MEM_ADDR + + (mtt_base + i) * REG_WIDTH_TO_STRIDE(MMC_MTT_TBL_MEM_WIDTH); + + IA_WRITE_REG_MR(dev, reg_addr, ptr, sizeof(pa) / sizeof(u32), + xsc_get_iae_idx(dev)); + + xsc_core_dbg(dev, "reg mr, write mtt: pa[%u]=%llx\n", i, pa); + } + + resp->hdr.status = 0; + return 0; +} + +int xsc_reg_mr(struct xsc_core_device *xdev, void *in, void *out) +{ + unsigned long flags; + struct xsc_resources *xres = get_xsc_res(xdev); + int ret; + + xsc_acquire_lock(&xres->lock, &flags); + ret = xsc_cmd_exec_reg_mr(xdev, in, out); + xsc_release_lock(&xres->lock, flags); + return ret; +} + +static int xsc_cmd_exec_dereg_mr(struct xsc_core_device *dev, void *in, void *out) +{ + struct xsc_unregister_mr_mbox_in *req; + struct xsc_unregister_mr_mbox_out *resp; + u32 mpt_idx; + u32 mtt_base; + int pages_num; + + req = in; + resp = out; + resp->hdr.status = -EINVAL; + + mpt_idx = be32_to_cpu(req->mkey); + xsc_core_dbg(dev, "mpt idx:%u\n", mpt_idx); + + pages_num = get_xsc_res(dev)->mpt_entry[mpt_idx].page_num; + mtt_base = get_xsc_res(dev)->mpt_entry[mpt_idx].mtt_base; + if (pages_num > 0) + dealloc_mtt_entry(dev, pages_num, mtt_base); + + resp->hdr.status = 0; + return 0; +} + +int xsc_dereg_mr(struct xsc_core_device *xdev, void *in, void *out) +{ + unsigned long flags; + struct xsc_resources *xres = get_xsc_res(xdev); + int ret; + + xsc_acquire_lock(&xres->lock, &flags); + ret = xsc_cmd_exec_dereg_mr(xdev, in, out); + xsc_release_lock(&xres->lock, flags); + return ret; +} + +static int xsc_cmd_exec_ioctl_flow(struct xsc_core_device *dev, + void *in, void *out) +{ + struct xsc_ioctl_mbox_in *req; + struct xsc_ioctl_mbox_out *resp; + struct xsc_ioctl_data_tl *tl; + char *data; + u16 datalen; + u16 tllen = sizeof(struct xsc_ioctl_data_tl); + int opmod; + int table; + int length; + int ret = -EINVAL; + + req = in; + resp = out; + resp->hdr.status = -EINVAL; + + data = (char *)req->data; + datalen = be16_to_cpu(req->len); + + if (datalen < tllen) + goto out; + + tl = (struct xsc_ioctl_data_tl *)data; + opmod = tl->opmod; + table = tl->table; + length = tl->length; + + switch (opmod) { + case XSC_IOCTL_OP_ADD: + ret = xsc_flow_add(dev, table, length, tl+1); + break; + default: + ret = -EINVAL; + break; + } + + xsc_core_dbg(dev, "table=%d, opcode=0x%x, ret=%d\n", table, opmod, ret); + +out: + resp->hdr.status = 0; + resp->error = cpu_to_be32(ret); + return ret; +} + +int xsc_cmd_write_reg_directly(struct xsc_core_device *dev, void *in, int in_size, void *out, + int out_size, int func_id) +{ + int opcode, ret = 0; + unsigned long flags; + struct xsc_inbox_hdr *hdr; + + hdr = (struct xsc_inbox_hdr *)in; + opcode = be16_to_cpu(hdr->opcode); + xsc_core_dbg(dev, "opcode: %x\n", opcode); + + xsc_acquire_lock(&dev->reg_access_lock, &flags); + switch (opcode) { + case XSC_CMD_OP_IOCTL_FLOW: + ret = xsc_cmd_exec_ioctl_flow(dev, in, out); + break; + default: + ret = -EINVAL; + break; + } + + /* ensure pci sequence */ + xsc_mmiowb(); + + xsc_release_lock(&dev->reg_access_lock, flags); + + return ret; +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/cmdq.h b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/cmdq.h new file mode 100644 index 0000000000000000000000000000000000000000..15fc8f1c706ec1471326c6ca229b2b769300cfa0 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/cmdq.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef CMDQ_H +#define CMDQ_H + +//hw will use this for some records(e.g. vf_id) +struct cmdq_rsv { + u16 func_id; + u8 rsv[2]; +}; + +//related with hw, won't change +#define CMDQ_ENTRY_SIZE 64 +#define CMD_FIRST_SIZE 8 +#define RSP_FIRST_SIZE 14 + +struct xsc_cmd_layout { + struct cmdq_rsv rsv0; + __be32 inlen; + __be64 in_ptr; + __be32 in[CMD_FIRST_SIZE]; + __be64 out_ptr; + __be32 outlen; + u8 token; + u8 sig; + u8 idx; + u8 type: 7; + u8 owner_bit: 1; //rsv for hw, arm will check this bit to make sure mem written +}; + +//static_assert(sizeof(struct xsc_cmd_layout) == CMDQ_ENTRY_SIZE, "xsc_cmd_layout wrong size"); + +struct xsc_rsp_layout { + struct cmdq_rsv rsv0; + __be32 out[RSP_FIRST_SIZE]; + u8 token; + u8 sig; + u8 idx; + u8 type: 7; + u8 owner_bit: 1; //rsv for hw, driver will check this bit to make sure mem written +}; + +//static_assert(sizeof(struct xsc_rsp_layout) == CMDQ_ENTRY_SIZE, "xsc_rsp_layout wrong size"); + +struct xsc_cmd_prot_block { + u8 data[512]; + u8 rsvd0[48]; + __be64 next; + __be32 block_num; + u8 owner_status; //fw should change this val to 1 + u8 token; + u8 ctrl_sig; + u8 sig; +}; + +#endif // XSC_CMD_H diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/osdep.c b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/osdep.c new file mode 100644 index 0000000000000000000000000000000000000000..cb04ca2351820dc3d29804e9c4c4e2a3f7a53bbd --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/osdep.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include + +void xsc_lock_init(struct xsc_lock *lock) +{ + spin_lock_init(&lock->lock); +} + +void xsc_acquire_lock(struct xsc_lock *lock, unsigned long *oflags) +{ + unsigned long flags; + + spin_lock_irqsave(&lock->lock, flags); + *oflags = flags; +} + +void xsc_release_lock(struct xsc_lock *lock, unsigned long flags) +{ + spin_unlock_irqrestore(&lock->lock, flags); +} + +void xsc_mmiowb(void) +{ + mmiowb(); +} + +void xsc_wmb(void) +{ + /* keep order */ + wmb(); +} + +void xsc_msleep(int timeout) +{ + msleep(timeout); +} + +void xsc_udelay(int timeout) +{ + udelay(timeout); +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/osdep.h b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/osdep.h new file mode 100644 index 0000000000000000000000000000000000000000..922a4467c7059a96c0e0b624bf3950c98e06fcf8 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/osdep.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef OSDEP_H +#define OSDEP_H + +#include + +#define xsc_print printk + +void xsc_msleep(int timeout); + +void xsc_udelay(int timeout); + +void xsc_lock_init(struct xsc_lock *lock); + +void xsc_acquire_lock(struct xsc_lock *lock, unsigned long *flags); + +void xsc_release_lock(struct xsc_lock *lock, unsigned long flags); + +void xsc_mmiowb(void); + +void xsc_wmb(void); + +void *xsc_malloc(unsigned int size); + +void xsc_free(void *addr); + +#endif + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_counters.h b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_counters.h new file mode 100644 index 0000000000000000000000000000000000000000..9c59657139a6c8ba4f342801e51ad206d811991c --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_counters.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef __XSC_COUNTERS_H__ +#define __XSC_COUNTERS_H__ + +/* From E-tile Hard User Guide */ +#define NIF_ETH_TX_PFC_LOW 0x83c +#define NIF_ETH_TX_PFC_HIGH 0x83d +#define NIF_ETH_RX_PFC_LOW 0x93c +#define NIF_ETH_RX_PFC_HIGH 0x93d +#define NIF_ETH_TX_CNTR_CONFIG 0x845 +#define NIF_ETH_RX_CNTR_CONFIG 0x945 +#define NIF_ETH_RX_FCSERR_LOW 0x904 +#define NIF_ETH_RX_FCSERR_HIGH 0x905 + +#define XSC_CNT_WIDTH_32_BIT 32 +#define XSC_CNT_WIDTH_64_BIT 64 +#define XSC_CNT_MASK_32 0xffffffff +#define XSC_CNT_MASK_64 0xffffffffffffffff + +struct cnt_value_64 { + u32 va_l; + u32 va_h; +}; + +struct cnt_value_96 { + u32 va_l; + u32 va_m; + u32 va_h; +}; + +enum { + XSC_CNT_TYPE_TX_PAUSE = 0, + XSC_CNT_TYPE_RX_PAUSE, +}; + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_flow.c b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_flow.c new file mode 100644 index 0000000000000000000000000000000000000000..6218f8b21dfee319d61ecde7fa11fcb29ca45d57 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_flow.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include + +#include "xsc_flow.h" + +static DECLARE_COMPLETION(dma_read_done); + +static inline int xsc_dma_wr_isbusy(struct xsc_core_device *xdev) +{ + u32 busy = 0; + + do { + busy = REG_RD32(xdev, HIF_TBL_TBL_DL_BUSY_REG_ADDR); + } while (busy != 0x0); + + return busy; +} + +static inline int xsc_dma_rd_isbusy(struct xsc_core_device *xdev) +{ + u32 busy = 0; + + do { + busy = REG_RD32(xdev, CLSF_DMA_DMA_UL_BUSY_REG_ADDR); + } while (busy != 0x0); + + return busy; +} + +static inline int xsc_dma_done(struct xsc_core_device *xdev) +{ + u32 done = 0; + + do { + done = REG_RD32(xdev, CLSF_DMA_DMA_DL_DONE_REG_ADDR); + } while ((done & 0x1) != 0x1); + + return done; +} + +static inline void xsc_dma_wr_success_get(struct xsc_core_device *xdev, u32 *success, u32 size) +{ + u32 *ptr = NULL; + + ptr = success; + IA_READ(xdev, CLSF_DMA_DMA_DL_SUCCESS_REG_ADDR, ptr, (size/sizeof(u32))); +} + +int xsc_flow_table_dma_write_add(struct xsc_core_device *xdev, + const struct tdi_dma_write_key_bits *key, + const struct tdi_dma_write_action_bits *action) +{ + u32 i = 0; + u32 busy = 0; + u32 dma_wr_num = 0; + u32 value = 0; + u32 done = 0; + u64 success[2]; + u32 data_len = 0; + u64 dma_wr_addr = 0; + + if (!xdev || !key || !action) + return -1; + + if (!action->entry_num) + return -1; + + dma_wr_num = ((action->entry_num + (XSC_DMA_WR_MAX - 1)) / XSC_DMA_WR_MAX); + + for (i = 0; i < dma_wr_num; i++) { + + if ((action->entry_num % XSC_DMA_WR_MAX) && (i == (dma_wr_num - 1))) + data_len = ((action->entry_num % XSC_DMA_WR_MAX) * XSC_DMA_LEN); + else + data_len = (XSC_DMA_WR_MAX * XSC_DMA_LEN); + + busy = xsc_dma_wr_isbusy(xdev); + if (busy) + return -1; + + REG_WR32(xdev, CLSF_DMA_ERR_CODE_CLR_REG_ADDR, 1); + + value = ((data_len << HIF_TBL_TBL_DL_REQ_REG_TBL_DL_LEN_SHIFT) | + (key->host_id << HIF_TBL_TBL_DL_REQ_REG_TBL_DL_HOST_ID_SHIFT) | + key->func_id); + + REG_WR32(xdev, HIF_TBL_TBL_DL_REQ_REG_ADDR, value); + + dma_wr_addr = (action->data_addr + ((i * XSC_DMA_WR_MAX) * XSC_DMA_LEN)); + value = (dma_wr_addr & HIF_TBL_TBL_DL_ADDR_L_REG_TBL_DL_ADDR_L_MASK); + REG_WR32(xdev, HIF_TBL_TBL_DL_ADDR_L_REG_ADDR, value); + + value = ((dma_wr_addr >> 32) & HIF_TBL_TBL_DL_ADDR_H_REG_TBL_DL_ADDR_H_MASK); + REG_WR32(xdev, HIF_TBL_TBL_DL_ADDR_H_REG_ADDR, value); + + REG_WR32(xdev, HIF_TBL_TBL_DL_START_REG_ADDR, 1); + + done = xsc_dma_done(xdev); + if (done != XSC_DMA_WR_SUCCESS) { + memset(success, 0, sizeof(success)); + xsc_dma_wr_success_get(xdev, (u32 *)&success, sizeof(success)); + xsc_core_err(xdev, "DMA write time %d status 0x%lx%lx fail.\n", i, + (unsigned long)success[1], (unsigned long)success[0]); + return -1; + } + } + + return 0; +} + +void xsc_dma_read_done_complete(void) +{ + complete(&dma_read_done); +} + +int xsc_flow_table_dma_read_add(struct xsc_core_device *xdev, + const struct tdi_dma_read_key_bits *key, + const struct tdi_dma_read_action_bits *action) +{ + u32 busy = 0; + u32 value = 0; + + if (!xdev || !key || !action) + return -1; + + if (!action->burst_num) + return -1; + + busy = xsc_dma_rd_isbusy(xdev); + if (busy) + return -1; + + value = ((key->host_id << HIF_TBL_TBL_UL_REQ_REG_TBL_UL_HOST_ID_SHIFT) | + key->func_id); + + REG_WR32(xdev, HIF_TBL_TBL_UL_REQ_REG_ADDR, value); + + value = (action->data_addr & HIF_TBL_TBL_UL_ADDR_L_REG_TBL_UL_ADDR_L_MASK); + REG_WR32(xdev, HIF_TBL_TBL_UL_ADDR_L_REG_ADDR, value); + + value = ((action->data_addr >> 32) & HIF_TBL_TBL_UL_ADDR_H_REG_TBL_UL_ADDR_H_MASK); + REG_WR32(xdev, HIF_TBL_TBL_UL_ADDR_H_REG_ADDR, value); + + REG_WR32(xdev, HIF_TBL_TBL_UL_START_REG_ADDR, 1); + + value = (key->tbl_id & CLSF_DMA_DMA_RD_TABLE_ID_REG_DMA_RD_TBL_ID_MASK); + REG_WR32(xdev, CLSF_DMA_DMA_RD_TABLE_ID_REG_ADDR, value); + + value = ((action->burst_num << CLSF_DMA_DMA_RD_ADDR_REG_DMA_RD_BURST_NUM_SHIFT) | + key->tbl_start_addr); + REG_WR32(xdev, CLSF_DMA_DMA_RD_ADDR_REG_ADDR, value); + + REG_WR32(xdev, CLSF_DMA_INDRW_RD_START_REG_ADDR, 1); + + /*wait msix interrupt */ + if (!wait_for_completion_timeout(&dma_read_done, msecs_to_jiffies(5000))) { + xsc_core_err(xdev, "wait for dma read done completion timeout.\n"); + return -ETIMEDOUT; + } + + REG_WR32(xdev, HIF_TBL_MSG_RDY_REG_ADDR, 1); + + return 0; +} + +int xsc_flow_add(struct xsc_core_device *xdev, + int table, int length, void *data) +{ + int ret = -EINVAL; + struct xsc_flow_dma_write_add *dma_wr; + struct xsc_flow_dma_read_add *dma_rd; + + switch (table) { + case XSC_FLOW_DMA_WR: + if (length == sizeof(struct xsc_flow_dma_write_add)) { + dma_wr = (struct xsc_flow_dma_write_add *)data; + ret = xsc_flow_table_dma_write_add(xdev, &dma_wr->key, &dma_wr->action); + } + break; + case XSC_FLOW_DMA_RD: + if (length == sizeof(struct xsc_flow_dma_read_add)) { + dma_rd = (struct xsc_flow_dma_read_add *)data; + ret = xsc_flow_table_dma_read_add(xdev, &dma_rd->key, &dma_rd->action); + } + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_flow.h b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_flow.h new file mode 100644 index 0000000000000000000000000000000000000000..1b3455e5a4fe6e8a12b1e4d43a96070daf6247a9 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_flow.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_FLOW_H +#define XSC_FLOW_H + +#include "osdep.h" + +#define XSC_DMA_LEN 64 +#define XSC_DMA_WR_MAX 128 +#define XSC_DMA_WR_SUCCESS 0x3 + +/* key */ +struct tdi_dma_write_key_bits { + u8 host_id:1; + u16 func_id:11; +} __packed; + +struct tdi_dma_read_key_bits { + u16 tbl_start_addr:16; + u8 tbl_id:7; + u8 host_id:1; + u16 func_id:11; +} __packed; + +/* action */ +struct tdi_dma_write_action_bits { + u32 entry_num:32; + u64 data_addr:64; +} __packed; + +struct tdi_dma_read_action_bits { + u16 burst_num:16; + u64 data_addr:64; +} __packed; + +/* ioctl data - add */ +struct xsc_flow_dma_write_add { + struct tdi_dma_write_key_bits key; + struct tdi_dma_write_action_bits action; +}; + +struct xsc_flow_dma_read_add { + struct tdi_dma_read_key_bits key; + struct tdi_dma_read_action_bits action; +}; + +struct xsc_logic_in_port_cfg_reg { + u32 phy_port_offset:11; + u32 resv0:5; + u32 func_id_offset:11; + u32 resv1:5; + u32 aps_port_offset:11; + u32 resv2:1; + u32 aps_port_rec_flg:1; + u32 resv3:19; +}; + +int xsc_flow_add(struct xsc_core_device *xdev, + int table, int length, void *data); + +void xsc_dma_read_done_complete(void); + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_fw.h b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_fw.h new file mode 100644 index 0000000000000000000000000000000000000000..2c2a0de06cb7ea4d2382ef0bd7f7007baf2d7b8f --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_fw.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_FW_H +#define XSC_FW_H + +#include "osdep.h" + +#include +#include + +struct xsc_free_list { + struct list_head list; + int start; + int end; +}; + +struct xsc_free_list_wl { + struct xsc_free_list head; + struct xsc_lock lock; +}; + +struct xsc_mpt_info { + u64 va; + u32 mtt_base; + u32 page_num; +}; + +struct xsc_resources { + int refcnt; + int iae_idx; +#define XSC_MAX_MPT_NUM MMC_MPT_TBL_MEM_DEPTH + struct xsc_mpt_info mpt_entry[XSC_MAX_MPT_NUM]; + int max_mpt_num; + u64 mpt_tbl[XSC_MAX_MPT_NUM >> 6]; +#define XSC_MAX_MTT_NUM MMC_MTT_TBL_MEM_DEPTH + int max_mtt_num; + struct xsc_free_list_wl mtt_list; + u16 msix_max_num; + u16 msix_vec_base; + u16 msix_vec_end; + unsigned long *msix_vec_tbl; + struct xsc_lock lock; +}; + +struct xsc_resources *get_xsc_res(struct xsc_core_device *dev); + +int xsc_alloc_res(u32 *res, u64 *res_tbl, u32 max); + +int xsc_dealloc_res(u32 *res, u64 *res_tbl); + +int alloc_from_free_list(struct xsc_free_list_wl *list, int required, u32 *alloc, + u32 base_align); + +int release_to_free_list(struct xsc_free_list_wl *list, u32 release, + u32 num_released); + +int alloc_mpt_entry(struct xsc_core_device *dev, u32 *mpt_idx); + +int dealloc_mpt_entry(struct xsc_core_device *dev, u32 *mpt_idx); + +int alloc_mtt_entry(struct xsc_core_device *dev, u32 pages_num, u32 *mtt_base); + +int dealloc_mtt_entry(struct xsc_core_device *dev, int pages_num, u32 mtt_base); + +int xsc_alloc_continuous_msix_vec(struct xsc_core_device *dev, u16 vec_num); +int xsc_free_continuous_msix_vec(struct xsc_core_device *dev); + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_mem.c b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_mem.c new file mode 100644 index 0000000000000000000000000000000000000000..59b75c3edb3401b79844ad193ca088d207bad495 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_mem.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include + +void *xsc_malloc(unsigned int size) +{ + return kmalloc(size, GFP_ATOMIC); +} + +void xsc_free(void *addr) +{ + kfree(addr); +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_reg_struct.h b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_reg_struct.h new file mode 100644 index 0000000000000000000000000000000000000000..fa3f51f49d0f433d2a426710590c08cdedb02773 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_reg_struct.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_REG_DEFINE_H +#define XSC_REG_DEFINE_H + +struct xsc_mpt_entry { + u32 va_l; + u32 va_h; + u32 mem_size; + u32 pdn:24; + u32 key:8; + u32 mtt_base:18; + u32 acc:4; + u32 page_mode:2; + u32 mem_map_en:1; + u32 rsv:7; +}; + +#endif diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_res.c b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_res.c new file mode 100644 index 0000000000000000000000000000000000000000..a3ba44e9fd323460597e9997fcec957d5a156e20 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_res.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include "xsc_fw.h" + +struct xsc_resources g_xres[MAX_BOARD_NUM]; + +static int xsc_alloc_free_list_res(struct xsc_free_list_wl *list, int max_num) +{ + struct xsc_free_list *free_node; + + xsc_lock_init(&list->lock); + INIT_LIST_HEAD(&list->head.list); + + free_node = xsc_malloc(sizeof(struct xsc_free_list)); + if (!free_node) + return -ENOMEM; + + free_node->start = 0; + free_node->end = free_node->start + max_num - 1; + list_add(&free_node->list, &list->head.list); + + return 0; +} + +static void xsc_destroy_free_list_res(struct xsc_free_list_wl *list) +{ + struct xsc_free_list *pos; + struct xsc_free_list *next; + + list_for_each_entry_safe(pos, next, &list->head.list, list) { + list_del(&pos->list); + xsc_free(pos); + } +} + +static int xsc_alloc_msix_res_tbl(unsigned long **res_tbl, u32 max) +{ + u32 bytes; + + bytes = max >> 3; + if (!bytes) + bytes = 1; + *res_tbl = kmalloc(bytes, GFP_KERNEL); + if (!*res_tbl) + return -ENOMEM; + memset((u8 *)(*res_tbl), 0xFF, bytes); + return 0; +} + +static void xsc_free_msix_res_tbl(unsigned long **res) +{ + if (*res) + kfree(*res); + *res = 0; +} + +int xsc_alloc_continuous_msix_vec(struct xsc_core_device *dev, u16 vec_num) +{ + struct xsc_resources *xres = get_xsc_res(dev); + int i, j, start, end; + int vec_base = 0; + int avail_vec = 0; + + start = xres->msix_vec_base; + end = xres->msix_vec_end; + + vec_base = -1; + for (i = start; i <= end; i++) { + if (test_bit(i, xres->msix_vec_tbl)) { + avail_vec++; + if (vec_base == -1) + vec_base = i; + } else { + avail_vec = 0; + vec_base = -1; + } + + if (avail_vec == vec_num) { + for (j = 0; j < vec_num; j++) + clear_bit(vec_base + j, xres->msix_vec_tbl); + + dev->msix_vec_base = vec_base; + return 0; + } + } + xsc_core_warn(dev, + "failed to alloc msix vec, vec_range=%d~%d, vec_num=%d\n", + start, end, vec_num); + return -EINVAL; +} + +int xsc_free_continuous_msix_vec(struct xsc_core_device *dev) +{ + struct xsc_resources *xres = get_xsc_res(dev); + struct xsc_eq_table *table = &dev->dev_res->eq_table; + u32 vec_base, vec_num; + int i, start, end; + + vec_base = dev->msix_vec_base; + vec_num = table->num_comp_vectors + table->eq_vec_comp_base; + + start = xres->msix_vec_base; + end = xres->msix_vec_end; + + if (vec_base < start || vec_base > end || + (vec_base + vec_num - 1) > end) { + xsc_core_warn(dev, + "failed to free msix vec, vec_base=%d, vec_num=%d, range=%d~%d\n", + vec_base, vec_num, start, end); + return -EINVAL; + } + + for (i = 0; i < vec_num; i++) + set_bit(vec_base + i, xres->msix_vec_tbl); + + return 0; +} + +int xsc_create_res(struct xsc_core_device *dev) +{ + int ret = 0; + struct xsc_resources *xres = get_xsc_res(dev); + + xres->refcnt++; + if (xres->refcnt > 1) + return 0; + + xsc_lock_init(&xres->lock); + xres->max_mpt_num = XSC_MAX_MPT_NUM; + memset(xres->mpt_tbl, 0xFF, XSC_MAX_MPT_NUM >> 3); + /* reserved for local dma lkey */ + clear_bit(0, (unsigned long *)xres->mpt_tbl); + + ret = xsc_alloc_iae_idx(dev, &xres->iae_idx); + if (ret) + return -EINVAL; + + xres->msix_max_num = 1 << dev->caps.log_max_msix; + xres->msix_vec_base = 0; + xres->msix_vec_end = (xres->msix_max_num - 1); + ret = xsc_alloc_msix_res_tbl(&xres->msix_vec_tbl, xres->msix_max_num); + if (ret) + goto err_msix; + + xres->max_mtt_num = XSC_MAX_MTT_NUM; + ret = xsc_alloc_free_list_res(&xres->mtt_list, xres->max_mtt_num); + if (ret) + goto err_mtt; + + return ret; + +err_mtt: + xsc_free_msix_res_tbl(&xres->msix_vec_tbl); +err_msix: + xsc_release_iae_idx(dev, &xres->iae_idx); + return ret; +} + +void xsc_destroy_res(struct xsc_core_device *dev) +{ + struct xsc_resources *xres = get_xsc_res(dev); + + xres->refcnt--; + if (xres->refcnt) + return; + + xsc_destroy_free_list_res(&xres->mtt_list); + xsc_free_msix_res_tbl(&xres->msix_vec_tbl); + xsc_release_iae_idx(dev, &xres->iae_idx); +} + +struct xsc_resources *get_xsc_res(struct xsc_core_device *dev) +{ + return &g_xres[dev->board_id]; +} + +int xsc_alloc_res(u32 *res, u64 *res_tbl, u32 max) +{ + u32 bit_num; + + bit_num = find_first_bit((unsigned long *)res_tbl, max); + if (bit_num == max) + return -ENOMEM; + clear_bit(bit_num, (unsigned long *)res_tbl); + *res = bit_num; + return 0; + +} + +int xsc_dealloc_res(u32 *res, u64 *res_tbl) +{ + if (test_and_set_bit(*res, (unsigned long *)res_tbl)) + return -EINVAL; + + *res = 0; + return 0; +} + +int alloc_from_free_list(struct xsc_free_list_wl *list, int required, u32 *alloc, + u32 base_align) +{ + struct xsc_free_list *free_node; + struct xsc_free_list *next; + struct xsc_free_list *new_node; + unsigned long flags; + + *alloc = -1; + xsc_acquire_lock(&list->lock, &flags); + list_for_each_entry_safe(free_node, next, &list->head.list, list) { + int start = round_up(free_node->start, base_align); + int avail_num = free_node->end - start + 1; + + if (required < avail_num) { + if (start > free_node->start) { + new_node = xsc_malloc(sizeof(struct xsc_free_list)); + if (!new_node) { + xsc_release_lock(&list->lock, flags); + return -ENOMEM; + } + new_node->start = free_node->start; + new_node->end = start - 1; + __list_add(&new_node->list, free_node->list.prev, + &free_node->list); + } + *alloc = start; + free_node->start = start + required; + break; + } else if (required == avail_num) { + *alloc = start; + if (start > free_node->start) { + free_node->end = start - 1; + } else { + list_del(&free_node->list); + xsc_free(free_node); + } + break; + } + } + xsc_release_lock(&list->lock, flags); + + if (*alloc == -1) + return -EINVAL; + + return 0; +} + +int release_to_free_list(struct xsc_free_list_wl *list, u32 release, + u32 num_released) +{ + struct xsc_free_list *free_node = NULL; + struct xsc_free_list *next, *prev; + struct xsc_free_list *new_node; + unsigned long flags; + bool new_flag = false; + bool end_merge = false; + int ret = 0; + + xsc_acquire_lock(&list->lock, &flags); + list_for_each_entry_safe(free_node, next, &list->head.list, list) { + if (release + num_released < free_node->start) { + new_flag = true; + } else if (release + num_released == free_node->start) { + /* backward merge */ + end_merge = true; + free_node->start = release; + } + + if (new_flag || end_merge) { + /* forward merge, and backward merge if possible */ + if (free_node->list.prev == &list->head.list) + goto create_node; + + prev = list_entry(free_node->list.prev, struct xsc_free_list, list); + if (release == prev->end + 1) { + if (end_merge) { + prev->end = free_node->end; + list_del(&free_node->list); + xsc_free(free_node); + free_node = NULL; + } else { + prev->end = release + num_released - 1; + new_flag = false; + } + } + + break; + } + } + + if (list_empty(&list->head.list)) { + new_flag = true; + free_node = &list->head; + } + +create_node: + if (new_flag && free_node) { + new_node = xsc_malloc(sizeof(struct xsc_free_list)); + if (!new_node) { + ret = -ENOMEM; + goto ret; + } + new_node->start = release; + new_node->end = release + num_released - 1; + __list_add(&new_node->list, free_node->list.prev, + &free_node->list); + } +ret: + xsc_release_lock(&list->lock, flags); + return ret; +} + +int alloc_mpt_entry(struct xsc_core_device *dev, u32 *mpt_idx) +{ + struct xsc_resources *xres = get_xsc_res(dev); + + if (xsc_alloc_res(mpt_idx, xres->mpt_tbl, xres->max_mpt_num)) + return -EINVAL; + + return 0; +} + +int dealloc_mpt_entry(struct xsc_core_device *dev, u32 *mpt_idx) +{ + struct xsc_resources *xres = get_xsc_res(dev); + + if (xsc_dealloc_res(mpt_idx, xres->mpt_tbl)) + return -EINVAL; + + return 0; +} + +int alloc_mtt_entry(struct xsc_core_device *dev, u32 pages_num, u32 *mtt_base) +{ + struct xsc_resources *xres = get_xsc_res(dev); + int ret = alloc_from_free_list(&xres->mtt_list, pages_num, mtt_base, 1); + + xsc_core_dbg(dev, "alloc mtt for %d pages start from %d\n", + pages_num, *mtt_base); + + return ret; +} + +int dealloc_mtt_entry(struct xsc_core_device *dev, int pages_num, u32 mtt_base) +{ + struct xsc_resources *xres = get_xsc_res(dev); + int ret = release_to_free_list(&xres->mtt_list, mtt_base, pages_num); + + xsc_core_dbg(dev, "mtt release %d pages start from %d\n", + pages_num, mtt_base); + + return ret; +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_tbm.h b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_tbm.h new file mode 100644 index 0000000000000000000000000000000000000000..c513d572b25b7a6f6d9afa1cd69fa7d1d9841eb1 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/fw/xsc_tbm.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_TBM_H +#define XSC_TBM_H + +#include + +#define XSC_LAG_NUM_MAX 0x30 + +#ifndef PCIE0_PF_NUM +#define PCIE0_PF_NUM 2 +#endif + +#ifndef PCIE1_PF_NUM +#define PCIE1_PF_NUM 0 +#endif + +enum { + XSC_VLAN_MODE_NONE = 0, + XSC_VLAN_MODE_TRUNK, + XSC_VLAN_MODE_ACCESS, + XSC_VLAN_MODE_TUNNEL, + XSC_VLAN_MODE_NATIVE_TAGGED, + XSC_VLAN_MODE_NATIVE_UNTAGGED, +}; + +struct xsc_vlan_config { + u32 mode; + u32 pvid; + u32 proto; + u32 prio; + u32 vid_allow_base; + u32 vid_allow_num; + u32 smac_filter_en; +}; + +enum { + XSC_HASH_FIELD_SEL_SRC_IP = 1 << 0, + XSC_HASH_FIELD_SEL_PROTO = 1 << 1, + XSC_HASH_FIELD_SEL_DST_IP = 1 << 2, + XSC_HASH_FIELD_SEL_SPORT = 1 << 3, + XSC_HASH_FIELD_SEL_DPORT = 1 << 4, + XSC_HASH_FIELD_SEL_SRC_IPV6 = 1 << 5, + XSC_HASH_FIELD_SEL_DST_IPV6 = 1 << 6, + XSC_HASH_FIELD_SEL_SPORT_V6 = 1 << 7, + XSC_HASH_FIELD_SEL_DPORT_V6 = 1 << 8, +}; + +#define XSC_HASH_IP (XSC_HASH_FIELD_SEL_SRC_IP |\ + XSC_HASH_FIELD_SEL_DST_IP |\ + XSC_HASH_FIELD_SEL_PROTO) +#define XSC_HASH_IP_PORTS (XSC_HASH_FIELD_SEL_SRC_IP |\ + XSC_HASH_FIELD_SEL_DST_IP |\ + XSC_HASH_FIELD_SEL_SPORT |\ + XSC_HASH_FIELD_SEL_DPORT |\ + XSC_HASH_FIELD_SEL_PROTO) +#define XSC_HASH_IP6 (XSC_HASH_FIELD_SEL_SRC_IPV6 |\ + XSC_HASH_FIELD_SEL_DST_IPV6 |\ + XSC_HASH_FIELD_SEL_PROTO) +#define XSC_HASH_IP6_PORTS (XSC_HASH_FIELD_SEL_SRC_IPV6 |\ + XSC_HASH_FIELD_SEL_DST_IPV6 |\ + XSC_HASH_FIELD_SEL_SPORT_V6 |\ + XSC_HASH_FIELD_SEL_DPORT_V6 |\ + XSC_HASH_FIELD_SEL_PROTO) + +int xsc_tbm_vlan_config(struct xsc_core_device *dev, + struct xsc_logic_port_info *info, + struct xsc_vlan_config *config); + +static inline bool xsc_vf_pp_init_enable(const struct xsc_core_device *dev) +{ + return dev->vf_pp_init; +} + +static inline void xsc_set_vf_pp_status(struct xsc_core_device *dev, bool status) +{ + dev->vf_pp_init = status; +} + +#endif /* XSC_TBM_H */ + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/intf.c b/drivers/net/ethernet/yunsilicon/xsc/pci/intf.c new file mode 100644 index 0000000000000000000000000000000000000000..2177d4b1948a4a751c50fc4a118681ab1298058b --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/intf.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include + +static void xsc_add_device(struct xsc_interface *intf, struct xsc_priv *priv) +{ + struct xsc_device_context *dev_ctx; + struct xsc_core_device *dev; + + dev = container_of(priv, struct xsc_core_device, priv); + dev_ctx = kzalloc(sizeof(*dev_ctx), GFP_KERNEL); + if (!dev_ctx) + return; + + dev_ctx->intf = intf; + + dev_ctx->context = intf->add(dev); + if (dev_ctx->context) { + set_bit(XSC_INTERFACE_ADDED, &dev_ctx->state); + if (intf->attach) + set_bit(XSC_INTERFACE_ATTACHED, &dev_ctx->state); + + spin_lock_irq(&priv->ctx_lock); + list_add_tail(&dev_ctx->list, &priv->ctx_list); + spin_unlock_irq(&priv->ctx_lock); + } else { + kfree(dev_ctx); + } +} + +static struct xsc_device_context *xsc_get_device(struct xsc_interface *intf, + struct xsc_priv *priv) +{ + struct xsc_device_context *dev_ctx; + + /* caller of this function has mutex protection */ + list_for_each_entry(dev_ctx, &priv->ctx_list, list) + if (dev_ctx->intf == intf) + return dev_ctx; + + return NULL; +} + +static void xsc_remove_device(struct xsc_interface *intf, struct xsc_priv *priv) +{ + struct xsc_device_context *dev_ctx; + struct xsc_core_device *dev = container_of(priv, struct xsc_core_device, priv); + + dev_ctx = xsc_get_device(intf, priv); + if (!dev_ctx) + return; + + spin_lock_irq(&priv->ctx_lock); + list_del(&dev_ctx->list); + spin_unlock_irq(&priv->ctx_lock); + + if (test_bit(XSC_INTERFACE_ADDED, &dev_ctx->state)) + intf->remove(dev, dev_ctx->context); + + kfree(dev_ctx); +} + +int xsc_register_interface(struct xsc_interface *intf) +{ + struct xsc_priv *priv; + + if (!intf->add || !intf->remove) + return -EINVAL; + + mutex_lock(&xsc_intf_mutex); + list_add_tail(&intf->list, &intf_list); + list_for_each_entry(priv, &xsc_dev_list, dev_list) { + xsc_add_device(intf, priv); + } + mutex_unlock(&xsc_intf_mutex); + + return 0; +} +EXPORT_SYMBOL(xsc_register_interface); + +void xsc_unregister_interface(struct xsc_interface *intf) +{ + struct xsc_priv *priv; + + mutex_lock(&xsc_intf_mutex); + list_for_each_entry(priv, &xsc_dev_list, dev_list) + xsc_remove_device(intf, priv); + list_del(&intf->list); + mutex_unlock(&xsc_intf_mutex); +} +EXPORT_SYMBOL(xsc_unregister_interface); + +static void xsc_attach_interface(struct xsc_interface *intf, + struct xsc_priv *priv) +{ + struct xsc_device_context *dev_ctx; + struct xsc_core_device *dev = container_of(priv, struct xsc_core_device, priv); + + dev_ctx = xsc_get_device(intf, priv); + if (!dev_ctx) + return; + + if (intf->attach) { + if (test_bit(XSC_INTERFACE_ATTACHED, &dev_ctx->state)) + return; + if (intf->attach(dev, dev_ctx->context)) + return; + set_bit(XSC_INTERFACE_ATTACHED, &dev_ctx->state); + } else { + if (test_bit(XSC_INTERFACE_ADDED, &dev_ctx->state)) + return; + dev_ctx->context = intf->add(dev); + if (!dev_ctx->context) + return; + set_bit(XSC_INTERFACE_ADDED, &dev_ctx->state); + } +} + +static void xsc_detach_interface(struct xsc_interface *intf, + struct xsc_priv *priv) +{ + struct xsc_device_context *dev_ctx; + struct xsc_core_device *dev = container_of(priv, struct xsc_core_device, priv); + + dev_ctx = xsc_get_device(intf, priv); + if (!dev_ctx) + return; + + if (intf->detach) { + if (!test_bit(XSC_INTERFACE_ATTACHED, &dev_ctx->state)) + return; + intf->detach(dev, dev_ctx->context); + clear_bit(XSC_INTERFACE_ATTACHED, &dev_ctx->state); + } else { + if (!test_bit(XSC_INTERFACE_ADDED, &dev_ctx->state)) + return; + intf->remove(dev, dev_ctx->context); + clear_bit(XSC_INTERFACE_ADDED, &dev_ctx->state); + } +} + +void xsc_attach_device(struct xsc_core_device *dev) +{ + struct xsc_priv *priv = &dev->priv; + struct xsc_interface *intf; + + mutex_lock(&xsc_intf_mutex); + list_for_each_entry(intf, &intf_list, list) { + xsc_attach_interface(intf, priv); + } + mutex_unlock(&xsc_intf_mutex); +} +EXPORT_SYMBOL(xsc_attach_device); + +void xsc_attach_device_by_protocol(struct xsc_core_device *dev, int protocol) +{ + struct xsc_priv *priv = &dev->priv; + struct xsc_interface *intf; + + mutex_lock(&xsc_intf_mutex); + list_for_each_entry(intf, &intf_list, list) + if (intf->protocol == protocol) + xsc_attach_interface(intf, priv); + mutex_unlock(&xsc_intf_mutex); +} + +void xsc_detach_device(struct xsc_core_device *dev) +{ + struct xsc_priv *priv = &dev->priv; + struct xsc_interface *intf; + + mutex_lock(&xsc_intf_mutex); + list_for_each_entry(intf, &intf_list, list) + xsc_detach_interface(intf, priv); + mutex_unlock(&xsc_intf_mutex); +} +EXPORT_SYMBOL(xsc_detach_device); + +bool xsc_device_registered(struct xsc_core_device *dev) +{ + struct xsc_priv *priv; + bool found = false; + + mutex_lock(&xsc_intf_mutex); + list_for_each_entry(priv, &xsc_dev_list, dev_list) + if (priv == &dev->priv) + found = true; + mutex_unlock(&xsc_intf_mutex); + + return found; +} + +int xsc_register_device(struct xsc_core_device *dev) +{ + struct xsc_priv *priv = &dev->priv; + struct xsc_interface *intf; + + mutex_lock(&xsc_intf_mutex); + list_add_tail(&priv->dev_list, &xsc_dev_list); + list_for_each_entry(intf, &intf_list, list) + xsc_add_device(intf, priv); + mutex_unlock(&xsc_intf_mutex); + + return 0; +} +EXPORT_SYMBOL(xsc_register_device); + +void xsc_unregister_device(struct xsc_core_device *dev) +{ + struct xsc_priv *priv = &dev->priv; + struct xsc_interface *intf; + + mutex_lock(&xsc_intf_mutex); + list_for_each_entry(intf, &intf_list, list) + xsc_remove_device(intf, priv); + list_del(&priv->dev_list); + mutex_unlock(&xsc_intf_mutex); +} +EXPORT_SYMBOL(xsc_unregister_device); + +/* Must be called with intf_mutex held */ +static bool xsc_has_added_dev_by_protocol(struct xsc_core_device *dev, int protocol) +{ + struct xsc_device_context *dev_ctx; + struct xsc_interface *intf; + bool found = false; + + list_for_each_entry(intf, &intf_list, list) { + if (intf->protocol == protocol) { + dev_ctx = xsc_get_device(intf, &dev->priv); + if (dev_ctx && test_bit(XSC_INTERFACE_ADDED, &dev_ctx->state)) + found = true; + break; + } + } + + return found; +} + +/* Must be called with intf_mutex held */ +void xsc_add_dev_by_protocol(struct xsc_core_device *dev, int protocol) +{ + struct xsc_interface *intf; + + list_for_each_entry(intf, &intf_list, list) + if (intf->protocol == protocol) { + xsc_add_device(intf, &dev->priv); + break; + } +} +EXPORT_SYMBOL(xsc_add_dev_by_protocol); + +/* Must be called with intf_mutex held */ +void xsc_remove_dev_by_protocol(struct xsc_core_device *dev, int protocol) +{ + struct xsc_interface *intf; + + list_for_each_entry(intf, &intf_list, list) + if (intf->protocol == protocol) { + xsc_remove_device(intf, &dev->priv); + break; + } +} +EXPORT_SYMBOL(xsc_remove_dev_by_protocol); + +void xsc_reload_interfaces(struct xsc_core_device *dev, + int protocol1, int protocol2, + bool valid1, bool valid2) +{ + bool reload1; + bool reload2; + + mutex_lock(&xsc_intf_mutex); + + reload1 = valid1 && xsc_has_added_dev_by_protocol(dev, protocol1); + reload2 = valid2 && xsc_has_added_dev_by_protocol(dev, protocol2); + + if (reload2) + xsc_remove_dev_by_protocol(dev, protocol2); + if (reload1) + xsc_remove_dev_by_protocol(dev, protocol1); + if (reload1) + xsc_add_dev_by_protocol(dev, protocol1); + if (reload2) + xsc_add_dev_by_protocol(dev, protocol2); + mutex_unlock(&xsc_intf_mutex); +} + +void xsc_reload_interface(struct xsc_core_device *dev, int protocol) +{ + mutex_lock(&xsc_intf_mutex); + if (xsc_has_added_dev_by_protocol(dev, protocol)) { + xsc_remove_dev_by_protocol(dev, protocol); + xsc_add_dev_by_protocol(dev, protocol); + } + mutex_unlock(&xsc_intf_mutex); +} + +static u32 xsc_gen_pci_id(struct xsc_core_device *dev) +{ + return (u32)((pci_domain_nr(dev->pdev->bus) << 16) | + (dev->pdev->bus->number << 8) | + PCI_SLOT(dev->pdev->devfn)); +} + +struct xsc_core_device *xsc_get_next_phys_dev(struct xsc_core_device *dev) +{ + struct xsc_core_device *res = NULL; + struct xsc_core_device *tmp_dev; + struct xsc_priv *priv; + u32 pci_id; + + if (!xsc_core_is_pf(dev)) + return NULL; + + pci_id = xsc_gen_pci_id(dev); + list_for_each_entry(priv, &xsc_dev_list, dev_list) { + tmp_dev = container_of(priv, struct xsc_core_device, priv); + if (!xsc_core_is_pf(tmp_dev)) + continue; + + if ((dev != tmp_dev) && (xsc_gen_pci_id(tmp_dev) == pci_id)) { + res = tmp_dev; + break; + } + } + + return res; +} +EXPORT_SYMBOL(xsc_get_next_phys_dev); + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/mad.c b/drivers/net/ethernet/yunsilicon/xsc/pci/mad.c new file mode 100644 index 0000000000000000000000000000000000000000..4ec070c13d852e7dd94aae9eb4c8145c0d785d5f --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/mad.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include + +int xsc_core_mad_ifc(struct xsc_core_device *xdev, void *inb, void *outb, + u16 opmod, int port) +{ + struct xsc_mad_ifc_mbox_in *in = NULL; + struct xsc_mad_ifc_mbox_out *out = NULL; + int err; + + in = kzalloc(sizeof(*in), GFP_KERNEL); + if (!in) + return -ENOMEM; + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) { + err = -ENOMEM; + goto out; + } + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_MAD_IFC); + in->hdr.opmod = cpu_to_be16(opmod); + in->port = port; + + memcpy(in->data, inb, sizeof(in->data)); + + err = xsc_cmd_exec(xdev, in, sizeof(*in), out, sizeof(*out)); + if (err) + goto out; + + if (out->hdr.status) { + err = xsc_cmd_status_to_err(&out->hdr); + goto out; + } + + memcpy(outb, out->data, sizeof(out->data)); + +out: + kfree(out); + kfree(in); + return err; +} +EXPORT_SYMBOL_GPL(xsc_core_mad_ifc); diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/main.c b/drivers/net/ethernet/yunsilicon/xsc/pci/main.c new file mode 100644 index 0000000000000000000000000000000000000000..dbad66b4285c5f4c80e5a3851b7c3f4c2ab1468c --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/main.c @@ -0,0 +1,981 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_XSC_ESWITCH +#include "devlink.h" +#include "eswitch.h" +#endif +#include "fw/xsc_counters.h" +#include "fw/xsc_tbm.h" +#include "xsc_pci_ctrl.h" + +#ifdef RUN_WITH_PSV +#include "../../../../xscale-fw/include/xscale-fw.h" +#endif + +unsigned int xsc_debug_mask; +module_param_named(debug_mask, xsc_debug_mask, uint, 0644); +MODULE_PARM_DESC(debug_mask, + "debug mask: 1=dump cmd data, 2=dump cmd exec time, 3=both. Default=0"); + +bool hw_init = 1; +int pcie_no; +module_param_named(hw_init, hw_init, bool, 0644); +module_param_named(pcie_no, pcie_no, int, 0644); + +static bool probe_vf = 1; +module_param_named(probe_vf, probe_vf, bool, 0644); +MODULE_PARM_DESC(probe_vf, "probe VFs or not, 0 = not probe, 1 = probe. Default = 1"); + +static bool xsc_hw_reset; + +#define DRIVER_NAME "xsc_pci" +#define DRIVER_VERSION "0.1.0" + +#define XSC_PCI_VENDOR_ID_OBSOLETE 0x1172 +#define XSC_PCI_VENDOR_ID 0x1f67 +#define XSC_PF1_DEVICE_ID 0x0001 +#define XSC_PF1_VF_DEVICE_ID 0x0002 + +#ifdef USE_VIRTIO +static const struct pci_device_id xsc_pci_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) }, + { 0 } +}; +#else +static const struct pci_device_id xsc_pci_id_table[] = { + { PCI_DEVICE(XSC_PCI_VENDOR_ID, XSC_PF1_DEVICE_ID) }, + { PCI_DEVICE(XSC_PCI_VENDOR_ID_OBSOLETE, XSC_PF1_DEVICE_ID) }, + { PCI_DEVICE(XSC_PCI_VENDOR_ID, XSC_PF1_VF_DEVICE_ID), + .driver_data = XSC_PCI_DEV_IS_VF}, /* PF1's VF */ + { 0 } +}; +#endif /* !USE_VIRTIO */ +MODULE_DEVICE_TABLE(pci, xsc_pci_id_table); + +#ifndef RUN_WITH_PSV +static bool need_write_reg_directly(void *in) +{ + struct xsc_inbox_hdr *hdr; + struct xsc_ioctl_mbox_in *req; + struct xsc_ioctl_data_tl *tl; + char *data; + + hdr = (struct xsc_inbox_hdr *)in; + if (unlikely(be16_to_cpu(hdr->opcode) == XSC_CMD_OP_IOCTL_FLOW)) { + req = (struct xsc_ioctl_mbox_in *)in; + data = (char *)req->data; + tl = (struct xsc_ioctl_data_tl *)data; + if (tl->opmod == XSC_IOCTL_OP_ADD) { + if (unlikely(tl->table == XSC_FLOW_DMA_WR || tl->table == XSC_FLOW_DMA_RD)) + return true; + } + } + return false; +} + +#else +static u8 phyport_num; + +static u32 xsc_get_glb_func_id(struct xsc_core_device *dev) +{ + u16 vf_id, vf_bdf; + struct pci_dev *pdev = dev->pdev; + struct xsc_core_device *pf_xdev; + + if (xsc_core_is_pf(dev)) { + if (xsc_get_pcie_no() == 0) + return XSC_PCIE0_PF_N_FUNC_ID(phyport_num++); + else + return XSC_PCIE1_PF_N_FUNC_ID(phyport_num++); + } else { + pf_xdev = pci_get_drvdata(pdev->physfn); + vf_bdf = (pdev->bus->number << 8) | pdev->devfn; + if (xsc_get_pcie_no() == 0 && vf_bdf >= pf_xdev->priv.sriov.vf_bdf_base) { + vf_id = vf_bdf - pf_xdev->priv.sriov.vf_bdf_base; + if (pf_xdev->pf_id == 0) + return XSC_PCIE0_PF0_VF_N_FUNC_ID(vf_id); + else + return XSC_PCIE0_PF1_VF_N_FUNC_ID(vf_id); + } + } + + return 0; +} +#endif + +int xsc_cmd_exec(struct xsc_core_device *dev, void *in, int in_size, void *out, + int out_size) +{ +#ifndef RUN_WITH_PSV + int ret; + + if (need_write_reg_directly(in)) + ret = xsc_cmd_write_reg_directly(dev, in, in_size, out, out_size, dev->glb_func_id); + else + ret = _xsc_cmd_exec(dev, in, in_size, out, out_size); + + return ret; +#else + return xsc_cmd_exec_psv(dev, in, in_size, out, out_size, dev->glb_func_id); +#endif +} +EXPORT_SYMBOL(xsc_cmd_exec); + +int xsc_get_pcie_no(void) +{ + return pcie_no; +} +EXPORT_SYMBOL(xsc_get_pcie_no); + +static int set_dma_caps(struct pci_dev *pdev) +{ + int err = 0; + + err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + } else { + err = dma_set_coherent_mask(&pdev->dev, +#ifdef USE_VIRTIO + DMA_BIT_MASK(32 + VIRTIO_PCI_QUEUE_ADDR_SHIFT)); +#else + DMA_BIT_MASK(64)); +#endif /* !USE_VIRTIO */ + } + + if (!err) + dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); + + return err; +} + +static void xsc_pci_get_bdf(struct xsc_core_device *dev) +{ + struct pci_dev *pci_dev = dev->pdev; + + dev->bus_id = pci_dev->bus->number; + dev->dev_id = PCI_SLOT(pci_dev->devfn); + dev->func_id = PCI_FUNC(pci_dev->devfn); + + xsc_core_info(dev, "%s: bdf=%04x.%04x.%04x\n", + __func__, dev->bus_id, dev->dev_id, dev->func_id); +} + +static void xsc_pci_calc_pf_port(struct xsc_core_device *dev) +{ + dev->pf = xsc_cal_pf_vf_id(dev->glb_func_id, &dev->pf_id, + &dev->pcie, &dev->vf_id) ? 1 : 0; + if (dev->pcie == 0) { + dev->pcie_port = XSC_PHY_PORT_PCIE_N(0); + dev->logic_port = XSC_PCIE0_PF_N_LOGIC_PORT(dev->pf_id); + + } else { + dev->pcie_port = XSC_PHY_PORT_PCIE_N(1); + dev->logic_port = XSC_PCIE1_PF_N_LOGIC_PORT(dev->pf_id); + } + dev->pf_logic_port = dev->logic_port; + dev->mac_logic_port = dev->mac_port = dev->caps.mac_port; + + xsc_core_dbg(dev, + "glb_func=%d, pcie_port=%d, pf_logic_port=%d, mac_port=%d, board_id=%d\n", + dev->glb_func_id, dev->pcie_port, dev->logic_port, + dev->mac_port, dev->board_id); +} + +void xsc_pci_get_vf_info(struct xsc_core_device *dev, struct xsc_vf_info *info) +{ + if (info->phy_port == 0) { + if (info->pf_id == 0) { + info->logic_port = XSC_PCIE0_PF0_VF_N_LOGIC_PORT(info->vf_id); + info->func_id = XSC_PCIE0_PF0_VF_N_FUNC_ID(info->vf_id); + } else { + info->logic_port = XSC_PCIE0_PF1_VF_N_LOGIC_PORT(info->vf_id); + info->func_id = XSC_PCIE0_PF1_VF_N_FUNC_ID(info->vf_id); + } + } else { + if (info->pf_id == 0) { + info->logic_port = XSC_PCIE1_PF0_VF_N_LOGIC_PORT(info->vf_id); + info->func_id = XSC_PCIE1_PF0_VF_N_FUNC_ID(info->vf_id); + } else { + info->logic_port = XSC_PCIE1_PF1_VF_N_LOGIC_PORT(info->vf_id); + info->func_id = XSC_PCIE1_PF1_VF_N_FUNC_ID(info->vf_id); + } + } +} +EXPORT_SYMBOL(xsc_pci_get_vf_info); + +static void xsc_pci_calc_vf_port(struct xsc_core_device *dev) +{ + dev->pf = xsc_cal_pf_vf_id(dev->glb_func_id, &dev->pf_id, &dev->pcie, &dev->vf_id); + if (unlikely(dev->pcie == 1)) + return; + + if (dev->pf_id == 0) + dev->logic_port = XSC_PCIE0_PF0_VF_N_LOGIC_PORT(dev->vf_id); + else + dev->logic_port = XSC_PCIE0_PF1_VF_N_LOGIC_PORT(dev->vf_id); + + dev->pcie_port = XSC_PHY_PORT_PCIE_N(0); + dev->pf_logic_port = XSC_PCIE0_PF_N_LOGIC_PORT(dev->pf_id); + dev->mac_logic_port = dev->mac_port = dev->caps.mac_port; + + xsc_core_dbg(dev, + "vf%d_logic_port=%d, glb_func_id=%d, pf%d_logic_port=%d, mac_logic_port=%d, board_id=%d\n", + dev->vf_id, dev->logic_port, dev->glb_func_id, dev->pf_id, + dev->pf_logic_port, dev->mac_logic_port, dev->board_id); +} + +static int xsc_pci_enable_device(struct xsc_core_device *dev) +{ + struct pci_dev *pdev = dev->pdev; + int err = 0; + + mutex_lock(&dev->pci_status_mutex); + if (dev->pci_status == XSC_PCI_STATUS_DISABLED) { + err = pci_enable_device(pdev); + if (!err) + dev->pci_status = XSC_PCI_STATUS_ENABLED; + } + mutex_unlock(&dev->pci_status_mutex); + + return err; +} + +static void xsc_pci_disable_device(struct xsc_core_device *dev) +{ + struct pci_dev *pdev = dev->pdev; + + mutex_lock(&dev->pci_status_mutex); + if (dev->pci_status == XSC_PCI_STATUS_ENABLED) { + pci_disable_device(pdev); + dev->pci_status = XSC_PCI_STATUS_DISABLED; + } + mutex_unlock(&dev->pci_status_mutex); +} + +int xsc_priv_init(struct xsc_core_device *dev) +{ + struct xsc_priv *priv = &dev->priv; + + strncpy(priv->name, dev_name(&dev->pdev->dev), XSC_MAX_NAME_LEN); + priv->name[XSC_MAX_NAME_LEN - 1] = 0; + + INIT_LIST_HEAD(&priv->ctx_list); + spin_lock_init(&priv->ctx_lock); + mutex_init(&dev->intf_state_mutex); + + return 0; +} + +int xsc_dev_res_init(struct xsc_core_device *dev) +{ + struct xsc_dev_resource *dev_res = NULL; + + dev_res = kvzalloc(sizeof(*dev_res), GFP_KERNEL); + if (!dev_res) + return -ENOMEM; + + dev->dev_res = dev_res; + /* init access lock */ + spin_lock_init(&dev->reg_access_lock.lock); + mutex_init(&dev_res->alloc_mutex); + mutex_init(&dev_res->pgdir_mutex); + INIT_LIST_HEAD(&dev_res->pgdir_list); + spin_lock_init(&dev_res->mkey_lock); + + return 0; +} + +void xsc_dev_res_cleanup(struct xsc_core_device *dev) +{ + kfree(dev->dev_res); + dev->dev_res = NULL; +} + +void xsc_init_reg_addr(struct xsc_core_device *dev) +{ + if (xsc_core_is_pf(dev)) { + dev->regs.cpm_get_lock = HIF_CPM_LOCK_GET_REG_ADDR; + dev->regs.cpm_put_lock = HIF_CPM_LOCK_PUT_REG_ADDR; + dev->regs.cpm_lock_avail = HIF_CPM_LOCK_AVAIL_REG_ADDR; + dev->regs.cpm_data_mem = HIF_CPM_IDA_DATA_MEM_ADDR; + dev->regs.cpm_cmd = HIF_CPM_IDA_CMD_REG_ADDR; + dev->regs.cpm_addr = HIF_CPM_IDA_ADDR_REG_ADDR; + dev->regs.cpm_busy = HIF_CPM_IDA_BUSY_REG_ADDR; + } else { + dev->regs.tx_db = TX_DB_FUNC_MEM_ADDR; + dev->regs.rx_db = RX_DB_FUNC_MEM_ADDR; + dev->regs.complete_db = DB_CQ_FUNC_MEM_ADDR; + dev->regs.complete_reg = DB_CQ_CID_DIRECT_MEM_ADDR; + dev->regs.event_db = DB_EQ_FUNC_MEM_ADDR; + dev->regs.cpm_get_lock = CPM_LOCK_GET_REG_ADDR; + dev->regs.cpm_put_lock = CPM_LOCK_PUT_REG_ADDR; + dev->regs.cpm_lock_avail = CPM_LOCK_AVAIL_REG_ADDR; + dev->regs.cpm_data_mem = CPM_IDA_DATA_MEM_ADDR; + dev->regs.cpm_cmd = CPM_IDA_CMD_REG_ADDR; + dev->regs.cpm_addr = CPM_IDA_ADDR_REG_ADDR; + dev->regs.cpm_busy = CPM_IDA_BUSY_REG_ADDR; + } +} + +int xsc_dev_init(struct xsc_core_device *dev) +{ + int err = 0; + + xsc_priv_init(dev); + + err = xsc_dev_res_init(dev); + if (err) { + xsc_core_err(dev, "xsc dev res init failed %d\n", err); + goto err_res_init; + } + + /* create debugfs */ + err = xsc_debugfs_init(dev); + if (err) { + xsc_core_err(dev, "xsc_debugfs_init failed %d\n", err); + goto err_debugfs_init; + } + + err = xsc_pagealloc_init(dev); + if (err) { + xsc_core_err(dev, "xsc_pagealloc_init failed %d\n", err); + goto err_pagealloc_init; + } + + xsc_init_reg_addr(dev); + + return 0; + +err_pagealloc_init: + xsc_debugfs_fini(dev); +err_debugfs_init: + xsc_dev_res_cleanup(dev); +err_res_init: + return err; +} + +void xsc_dev_cleanup(struct xsc_core_device *dev) +{ +// xsc_pagealloc_stop(dev); +// xsc_reclaim_startup_pages(dev); + xsc_pagealloc_cleanup(dev); +// iounmap(dev->iseg); + xsc_debugfs_fini(dev); + xsc_dev_res_cleanup(dev); +} + +static int xsc_pci_init(struct xsc_core_device *dev, const struct pci_device_id *id) +{ + struct pci_dev *pdev = dev->pdev; + int err = 0; + int bar_num = 0; + void __iomem *bar_base = NULL; + char name[16]; + +#ifdef USE_VIRTIO + snprintf(name, sizeof(name), "%s", "xsc-virtio"); +#else + snprintf(name, sizeof(name), "%s", "xsc-pci"); + if (id->vendor == XSC_PCI_VENDOR_ID_OBSOLETE) + bar_num = 2; +#endif + + mutex_init(&dev->pci_status_mutex); + dev->priv.numa_node = dev_to_node(&pdev->dev); + if (dev->priv.numa_node == -1) + dev->priv.numa_node = 0; + + /* enable the device */ + err = xsc_pci_enable_device(dev); + if (err) { + xsc_core_err(dev, "failed to enable PCI device: err=%d\n", err); + goto err_ret; + } + + err = pci_request_region(pdev, bar_num, name); + if (err) { + xsc_core_err(dev, "failed to request %s pci_region=%d: err=%d\n", + name, bar_num, err); + goto err_disable; + } + + pci_set_master(pdev); + + err = set_dma_caps(pdev); + if (err) { + xsc_core_err(dev, "failed to set DMA capabilities mask: err=%d\n", err); + goto err_clr_master; + } + + bar_base = pci_ioremap_bar(pdev, bar_num); + if (!bar_base) { + xsc_core_err(dev, "failed to ioremap %s bar%d\n", name, bar_num); + goto err_clr_master; + } else { + xsc_core_info(dev, "ioremap bar%d base address=0x%llx\n", bar_num, + (unsigned long long)bar_base); + } + + err = pci_save_state(pdev); + if (err) { + xsc_core_err(dev, "pci_save_state failed: err=%d\n", err); + goto err_io_unmap; + } + + dev->bar_num = bar_num; +#ifdef USE_VIRTIO + dev->bar0 = bar_base; +#else + dev->bar2 = bar_base; +#endif /* !USE_VIRTIO */ + xsc_pci_get_bdf(dev); + + xsc_init_reg_addr(dev); + + return 0; + +err_io_unmap: + pci_iounmap(pdev, bar_base); +err_clr_master: + pci_clear_master(pdev); + pci_release_region(pdev, bar_num); +err_disable: + xsc_pci_disable_device(dev); +err_ret: + return err; +} + +static void xsc_pci_fini(struct xsc_core_device *dev) +{ + struct pci_dev *pdev = dev->pdev; + void __iomem *bar_base = NULL; + +#ifdef USE_VIRTIO + bar_base = dev->bar0; +#else + bar_base = dev->bar2; +#endif + + if (bar_base) + pci_iounmap(pdev, bar_base); + pci_clear_master(pdev); + pci_release_region(pdev, dev->bar_num); + xsc_pci_disable_device(dev); +} + +static int xsc_check_cmdq_version(struct xsc_core_device *dev) +{ + struct xsc_cmd_query_cmdq_ver_mbox_out *out; + struct xsc_cmd_query_cmdq_ver_mbox_in in; + + int err; + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) { + err = -ENOMEM; + goto no_mem_out; + } + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_CMDQ_VERSION); + in.hdr.opmod = cpu_to_be16(0x1); + err = xsc_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); + if (err) + goto out_out; + + if (out->hdr.status) { + err = xsc_cmd_status_to_err(&out->hdr); + goto out_out; + } + + if (be16_to_cpu(out->cmdq_ver) != CMDQ_VERSION) { + xsc_core_err(dev, "cmdq version check failed, expecting version %d, actual version %d\n", + CMDQ_VERSION, be16_to_cpu(out->cmdq_ver)); + err = -EINVAL; + goto out_out; + } + dev->cmdq_ver = CMDQ_VERSION; + +out_out: + kfree(out); +no_mem_out: + return err; +} + +int xsc_reset_function_resource(struct xsc_core_device *dev) +{ + struct xsc_function_reset_mbox_in in; + struct xsc_function_reset_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_FUNCTION_RESET); + in.glb_func_id = cpu_to_be16(dev->glb_func_id); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); + if (err || out.hdr.status) + return -EINVAL; + + return 0; +} + +static int xsc_fpga_not_supported(struct xsc_core_device *dev) +{ +#define FPGA_VERSION_H 0x100 + u32 ver_h; + + if (!xsc_core_is_pf(dev)) + return 0; + + ver_h = REG_RD32(dev, HIF_CPM_CHIP_VERSION_H_REG_ADDR); + if (ver_h != FPGA_VERSION_H) { + xsc_core_err(dev, "fpga version 0x%x not supported\n", ver_h); + return 1; + } + + return 0; +} + +static int xsc_init_once(struct xsc_core_device *dev) +{ + int err; + + err = xsc_cmd_init(dev); + if (err) { + xsc_core_err(dev, "Failed initializing command interface, aborting\n"); + goto err_cmd_init; + } + +#ifdef RUN_WITH_PSV + dev->glb_func_id = xsc_get_glb_func_id(dev); +#endif + + err = xsc_check_cmdq_version(dev); + if (err) { + xsc_core_err(dev, "Failed to check cmdq version\n"); + goto err_cmdq_ver_chk; + } + + err = xsc_cmd_query_hca_cap(dev, &dev->caps); + if (err) { + xsc_core_err(dev, "Failed to query hca, err=%d\n", err); + goto err_cmdq_ver_chk; + } + + err = xsc_reset_function_resource(dev); + if (err) { + xsc_core_err(dev, "Failed to reset function resource\n"); + goto err_cmdq_ver_chk; + } + + err = xsc_get_board_id(dev); + if (err) { + xsc_core_err(dev, "Failed to get board id, err=%d\n", err); + goto err_cmdq_ver_chk; + } + if (xsc_core_is_pf(dev)) { + xsc_pci_calc_pf_port(dev); + err = xsc_create_res(dev); + if (err) { + xsc_core_err(dev, "Failed to create resource, err=%d\n", err); + goto err_cmdq_ver_chk; + } + } else { + xsc_pci_calc_vf_port(dev); + } + + xsc_init_cq_table(dev); + xsc_init_qp_table(dev); + xsc_eq_init(dev); + +#ifdef CONFIG_XSC_SRIOV + err = xsc_sriov_init(dev); + if (err) { + xsc_core_err(dev, "Failed to init sriov %d\n", err); + goto err_sriov_init; + } +#ifdef CONFIG_XSC_ESWITCH + err = xsc_eswitch_init(dev); + if (err) { + xsc_core_err(dev, "Failed to init eswitch %d\n", err); + goto err_eswitch_init; + } +#endif +#endif + return 0; + +#ifdef CONFIG_XSC_ESWITCH +err_eswitch_init: + xsc_sriov_cleanup(dev); +#endif +#ifdef CONFIG_XSC_SRIOV +err_sriov_init: +#endif + xsc_eq_cleanup(dev); + xsc_cleanup_qp_table(dev); + xsc_cleanup_cq_table(dev); + if (xsc_core_is_pf(dev)) + xsc_destroy_res(dev); +err_cmdq_ver_chk: + xsc_cmd_cleanup(dev); +err_cmd_init: + return err; +} + +static int xsc_cleanup_once(struct xsc_core_device *dev) +{ +#ifdef CONFIG_XSC_SRIOV +#ifdef CONFIG_XSC_ESWITCH + xsc_eswitch_cleanup(dev); +#endif + xsc_sriov_cleanup(dev); +#endif + xsc_eq_cleanup(dev); + xsc_cleanup_qp_table(dev); + xsc_cleanup_cq_table(dev); + if (xsc_core_is_pf(dev)) + xsc_destroy_res(dev); + xsc_cmd_cleanup(dev); + return 0; +} + +static int xsc_load(struct xsc_core_device *dev) +{ + int err; + + xsc_pagealloc_start(dev); + + err = xsc_irq_eq_create(dev); + if (err) { + xsc_core_err(dev, "xsc_irq_eq_create failed %d\n", err); + goto err_irq_eq_create; + } + +#ifdef CONFIG_XSC_SRIOV + err = xsc_sriov_attach(dev); + if (err) { + xsc_core_err(dev, "sriov init failed %d\n", err); + goto err_sriov; + } +#endif + + return 0; + +#ifdef CONFIG_XSC_SRIOV +err_sriov: + xsc_irq_eq_destroy(dev); +#endif +err_irq_eq_create: + xsc_pagealloc_stop(dev); + return err; +} + +static int xsc_unload(struct xsc_core_device *dev) +{ + xsc_lag_remove_xdev(dev); +#ifdef CONFIG_XSC_SRIOV + xsc_sriov_detach(dev); +#endif + xsc_irq_eq_destroy(dev); + xsc_pagealloc_stop(dev); + + return 0; +} + +int xsc_load_one(struct xsc_core_device *dev, bool boot) +{ + int err = 0; + + mutex_lock(&dev->intf_state_mutex); + if (test_bit(XSC_INTERFACE_STATE_UP, &dev->intf_state)) { + xsc_core_warn(dev, "interface is up, NOP\n"); + goto out; + } + + if (test_bit(XSC_INTERFACE_STATE_TEARDOWN, &dev->intf_state)) { + xsc_core_warn(dev, "device is being removed, stop load\n"); + err = -ENODEV; + goto out; + } + + if (boot) { + err = xsc_init_once(dev); + if (err) { + xsc_core_err(dev, "xsc_init_once failed %d\n", err); + goto err_dev_init; + } + } + + err = xsc_load(dev); + if (err) { + xsc_core_err(dev, "xsc_load failed %d\n", err); + goto err_load; + } + + if (boot) { +#ifdef CONFIG_XSC_ESWITCH + err = xsc_devlink_register(priv_to_devlink(dev), dev->device); + if (err) + goto err_devlink_reg; +#endif + } + + if (dev->pf && dev->pf_id < 2) + xsc_lag_add_xdev(dev); + + if (xsc_device_registered(dev)) { + xsc_attach_device(dev); + } else { + err = xsc_register_device(dev); + if (err) { + xsc_core_err(dev, "register device failed %d\n", err); + goto err_reg_dev; + } + } + + err = xsc_port_ctrl_probe(dev); + if (err) { + xsc_core_err(dev, "failed to probe port control node\n"); + goto err_port_ctrl; + } + + set_bit(XSC_INTERFACE_STATE_UP, &dev->intf_state); + mutex_unlock(&dev->intf_state_mutex); + + return err; + +err_port_ctrl: + xsc_unregister_device(dev); +err_reg_dev: + xsc_lag_remove_xdev(dev); +#ifdef CONFIG_XSC_ESWITCH + if (boot) + xsc_devlink_unregister(priv_to_devlink(dev)); +err_devlink_reg: +#endif + xsc_unload(dev); + +err_load: + if (boot) + xsc_cleanup_once(dev); +err_dev_init: +out: + mutex_unlock(&dev->intf_state_mutex); + return err; +} + +int xsc_unload_one(struct xsc_core_device *dev, bool cleanup) +{ + xsc_port_ctrl_remove(dev); +#ifdef CONFIG_XSC_ESWITCH + xsc_devlink_unregister(priv_to_devlink(dev)); +#endif + if (cleanup) + xsc_unregister_device(dev); + mutex_lock(&dev->intf_state_mutex); + if (!test_bit(XSC_INTERFACE_STATE_UP, &dev->intf_state)) { + xsc_core_warn(dev, "%s: interface is down, NOP\n", + __func__); + if (cleanup) + xsc_cleanup_once(dev); + goto out; + } + + clear_bit(XSC_INTERFACE_STATE_UP, &dev->intf_state); + if (xsc_device_registered(dev)) + xsc_detach_device(dev); + + xsc_unload(dev); + + if (cleanup) + xsc_cleanup_once(dev); + +out: + mutex_unlock(&dev->intf_state_mutex); + + return 0; +} + +static int xsc_pci_probe(struct pci_dev *pci_dev, + const struct pci_device_id *id) +{ + struct xsc_core_device *xdev; + struct xsc_priv *priv; + int err; +#ifdef CONFIG_XSC_ESWITCH + struct devlink *devlink; + + devlink = xsc_devlink_alloc(); + if (!devlink) { + dev_err(&pci_dev->dev, "devlink alloc failed\n"); + return -ENOMEM; + } + xdev = devlink_priv(devlink); +#else + /* allocate core structure and fill it out */ + xdev = kzalloc(sizeof(struct xsc_core_device), GFP_KERNEL); + if (!xdev) + return -ENOMEM; +#endif + + xdev->pdev = pci_dev; + xdev->device = &pci_dev->dev; + priv = &xdev->priv; + xdev->coredev_type = pci_dev->is_virtfn ? + XSC_COREDEV_VF : XSC_COREDEV_PF; + xsc_core_info(xdev, "%s: dev_type=%d is_vf=%d\n", + __func__, xdev->coredev_type, pci_dev->is_virtfn); + +#ifdef CONFIG_XSC_SRIOV + priv->sriov.probe_vf = probe_vf; + if (pci_dev->is_virtfn && !probe_vf) { + xsc_core_err(xdev, "VFs are not binded to xsc driver\n"); + return 0; + } +#endif + + /* init pcie device */ + pci_set_drvdata(pci_dev, xdev); + err = xsc_pci_init(xdev, id); + if (err) { + xsc_core_err(xdev, "xsc_pci_init failed %d\n", err); + goto err_pci_init; + } + +#ifdef RUN_WITH_PSV + if (!xsc_hw_reset) { + xsc_hw_reset = true; + err = xsc_start_fw(xdev); + if (err) { + xsc_core_err(xdev, "PSV: failed to start fw.\n"); + goto err_start_fw; + } + } +#endif + + err = xsc_dev_init(xdev); + if (err) { + xsc_core_err(xdev, "xsc_dev_init failed %d\n", err); + goto err_dev_init; + } + + if (xsc_fpga_not_supported(xdev)) { + err = -EOPNOTSUPP; + goto err_dev_init; + } + + err = xsc_load_one(xdev, true); + if (err) { + xsc_core_err(xdev, "xsc_load_one failed %d\n", err); + goto err_load; + } + + return 0; + +err_load: + xsc_dev_cleanup(xdev); +err_dev_init: +#ifdef RUN_WITH_PSV +err_start_fw: +#endif + xsc_pci_fini(xdev); +err_pci_init: + pci_set_drvdata(pci_dev, NULL); +#ifdef CONFIG_XSC_ESWITCH + xsc_devlink_free(devlink); +#else + kfree(xdev); +#endif + return err; +} + +static void xsc_pci_remove(struct pci_dev *pci_dev) +{ + struct xsc_core_device *xdev = pci_get_drvdata(pci_dev); + + xsc_core_info(xdev, "%s: enter\n", __func__); + + set_bit(XSC_INTERFACE_STATE_TEARDOWN, &xdev->intf_state); + xsc_unload_one(xdev, true); + xsc_dev_cleanup(xdev); + + xsc_pci_fini(xdev); + pci_set_drvdata(pci_dev, NULL); +#ifdef CONFIG_XSC_ESWITCH + xsc_devlink_free(priv_to_devlink(xdev)); +#else + kfree(xdev); +#endif +} + +static struct pci_driver xsc_pci_driver = { + .name = "xsc-pci", + .id_table = xsc_pci_id_table, + .probe = xsc_pci_probe, + .remove = xsc_pci_remove, + +#ifdef CONFIG_XSC_SRIOV + .sriov_configure = xsc_core_sriov_configure, +#endif +}; + +static int __init xsc_init(void) +{ + int err; + + xsc_register_debugfs(); + + err = xsc_port_ctrl_init(); + if (err) { + pr_err("failed to initialize port control\n"); + goto err_port_ctrl; + } + + err = xsc_pci_ctrl_init(); + if (err) { + pr_err("failed to initialize dpdk ctrl\n"); + goto err_pci_ctrl; + } + + xsc_hw_reset = false; + err = pci_register_driver(&xsc_pci_driver); + if (err) { + pr_err("failed to register pci driver\n"); + goto err_register; + } + + return 0; + +err_register: + xsc_pci_ctrl_fini(); +err_pci_ctrl: + xsc_port_ctrl_fini(); +err_port_ctrl: + xsc_unregister_debugfs(); + return err; +} + +static void __exit xsc_fini(void) +{ + pci_unregister_driver(&xsc_pci_driver); + xsc_pci_ctrl_fini(); + xsc_port_ctrl_fini(); + xsc_unregister_debugfs(); +} + +module_init(xsc_init); +module_exit(xsc_fini); + +MODULE_DESCRIPTION("Yunsilicon XSC PCI driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0.0"); diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/mr.c b/drivers/net/ethernet/yunsilicon/xsc/pci/mr.c new file mode 100644 index 0000000000000000000000000000000000000000..64317142fa9a3e0317f3b23ddd4070284bafee82 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/mr.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include + +int xsc_core_create_mkey(struct xsc_core_device *dev, struct xsc_core_mr *mr) +{ + struct xsc_create_mkey_mbox_in in; + struct xsc_create_mkey_mbox_out out; + int err; + u8 key; + + memset(&out, 0, sizeof(out)); + spin_lock(&dev->dev_res->mkey_lock); + key = 0x80 + dev->dev_res->mkey_key++; + spin_unlock(&dev->dev_res->mkey_lock); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_CREATE_MKEY); +#ifdef REG_MR_VIA_CMDQ + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); +#else + err = xsc_create_mkey(dev, &in, &out); +#endif + if (err) { + xsc_core_dbg(dev, "cmd exec faile %d\n", err); + return err; + } + + if (out.hdr.status) { + xsc_core_dbg(dev, "status %d\n", out.hdr.status); + return xsc_cmd_status_to_err(&out.hdr); + } + + mr->key = xsc_idx_to_mkey(be32_to_cpu(out.mkey) & 0xffffff) | key; + xsc_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n", be32_to_cpu(out.mkey), key, mr->key); + + return err; +} +EXPORT_SYMBOL(xsc_core_create_mkey); + +int xsc_core_destroy_mkey(struct xsc_core_device *dev, struct xsc_core_mr *mr) +{ + struct xsc_destroy_mkey_mbox_in in; + struct xsc_destroy_mkey_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_MKEY); + in.mkey = cpu_to_be32(mr->key); +#ifdef REG_MR_VIA_CMDQ + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); +#else + err = xsc_destroy_mkey(dev, &in, &out); +#endif + if (err) + return err; + + if (out.hdr.status) + return xsc_cmd_status_to_err(&out.hdr); + + return err; +} +EXPORT_SYMBOL(xsc_core_destroy_mkey); + +int xsc_core_register_mr(struct xsc_core_device *dev, struct xsc_core_mr *mr, + struct xsc_register_mr_mbox_in *in, int inlen) +{ + struct xsc_register_mr_mbox_out out; + int err; + + memset(&out, 0, sizeof(out)); + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_REG_MR); +#ifdef REG_MR_VIA_CMDQ + err = xsc_cmd_exec(dev, in, inlen, &out, sizeof(out)); +#else + err = xsc_reg_mr(dev, in, &out); +#endif + if (err) { + xsc_core_dbg(dev, "cmd exec failed %d\n", err); + return err; + } + if (out.hdr.status) { + xsc_core_dbg(dev, "status %d\n", out.hdr.status); + return xsc_cmd_status_to_err(&out.hdr); + } + + return 0; +} +EXPORT_SYMBOL(xsc_core_register_mr); + +int xsc_core_dereg_mr(struct xsc_core_device *dev, struct xsc_core_mr *mr) +{ + struct xsc_unregister_mr_mbox_in in; + struct xsc_unregister_mr_mbox_out out; + int err; + + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DEREG_MR); + in.mkey = cpu_to_be32(xsc_mkey_to_idx(mr->key)); +#ifdef REG_MR_VIA_CMDQ + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); +#else + err = xsc_dereg_mr(dev, &in, &out); +#endif + if (err) { + xsc_core_dbg(dev, "cmd exec failed %d\n", err); + return err; + } + if (out.hdr.status) { + xsc_core_dbg(dev, "status %d\n", out.hdr.status); + return xsc_cmd_status_to_err(&out.hdr); + } + + return 0; +} +EXPORT_SYMBOL(xsc_core_dereg_mr); + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/pagealloc.c b/drivers/net/ethernet/yunsilicon/xsc/pci/pagealloc.c new file mode 100644 index 0000000000000000000000000000000000000000..e393670cffe4c7dc2f7e4b704ba000c925bb5186 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/pagealloc.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include + +enum { + XSC_PAGES_CANT_GIVE = 0, + XSC_PAGES_GIVE = 1, + XSC_PAGES_TAKE = 2 +}; + +struct xsc_pages_req { + struct xsc_core_device *xdev; + u32 func_id; + s16 npages; + struct work_struct work; +}; + +struct fw_page { + struct rb_node rb_node; + u64 addr; + struct page *page; + u16 func_id; +}; + +struct xsc_query_pages_inbox { + struct xsc_inbox_hdr hdr; + u8 rsvd[8]; +}; + +struct xsc_query_pages_outbox { + struct xsc_outbox_hdr hdr; + u8 reserved[2]; + __be16 func_id; + __be16 init_pages; + __be16 num_pages; +}; + +struct xsc_manage_pages_inbox { + struct xsc_inbox_hdr hdr; + __be16 rsvd0; + __be16 func_id; + __be16 rsvd1; + __be16 num_entries; + u8 rsvd2[16]; + __be64 pas[0]; +}; + +struct xsc_manage_pages_outbox { + struct xsc_outbox_hdr hdr; + u8 rsvd0[2]; + __be16 num_entries; + u8 rsvd1[20]; + __be64 pas[0]; +}; + +static int insert_page(struct xsc_core_device *xdev, u64 addr, struct page *page, u16 func_id) +{ + struct rb_root *root = &xdev->dev_res->page_root; + struct rb_node **new = &root->rb_node; + struct rb_node *parent = NULL; + struct fw_page *nfp; + struct fw_page *tfp; + + while (*new) { + parent = *new; + tfp = rb_entry(parent, struct fw_page, rb_node); + if (tfp->addr < addr) + new = &parent->rb_left; + else if (tfp->addr > addr) + new = &parent->rb_right; + else + return -EEXIST; + } + + nfp = kmalloc(sizeof(*nfp), GFP_KERNEL); + if (!nfp) + return -ENOMEM; + + nfp->addr = addr; + nfp->page = page; + nfp->func_id = func_id; + + rb_link_node(&nfp->rb_node, parent, new); + rb_insert_color(&nfp->rb_node, root); + + return 0; +} + +static struct page *remove_page(struct xsc_core_device *xdev, u64 addr) +{ + struct rb_root *root = &xdev->dev_res->page_root; + struct rb_node *tmp = root->rb_node; + struct page *result = NULL; + struct fw_page *tfp; + + while (tmp) { + tfp = rb_entry(tmp, struct fw_page, rb_node); + if (tfp->addr < addr) { + tmp = tmp->rb_left; + } else if (tfp->addr > addr) { + tmp = tmp->rb_right; + } else { + rb_erase(&tfp->rb_node, root); + result = tfp->page; + kfree(tfp); + break; + } + } + + return result; +} + +static int xsc_cmd_query_pages(struct xsc_core_device *xdev, u16 *func_id, + s16 *pages, s16 *init_pages) +{ + struct xsc_query_pages_inbox in; + struct xsc_query_pages_outbox out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_PAGES); + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + //return xsc_cmd_status_to_err(&out.hdr); + return -EIO; + + if (pages) + *pages = be16_to_cpu(out.num_pages); + if (init_pages) + *init_pages = be16_to_cpu(out.init_pages); + *func_id = be16_to_cpu(out.func_id); + + return err; +} + +static int give_pages(struct xsc_core_device *xdev, u16 func_id, int npages, + int notify_fail) +{ + struct xsc_manage_pages_inbox *in; + struct xsc_manage_pages_outbox out; + struct page *page; + int inlen; + u64 addr; + int err; + int i; + + inlen = sizeof(*in) + npages * sizeof(in->pas[0]); + in = xsc_vzalloc(inlen); + if (!in) { + xsc_core_warn(xdev, "vzalloc failed %d\n", inlen); + return -ENOMEM; + } + memset(&out, 0, sizeof(out)); + + for (i = 0; i < npages; i++) { + page = alloc_page(GFP_HIGHUSER); + if (!page) { + err = -ENOMEM; + xsc_core_warn(xdev, "failed to allocate page\n"); + goto out_alloc; + } + addr = dma_map_page(&xdev->pdev->dev, page, 0, + PAGE_SIZE, DMA_BIDIRECTIONAL); + if (dma_mapping_error(&xdev->pdev->dev, addr)) { + xsc_core_warn(xdev, "failed dma mapping page\n"); + __free_page(page); + err = -ENOMEM; + goto out_alloc; + } + err = insert_page(xdev, addr, page, func_id); + if (err) { + xsc_core_err(xdev, "failed to track allocated page\n"); + dma_unmap_page(&xdev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(page); + err = -ENOMEM; + goto out_alloc; + } + in->pas[i] = cpu_to_be64(addr); + } + + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_MANAGE_PAGES); + in->hdr.opmod = cpu_to_be16(XSC_PAGES_GIVE); + in->func_id = cpu_to_be16(func_id); + in->num_entries = cpu_to_be16(npages); + err = xsc_cmd_exec(xdev, in, inlen, &out, sizeof(out)); + xsc_core_dbg(xdev, "err %d\n", err); + if (err) { + xsc_core_warn(xdev, "func_id 0x%x, npages %d, err %d\n", func_id, npages, err); + goto out_alloc; + } + xdev->dev_res->fw_pages += npages; + + if (out.hdr.status) { + //err = xsc_cmd_status_to_err(&out.hdr); + err = -EIO; + if (err) { + xsc_core_warn(xdev, + "func_id 0x%x, npages %d, status %d\n", + func_id, npages, out.hdr.status); + goto out_alloc; + } + } + + xsc_core_dbg(xdev, "err %d\n", err); + + goto out_free; + +out_alloc: + if (notify_fail) { + memset(in, 0, inlen); + memset(&out, 0, sizeof(out)); + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_MANAGE_PAGES); + in->hdr.opmod = cpu_to_be16(XSC_PAGES_CANT_GIVE); + if (xsc_cmd_exec(xdev, in, sizeof(*in), &out, sizeof(out))) + xsc_core_warn(xdev, "\n"); + } + for (i--; i >= 0; i--) { + addr = be64_to_cpu(in->pas[i]); + page = remove_page(xdev, addr); + if (!page) { + xsc_core_err(xdev, "BUG: can't remove page at addr 0x%llx\n", + addr); + continue; + } + dma_unmap_page(&xdev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(page); + } + +out_free: + xsc_vfree(in); + return err; +} + +static int reclaim_pages(struct xsc_core_device *xdev, u32 func_id, int npages, + int *nclaimed) +{ + struct xsc_manage_pages_inbox in; + struct xsc_manage_pages_outbox *out; + struct page *page; + int num_claimed; + int outlen; + u64 addr; + int err; + int i; + + memset(&in, 0, sizeof(in)); + outlen = sizeof(*out) + npages * sizeof(out->pas[0]); + out = xsc_vzalloc(outlen); + if (!out) + return -ENOMEM; + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_MANAGE_PAGES); + in.hdr.opmod = cpu_to_be16(XSC_PAGES_TAKE); + in.func_id = cpu_to_be16(func_id); + in.num_entries = cpu_to_be16(npages); + xsc_core_dbg(xdev, "npages %d, outlen %d\n", npages, outlen); + err = xsc_cmd_exec(xdev, &in, sizeof(in), out, outlen); + if (err) { + xsc_core_err(xdev, "failed recliaming pages\n"); + goto out_free; + } + xdev->dev_res->fw_pages -= npages; + + if (out->hdr.status) { + //err = xsc_cmd_status_to_err(&out->hdr); + err = -EIO; + goto out_free; + } + + num_claimed = be16_to_cpu(out->num_entries); + if (nclaimed) + *nclaimed = num_claimed; + + for (i = 0; i < num_claimed; i++) { + addr = be64_to_cpu(out->pas[i]); + page = remove_page(xdev, addr); + if (!page) { + xsc_core_warn(xdev, "FW reported unknown DMA address 0x%llx\n", addr); + } else { + dma_unmap_page(&xdev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(page); + } + } + +out_free: + xsc_vfree(out); + return err; +} + +static void pages_work_handler(struct work_struct *work) +{ + struct xsc_pages_req *req = container_of(work, struct xsc_pages_req, work); + struct xsc_core_device *xdev = req->xdev; + int err = 0; + + if (req->npages < 0) + err = reclaim_pages(xdev, req->func_id, -1 * req->npages, NULL); + else if (req->npages > 0) + err = give_pages(xdev, req->func_id, req->npages, 1); + + if (err) + xsc_core_warn(xdev, "%s fail %d\n", req->npages < 0 ? + "reclaim" : "give", err); + + kfree(req); +} + +void xsc_core_req_pages_handler(struct xsc_core_device *xdev, u16 func_id, + s16 npages) +{ + struct xsc_pages_req *req; + + req = kzalloc(sizeof(*req), GFP_ATOMIC); + if (!req) + return; + + req->xdev = xdev; + req->func_id = func_id; + req->npages = npages; + INIT_WORK(&req->work, pages_work_handler); + queue_work(xdev->dev_res->pg_wq, &req->work); +} + +int xsc_satisfy_startup_pages(struct xsc_core_device *xdev) +{ + s16 init_pages; + u16 func_id; + int err; + + err = xsc_cmd_query_pages(xdev, &func_id, NULL, &init_pages); + if (err) + return err; + + xsc_core_dbg(xdev, "requested %d init pages for func_id 0x%x\n", init_pages, func_id); + + return give_pages(xdev, func_id, init_pages, 0); +} + +static int optimal_reclaimed_pages(void) +{ + struct xsc_cmd_prot_block *block; + struct xsc_cmd_layout *lay; + int ret; + + ret = (sizeof(lay->in) + sizeof(block->data) - + sizeof(struct xsc_manage_pages_outbox)) / 8; + + return ret; +} + +int xsc_reclaim_startup_pages(struct xsc_core_device *xdev) +{ + unsigned long end = jiffies + msecs_to_jiffies(5000); + struct fw_page *fwp; + struct rb_node *p; + int err; + + do { + p = rb_first(&xdev->dev_res->page_root); + if (p) { + fwp = rb_entry(p, struct fw_page, rb_node); + err = reclaim_pages(xdev, fwp->func_id, optimal_reclaimed_pages(), NULL); + if (err) { + xsc_core_warn(xdev, "failed reclaiming pages (%d)\n", err); + return err; + } + } + if (time_after(jiffies, end)) { + xsc_core_warn(xdev, "FW did not return all pages. giving up...\n"); + break; + } + } while (p); + + return 0; +} + +int xsc_pagealloc_init(struct xsc_core_device *xdev) +{ + xdev->dev_res->page_root = RB_ROOT; + xdev->dev_res->pg_wq = create_singlethread_workqueue("xsc_page_allocator"); + if (!xdev->dev_res->pg_wq) + return -ENOMEM; + + return 0; +} + +void xsc_pagealloc_cleanup(struct xsc_core_device *xdev) +{ + destroy_workqueue(xdev->dev_res->pg_wq); +} + +int xsc_pagealloc_start(struct xsc_core_device *xdev) +{ + return 0; +} + +void xsc_pagealloc_stop(struct xsc_core_device *xdev) +{ + flush_workqueue(xdev->dev_res->pg_wq); +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/pci_irq.c b/drivers/net/ethernet/yunsilicon/xsc/pci/pci_irq.c new file mode 100644 index 0000000000000000000000000000000000000000..8b667bf13d7184678128a87b4a59bd776266f371 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/pci_irq.c @@ -0,0 +1,904 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_RFS_ACCEL +#include +#endif +#include "fw/xsc_flow.h" +#include "fw/xsc_fw.h" + +enum xsc_eq_type { + XSC_EQ_TYPE_COMP, + XSC_EQ_TYPE_ASYNC, +#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING + XSC_EQ_TYPE_PF, +#endif +}; + +struct xsc_irq { + struct atomic_notifier_head nh; + cpumask_var_t mask; + char name[XSC_MAX_IRQ_NAME]; +}; + +struct xsc_irq_table { + struct xsc_irq *irq; + int nvec; +#ifdef CONFIG_RFS_ACCEL + struct cpu_rmap *rmap; +#endif +}; + +struct xsc_msix_resource { + u16 msix_max_num; + u16 msix_vec_base; + u16 msix_vec_end; + unsigned long *msix_vec_tbl; + atomic_t refcount; +}; + +struct xsc_msix_resource *g_msix_xres; + +static irqreturn_t xsc_dma_read_msix_handler(int irq, void *dev_id) +{ + xsc_dma_read_done_complete(); + return IRQ_HANDLED; +} + +static int xsc_dma_read_msix_init(struct xsc_core_device *xdev) +{ + int err; + char *name = "xsc_dma_read_done"; + struct xsc_dev_resource *dev_res = xdev->dev_res; + int irqn; + u32 value = 0; + int vecid = 0; + + snprintf(dev_res->irq_info[XSC_DMA_READ_DONE_VEC].name, XSC_MAX_IRQ_NAME, "%s@pci:%s", + name, pci_name(xdev->pdev)); + irqn = pci_irq_vector(xdev->pdev, XSC_DMA_READ_DONE_VEC); + err = request_irq(irqn, xsc_dma_read_msix_handler, 0, + dev_res->irq_info[XSC_DMA_READ_DONE_VEC].name, (void *)xdev); + + vecid = (xdev->msix_vec_base + XSC_DMA_READ_DONE_VEC); + value = ((1 << 12) | (vecid & 0xfff)); + REG_WR32(xdev, HIF_IRQ_TBL2IRQ_TBL_RD_DONE_INT_MSIX_REG_ADDR, value); + + return err; +} + +static void xsc_dma_read_msix_fini(struct xsc_core_device *xdev) +{ + if ((xdev->caps.msix_enable) && xsc_core_is_pf(xdev)) + free_irq(pci_irq_vector(xdev->pdev, XSC_DMA_READ_DONE_VEC), xdev); +} + +#if defined(MSIX_SUPPORT) && !defined(XSC_MSIX_BAR_EMUL) +static void xsc_write_msix_ctrl_tbl(struct xsc_core_device *xdev, u32 func_idx, u32 func_mask) +{ + struct xsc_core_device *pf_xdev; + + if (!xsc_core_is_pf(xdev)) + pf_xdev = pci_get_drvdata(xdev->pdev->physfn); + else + pf_xdev = xdev; + + REG_WR32(pf_xdev, HIF_IRQ_CONTROL_TBL_MEM_ADDR + func_idx * 4, func_mask); +} + +static void xsc_msix_ctrl_tbl_func_fini(struct xsc_core_device *xdev, u32 func_id) +{ + return xsc_write_msix_ctrl_tbl(xdev, func_id, 1); +} + +static void xsc_msix_ctrl_tbl_func_init(struct xsc_core_device *xdev, u32 func_id) +{ + xsc_write_msix_ctrl_tbl(xdev, func_id, 0); +} + +static int xsc_msix_mask_tbl_bit_write(struct xsc_core_device *xdev, + u32 vector_id, + u32 mask_or_unmask) +{ + u32 v; + struct xsc_core_device *pf_xdev; + + if (!xsc_core_is_pf(xdev)) + pf_xdev = pci_get_drvdata(xdev->pdev->physfn); + else + pf_xdev = xdev; + + if (vector_id >= BIT(xdev->caps.log_max_msix)) + return -1; + if (!(mask_or_unmask == 0 || mask_or_unmask == 1)) + return -1; + + v = (((mask_or_unmask == 1) ? 0x3 : 0x2) << 12) | vector_id; + + REG_WR32(pf_xdev, HIF_IRQ_INT_DB_REG_ADDR, v); + + return 0; +} + +static void xsc_msix_check_vtr_tbl_idle(struct xsc_core_device *xdev, u32 *idle) +{ + u32 is_busy; + struct xsc_core_device *pf_xdev; + + if (!xsc_core_is_pf(xdev)) + pf_xdev = pci_get_drvdata(xdev->pdev->physfn); + else + pf_xdev = xdev; + + *idle = 0; + do { + is_busy = REG_RD32(pf_xdev, HIF_IRQ_CFG_VECTOR_TABLE_BUSY_REG_ADDR); + if (is_busy == 0) + break; + + usleep_range(2000, 3000); + } while (is_busy); + + *idle = 1; +} + +int xsc_msix_vector_tbl_write(struct xsc_core_device *xdev, u32 vector_id, + u32 laddr, u32 uaddr, u32 data, + u32 func_id, u32 vector_en) +{ + int ret = 0; + u32 idle = 0; + u32 tmp; + struct xsc_core_device *pf_xdev; + + if (!xsc_core_is_pf(xdev)) + pf_xdev = pci_get_drvdata(xdev->pdev->physfn); + else + pf_xdev = xdev; + + if (vector_id >= BIT(xdev->caps.log_max_msix) || + func_id > XSC_FUNC_ID_END || vector_en > 1) { + xsc_core_err(xdev, "%s: invalid input params: func_id=%d\n", + __func__, func_id); + return -EINVAL; + } + + REG_WR32(pf_xdev, HIF_IRQ_CFG_VECTOR_TABLE_ADDR_REG_ADDR, vector_id); + xsc_msix_check_vtr_tbl_idle(pf_xdev, &idle); + if (idle) { + REG_WR32(pf_xdev, HIF_IRQ_CFG_VECTOR_TABLE_CMD_REG_ADDR, 0); + REG_WR32(pf_xdev, HIF_IRQ_CFG_VECTOR_TABLE_MSG_LADDR_REG_ADDR, laddr); + REG_WR32(pf_xdev, HIF_IRQ_CFG_VECTOR_TABLE_MSG_UADDR_REG_ADDR, uaddr); + REG_WR32(pf_xdev, HIF_IRQ_CFG_VECTOR_TABLE_MSG_DATA_REG_ADDR, data); + + tmp = ((vector_en & 0x1) << 11) | (func_id & 0x7FF); + REG_WR32(pf_xdev, HIF_IRQ_CFG_VECTOR_TABLE_CTRL_REG_ADDR, tmp); + REG_WR32(pf_xdev, HIF_IRQ_CFG_VECTOR_TABLE_START_REG_ADDR, 1); + } else { + xsc_core_err(xdev, "VTR tbl is busy.\n"); + ret = -1; + } + + return ret; +} + +int xsc_read_msix_tbl_info(struct xsc_core_device *xdev, u16 index, struct msi_msg *msg) +{ + struct xsc_msix_table_info_mbox_in in; + struct xsc_msix_table_info_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_MSIX_TBL_INFO); + in.index = cpu_to_be16(index); + + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) { + xsc_core_err(xdev, "xsc cmd ops get msix tbl failed %d\n", err); + return err; + } + msg->address_lo = be32_to_cpu(out.addr_lo); + msg->address_hi = be32_to_cpu(out.addr_hi); + msg->data = be32_to_cpu(out.data); + return 0; +} + +static int xsc_msix_ctrl_tbl_init(struct xsc_core_device *xdev) +{ + if (xdev->glb_func_id >= XSC_FUNC_ID_END) + return -1; + + xsc_msix_ctrl_tbl_func_init(xdev, xdev->glb_func_id); + + xsc_core_dbg(xdev, "glb_funcid=%d\n", xdev->glb_func_id); + return 0; +} + +static int xsc_msix_ctrl_tbl_fini(struct xsc_core_device *xdev) +{ + if (xdev->glb_func_id >= XSC_FUNC_ID_END) + return -1; + + xsc_msix_ctrl_tbl_func_fini(xdev, xdev->glb_func_id); + + xsc_core_dbg(xdev, "glb_funcid=%d\n", xdev->glb_func_id); + return 0; +} + +static int xsc_msix_mask_tbl_ops(struct xsc_core_device *xdev, u32 value) +{ + int err; + int i; + int vec_offset; + struct xsc_eq_table *table = &xdev->dev_res->eq_table; + + if (table->num_comp_vectors == 0) + return -1; + + xsc_core_dbg(xdev, "num_comp_vectors=%d\n", table->num_comp_vectors); + + for (i = 0; i < table->num_comp_vectors + table->eq_vec_comp_base; i++) { + vec_offset = xdev->msix_vec_base + i; + err = xsc_msix_mask_tbl_bit_write(xdev, vec_offset, value); + if (err != 0) + return err; + } + + return 0; +} + +static int xsc_msix_mask_tbl_init(struct xsc_core_device *xdev) +{ + return xsc_msix_mask_tbl_ops(xdev, 0); +} + +static int xsc_msix_mask_tbl_fini(struct xsc_core_device *xdev) +{ + return xsc_msix_mask_tbl_ops(xdev, 1); +} + +static int xsc_msix_vector_tbl_ops(struct xsc_core_device *xdev, u32 vector_en) +{ + int err; + int i; + int vec_size; + u16 index; + struct pci_dev *pdev = xdev->pdev; + struct msi_desc *entry; + struct msi_msg *msgs; + + struct xsc_eq_table *table = &xdev->dev_res->eq_table; + + if (table->num_comp_vectors == 0) + return -1; + + xsc_core_dbg(xdev, "num_comp_vectors=%d\n", table->num_comp_vectors); + vec_size = table->num_comp_vectors + table->eq_vec_comp_base; + msgs = kcalloc(vec_size, sizeof(struct msi_msg), GFP_KERNEL); + err = 0; + + i = xdev->msix_vec_base; + index = 0; + for_each_pci_msi_entry(entry, pdev) { + if (vector_en == 1) { + err = xsc_read_msix_tbl_info(xdev, index, &msgs[index]); + if (err) { + xsc_core_err(xdev, "failed to get msix tbl %d\n", err); + goto out; + } + } + i++; + index++; + } + + i = xdev->msix_vec_base; + index = 0; + for_each_pci_msi_entry(entry, pdev) { + err = xsc_msix_vector_tbl_write(xdev, i, + msgs[index].address_lo, msgs[index].address_hi, msgs[index].data, + xdev->glb_func_id, vector_en); + i++; + index++; + } + +out: + kfree(msgs); + return err; +} + +static int xsc_msix_vector_tbl_init(struct xsc_core_device *xdev) +{ + return xsc_msix_vector_tbl_ops(xdev, 1); +} + +static int xsc_msix_vector_tbl_fini(struct xsc_core_device *xdev) +{ + return xsc_msix_vector_tbl_ops(xdev, 0); +} + +int xsc_msix_mask_set_vec(struct xsc_core_device *xdev, u32 vec, u32 value) +{ + int err = 0; + + int max_msix = 1 << xdev->caps.log_max_msix; + + if (vec >= max_msix) { + xsc_core_err(xdev, "failed to set msix mask.vec=%d\n", vec); + return -1; + } + + err = xsc_msix_mask_tbl_bit_write(xdev, vec, value); + + return err; +} +#endif + +static int xsc_msix_tbl_init(struct xsc_core_device *xdev) +{ +#if defined(MSIX_SUPPORT) && !defined(XSC_MSIX_BAR_EMUL) + int err; + + err = xsc_msix_vector_tbl_init(xdev); + if (err) { + xsc_core_err(xdev, "xsc_msix_vector_tbl_init failed: err=%d\n", err); + return -1; + } + + err = xsc_msix_ctrl_tbl_init(xdev); + if (err) { + xsc_core_err(xdev, "xsc_msix_ctrl_tbl_init failed: err=%d\n", err); + return -1; + } + + err = xsc_msix_mask_tbl_init(xdev); + if (err) { + xsc_core_err(xdev, "xsc_msix_mask_tbl_init failed: err=%d\n", err); + return -1; + } +#endif + return 0; +} + +static int xsc_msix_tbl_fini(struct xsc_core_device *xdev) +{ +#if defined(MSIX_SUPPORT) && !defined(XSC_MSIX_BAR_EMUL) + int err; + + err = xsc_msix_vector_tbl_fini(xdev); + if (err) { + xsc_core_err(xdev, "xsc_msix_vector_tbl_fini failed: err=%d\n", err); + return -1; + } + + err = xsc_msix_mask_tbl_fini(xdev); + if (err) { + xsc_core_err(xdev, "xsc_msix_mask_tbl_fini failed: err=%d\n", err); + return -1; + } + + err = xsc_msix_ctrl_tbl_fini(xdev); + if (err) { + xsc_core_err(xdev, "xsc_msix_ctrl_tbl_fini failed: err=%d\n", err); + return -1; + } +#endif + return 0; +} + +static struct xsc_eq *xsc_eq_get(struct xsc_core_device *dev, int i) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + struct xsc_eq *eq, *n; + struct xsc_eq *eq_ret = NULL; + + spin_lock(&table->lock); + list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { + if (eq->index == i) { + eq_ret = eq; + break; + } + } + spin_unlock(&table->lock); + + return eq_ret; +} + +void mask_cpu_by_node(int node, struct cpumask *dstp) +{ + int i; + + for (i = 0; i < nr_cpu_ids; i++) { + if (node == cpu_to_node(i)) + cpumask_set_cpu(i, dstp); + } +} +EXPORT_SYMBOL(mask_cpu_by_node); + +static int set_comp_irq_affinity_hint(struct xsc_core_device *dev, int i) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + int vecidx = table->eq_vec_comp_base + i; + struct xsc_eq *eq = xsc_eq_get(dev, i); + unsigned int irqn; + int ret; + + irqn = pci_irq_vector(dev->pdev, vecidx); + if (!zalloc_cpumask_var(&eq->mask, GFP_KERNEL)) { + xsc_core_err(dev, "zalloc_cpumask_var rx cpumask failed"); + return -ENOMEM; + } + + if (!zalloc_cpumask_var(&dev->xps_cpumask, GFP_KERNEL)) { + xsc_core_err(dev, "zalloc_cpumask_var tx cpumask failed"); + return -ENOMEM; + } + + mask_cpu_by_node(dev->priv.numa_node, eq->mask); + ret = irq_set_affinity_hint(irqn, eq->mask); + + return ret; +} + +static void clear_comp_irq_affinity_hint(struct xsc_core_device *dev, int i) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + int vecidx = table->eq_vec_comp_base + i; + struct xsc_eq *eq = xsc_eq_get(dev, i); + int irqn; + + irqn = pci_irq_vector(dev->pdev, vecidx); + irq_set_affinity_hint(irqn, NULL); + free_cpumask_var(eq->mask); +} + +static int set_comp_irq_affinity_hints(struct xsc_core_device *dev) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + int nvec = table->num_comp_vectors; + int err; + int i; + + for (i = 0; i < nvec; i++) { + err = set_comp_irq_affinity_hint(dev, i); + if (err) + goto err_out; + } + + return 0; + +err_out: + for (i--; i >= 0; i--) + clear_comp_irq_affinity_hint(dev, i); + free_cpumask_var(dev->xps_cpumask); + + return err; +} + +static void clear_comp_irq_affinity_hints(struct xsc_core_device *dev) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + int nvec = table->num_comp_vectors; + int i; + + for (i = 0; i < nvec; i++) + clear_comp_irq_affinity_hint(dev, i); + free_cpumask_var(dev->xps_cpumask); +} + +struct cpumask * +xsc_comp_irq_get_affinity_mask(struct xsc_core_device *dev, int vector) +{ + struct xsc_eq *eq = xsc_eq_get(dev, vector); + + if (!unlikely(eq)) + return NULL; + + return eq->mask; +} +EXPORT_SYMBOL(xsc_comp_irq_get_affinity_mask); + +#ifdef XSC_MSIX_BAR_EMUL +static int xsc_alloc_irq_vectors(struct xsc_core_device *dev) +{ + struct xsc_dev_resource *dev_res = dev->dev_res; + struct xsc_eq_table *table = &dev_res->eq_table; + int nvec = dev->caps.msix_num; + int nvec_base; + int err; + + if (xsc_core_is_pf(dev)) + nvec_base = XSC_EQ_VEC_COMP_BASE; + else + /*VF device not need dma read done vector.*/ + nvec_base = (XSC_EQ_VEC_COMP_BASE - 1); + + if (nvec <= nvec_base) { + xsc_core_warn(dev, "failed to alloc irq vector(%d)\n", nvec); + return -ENOMEM; + } + + dev_res->irq_info = kcalloc(nvec, sizeof(*dev_res->irq_info), GFP_KERNEL); + if (!dev_res->irq_info) + return -ENOMEM; + + nvec = pci_alloc_irq_vectors(dev->pdev, nvec_base + 1, nvec, PCI_IRQ_MSIX); + if (nvec < 0) { + err = nvec; + goto err_free_irq_info; + } + + table->eq_vec_comp_base = nvec_base; + table->num_comp_vectors = nvec - nvec_base; + dev->msix_vec_base = dev->caps.msix_base; + xsc_core_info(dev, "alloc msix_vec_num=%d, vec_base_num=%d, max_msix_num=%d, msix_vec_base=%d\n", + nvec, nvec_base, dev->caps.msix_num, dev->msix_vec_base); + + return 0; + +err_free_irq_info: + pci_free_irq_vectors(dev->pdev); + kfree(dev_res->irq_info); + return err; +} + +#else + +static int xsc_alloc_irq_vectors(struct xsc_core_device *dev) +{ + struct xsc_dev_resource *dev_res = dev->dev_res; + struct xsc_eq_table *table = &dev_res->eq_table; + int num_eqs = (dev->caps.max_num_eqs ? + dev->caps.max_num_eqs : 1 << dev->caps.log_max_eq); + int nvec, nvec_base; + int err; + + if (xsc_core_is_pf(dev)) + nvec_base = XSC_EQ_VEC_COMP_BASE; + else + /*VF device not need dma read done vector.*/ + nvec_base = (XSC_EQ_VEC_COMP_BASE - 1); + nvec = XSC_MAX_PORTS * num_online_cpus() + nvec_base; + + nvec = min_t(int, nvec, (num_eqs + nvec_base)); + if (nvec <= nvec_base) { + xsc_core_warn(dev, "failed to alloc irq vector(%d)\n", nvec); + return -ENOMEM; + } + + dev_res->irq_info = kcalloc(nvec, sizeof(*dev_res->irq_info), GFP_KERNEL); + if (!dev_res->irq_info) + return -ENOMEM; + + nvec = pci_alloc_irq_vectors(dev->pdev, nvec_base + 1, nvec, PCI_IRQ_MSIX); + if (nvec < 0) { + err = nvec; + goto err_free_irq_info; + } + + table->eq_vec_comp_base = nvec_base; + table->num_comp_vectors = nvec - nvec_base; + xsc_core_info(dev, "alloc irq vector=%d, vec_base=%d, max_eq_nums=%d, log_max_eq=%d\n", + nvec, nvec_base, dev->caps.max_num_eqs, dev->caps.log_max_eq); + + return 0; + +err_free_irq_info: + pci_free_irq_vectors(dev->pdev); + kfree(dev_res->irq_info); + return err; +} + +#endif + +static void xsc_free_irq_vectors(struct xsc_core_device *dev) +{ + struct xsc_dev_resource *dev_res = dev->dev_res; + + pci_free_irq_vectors(dev->pdev); + kfree(dev_res->irq_info); +} + +int xsc_vector2eqn(struct xsc_core_device *dev, int vector, int *eqn, + unsigned int *irqn) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + struct xsc_eq *eq, *n; + int err = -ENOENT; + + if (!dev->caps.msix_enable) + return 0; + + spin_lock(&table->lock); + list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { + if (eq->index == vector) { + *eqn = eq->eqn; + *irqn = eq->irqn; + err = 0; + break; + } + } + spin_unlock(&table->lock); + + return err; +} +EXPORT_SYMBOL(xsc_vector2eqn); + +static void free_comp_eqs(struct xsc_core_device *dev) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + struct xsc_eq *eq, *n; + + spin_lock(&table->lock); + list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { + list_del(&eq->list); + spin_unlock(&table->lock); + if (xsc_destroy_unmap_eq(dev, eq)) + xsc_core_warn(dev, "failed to destroy EQ 0x%x\n", eq->eqn); + kfree(eq); + spin_lock(&table->lock); + } + spin_unlock(&table->lock); +} + +static int alloc_comp_eqs(struct xsc_core_device *dev) +{ + struct xsc_eq_table *table = &dev->dev_res->eq_table; + char name[XSC_MAX_IRQ_NAME]; + struct xsc_eq *eq; + int ncomp_vec; + int nent; + int err; + int i; + + INIT_LIST_HEAD(&table->comp_eqs_list); + ncomp_vec = table->num_comp_vectors; + nent = XSC_COMP_EQ_SIZE; + + for (i = 0; i < ncomp_vec; i++) { + eq = kzalloc(sizeof(*eq), GFP_KERNEL); + if (!eq) { + err = -ENOMEM; + goto clean; + } + + snprintf(name, XSC_MAX_IRQ_NAME, "xsc_comp%d", i); + err = xsc_create_map_eq(dev, eq, + i + table->eq_vec_comp_base, nent, name); + if (err) { + kfree(eq); + goto clean; + } + + eq->index = i; + spin_lock(&table->lock); + list_add_tail(&eq->list, &table->comp_eqs_list); + spin_unlock(&table->lock); + } + + return 0; + +clean: + free_comp_eqs(dev); + return err; +} + +static irqreturn_t xsc_cmd_handler(int irq, void *arg) +{ + struct xsc_core_device *dev = (struct xsc_core_device *)arg; + int err; + +#ifdef XSC_DEBUG + xsc_core_dbg(dev, "cmdq hint irq: %d\n", irq); +#endif + disable_irq_nosync(dev->cmd.irqn); + err = xsc_cmd_err_handler(dev); + if (!err) + xsc_cmd_resp_handler(dev); + enable_irq(dev->cmd.irqn); + + return IRQ_HANDLED; +} + +int xsc_request_irq_for_cmdq(struct xsc_core_device *dev, u8 vecidx) +{ + struct xsc_dev_resource *dev_res = dev->dev_res; + + writel(dev->msix_vec_base + vecidx, REG_ADDR(dev, dev->cmd.reg.msix_vec_addr)); + + snprintf(dev_res->irq_info[vecidx].name, XSC_MAX_IRQ_NAME, "%s@pci:%s", + "xsc_cmd", pci_name(dev->pdev)); + dev->cmd.irqn = pci_irq_vector(dev->pdev, vecidx); + return request_irq(dev->cmd.irqn, xsc_cmd_handler, 0, + dev_res->irq_info[vecidx].name, dev); +} + +void xsc_free_irq_for_cmdq(struct xsc_core_device *dev) +{ + free_irq(dev->cmd.irqn, dev); +} + +static irqreturn_t xsc_event_handler(int irq, void *arg) +{ + struct xsc_core_device *dev = (struct xsc_core_device *)arg; + + xsc_core_dbg(dev, "cmd event hint irq: %d\n", irq); + + if (!dev->eth_priv) + return IRQ_NONE; + + if (!dev->event_handler) + return IRQ_NONE; + + dev->event_handler(dev->eth_priv); + + return IRQ_HANDLED; +} + +int xsc_request_irq_for_event(struct xsc_core_device *dev) +{ + struct xsc_dev_resource *dev_res = dev->dev_res; + + snprintf(dev_res->irq_info[XSC_VEC_CMD_EVENT].name, XSC_MAX_IRQ_NAME, "%s@pci:%s", + "xsc_eth_event", pci_name(dev->pdev)); + + return request_irq(pci_irq_vector(dev->pdev, XSC_VEC_CMD_EVENT), xsc_event_handler, 0, + dev_res->irq_info[XSC_VEC_CMD_EVENT].name, dev); +} + +void xsc_free_irq_for_event(struct xsc_core_device *dev) +{ + free_irq(pci_irq_vector(dev->pdev, XSC_VEC_CMD_EVENT), dev); +} + +#ifdef XSC_MSIX_BAR_EMUL +int xsc_cmd_enable_msix(struct xsc_core_device *xdev) +{ + struct xsc_msix_table_info_mbox_in in; + struct xsc_msix_table_info_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_ENABLE_MSIX); + + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) { + xsc_core_err(xdev, "xsc_cmd_exec enable msix failed %d\n", err); + return err; + } + + return 0; +} +#endif + +int xsc_irq_eq_create(struct xsc_core_device *dev) +{ + int err; +#if !defined XSC_MSIX_BAR_EMUL + struct xsc_eq_table *table = &dev->dev_res->eq_table; +#endif + + if (dev->caps.msix_enable == 0) + return 0; + + err = xsc_alloc_irq_vectors(dev); + if (err) { + xsc_core_err(dev, "enable msix failed, err=%d\n", err); + goto err_alloc_irq; + } + +#if !defined XSC_MSIX_BAR_EMUL + /*MUST place afer xsc_alloc_irq_vectors, and MUST place before xsc_start_eqs*/ + err = xsc_alloc_continuous_msix_vec(dev, table->num_comp_vectors + table->eq_vec_comp_base); + if (err) { + xsc_core_err(dev, "alloc msix vec res failed, err=%d\n", err); + goto err_alloc_msix_vec; + } +#endif + + err = xsc_start_eqs(dev); + if (err) { + xsc_core_err(dev, "failed to start EQs, err=%d\n", err); + goto err_start_eqs; + } + + err = alloc_comp_eqs(dev); + if (err) { + xsc_core_err(dev, "failed to alloc comp EQs, err=%d\n", err); + goto err_alloc_comp_eqs; + } + + err = xsc_request_irq_for_cmdq(dev, XSC_VEC_CMD); + if (err) { + xsc_core_err(dev, "failed to request irq for cmdq, err=%d\n", err); + goto err_request_cmd_irq; + } + + err = xsc_request_irq_for_event(dev); + if (err) { + xsc_core_err(dev, "failed to request irq for event, err=%d\n", err); + goto err_request_event_irq; + } + + if ((dev->caps.msix_enable) && xsc_core_is_pf(dev)) { + err = xsc_dma_read_msix_init(dev); + if (err) { + xsc_core_err(dev, "dma read msix init failed %d.\n", err); + goto err_dma_read_msix; + } + } + + err = xsc_msix_tbl_init(dev); + if (err) { + xsc_core_err(dev, "failed to init msix tbl, err=%d\n", err); + goto err_msix_tbl_init; + } + + err = set_comp_irq_affinity_hints(dev); + if (err) { + xsc_core_err(dev, "failed to alloc affinity hint cpumask, err=%d\n", err); + goto err_set_affinity; + } + + xsc_cmd_use_events(dev); +#ifdef XSC_MSIX_BAR_EMUL + err = xsc_cmd_enable_msix(dev); + if (err) { + xsc_core_err(dev, "xsc_cmd_enable_msix failed %d.\n", err); + xsc_cmd_use_polling(dev); + goto err_set_affinity; + } +#endif + return 0; + +err_set_affinity: + xsc_msix_tbl_fini(dev); +err_msix_tbl_init: + xsc_dma_read_msix_fini(dev); +err_dma_read_msix: + xsc_free_irq_for_event(dev); +err_request_event_irq: + xsc_free_irq_for_cmdq(dev); +err_request_cmd_irq: + free_comp_eqs(dev); +err_alloc_comp_eqs: + xsc_stop_eqs(dev); +err_start_eqs: +#if !defined XSC_MSIX_BAR_EMUL + xsc_free_continuous_msix_vec(dev); +err_alloc_msix_vec: +#endif + xsc_free_irq_vectors(dev); +err_alloc_irq: + return err; +} + +int xsc_irq_eq_destroy(struct xsc_core_device *dev) +{ + if (dev->caps.msix_enable == 0) + return 0; + + xsc_stop_eqs(dev); + clear_comp_irq_affinity_hints(dev); + free_comp_eqs(dev); + + xsc_dma_read_msix_fini(dev); + xsc_free_irq_for_event(dev); + xsc_free_irq_for_cmdq(dev); + xsc_free_continuous_msix_vec(dev); + xsc_free_irq_vectors(dev); + xsc_msix_tbl_fini(dev); + + return 0; +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/pd.c b/drivers/net/ethernet/yunsilicon/xsc/pci/pd.c new file mode 100644 index 0000000000000000000000000000000000000000..a997bcbe785617360ac1b779635b1da90c556ada --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/pd.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include + +int xsc_core_alloc_pd(struct xsc_core_device *xdev, u32 *pdn) +{ + struct xsc_alloc_pd_mbox_in in; + struct xsc_alloc_pd_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_ALLOC_PD); + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + return xsc_cmd_status_to_err(&out.hdr); + + *pdn = be32_to_cpu(out.pdn) & 0xffffff; + return err; +} +EXPORT_SYMBOL(xsc_core_alloc_pd); + +int xsc_core_dealloc_pd(struct xsc_core_device *xdev, u32 pdn) +{ + struct xsc_dealloc_pd_mbox_in in; + struct xsc_dealloc_pd_mbox_out out; + int err; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DEALLOC_PD); + in.pdn = cpu_to_be32(pdn); + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + return xsc_cmd_status_to_err(&out.hdr); + + return err; +} +EXPORT_SYMBOL(xsc_core_dealloc_pd); diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/port.c b/drivers/net/ethernet/yunsilicon/xsc/pci/port.c new file mode 100644 index 0000000000000000000000000000000000000000..2beb815b5f6a24416e440250139176e85b9561c4 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/port.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include + +int xsc_core_access_reg(struct xsc_core_device *xdev, void *data_in, + int size_in, void *data_out, int size_out, + u16 reg_num, int arg, int write) +{ + struct xsc_access_reg_mbox_in *in = NULL; + struct xsc_access_reg_mbox_out *out = NULL; + int err = -ENOMEM; + + in = xsc_vzalloc(sizeof(*in) + size_in); + if (!in) + return -ENOMEM; + + out = xsc_vzalloc(sizeof(*out) + size_out); + if (!out) + goto ex1; + + memcpy(in->data, data_in, size_in); + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_ACCESS_REG); + in->hdr.opmod = cpu_to_be16(!write); + in->arg = cpu_to_be32(arg); + in->register_id = cpu_to_be16(reg_num); + err = xsc_cmd_exec(xdev, in, sizeof(*in) + size_in, out, + sizeof(*out) + size_out); + if (err) + goto ex2; + + if (out->hdr.status) + return xsc_cmd_status_to_err(&out->hdr); + + if (!err) + memcpy(data_out, out->data, size_out); + +ex2: + xsc_vfree(out); +ex1: + xsc_vfree(in); + return err; +} +EXPORT_SYMBOL_GPL(xsc_core_access_reg); + +struct xsc_reg_pcap { + u8 rsvd0; + u8 port_num; + u8 rsvd1[2]; + __be32 caps_127_96; + __be32 caps_95_64; + __be32 caps_63_32; + __be32 caps_31_0; +}; + +int xsc_set_port_caps(struct xsc_core_device *xdev, int port_num, u32 caps) +{ + struct xsc_reg_pcap in; + struct xsc_reg_pcap out; + int err; + + memset(&in, 0, sizeof(in)); + in.caps_127_96 = cpu_to_be32(caps); + in.port_num = port_num; + + err = xsc_core_access_reg(xdev, &in, sizeof(in), &out, + sizeof(out), XSC_REG_PCAP, 0, 1); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_set_port_caps); + +static int xsc_query_module_num(struct xsc_core_device *dev, int *module_num) +{ + *module_num = dev->mac_port; + return 0; +} + +static int xsc_query_module_id(struct xsc_core_device *dev, int module_num, + u8 *module_id) +{ + struct xsc_reg_mcia in; + struct xsc_reg_mcia out; + int err, status; + u8 *ptr; + + in.i2c_device_address = XSC_I2C_ADDR_LOW; + in.module = module_num; + in.device_address = 0; + in.page_number = 0; + in.size = 1; + + err = xsc_core_access_reg(dev, &in, sizeof(in), &out, + sizeof(out), XSC_REG_MCIA, 0, 0); + if (err) + return err; + + status = out.status; + if (status) { + xsc_core_err(dev, "query_mcia_reg failed: status: 0x%x\n", + status); + return -EIO; + } + ptr = out.dword_0; + + *module_id = ptr[0]; + + return 0; +} + +static int xsc_qsfp_eeprom_page(u16 offset) +{ + if (offset < XSC_EEPROM_PAGE_LENGTH) + /* Addresses between 0-255 - page 00 */ + return 0; + + /* Addresses between 256 - 639 belongs to pages 01, 02 and 03 + * For example, offset = 400 belongs to page 02: + * 1 + ((400 - 256)/128) = 2 + */ + return 1 + ((offset - XSC_EEPROM_PAGE_LENGTH) / + XSC_EEPROM_HIGH_PAGE_LENGTH); +} + +static int xsc_qsfp_eeprom_high_page_offset(int page_num) +{ + if (!page_num) /* Page 0 always start from low page */ + return 0; + + /* High page */ + return page_num * XSC_EEPROM_HIGH_PAGE_LENGTH; +} + +static void xsc_qsfp_eeprom_params_set(u16 *i2c_addr, int *page_num, u16 *offset) +{ + *i2c_addr = XSC_I2C_ADDR_LOW; + *page_num = xsc_qsfp_eeprom_page(*offset); + *offset -= xsc_qsfp_eeprom_high_page_offset(*page_num); +} + +static void xsc_sfp_eeprom_params_set(u16 *i2c_addr, int *page_num, u16 *offset) +{ + *i2c_addr = XSC_I2C_ADDR_LOW; + *page_num = 0; + + if (*offset < XSC_EEPROM_PAGE_LENGTH) + return; + + *i2c_addr = XSC_I2C_ADDR_HIGH; + *offset -= XSC_EEPROM_PAGE_LENGTH; +} + +static int xsc_query_mcia(struct xsc_core_device *dev, + struct xsc_module_eeprom_query_params *params, u8 *data) +{ + struct xsc_reg_mcia in; + struct xsc_reg_mcia out; + int status, err; + void *ptr; + u16 size; + + size = min_t(int, params->size, XSC_EEPROM_MAX_BYTES); + + in.i2c_device_address = params->i2c_address; + in.module = params->module_number; + in.device_address = params->offset; + in.page_number = params->page; + in.size = size; + + err = xsc_core_access_reg(dev, &in, sizeof(in), &out, + sizeof(out), XSC_REG_MCIA, 0, 0); + if (err) + return err; + + status = out.status; + if (status) { + xsc_core_err(dev, "query_mcia_reg failed: status: 0x%x\n", + status); + return -EIO; + } + + ptr = out.dword_0; + memcpy(data, ptr, size); + + return size; +} + +int xsc_query_module_eeprom(struct xsc_core_device *dev, + u16 offset, u16 size, u8 *data) +{ + struct xsc_module_eeprom_query_params query = {0}; + u8 module_id; + int err; + + err = xsc_query_module_num(dev, &query.module_number); + if (err) + return err; + + err = xsc_query_module_id(dev, query.module_number, &module_id); + if (err) + return err; + + switch (module_id) { + case XSC_MODULE_ID_SFP: + xsc_sfp_eeprom_params_set(&query.i2c_address, &query.page, &offset); + break; + case XSC_MODULE_ID_QSFP: + case XSC_MODULE_ID_QSFP_PLUS: + case XSC_MODULE_ID_QSFP28: + xsc_qsfp_eeprom_params_set(&query.i2c_address, &query.page, &offset); + break; + default: + xsc_core_err(dev, "Module ID not recognized: 0x%x\n", module_id); + return -EINVAL; + } + + if (offset + size > XSC_EEPROM_PAGE_LENGTH) + /* Cross pages read, read until offset 256 in low page */ + size = XSC_EEPROM_PAGE_LENGTH - offset; + + query.size = size; + query.offset = offset; + + return xsc_query_mcia(dev, &query, data); +} +EXPORT_SYMBOL_GPL(xsc_query_module_eeprom); + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/qp.c b/drivers/net/ethernet/yunsilicon/xsc/pci/qp.c new file mode 100644 index 0000000000000000000000000000000000000000..37d4460adbec641797b3c75b8fa7923d299ae727 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/qp.c @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#define GROUP_DESTROY_FLAG_SHFIT 15 +#define GROUP_DESTROY_FLAG_MASK (1 << (GROUP_DESTROY_FLAG_SHFIT)) + +#define GROUP_OTHER_HASH_SIZE 16 +#define GROUP_CC_HASH_SIZE (1024 - GROUP_OTHER_HASH_SIZE) + +enum { + GROUP_MODE_PER_QP = 0, + GROUP_MODE_PER_DEST_IP, +}; + +int create_resource_common(struct xsc_core_device *xdev, + struct xsc_core_qp *qp) +{ + struct xsc_qp_table *table = &xdev->dev_res->qp_table; + int err; + + spin_lock_irq(&table->lock); + err = radix_tree_insert(&table->tree, qp->qpn, qp); + spin_unlock_irq(&table->lock); + if (err) + return err; + + atomic_set(&qp->refcount, 1); + init_completion(&qp->free); + qp->pid = current->pid; + + return 0; +} +EXPORT_SYMBOL_GPL(create_resource_common); + +void destroy_resource_common(struct xsc_core_device *xdev, + struct xsc_core_qp *qp) +{ + struct xsc_qp_table *table = &xdev->dev_res->qp_table; + unsigned long flags; + + spin_lock_irqsave(&table->lock, flags); + radix_tree_delete(&table->tree, qp->qpn); + spin_unlock_irqrestore(&table->lock, flags); + + if (atomic_dec_and_test(&qp->refcount)) + complete(&qp->free); + wait_for_completion(&qp->free); +} +EXPORT_SYMBOL_GPL(destroy_resource_common); + +void xsc_qp_event(struct xsc_core_device *xdev, u32 qpn, int event_type) +{ + struct xsc_qp_table *table = &xdev->dev_res->qp_table; + struct xsc_core_qp *qp; + + spin_lock(&table->lock); + + qp = radix_tree_lookup(&table->tree, qpn); + if (qp) + atomic_inc(&qp->refcount); + + spin_unlock(&table->lock); + + if (!qp) { + xsc_core_warn(xdev, "Async event for bogus QP 0x%x\n", qpn); + return; + } + + qp->event(qp, event_type); + + if (atomic_dec_and_test(&qp->refcount)) + complete(&qp->free); +} + +int xsc_core_create_qp(struct xsc_core_device *xdev, + struct xsc_core_qp *qp, + struct xsc_create_qp_mbox_in *in, + int inlen) +{ + struct xsc_create_qp_mbox_out out; + struct xsc_destroy_qp_mbox_in din; + struct xsc_destroy_qp_mbox_out dout; + int err; + struct timespec64 ts; + int exec = 1; + + ktime_get_boottime_ts64(&ts); + + memset(&dout, 0, sizeof(dout)); + in->hdr.opcode = cpu_to_be16(XSC_CMD_OP_CREATE_QP); + +#ifdef XSC_CHIP_RDMA_UNSUPPORTED + if ((in->req.qp_type == XSC_QUEUE_TYPE_RDMA_MAD) || + (in->req.qp_type == XSC_QUEUE_TYPE_RDMA_RC)) { + exec = 0; + qp->qpn = 0; + } +#endif + if (exec) { + err = xsc_cmd_exec(xdev, in, inlen, &out, sizeof(out)); + if (err) { + xsc_core_warn(xdev, "ret %d", err); + return err; + } + + if (out.hdr.status) { + pr_warn("current num of QPs 0x%x\n", atomic_read(&xdev->num_qps)); + return xsc_cmd_status_to_err(&out.hdr); + } + qp->qpn = be32_to_cpu(out.qpn) & 0xffffff; + xsc_core_dbg(xdev, "qpn = %x\n", qp->qpn); + } + + qp->trace_info = kzalloc(sizeof(struct xsc_qp_trace), GFP_KERNEL); + if (!qp->trace_info) { + err = -ENOMEM; + goto err_cmd; + } + qp->trace_info->pid = current->pid; + qp->trace_info->timestamp = (u64)(u32)ts.tv_sec * MSEC_PER_SEC + + ts.tv_nsec / NSEC_PER_MSEC; + + err = create_resource_common(xdev, qp); + if (err) { + xsc_core_warn(xdev, "err %d", err); + goto err_trace; + } + + err = xsc_debug_qp_add(xdev, qp); + if (err) + xsc_core_dbg(xdev, "failed adding QP 0x%x to debug file system\n", + qp->qpn); + + atomic_inc(&xdev->num_qps); + return 0; +err_trace: + kfree(qp->trace_info); +err_cmd: + memset(&din, 0, sizeof(din)); + memset(&dout, 0, sizeof(dout)); + din.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_QP); + din.qpn = cpu_to_be32(qp->qpn); + xsc_cmd_exec(xdev, &din, sizeof(din), &out, sizeof(dout)); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_core_create_qp); + +int xsc_core_destroy_qp(struct xsc_core_device *xdev, + struct xsc_core_qp *qp) +{ + struct xsc_destroy_qp_mbox_in in; + struct xsc_destroy_qp_mbox_out out; + int err; + int exec = 1; + + xsc_debug_qp_remove(xdev, qp); + xsc_remove_qptrace(xdev, qp); + kfree(qp->trace_info); + + destroy_resource_common(xdev, qp); + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_DESTROY_QP); + in.qpn = cpu_to_be32(qp->qpn); +#ifdef XSC_CHIP_RDMA_UNSUPPORTED + if ((qp->qp_type == XSC_QUEUE_TYPE_RDMA_MAD) || + (qp->qp_type == XSC_QUEUE_TYPE_RDMA_RC)) { + exec = 0; + } +#endif + if (exec) { + err = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + if (err) + return err; + + if (out.hdr.status) + return xsc_cmd_status_to_err(&out.hdr); + } + atomic_dec(&xdev->num_qps); + return 0; +} +EXPORT_SYMBOL_GPL(xsc_core_destroy_qp); + +int xsc_core_qp_modify(struct xsc_core_device *xdev, enum xsc_qp_state cur_state, + enum xsc_qp_state new_state, + struct xsc_modify_qp_mbox_in *in, int sqd_event, + struct xsc_core_qp *qp) +{ + static const u16 optab[XSC_QP_NUM_STATE][XSC_QP_NUM_STATE] = { + [XSC_QP_STATE_RST] = { + [XSC_QP_STATE_RST] = XSC_CMD_OP_2RST_QP, + [XSC_QP_STATE_ERR] = XSC_CMD_OP_2ERR_QP, + [XSC_QP_STATE_INIT] = XSC_CMD_OP_RST2INIT_QP, + }, + [XSC_QP_STATE_INIT] = { + [XSC_QP_STATE_RST] = XSC_CMD_OP_2RST_QP, + [XSC_QP_STATE_ERR] = XSC_CMD_OP_2ERR_QP, + [XSC_QP_STATE_INIT] = XSC_CMD_OP_INIT2INIT_QP, + [XSC_QP_STATE_RTR] = XSC_CMD_OP_INIT2RTR_QP, + }, + [XSC_QP_STATE_RTR] = { + [XSC_QP_STATE_RST] = XSC_CMD_OP_2RST_QP, + [XSC_QP_STATE_ERR] = XSC_CMD_OP_2ERR_QP, + [XSC_QP_STATE_RTS] = XSC_CMD_OP_RTR2RTS_QP, + }, + [XSC_QP_STATE_RTS] = { + [XSC_QP_STATE_RST] = XSC_CMD_OP_2RST_QP, + [XSC_QP_STATE_ERR] = XSC_CMD_OP_2ERR_QP, + [XSC_QP_STATE_RTS] = XSC_CMD_OP_RTS2RTS_QP, + [XSC_QP_STATE_SQD] = XSC_CMD_OP_RTS2SQD_QP, + }, + [XSC_QP_STATE_SQD] = { + [XSC_QP_STATE_RST] = XSC_CMD_OP_2RST_QP, + [XSC_QP_STATE_ERR] = XSC_CMD_OP_2ERR_QP, + [XSC_QP_STATE_RTS] = XSC_CMD_OP_SQD2RTS_QP, + [XSC_QP_STATE_SQD] = XSC_CMD_OP_SQD2SQD_QP, + }, + [XSC_QP_STATE_SQER] = { + [XSC_QP_STATE_RST] = XSC_CMD_OP_2RST_QP, + [XSC_QP_STATE_ERR] = XSC_CMD_OP_2ERR_QP, + [XSC_QP_STATE_RTS] = XSC_CMD_OP_SQERR2RTS_QP, + }, + [XSC_QP_STATE_ERR] = { + [XSC_QP_STATE_RST] = XSC_CMD_OP_2RST_QP, + [XSC_QP_STATE_ERR] = XSC_CMD_OP_2ERR_QP, + } + }; + + struct xsc_modify_qp_mbox_out out; + int err = 0; + u16 op; + + if (cur_state >= XSC_QP_NUM_STATE || new_state >= XSC_QP_NUM_STATE || + !optab[cur_state][new_state]) + return -EINVAL; + + memset(&out, 0, sizeof(out)); + op = optab[cur_state][new_state]; + in->hdr.opcode = cpu_to_be16(op); + in->qpn = cpu_to_be32(qp->qpn); + // TODO not support host2soc qp group + + if (new_state == XSC_QP_STATE_RTR) { + if (qp->qp_type_internal == XSC_QUEUE_TYPE_RDMA_RC && + ((in->ctx.ip_type == 0 && in->ctx.dip[0] == in->ctx.sip[0]) || + (in->ctx.ip_type != 0 && + memcmp(in->ctx.dip, in->ctx.sip, sizeof(in->ctx.sip)) == 0))) + in->ctx.qp_out_port = NIF_PORT_NUM + xsc_get_pcie_no(); + else if (in->ctx.lag_sel_en == 0) + in->ctx.qp_out_port = XSC_PF_VF_GET_PF_ID(xdev->glb_func_id); + else + in->ctx.qp_out_port = in->ctx.lag_sel; + + in->ctx.pcie_no = xsc_get_pcie_no(); + in->ctx.func_id = cpu_to_be16(xdev->glb_func_id); + } + + err = xsc_cmd_exec(xdev, in, sizeof(*in), &out, sizeof(out)); + if (err) + return err; + + if (new_state == XSC_QP_STATE_RTR) { + qp->trace_info->main_ver = 1; + qp->trace_info->sub_ver = 0; + qp->trace_info->qp_type = qp->qp_type; + qp->trace_info->s_port = in->ctx.src_udp_port; + qp->trace_info->d_port = cpu_to_be16(4791); + qp->trace_info->lqpn = qp->qpn; + qp->trace_info->rqpn = be32_to_cpu(in->ctx.remote_qpn); + qp->trace_info->affinity_idx = 0; + qp->trace_info->af_type = (in->ctx.ip_type == 0 ? AF_INET : AF_INET6); + + if (in->ctx.ip_type == 0) { + qp->trace_info->s_addr.s_addr4 = in->ctx.sip[0]; + qp->trace_info->d_addr.d_addr4 = in->ctx.dip[0]; + } else { + memcpy(qp->trace_info->s_addr.s_addr6, in->ctx.sip, + sizeof(qp->trace_info->s_addr.s_addr6)); + memcpy(qp->trace_info->d_addr.d_addr6, in->ctx.dip, + sizeof(qp->trace_info->d_addr.d_addr6)); + } + + err = xsc_create_qptrace(xdev, qp); + if (err) + return err; + + } + + return xsc_cmd_status_to_err(&out.hdr); +} +EXPORT_SYMBOL_GPL(xsc_core_qp_modify); + +int xsc_core_qp_query(struct xsc_core_device *xdev, struct xsc_core_qp *qp, + struct xsc_query_qp_mbox_out *out, int outlen) +{ + struct xsc_query_qp_mbox_in in; + int err; + + memset(&in, 0, sizeof(in)); + memset(out, 0, outlen); + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_QUERY_QP); + in.qpn = cpu_to_be32(qp->qpn); + err = xsc_cmd_exec(xdev, &in, sizeof(in), out, outlen); + if (err) + return err; + + if (out->hdr.status) + return xsc_cmd_status_to_err(&out->hdr); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_core_qp_query); + +void xsc_init_qp_table(struct xsc_core_device *xdev) +{ + struct xsc_qp_table *table = &xdev->dev_res->qp_table; + + spin_lock_init(&table->lock); + INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); + + xsc_qp_debugfs_init(xdev); + xsc_qptrace_debugfs_init(xdev); +} + +void xsc_cleanup_qp_table(struct xsc_core_device *xdev) +{ + xsc_qp_debugfs_cleanup(xdev); + xsc_qptrace_debugfs_cleanup(xdev); +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/sriov.c b/drivers/net/ethernet/yunsilicon/xsc/pci/sriov.c new file mode 100644 index 0000000000000000000000000000000000000000..4011e998cb1465f7fd7671437be0c9f119ba8fe1 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/sriov.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#ifdef CONFIG_XSC_ESWITCH +#include "eswitch.h" +#endif +#include "fw/xsc_tbm.h" + +static int xsc_device_enable_sriov(struct xsc_core_device *dev, int num_vfs) +{ + struct xsc_core_sriov *sriov = &dev->priv.sriov; + int err; + u16 vf; + + if (!XSC_ESWITCH_MANAGER(dev)) + goto enable_vfs; + +#ifdef CONFIG_XSC_ESWITCH + err = xsc_eswitch_enable(dev->priv.eswitch, XSC_ESWITCH_LEGACY, + num_vfs); + if (err) { + xsc_core_warn(dev, + "failed to enable eswitch SRIOV (%d)\n", err); + return err; + } +#endif + +enable_vfs: + err = xsc_create_vfs_sysfs(dev, num_vfs); + if (err) { + xsc_core_warn(dev, "failed to create SRIOV sysfs (%d)\n", err); +#ifdef CONFIG_XSC_ESWITCH + if (XSC_ESWITCH_MANAGER(dev)) + xsc_eswitch_disable(dev->priv.eswitch, true); +#endif + return err; + } + + xsc_lag_disable(dev); + for (vf = 0; vf < num_vfs; vf++) { + err = xsc_cmd_enable_hca(dev, vf); + if (err) { + xsc_core_warn(dev, "failed to enable VF %d (%d)\n", vf, err); + continue; + } + + sriov->vfs_ctx[vf].enabled = 1; + xsc_core_info(dev, "enabled VF%d ok\n", vf); + } + xsc_lag_enable(dev); + + return 0; +} + +static void xsc_device_disable_sriov(struct xsc_core_device *dev, + int num_vfs, bool clear_vf) +{ + struct xsc_core_sriov *sriov = &dev->priv.sriov; + int vf, err; + + for (vf = num_vfs - 1; vf >= 0; vf--) { + if (!sriov->vfs_ctx[vf].enabled) + continue; + + err = xsc_cmd_disable_hca(dev, (u16)vf); + if (err) { + xsc_core_warn(dev, "failed to disable VF %d\n", vf); + continue; + } + sriov->vfs_ctx[vf].enabled = 0; + } + +#ifdef CONFIG_XSC_ESWITCH + if (XSC_ESWITCH_MANAGER(dev)) { + xsc_lag_disable(dev); + xsc_eswitch_disable(dev->priv.eswitch, clear_vf); + xsc_lag_enable(dev); + } +#endif + + xsc_destroy_vfs_sysfs(dev, num_vfs); +} + +static int xsc_sriov_enable(struct pci_dev *pdev, int num_vfs) +{ + struct xsc_core_device *dev = pci_get_drvdata(pdev); + int err; + + if (num_vfs > dev->caps.max_vfs) { + xsc_core_warn(dev, + "invalid sriov param, num_vfs(%d) > total_vfs(%d)\n", + num_vfs, dev->caps.max_vfs); + return -EINVAL; + } + + if (num_vfs && pci_num_vf(dev->pdev)) { + if (num_vfs == pci_num_vf(dev->pdev)) + return 0; + + xsc_core_warn(dev, + "VFs already enabled. Disable before enabling %d VFs\n", + num_vfs); + return -EBUSY; + } + + xsc_core_info(dev, "%s: num_vfs=%d\n", __func__, num_vfs); + + err = xsc_device_enable_sriov(dev, num_vfs); + if (err) { + xsc_core_warn(dev, "xsc_device_enable_sriov failed : %d\n", err); + return err; + } + + err = pci_enable_sriov(pdev, num_vfs); + if (err) { + xsc_core_warn(dev, "pci_enable_sriov failed : %d\n", err); + xsc_device_disable_sriov(dev, num_vfs, true); + } + xsc_set_vf_pp_status(dev, true); + return err; +} + +static void xsc_sriov_disable(struct pci_dev *pdev) +{ + struct xsc_core_device *dev = pci_get_drvdata(pdev); + int num_vfs = pci_num_vf(dev->pdev); + + xsc_core_info(dev, "%s: num_vfs=%d\n", __func__, num_vfs); + pci_disable_sriov(pdev); + xsc_device_disable_sriov(dev, num_vfs, true); + xsc_set_vf_pp_status(dev, true); +} + +int xsc_core_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + struct xsc_core_device *dev = pci_get_drvdata(pdev); + struct xsc_core_sriov *sriov = &dev->priv.sriov; + int err = 0; + + xsc_core_info(dev, "%s: requested num_vfs %d\n", + __func__, num_vfs); + + if (num_vfs) + err = xsc_sriov_enable(pdev, num_vfs); + else + xsc_sriov_disable(pdev); + + if (!err) + sriov->num_vfs = num_vfs; + return err ? err : num_vfs; +} + +int xsc_sriov_attach(struct xsc_core_device *dev) +{ + struct pci_dev *pdev = dev->pdev; + struct xsc_core_device *pf_xdev; + struct xsc_core_sriov *sriov; + + if (!xsc_core_is_pf(dev)) { + pf_xdev = pci_get_drvdata(pdev->physfn); + sriov = &pf_xdev->priv.sriov; + + sriov->vfs[dev->vf_id].vf = dev->vf_id; + sriov->vfs[dev->vf_id].dev = dev; + return 0; + } + + if (!dev->priv.sriov.num_vfs) + return 0; + + /* If sriov VFs exist in PCI level, enable them in device level */ + return xsc_device_enable_sriov(dev, pci_num_vf(dev->pdev)); +} + +void xsc_sriov_detach(struct xsc_core_device *dev) +{ + if (!xsc_core_is_pf(dev) || !dev->priv.sriov.num_vfs) + return; + + xsc_device_disable_sriov(dev, pci_num_vf(dev->pdev), false); +} + +static u16 xsc_get_max_vfs(struct xsc_core_device *dev) +{ + /* In RH6.8 and lower pci_sriov_get_totalvfs might return -EINVAL + * return in that case 1 + */ + return (pci_sriov_get_totalvfs(dev->pdev) < 0) ? 0 : + pci_sriov_get_totalvfs(dev->pdev); +} + +static int xsc_sriov_pci_cfg_info(struct xsc_core_device *dev, + struct xsc_pci_sriov *iov) +{ + int pos; + struct pci_dev *pdev = dev->pdev; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); + if (!pos) { + xsc_core_err(dev, "%s: failed to find SRIOV capability in device\n", + __func__); + return -ENODEV; + } + + iov->pos = pos; + pci_read_config_dword(pdev, pos + PCI_SRIOV_CAP, &iov->cap); + pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &iov->ctrl); + pci_read_config_word(pdev, pos + PCI_SRIOV_TOTAL_VF, &iov->total_vfs); + pci_read_config_word(pdev, pos + PCI_SRIOV_INITIAL_VF, &iov->initial_vfs); + pci_read_config_word(pdev, pos + PCI_SRIOV_NUM_VF, &iov->num_vfs); + pci_read_config_word(pdev, pos + PCI_SRIOV_VF_OFFSET, &iov->offset); + pci_read_config_word(pdev, pos + PCI_SRIOV_VF_STRIDE, &iov->stride); + pci_read_config_word(pdev, pos + PCI_SRIOV_VF_DID, &iov->vf_device); + pci_read_config_dword(pdev, pos + PCI_SRIOV_SUP_PGSIZE, &iov->pgsz); + pci_read_config_byte(pdev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); + + return 0; +} + +int xsc_sriov_init(struct xsc_core_device *dev) +{ + struct xsc_core_sriov *sriov = &dev->priv.sriov; + struct pci_dev *pdev = dev->pdev; + struct xsc_pci_sriov *iov = &sriov->pci_sriov; + int total_vfs; + u32 vf_bus, vf_devfn; + int err; + + if (!xsc_core_is_pf(dev)) + return 0; + + err = xsc_sriov_pci_cfg_info(dev, iov); + if (err) { + xsc_core_warn(dev, "%s: pci not support sriov, err=%d\n", + __func__, err); + return 0; + } + + total_vfs = pci_sriov_get_totalvfs(pdev); + if (unlikely(iov->total_vfs == 0)) { + xsc_core_warn(dev, "%s: pci not support sriov, total_vfs=%d, cur_vfs=%d\n", + __func__, iov->total_vfs, sriov->num_vfs); + return 0; + } + sriov->max_vfs = xsc_get_max_vfs(dev); + sriov->num_vfs = pci_num_vf(pdev); + + vf_bus = pdev->bus->number + ((pdev->devfn + iov->offset) >> 8); + vf_devfn = (pdev->devfn + iov->offset) & 0xff; + sriov->vf_bdf_base = (u16)((vf_bus << 8) | vf_devfn); + + sriov->vfs_ctx = kcalloc(total_vfs, sizeof(*sriov->vfs_ctx), GFP_KERNEL); + if (!sriov->vfs_ctx) + return -ENOMEM; + + xsc_core_info(dev, "%s: total_vfs=%d, cur_vfs=%d, vf_bdf_base=0x%02x\n", + __func__, total_vfs, sriov->num_vfs, sriov->vf_bdf_base); + xsc_core_info(dev, "%s: vf_offset=%d, stride=%d, vf_device_id=0x%x\n", + __func__, iov->offset, iov->stride, iov->vf_device); + err = xsc_sriov_sysfs_init(dev); + if (err) { + xsc_core_warn(dev, "failed to init SRIOV sysfs (%d)\n", err); + kfree(sriov->vfs_ctx); + return err; + } + + return 0; +} + +void xsc_sriov_cleanup(struct xsc_core_device *dev) +{ + struct xsc_core_sriov *sriov = &dev->priv.sriov; + + if (!xsc_core_is_pf(dev)) + return; + + xsc_sriov_sysfs_cleanup(dev); + kfree(sriov->vfs_ctx); +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/sriov_sysfs.c b/drivers/net/ethernet/yunsilicon/xsc/pci/sriov_sysfs.c new file mode 100644 index 0000000000000000000000000000000000000000..545c5deae255d3cd4177ba082176ea39f93ea587 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/sriov_sysfs.c @@ -0,0 +1,1082 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#ifdef CONFIG_XSC_ESWITCH +#include "eswitch.h" +#endif + +struct vf_attributes { + struct attribute attr; + ssize_t (*show)(struct xsc_sriov_vf *vf, struct vf_attributes *attr, + char *buf); + ssize_t (*store)(struct xsc_sriov_vf *vf, struct vf_attributes *attr, + const char *buf, size_t count); +}; + +static ssize_t vf_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct vf_attributes *ga = + container_of(attr, struct vf_attributes, attr); + struct xsc_sriov_vf *g = container_of(kobj, struct xsc_sriov_vf, kobj); + + if (!ga->show) + return -EIO; + + return ga->show(g, ga, buf); +} + +static ssize_t vf_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t size) +{ + struct vf_attributes *ga = + container_of(attr, struct vf_attributes, attr); + struct xsc_sriov_vf *g = container_of(kobj, struct xsc_sriov_vf, kobj); + + if (!ga->store) + return -EIO; + + return ga->store(g, ga, buf, size); +} + +struct vf_group_attributes { + struct attribute attr; + ssize_t (*show)(struct xsc_vgroup *group, struct vf_group_attributes *attr, + char *buf); + ssize_t (*store)(struct xsc_vgroup *group, struct vf_group_attributes *attr, + const char *buf, size_t count); +}; + +static ssize_t vf_group_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct vf_group_attributes *ga = + container_of(attr, struct vf_group_attributes, attr); + struct xsc_vgroup *g = container_of(kobj, struct xsc_vgroup, kobj); + + if (!ga->show) + return -EIO; + + return ga->show(g, ga, buf); +} + +static ssize_t vf_group_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t size) +{ + struct vf_group_attributes *ga = + container_of(attr, struct vf_group_attributes, attr); + struct xsc_vgroup *g = container_of(kobj, struct xsc_vgroup, kobj); + + if (!ga->store) + return -EIO; + + return ga->store(g, ga, buf, size); +} + +static ssize_t port_show(struct xsc_sriov_vf *g, struct vf_attributes *oa, + char *buf) +{ + struct xsc_core_device *dev = g->dev; + union ib_gid gid; + int err; + u8 *p; + + err = xsc_query_hca_vport_gid(dev, 1, 1, g->vf, 0, &gid); + if (err) { + xsc_core_warn(dev, "failed to query gid at index 0 for vf %d\n", g->vf); + return err; + } + + p = &gid.raw[8]; + err = sprintf(buf, "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n", + p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]); + return err; +} + +static ssize_t port_store(struct xsc_sriov_vf *g, struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_vf_context *vfs_ctx = dev->priv.sriov.vfs_ctx; + struct xsc_hca_vport_context *in; + u64 guid = 0; + int err; + int tmp[8]; + int i; + + err = sscanf(buf, "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n", + &tmp[0], &tmp[1], &tmp[2], &tmp[3], &tmp[4], &tmp[5], &tmp[6], &tmp[7]); + if (err != 8) + return -EINVAL; + + for (i = 0; i < 8; i++) + guid += ((u64)tmp[i] << ((7 - i)*8)); + + in = kzalloc(sizeof(*in), GFP_KERNEL); + if (!in) + return -ENOMEM; + + in->field_select = XSC_HCA_VPORT_SEL_PORT_GUID; + in->port_guid = guid; + err = xsc_modify_hca_vport_context(dev, 1, 1, g->vf + 1, in); + kfree(in); + if (err) + return err; + + vfs_ctx[g->vf].port_guid = guid; + + return count; +} + +static int show_nic_node_guid(struct xsc_core_device *dev, u16 vf, + __be64 *node_guid) +{ + int err; + + err = xsc_query_nic_vport_node_guid(dev, vf + 1, node_guid); + if (!err) + *node_guid = cpu_to_be64(*node_guid); + + return err; +} + +static ssize_t node_show(struct xsc_sriov_vf *g, struct vf_attributes *oa, + char *buf) +{ + struct xsc_core_device *dev = g->dev; + __be64 guid; + + int err; + u8 *p; + + err = show_nic_node_guid(dev, g->vf, &guid); + if (err) { + xsc_core_warn(dev, "failed to query node guid for vf %d (%d)\n", + g->vf, err); + return err; + } + + p = (u8 *)&guid; + err = sprintf(buf, "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n", + p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]); + + return err; +} + +static int modify_nic_node_guid(struct xsc_core_device *dev, u16 vf, + u64 node_guid) +{ + return xsc_modify_other_nic_vport_node_guid(dev, vf + 1, node_guid); +} + +static ssize_t node_store(struct xsc_sriov_vf *g, struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + u64 guid = 0; + int err; + int tmp[8]; + int i; + + err = sscanf(buf, "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n", + &tmp[0], &tmp[1], &tmp[2], &tmp[3], &tmp[4], &tmp[5], &tmp[6], &tmp[7]); + if (err != 8) + return -EINVAL; + + for (i = 0; i < 8; i++) + guid += ((u64)tmp[i] << ((7 - i)*8)); + + err = modify_nic_node_guid(dev, g->vf, guid); + if (err) { + xsc_core_warn(dev, "failed to modify node guid for vf %d (%d)\n", + g->vf, err); + return err; + } + + return count; +} + +static const char *policy_str(enum port_state_policy policy) +{ + switch (policy) { + case XSC_POLICY_DOWN: return "Down\n"; + case XSC_POLICY_UP: return "Up\n"; + case XSC_POLICY_FOLLOW: return "Follow\n"; + default: return "Invalid policy\n"; + } +} + +static ssize_t policy_show(struct xsc_sriov_vf *g, struct vf_attributes *oa, + char *buf) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_hca_vport_context *rep; + const char *p = NULL; + int err; + + rep = kzalloc(sizeof(*rep), GFP_KERNEL); + if (!rep) + return -ENOMEM; + + err = xsc_query_hca_vport_context(dev, 1, 1, g->vf, rep); + if (err) { + xsc_core_warn(dev, "failed to query port policy for vf %d (%d)\n", + g->vf, err); + goto free; + } + p = policy_str(rep->vport_state_policy); + strcpy(buf, p); + +free: + kfree(rep); + return p ? strlen(p) : err; +} + +static int strpolicy(const char *buf, enum port_state_policy *policy) +{ + if (sysfs_streq(buf, "Down")) { + *policy = XSC_POLICY_DOWN; + return 0; + } + + if (sysfs_streq(buf, "Up")) { + *policy = XSC_POLICY_UP; + return 0; + } + + if (sysfs_streq(buf, "Follow")) { + *policy = XSC_POLICY_FOLLOW; + return 0; + } + return -EINVAL; +} + +static ssize_t policy_store(struct xsc_sriov_vf *g, struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_vf_context *vfs_ctx = dev->priv.sriov.vfs_ctx; + struct xsc_hca_vport_context *in; + enum port_state_policy policy; + int err; + + err = strpolicy(buf, &policy); + if (err) + return err; + + in = kzalloc(sizeof(*in), GFP_KERNEL); + if (!in) + return -ENOMEM; + + in->vport_state_policy = policy; + in->field_select = XSC_HCA_VPORT_SEL_STATE_POLICY; + err = xsc_modify_hca_vport_context(dev, 1, 1, g->vf + 1, in); + kfree(in); + if (err) + return err; + + vfs_ctx[g->vf].policy = policy; + + return count; +} + +#ifdef CONFIG_XSC_ESWITCH +/* ETH SRIOV SYSFS */ +static ssize_t mac_show(struct xsc_sriov_vf *g, struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to set VF Mac Address\n"); +} + +static ssize_t mac_store(struct xsc_sriov_vf *g, struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + u8 mac[ETH_ALEN]; + int err; + + err = sscanf(buf, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", + &mac[0], &mac[1], &mac[2], &mac[3], &mac[4], &mac[5]); + if (err == 6) + goto set_mac; + + if (sysfs_streq(buf, "Random")) + eth_random_addr(mac); + else + return -EINVAL; + +set_mac: + err = xsc_eswitch_set_vport_mac(dev->priv.eswitch, g->vf + 1, mac); + return err ? err : count; +} + +static ssize_t vlan_show(struct xsc_sriov_vf *g, struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to set VF Vlan,Qos and Vlan Protocol\n"); +} + +static ssize_t vlan_store(struct xsc_sriov_vf *g, struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + char vproto_ext[5] = {'\0'}; + __be16 vlan_proto; + u16 vlan_id; + u8 qos; + int err; + + err = sscanf(buf, "%hu:%hhu:802.%4s", &vlan_id, &qos, vproto_ext); + if (err == 3) { + if ((strcmp(vproto_ext, "1AD") == 0) || + (strcmp(vproto_ext, "1ad") == 0)) + vlan_proto = htons(ETH_P_8021AD); + else if ((strcmp(vproto_ext, "1Q") == 0) || + (strcmp(vproto_ext, "1q") == 0)) + vlan_proto = htons(ETH_P_8021Q); + else + return -EINVAL; + } else { + err = sscanf(buf, "%hu:%hhu", &vlan_id, &qos); + if (err != 2) + return -EINVAL; + vlan_proto = htons(ETH_P_8021Q); + } + + err = xsc_eswitch_set_vport_vlan(dev->priv.eswitch, g->vf + 1, + vlan_id, qos, vlan_proto); + return err ? err : count; +} + +static const char *vlan_proto_str(u16 vlan, u8 qos, __be16 vlan_proto) +{ + if (!vlan && !qos) + return "N/A"; + + switch (vlan_proto) { + case htons(ETH_P_8021AD): return "802.1ad"; + case htons(ETH_P_8021Q): return "802.1Q"; + default: return "Invalid vlan protocol"; + } +} + +static ssize_t spoofcheck_show(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to enable|disable VF SpoofCheck\n" + ); +} + +static ssize_t spoofcheck_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, + size_t count) +{ + struct xsc_core_device *dev = g->dev; + bool settings; + int err; + + if (sysfs_streq(buf, "ON")) + settings = true; + else if (sysfs_streq(buf, "OFF")) + settings = false; + else + return -EINVAL; + + err = xsc_eswitch_set_vport_spoofchk(dev->priv.eswitch, g->vf + 1, settings); + return err ? err : count; +} + +static ssize_t trust_show(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to trust|untrust VF\n" + ); +} + +static ssize_t trust_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, + size_t count) +{ + struct xsc_core_device *dev = g->dev; + bool settings; + int err; + + if (sysfs_streq(buf, "ON")) + settings = true; + else if (sysfs_streq(buf, "OFF")) + settings = false; + else + return -EINVAL; + + err = xsc_eswitch_set_vport_trust(dev->priv.eswitch, g->vf + 1, settings); + return err ? err : count; +} + +static ssize_t link_state_show(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, "usage: write to set VF State\n"); +} + +static ssize_t link_state_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, + size_t count) +{ + struct xsc_core_device *dev = g->dev; + enum port_state_policy policy; + int err; + + err = strpolicy(buf, &policy); + if (err) + return err; + + err = xsc_eswitch_set_vport_state(dev->priv.eswitch, g->vf + 1, policy); + return err ? err : count; +} + +static ssize_t max_tx_rate_show(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to set VF max rate\n"); +} + +static ssize_t max_tx_rate_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + u32 max_tx_rate; + u32 min_tx_rate; + int err; + + mutex_lock(&esw->state_lock); + min_tx_rate = esw->vports[g->vf + 1].info.min_rate; + mutex_unlock(&esw->state_lock); + + err = kstrtou32(buf, 10, &max_tx_rate); + if (err != 1) + return -EINVAL; + + err = xsc_eswitch_set_vport_rate(dev->priv.eswitch, g->vf + 1, + max_tx_rate, min_tx_rate); + return err ? err : count; +} + +static ssize_t min_tx_rate_show(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to set VF min rate\n"); +} + +static ssize_t min_tx_rate_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + u32 min_tx_rate; + u32 max_tx_rate; + int err; + + mutex_lock(&esw->state_lock); + max_tx_rate = esw->vports[g->vf + 1].info.max_rate; + mutex_unlock(&esw->state_lock); + + err = kstrtou32(buf, 10, &min_tx_rate); + if (err != 1) + return -EINVAL; + + err = xsc_eswitch_set_vport_rate(dev->priv.eswitch, g->vf + 1, + max_tx_rate, min_tx_rate); + return err ? err : count; +} + +static ssize_t min_pf_tx_rate_show(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to set PF min rate\n"); +} + +static ssize_t min_pf_tx_rate_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + u32 min_tx_rate; + u32 max_tx_rate; + int err; + + mutex_lock(&esw->state_lock); + max_tx_rate = esw->vports[g->vf].info.max_rate; + mutex_unlock(&esw->state_lock); + + err = kstrtou32(buf, 10, &min_tx_rate); + if (err != 1) + return -EINVAL; + + err = xsc_eswitch_set_vport_rate(dev->priv.eswitch, g->vf, + max_tx_rate, min_tx_rate); + return err ? err : count; +} + +static ssize_t group_show(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to set VF vport group\n"); +} + +static ssize_t group_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + u32 group_id; + int err; + + err = kstrtou32(buf, 10, &group_id); + if (err != 1) + return -EINVAL; + + if (group_id > 255) + return -EINVAL; + + err = xsc_eswitch_vport_update_group(esw, g->vf + 1, group_id); + + return err ? err : count; +} + +static ssize_t max_tx_rate_group_show(struct xsc_vgroup *g, + struct vf_group_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to set VF group max rate\n"); +} + +static ssize_t max_tx_rate_group_store(struct xsc_vgroup *g, + struct vf_group_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + u32 max_rate; + int err; + + err = kstrtou32(buf, 10, &max_rate); + if (err != 1) + return -EINVAL; + + err = xsc_eswitch_set_vgroup_max_rate(esw, g->group_id, max_rate); + + return err ? err : count; +} + +static ssize_t min_tx_rate_group_show(struct xsc_vgroup *g, + struct vf_group_attributes *oa, + char *buf) +{ + return sprintf(buf, + "usage: write to set VF group min rate\n"); +} + +static ssize_t min_tx_rate_group_store(struct xsc_vgroup *g, + struct vf_group_attributes *oa, + const char *buf, size_t count) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + u32 min_rate; + int err; + + err = kstrtou32(buf, 10, &min_rate); + if (err != 1) + return -EINVAL; + + err = xsc_eswitch_set_vgroup_min_rate(esw, g->group_id, min_rate); + + return err ? err : count; +} + +#define _sprintf(p, buf, format, arg...) \ + ((PAGE_SIZE - (int)(p - buf)) <= 0 ? 0 : \ + scnprintf(p, PAGE_SIZE - (int)(p - buf), format, ## arg)) + +static ssize_t trunk_show(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + char *buf) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + struct xsc_vport *vport = &esw->vports[g->vf + 1]; + u16 vlan_id = 0; + char *ret = buf; + + mutex_lock(&esw->state_lock); + if (!!bitmap_weight(vport->info.vlan_trunk_8021q_bitmap, VLAN_N_VID)) { + ret += _sprintf(ret, buf, "Allowed 802.1Q VLANs:"); + for_each_set_bit(vlan_id, vport->info.vlan_trunk_8021q_bitmap, VLAN_N_VID) + ret += _sprintf(ret, buf, " %d", vlan_id); + ret += _sprintf(ret, buf, "\n"); + } + mutex_unlock(&esw->state_lock); + + return (ssize_t)(ret - buf); +} + +static ssize_t trunk_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, + size_t count) +{ + struct xsc_core_device *dev = g->dev; + u16 start_vid, end_vid; + char op[5]; + int err; + + err = sscanf(buf, "%4s %hu %hu", op, &start_vid, &end_vid); + if (err != 3) + return -EINVAL; + + if (!strcmp(op, "add")) + err = xsc_eswitch_add_vport_trunk_range(dev->priv.eswitch, + g->vf + 1, + start_vid, end_vid); + else if (!strcmp(op, "rem")) + err = xsc_eswitch_del_vport_trunk_range(dev->priv.eswitch, + g->vf + 1, + start_vid, end_vid); + else + return -EINVAL; + + return err ? err : count; +} + +static ssize_t config_show(struct xsc_sriov_vf *g, struct vf_attributes *oa, + char *buf) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + struct xsc_vport_info *ivi; + int vport = g->vf + 1; + char *p = buf; + + if (!esw || !xsc_core_is_vport_manager(dev)) + return -EPERM; + if (!(vport >= 0 && vport < esw->total_vports)) + return -EINVAL; + + mutex_lock(&esw->state_lock); + ivi = &esw->vports[vport].info; + p += _sprintf(p, buf, "VF : %d\n", g->vf); + p += _sprintf(p, buf, "MAC : %pM\n", ivi->mac); + p += _sprintf(p, buf, "VLAN : %d\n", ivi->vlan); + p += _sprintf(p, buf, "QoS : %d\n", ivi->qos); + p += _sprintf(p, buf, "VLAN Proto : %s\n", + vlan_proto_str(ivi->vlan, ivi->qos, ivi->vlan_proto)); + p += _sprintf(p, buf, "SpoofCheck : %s\n", ivi->spoofchk ? "ON" : "OFF"); + p += _sprintf(p, buf, "Trust : %s\n", ivi->trusted ? "ON" : "OFF"); + p += _sprintf(p, buf, "LinkState : %s", policy_str(ivi->link_state)); + p += _sprintf(p, buf, "MinTxRate : %d\n", ivi->min_rate); + p += _sprintf(p, buf, "MaxTxRate : %d\n", ivi->max_rate); + p += _sprintf(p, buf, "VGT+ : %s\n", + !!bitmap_weight(ivi->vlan_trunk_8021q_bitmap, VLAN_N_VID) ? + "ON" : "OFF"); + p += _sprintf(p, buf, "RateGroup : %d\n", ivi->group); + mutex_unlock(&esw->state_lock); + + return (ssize_t)(p - buf); +} + +static ssize_t config_store(struct xsc_sriov_vf *g, + struct vf_attributes *oa, + const char *buf, size_t count) +{ + return -EOPNOTSUPP; +} + +static ssize_t config_group_show(struct xsc_vgroup *g, + struct vf_group_attributes *oa, + char *buf) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_eswitch *esw = dev->priv.eswitch; + char *p = buf; + + if (!esw || !xsc_core_is_vport_manager(dev)) + return -EPERM; + + mutex_lock(&esw->state_lock); + p += _sprintf(p, buf, "Num VFs : %d\n", g->num_vports); + p += _sprintf(p, buf, "MaxRate : %d\n", g->max_rate); + p += _sprintf(p, buf, "MinRate : %d\n", g->min_rate); + p += _sprintf(p, buf, "BWShare(Indirect cfg) : %d\n", g->bw_share); + mutex_unlock(&esw->state_lock); + + return (ssize_t)(p - buf); +} + +static ssize_t config_group_store(struct xsc_vgroup *g, + struct vf_group_attributes *oa, + const char *buf, size_t count) +{ + return -EOPNOTSUPP; +} + +static ssize_t stats_show(struct xsc_sriov_vf *g, struct vf_attributes *oa, + char *buf) +{ + struct xsc_core_device *dev = g->dev; + struct xsc_vport *vport = xsc_eswitch_get_vport(dev->priv.eswitch, g->vf + 1); + struct ifla_vf_stats ifi; + struct xsc_vport_drop_stats stats = {}; + int err; + char *p = buf; + + err = xsc_eswitch_get_vport_stats(dev->priv.eswitch, g->vf + 1, &ifi); + if (err) + return -EINVAL; + + err = xsc_eswitch_query_vport_drop_stats(dev, vport, &stats); + if (err) + return -EINVAL; + + p += _sprintf(p, buf, "tx_packets : %llu\n", ifi.tx_packets); + p += _sprintf(p, buf, "tx_bytes : %llu\n", ifi.tx_bytes); + p += _sprintf(p, buf, "tx_dropped : %llu\n", stats.tx_dropped); + p += _sprintf(p, buf, "rx_packets : %llu\n", ifi.rx_packets); + p += _sprintf(p, buf, "rx_bytes : %llu\n", ifi.rx_bytes); + p += _sprintf(p, buf, "rx_broadcast : %llu\n", ifi.broadcast); + p += _sprintf(p, buf, "rx_multicast : %llu\n", ifi.multicast); + p += _sprintf(p, buf, "rx_dropped : %llu\n", stats.rx_dropped); + + return (ssize_t)(p - buf); +} + +static ssize_t stats_store(struct xsc_sriov_vf *g, struct vf_attributes *oa, + const char *buf, size_t count) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_XSC_ESWITCH */ + +static ssize_t num_vfs_store(struct device *device, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct pci_dev *pdev = container_of(device, struct pci_dev, dev); + int req_vfs; + int err; + + if (kstrtoint(buf, 0, &req_vfs) || req_vfs < 0 || + req_vfs > pci_sriov_get_totalvfs(pdev)) + return -EINVAL; + + err = xsc_core_sriov_configure(pdev, req_vfs); + if (err < 0) + return err; + + return count; +} + +static ssize_t num_vfs_show(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct pci_dev *pdev = container_of(device, struct pci_dev, dev); + struct xsc_core_device *dev = pci_get_drvdata(pdev); + struct xsc_core_sriov *sriov = &dev->priv.sriov; + + return sprintf(buf, "%d\n", sriov->num_vfs); +} + +static DEVICE_ATTR_RW(num_vfs); + +static const struct sysfs_ops vf_sysfs_ops = { + .show = vf_attr_show, + .store = vf_attr_store, +}; + +static const struct sysfs_ops vf_group_sysfs_ops = { + .show = vf_group_attr_show, + .store = vf_group_attr_store, +}; + +#define VF_RATE_GROUP_ATTR(_name) struct vf_group_attributes vf_group_attr_##_name = \ + __ATTR(_name, 0644, _name##_group_show, _name##_group_store) +#define VF_ATTR(_name) struct vf_attributes vf_attr_##_name = \ + __ATTR(_name, 0644, _name##_show, _name##_store) + +VF_ATTR(node); +VF_ATTR(port); +VF_ATTR(policy); + +#ifdef CONFIG_XSC_ESWITCH +VF_ATTR(mac); +VF_ATTR(vlan); +VF_ATTR(link_state); +VF_ATTR(spoofcheck); +VF_ATTR(trust); +VF_ATTR(max_tx_rate); +VF_ATTR(min_tx_rate); +VF_ATTR(config); +VF_ATTR(trunk); +VF_ATTR(stats); +VF_ATTR(group); +VF_RATE_GROUP_ATTR(max_tx_rate); +VF_RATE_GROUP_ATTR(min_tx_rate); +VF_RATE_GROUP_ATTR(config); + +static struct attribute *vf_eth_attrs[] = { + &vf_attr_node.attr, + &vf_attr_mac.attr, + &vf_attr_vlan.attr, + &vf_attr_link_state.attr, + &vf_attr_spoofcheck.attr, + &vf_attr_trust.attr, + &vf_attr_max_tx_rate.attr, + &vf_attr_min_tx_rate.attr, + &vf_attr_config.attr, + &vf_attr_trunk.attr, + &vf_attr_stats.attr, + &vf_attr_group.attr, + NULL +}; + +static struct attribute *vf_group_attrs[] = { + &vf_group_attr_max_tx_rate.attr, + &vf_group_attr_min_tx_rate.attr, + &vf_group_attr_config.attr, + NULL +}; + +static struct kobj_type vf_type_eth = { + .sysfs_ops = &vf_sysfs_ops, + .default_attrs = vf_eth_attrs +}; + +static struct kobj_type vf_group = { + .sysfs_ops = &vf_group_sysfs_ops, + .default_attrs = vf_group_attrs +}; + +static struct vf_attributes pf_attr_min_pf_tx_rate = + __ATTR(min_tx_rate, 0644, min_pf_tx_rate_show, min_pf_tx_rate_store); + +static struct attribute *pf_eth_attrs[] = { + &pf_attr_min_pf_tx_rate.attr, + NULL, +}; + +static struct kobj_type pf_type_eth = { + .sysfs_ops = &vf_sysfs_ops, + .default_attrs = pf_eth_attrs +}; +#endif /* CONFIG_XSC_ESWITCH */ + +static struct attribute *vf_ib_attrs[] = { + &vf_attr_node.attr, + &vf_attr_port.attr, + &vf_attr_policy.attr, + NULL +}; + +static struct kobj_type vf_type_ib = { + .sysfs_ops = &vf_sysfs_ops, + .default_attrs = vf_ib_attrs +}; + +static struct device_attribute *xsc_class_attributes[] = { + &dev_attr_num_vfs, +}; + +int xsc_sriov_sysfs_init(struct xsc_core_device *dev) +{ + struct xsc_core_sriov *sriov = &dev->priv.sriov; + struct device *device = &dev->pdev->dev; + int err; + int i; + + sriov->config = kobject_create_and_add("sriov", &device->kobj); + if (!sriov->config) + return -ENOMEM; + +#ifdef CONFIG_XSC_ESWITCH + if (dev->caps.log_esw_max_sched_depth) { + sriov->groups_config = kobject_create_and_add("groups", + sriov->config); + if (!sriov->groups_config) { + err = -ENOMEM; + goto err_groups; + } + } +#endif + + for (i = 0; i < ARRAY_SIZE(xsc_class_attributes); i++) { + err = device_create_file(device, xsc_class_attributes[i]); + if (err) + goto err_attr; + } + + return 0; + +err_attr: +#ifdef CONFIG_XSC_ESWITCH + if (sriov->groups_config) { + kobject_put(sriov->groups_config); + sriov->groups_config = NULL; + } + +err_groups: +#endif + kobject_put(sriov->config); + sriov->config = NULL; + return err; +} + +void xsc_sriov_sysfs_cleanup(struct xsc_core_device *dev) +{ + struct xsc_core_sriov *sriov = &dev->priv.sriov; + struct device *device = &dev->pdev->dev; + int i; + + for (i = 0; i < ARRAY_SIZE(xsc_class_attributes); i++) + device_remove_file(device, xsc_class_attributes[i]); + + if (dev->caps.log_esw_max_sched_depth) + kobject_put(sriov->groups_config); + kobject_put(sriov->config); + sriov->config = NULL; +} + +int xsc_create_vf_group_sysfs(struct xsc_core_device *dev, + u32 group_id, struct kobject *group_kobj) +{ +#ifdef CONFIG_XSC_ESWITCH + struct xsc_core_sriov *sriov = &dev->priv.sriov; + int err; + + err = kobject_init_and_add(group_kobj, &vf_group, sriov->groups_config, + "%d", group_id); + if (err) + return err; + + kobject_uevent(group_kobj, KOBJ_ADD); +#endif + + return 0; +} + +void xsc_destroy_vf_group_sysfs(struct xsc_core_device *dev, + struct kobject *group_kobj) +{ +#ifdef CONFIG_XSC_ESWITCH + kobject_put(group_kobj); +#endif +} + +int xsc_create_vfs_sysfs(struct xsc_core_device *dev, int num_vfs) +{ + struct xsc_core_sriov *sriov = &dev->priv.sriov; + struct xsc_sriov_vf *tmp; + static struct kobj_type *sysfs; + int err; + int vf; + + sysfs = &vf_type_ib; + +#ifdef CONFIG_XSC_ESWITCH + sysfs = &vf_type_eth; +#endif + + sriov->vfs = kcalloc(num_vfs + 1, sizeof(*sriov->vfs), GFP_KERNEL); + if (!sriov->vfs) + return -ENOMEM; + + for (vf = 0; vf < num_vfs; vf++) { + tmp = &sriov->vfs[vf]; + tmp->dev = dev; + tmp->vf = vf; + err = kobject_init_and_add(&tmp->kobj, sysfs, sriov->config, + "%d", vf); + if (err) + goto err_vf; + + kobject_uevent(&tmp->kobj, KOBJ_ADD); + } + +#ifdef CONFIG_XSC_ESWITCH + tmp = &sriov->vfs[vf]; + tmp->dev = dev; + tmp->vf = 0; + err = kobject_init_and_add(&tmp->kobj, &pf_type_eth, + sriov->config, "%s", "pf"); + if (err) { + --vf; + goto err_vf; + } + + kobject_uevent(&tmp->kobj, KOBJ_ADD); +#endif + + return 0; + +err_vf: + for (; vf >= 0; vf--) { + tmp = &sriov->vfs[vf]; + kobject_put(&tmp->kobj); + } + + kfree(sriov->vfs); + sriov->vfs = NULL; + return err; +} + +void xsc_destroy_vfs_sysfs(struct xsc_core_device *dev, int num_vfs) +{ + struct xsc_core_sriov *sriov = &dev->priv.sriov; + struct xsc_sriov_vf *tmp; + int vf; + +#ifdef CONFIG_XSC_ESWITCH + if (num_vfs) { + tmp = &sriov->vfs[num_vfs]; + kobject_put(&tmp->kobj); + } +#endif + for (vf = 0; vf < num_vfs; vf++) { + tmp = &sriov->vfs[vf]; + kobject_put(&tmp->kobj); + } + + kfree(sriov->vfs); + sriov->vfs = NULL; +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/tmp_cmdq_defines.h b/drivers/net/ethernet/yunsilicon/xsc/pci/tmp_cmdq_defines.h new file mode 100644 index 0000000000000000000000000000000000000000..d8e5e5f97034f9e9eaff7fa96ef12a06bdf3e875 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/tmp_cmdq_defines.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef CMDQ_DEFINE_H +#define CMDQ_DEFINE_H + +#define CMDQ_PA_REG_ADDR 0xFC00000 +#define CMDQ_PA_REG_WIDTH 64 + +#define CMDQ_LOG_SIZE_REG_ADDR 0xFC00008 +#define CMDQ_LOG_SIZE_WIDTH 4 + +#define CMDQ_DB_REG_ADDR 0xFC0000C +#define CMDQ_DB_REG_WIDTH 32 + +#endif + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/vport.c b/drivers/net/ethernet/yunsilicon/xsc/pci/vport.c new file mode 100644 index 0000000000000000000000000000000000000000..4915fe2b1a6be3d12983dca0dd54ee6b48136a3d --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/vport.c @@ -0,0 +1,826 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include + +static int _xsc_query_vport_state(struct xsc_core_device *dev, u8 opmod, + u16 vport, void *out, int outlen) +{ + struct xsc_query_vport_state_in in; + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = XSC_CMD_OP_QUERY_VPORT_STATE; + in.hdr.opmod = opmod; + in.vport_number = vport; + if (vport) + in.other_vport = 1; + + return xsc_cmd_exec(dev, &in, sizeof(in), out, outlen); +} + +u8 xsc_query_vport_state(struct xsc_core_device *dev, u8 opmod, u16 vport) +{ + struct xsc_query_vport_state_out out; + + memset(&out, 0, sizeof(out)); + _xsc_query_vport_state(dev, opmod, vport, &out, sizeof(out)); + + return out.state; +} + +int xsc_modify_vport_admin_state(struct xsc_core_device *dev, u8 opmod, + u16 vport, u8 other_vport, u8 state) +{ + struct xsc_modify_vport_state_in in; + struct xsc_modify_vport_state_out out; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = XSC_CMD_OP_MODIFY_VPORT_STATE; + in.hdr.opmod = opmod; + in.vport_number = vport; + in.other_vport = other_vport; + in.admin_state = state; + + return xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); +} + +static int __xsc_query_nic_vport_context(struct xsc_core_device *dev, + u16 vport, void *out, int outlen, + int force_other) +{ + struct xsc_query_nic_vport_context_in in; + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = XSC_CMD_OP_QUERY_NIC_VPORT_CONTEXT; + in.vport_number = vport; + if (vport || force_other) + in.other_vport = 1; + + return xsc_cmd_exec(dev, &in, sizeof(in), out, outlen); +} + +static int xsc_query_nic_vport_context(struct xsc_core_device *dev, u16 vport, + void *out, int outlen) +{ + return __xsc_query_nic_vport_context(dev, vport, out, outlen, 0); +} + +static int xsc_modify_nic_vport_context(struct xsc_core_device *dev, void *in, + int inlen) +{ + struct xsc_modify_nic_vport_context_out out; + struct xsc_modify_nic_vport_context_in *tmp; + + memset(&out, 0, sizeof(out)); + tmp = (struct xsc_modify_nic_vport_context_in *)in; + tmp->hdr.opcode = XSC_CMD_OP_MODIFY_NIC_VPORT_CONTEXT; + return xsc_cmd_exec(dev, in, inlen, &out, sizeof(out)); +} + +int xsc_query_nic_vport_min_inline(struct xsc_core_device *dev, + u16 vport, u8 *min_inline) +{ + struct xsc_query_nic_vport_context_out out; + int err; + + memset(&out, 0, sizeof(out)); + err = xsc_query_nic_vport_context(dev, vport, &out, sizeof(out)); + if (!err) + *min_inline = out.nic_vport_ctx.min_wqe_inline_mode; + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_min_inline); + +void xsc_query_min_inline(struct xsc_core_device *dev, + u8 *min_inline_mode) +{ + switch (dev->caps.wqe_inline_mode) { + case XSC_CAP_INLINE_MODE_VPORT_CONTEXT: + if (!xsc_query_nic_vport_min_inline(dev, 0, min_inline_mode)) + break; + case XSC_CAP_INLINE_MODE_L2: + *min_inline_mode = XSC_INLINE_MODE_L2; + break; + case XSC_CAP_INLINE_MODE_NOT_REQUIRED: + *min_inline_mode = XSC_INLINE_MODE_NONE; + break; + } +} +EXPORT_SYMBOL_GPL(xsc_query_min_inline); + +int xsc_modify_nic_vport_min_inline(struct xsc_core_device *dev, + u16 vport, u8 min_inline) +{ + struct xsc_modify_nic_vport_context_in in; + + memset(&in, 0, sizeof(in)); + in.field_select.min_inline = 1; + in.vport_number = vport; + in.other_vport = 1; + in.nic_vport_ctx.min_wqe_inline_mode = min_inline; + + return xsc_modify_nic_vport_context(dev, &in, sizeof(in)); +} + +static int __xsc_query_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr, + int force_other) +{ + struct xsc_query_nic_vport_context_out out; + u8 *out_addr; + int err; + + memset(&out, 0, sizeof(out)); + out_addr = out.nic_vport_ctx.permanent_address; + + err = __xsc_query_nic_vport_context(dev, vport, &out, sizeof(out), + force_other); + if (!err) + ether_addr_copy(addr, out_addr); + + return err; +} + +int xsc_query_other_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr) +{ + return __xsc_query_nic_vport_mac_address(dev, vport, addr, 1); +} +EXPORT_SYMBOL_GPL(xsc_query_other_nic_vport_mac_address); + +int xsc_query_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr) +{ + return __xsc_query_nic_vport_mac_address(dev, vport, addr, 0); +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_mac_address); + +static int __xsc_modify_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr, + int force_other) +{ + struct xsc_modify_nic_vport_context_in in; + int err; + u8 *perm_mac; + + memset(&in, 0, sizeof(in)); + in.field_select.permanent_address = 1; + in.vport_number = vport; + + if (vport || force_other) + in.other_vport = 1; + + perm_mac = in.nic_vport_ctx.permanent_address; + + ether_addr_copy(perm_mac, addr); + + err = xsc_modify_nic_vport_context(dev, &in, sizeof(in)); + + return err; +} + +int xsc_modify_other_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr) +{ + return __xsc_modify_nic_vport_mac_address(dev, vport, addr, 1); +} +EXPORT_SYMBOL_GPL(xsc_modify_other_nic_vport_mac_address); + +int xsc_modify_nic_vport_mac_address(struct xsc_core_device *dev, + u16 vport, u8 *addr) +{ + return __xsc_modify_nic_vport_mac_address(dev, vport, addr, 0); +} +EXPORT_SYMBOL_GPL(xsc_modify_nic_vport_mac_address); + +int xsc_query_nic_vport_mtu(struct xsc_core_device *dev, u16 *mtu) +{ + struct xsc_query_nic_vport_context_out out; + int err; + + memset(&out, 0, sizeof(out)); + err = xsc_query_nic_vport_context(dev, 0, &out, sizeof(out)); + if (!err) + *mtu = out.nic_vport_ctx.mtu; + + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_mtu); + +int xsc_modify_nic_vport_mtu(struct xsc_core_device *dev, u16 mtu) +{ + struct xsc_modify_nic_vport_context_in in; + int err; + + memset(&in, 0, sizeof(in)); + in.field_select.mtu = 1; + in.nic_vport_ctx.mtu = mtu; + + err = xsc_modify_nic_vport_context(dev, &in, sizeof(in)); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_modify_nic_vport_mtu); + +int xsc_query_nic_vport_mac_list(struct xsc_core_device *dev, + u16 vport, + enum xsc_list_type list_type, + u8 addr_list[][ETH_ALEN], + int *list_size) +{ + struct xsc_query_nic_vport_context_in in; + struct xsc_query_nic_vport_context_out *out; + int max_list_size; + int req_list_size; + int out_sz; + int err; + int i; + + req_list_size = *list_size; + + max_list_size = list_type == XSC_NVPRT_LIST_TYPE_UC ? + 1 << dev->caps.log_max_current_uc_list : + 1 << dev->caps.log_max_current_mc_list; + + if (req_list_size > max_list_size) { + xsc_core_warn(dev, "Requested list size (%d) > (%d) max_list_size\n", + req_list_size, max_list_size); + req_list_size = max_list_size; + } + + out_sz = sizeof(struct xsc_query_nic_vport_context_out) + + req_list_size * 8; + + memset(&in, 0, sizeof(in)); + out = kzalloc(out_sz, GFP_KERNEL); + if (!out) + return -ENOMEM; + + in.hdr.opcode = XSC_CMD_OP_QUERY_NIC_VPORT_CONTEXT; + in.allowed_list_type = list_type; + in.vport_number = vport; + in.other_vport = 1; + + err = xsc_cmd_exec(dev, &in, sizeof(in), out, out_sz); + if (err) + goto out; + + req_list_size = out->nic_vport_ctx.allowed_list_size; + *list_size = req_list_size; + for (i = 0; i < req_list_size; i++) { + u8 *mac_addr = (u8 *)out->nic_vport_ctx.current_uc_mac_address[i]; + + ether_addr_copy(addr_list[i], mac_addr); + } +out: + kfree(out); + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_mac_list); + +int xsc_modify_nic_vport_mac_list(struct xsc_core_device *dev, + enum xsc_list_type list_type, + u8 addr_list[][ETH_ALEN], + int list_size) +{ + struct xsc_modify_nic_vport_context_out out; + struct xsc_modify_nic_vport_context_in *in; + int max_list_size; + int in_sz; + int err; + int i; + + max_list_size = list_type == XSC_NVPRT_LIST_TYPE_UC ? + 1 << dev->caps.log_max_current_uc_list : + 1 << dev->caps.log_max_current_mc_list; + + if (list_size > max_list_size) + return -ENOSPC; + + in_sz = sizeof(struct xsc_modify_nic_vport_context_in) + + list_size * 8; + in = kzalloc(in_sz, GFP_KERNEL); + if (!in) + return -ENOMEM; + + in->hdr.opcode = XSC_CMD_OP_MODIFY_NIC_VPORT_CONTEXT; + in->field_select.addresses_list = 1; + in->nic_vport_ctx.allowed_list_type = list_type; + in->nic_vport_ctx.allowed_list_size = list_size; + + for (i = 0; i < list_size; i++) { + u8 *curr_mac = + (u8 *)(in->nic_vport_ctx.current_uc_mac_address[i]); + ether_addr_copy(curr_mac, addr_list[i]); + } + + memset(&out, 0, sizeof(out)); + err = xsc_cmd_exec(dev, in, in_sz, &out, sizeof(out)); + kfree(in); + return err; +} +EXPORT_SYMBOL_GPL(xsc_modify_nic_vport_mac_list); + +int xsc_query_nic_vport_vlans(struct xsc_core_device *dev, u32 vport, + unsigned long *vlans) +{ + struct xsc_query_nic_vport_context_in in; + struct xsc_query_nic_vport_context_out *out; + int req_list_size; + int out_sz; + int err; + int i; + + req_list_size = 1 << dev->caps.log_max_vlan_list; + out_sz = sizeof(*out) + req_list_size * 8; + + out = kzalloc(out_sz, GFP_KERNEL); + if (!out) + return -ENOMEM; + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = XSC_CMD_OP_QUERY_NIC_VPORT_CONTEXT; + in.allowed_list_type = XSC_NVPRT_LIST_TYPE_VLAN; + in.vport_number = vport; + + if (vport) + in.other_vport = 1; + + err = xsc_cmd_exec(dev, &in, sizeof(in), out, out_sz); + if (err) + goto out; + + req_list_size = out->nic_vport_ctx.allowed_list_size; + + for (i = 0; i < req_list_size; i++) { + u16 *vlan_addr = (u16 *)out->nic_vport_ctx.current_uc_mac_address[i]; + + bitmap_set(vlans, (*vlan_addr & 0xfff), 1); + } +out: + kfree(out); + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_vlans); + +int xsc_modify_nic_vport_vlans(struct xsc_core_device *dev, + u16 vlans[], + int list_size) +{ + struct xsc_modify_nic_vport_context_out out; + struct xsc_modify_nic_vport_context_in *in; + int max_list_size; + int in_sz; + int err; + int i; + + max_list_size = 1 << dev->caps.log_max_vlan_list; + + if (list_size > max_list_size) + return -ENOSPC; + + in_sz = sizeof(struct xsc_modify_nic_vport_context_in) + + list_size * 8; + + in = kzalloc(in_sz, GFP_KERNEL); + if (!in) + return -ENOMEM; + + in->hdr.opcode = XSC_CMD_OP_MODIFY_NIC_VPORT_CONTEXT; + in->field_select.addresses_list = 1; + + in->nic_vport_ctx.allowed_list_type = XSC_NVPRT_LIST_TYPE_VLAN; + in->nic_vport_ctx.allowed_list_size = list_size; + + for (i = 0; i < list_size; i++) { + u16 *vlan_addr = (u16 *)in->nic_vport_ctx.current_uc_mac_address[i]; + *vlan_addr = vlans[i]; + } + + memset(&out, 0, sizeof(out)); + err = xsc_cmd_exec(dev, in, in_sz, &out, sizeof(out)); + kfree(in); + return err; +} +EXPORT_SYMBOL_GPL(xsc_modify_nic_vport_vlans); + +int xsc_query_nic_vport_system_image_guid(struct xsc_core_device *dev, + u64 *system_image_guid) +{ + struct xsc_query_nic_vport_context_out out; + + memset(&out, 0, sizeof(out)); + xsc_query_nic_vport_context(dev, 0, &out, sizeof(out)); + + *system_image_guid = out.nic_vport_ctx.system_image_guid; + + return 0; +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_system_image_guid); + +int xsc_query_nic_vport_node_guid(struct xsc_core_device *dev, u32 vport, + u64 *node_guid) +{ + struct xsc_query_nic_vport_context_out out; + + memset(&out, 0, sizeof(out)); + xsc_query_nic_vport_context(dev, vport, &out, sizeof(out)); + + *node_guid = out.nic_vport_ctx.node_guid; + + return 0; +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_node_guid); + +static int __xsc_modify_nic_vport_node_guid(struct xsc_core_device *dev, + u16 vport, u64 node_guid, + int force_other) +{ + struct xsc_modify_nic_vport_context_in in; + int err; + + /* vport = 0 only if ECPF modifying Host PF */ + if (!vport && !force_other) + return -EINVAL; + if (!dev->caps.vport_group_manager) + return -EACCES; + + memset(&in, 0, sizeof(in)); + in.field_select.node_guid = 1; + in.vport_number = vport; + if (vport || force_other) + in.other_vport = 1; + + in.nic_vport_ctx.node_guid = node_guid; + + err = xsc_modify_nic_vport_context(dev, &in, sizeof(in)); + + return err; +} + +int xsc_modify_nic_vport_node_guid(struct xsc_core_device *dev, + u16 vport, u64 node_guid) +{ + return __xsc_modify_nic_vport_node_guid(dev, vport, node_guid, 0); +} + +int xsc_modify_other_nic_vport_node_guid(struct xsc_core_device *dev, + u16 vport, u64 node_guid) +{ + return __xsc_modify_nic_vport_node_guid(dev, vport, node_guid, 1); +} + +int xsc_query_nic_vport_qkey_viol_cntr(struct xsc_core_device *dev, + u16 *qkey_viol_cntr) +{ + struct xsc_query_nic_vport_context_out out; + + memset(&out, 0, sizeof(out)); + xsc_query_nic_vport_context(dev, 0, &out, sizeof(out)); + + *qkey_viol_cntr = out.nic_vport_ctx.qkey_violation_counter; + + return 0; +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_qkey_viol_cntr); + +int xsc_query_hca_vport_gid(struct xsc_core_device *dev, u8 other_vport, + u8 port_num, u16 vf_num, u16 gid_index, + union ib_gid *gid) +{ + int in_sz = sizeof(struct xsc_query_hca_vport_gid_in); + int out_sz = sizeof(struct xsc_query_hca_vport_gid_out); + struct xsc_query_hca_vport_gid_in *in; + struct xsc_query_hca_vport_gid_out *out; + int is_group_manager; + union ib_gid *tmp; + int tbsz; + int nout; + int err; + + is_group_manager = dev->caps.vport_group_manager; + tbsz = dev->caps.port[port_num].gid_table_len; + xsc_core_dbg(dev, "vf_num %d, index %d, gid_table_size %d\n", + vf_num, gid_index, tbsz); + + if (gid_index > tbsz && gid_index != 0xffff) + return -EINVAL; + + if (gid_index == 0xffff) + nout = tbsz; + else + nout = 1; + + out_sz += nout * sizeof(*gid); + + in = kzalloc(in_sz, GFP_KERNEL); + out = kzalloc(out_sz, GFP_KERNEL); + if (!in || !out) { + err = -ENOMEM; + goto out; + } + + in->hdr.opcode = XSC_CMD_OP_QUERY_HCA_VPORT_GID; + if (other_vport) { + if (is_group_manager) { + in->vport_number = vf_num; + in->other_vport = 1; + } else { + err = -EPERM; + goto out; + } + } + + in->gid_index = gid_index; + in->port_num = port_num; + + err = xsc_cmd_exec(dev, in, in_sz, out, out_sz); + if (err) + goto out; + + tmp = (union ib_gid *)((void *)out + + sizeof(struct xsc_query_hca_vport_gid_out)); + gid->global.subnet_prefix = tmp->global.subnet_prefix; + gid->global.interface_id = tmp->global.interface_id; + +out: + kfree(in); + kfree(out); + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_hca_vport_gid); + +int xsc_query_hca_vport_pkey(struct xsc_core_device *dev, u8 other_vport, + u8 port_num, u16 vf_num, u16 pkey_index, + u16 *pkey) +{ + int in_sz = sizeof(struct xsc_query_hca_vport_pkey_in); + int out_sz = sizeof(struct xsc_query_hca_vport_pkey_out); + struct xsc_query_hca_vport_pkey_in *in; + struct xsc_query_hca_vport_pkey_out *out; + int is_group_manager; + void *pkarr; + int nout; + int tbsz; + int err; + int i; + + is_group_manager = dev->caps.vport_group_manager; + + tbsz = dev->caps.port[port_num].pkey_table_len; + if (pkey_index > tbsz && pkey_index != 0xffff) + return -EINVAL; + + if (pkey_index == 0xffff) + nout = tbsz; + else + nout = 1; + + out_sz += nout*sizeof(*pkey); + + in = kzalloc(in_sz, GFP_KERNEL); + out = kzalloc(out_sz, GFP_KERNEL); + if (!in || !out) { + err = -ENOMEM; + goto out; + } + + in->hdr.opcode = XSC_CMD_OP_QUERY_HCA_VPORT_PKEY; + if (other_vport) { + if (is_group_manager) { + in->vport_number = vf_num; + in->other_vport = 1; + } else { + err = -EPERM; + goto out; + } + } + in->pkey_index = pkey_index; + + if (dev->caps.num_ports == 2) + in->port_num = port_num; + + err = xsc_cmd_exec(dev, in, in_sz, out, out_sz); + if (err) + goto out; + + pkarr = out->pkey; + for (i = 0; i < nout; i++, pkey++, pkarr += sizeof(*pkey)) + *pkey = *(u16 *)pkarr; + +out: + kfree(in); + kfree(out); + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_hca_vport_pkey); + +int xsc_query_hca_vport_context(struct xsc_core_device *dev, + u8 other_vport, u8 port_num, + u16 vf_num, + struct xsc_hca_vport_context *rep) +{ + struct xsc_query_hca_vport_context_out *out = NULL; + struct xsc_query_hca_vport_context_in in; + int is_group_manager; + void *ctx; + int err; + + is_group_manager = dev->caps.vport_group_manager; + + memset(&in, 0, sizeof(in)); + in.hdr.opcode = XSC_CMD_OP_QUERY_HCA_VPORT_CONTEXT; + + if (other_vport) { + if (is_group_manager) { + in.other_vport = 1; + in.vport_number = vf_num; + } else { + err = -EPERM; + goto ex; + } + } + + out = kzalloc(sizeof(*out), GFP_KERNEL); + if (!out) + return -ENOMEM; + + if (dev->caps.num_ports == 2) + in.port_num = port_num; + + err = xsc_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); + if (err) + goto ex; + + ctx = &out->hca_vport_ctx; + memcpy(rep, ctx, sizeof(struct xsc_hca_vport_context)); + +ex: + kfree(out); + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_hca_vport_context); + +int xsc_query_hca_vport_node_guid(struct xsc_core_device *dev, + u64 *node_guid) +{ + struct xsc_hca_vport_context *rep; + int err; + + rep = kzalloc(sizeof(*rep), GFP_KERNEL); + if (!rep) + return -ENOMEM; + + err = xsc_query_hca_vport_context(dev, 0, 1, 0, rep); + if (!err) + *node_guid = rep->node_guid; + + kfree(rep); + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_hca_vport_node_guid); + +int xsc_query_nic_vport_promisc(struct xsc_core_device *dev, + u16 vport, + int *promisc_uc, + int *promisc_mc, + int *promisc_all) +{ + struct xsc_query_nic_vport_context_out *out; + int err; + + out = kzalloc(sizeof(out), GFP_KERNEL); + if (!out) + return -ENOMEM; + + err = xsc_query_nic_vport_context(dev, vport, out, sizeof(*out)); + if (err) + goto out; + + *promisc_uc = out->nic_vport_ctx.promisc_uc; + *promisc_mc = out->nic_vport_ctx.promisc_mc; + *promisc_all = out->nic_vport_ctx.promisc_all; + +out: + kfree(out); + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_nic_vport_promisc); + +int xsc_modify_nic_vport_promisc(struct xsc_core_device *dev, + int promisc_uc, + int promisc_mc, + int promisc_all) +{ + struct xsc_modify_nic_vport_context_in *in; + int err; + + in = kvzalloc(sizeof(*in), GFP_KERNEL); + if (!in) + return -ENOMEM; + + in->field_select.promisc = 1; + in->nic_vport_ctx.promisc_uc = promisc_uc; + in->nic_vport_ctx.promisc_mc = promisc_mc; + in->nic_vport_ctx.promisc_all = promisc_all; + + err = xsc_modify_nic_vport_context(dev, in, sizeof(*in)); + + kvfree(in); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_modify_nic_vport_promisc); + +int xsc_query_vport_counter(struct xsc_core_device *dev, u8 other_vport, + int vf, u8 port_num, void *out, + size_t out_sz) +{ + struct xsc_query_vport_counter_in *in; + int is_group_manager; + int err; + + is_group_manager = dev->caps.vport_group_manager; + in = kvzalloc(sizeof(*in), GFP_KERNEL); + if (!in) { + err = -ENOMEM; + return err; + } + + in->hdr.opcode = XSC_CMD_OP_QUERY_VPORT_COUNTER; + if (other_vport) { + if (is_group_manager) { + in->other_vport = 1; + in->vport_number = (vf + 1); + } else { + err = -EPERM; + goto free; + } + } + + if (dev->caps.num_ports == 2) + in->port_num = port_num; + + err = xsc_cmd_exec(dev, in, sizeof(*in), out, out_sz); +free: + kvfree(in); + return err; +} +EXPORT_SYMBOL_GPL(xsc_query_vport_counter); + +int xsc_modify_hca_vport_context(struct xsc_core_device *dev, + u8 other_vport, u8 port_num, + int vf, + struct xsc_hca_vport_context *req) +{ + struct xsc_modify_hca_vport_context_in in; + struct xsc_modify_hca_vport_context_out out; + int is_group_manager; + int err; + + xsc_core_dbg(dev, "vf %d\n", vf); + is_group_manager = dev->caps.vport_group_manager; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + in.hdr.opcode = XSC_CMD_OP_MODIFY_HCA_VPORT_CONTEXT; + if (other_vport) { + if (is_group_manager) { + in.other_vport = 1; + in.vport_number = vf; + } else { + err = -EPERM; + goto err; + } + } + + if (dev->caps.num_ports > 1) + in.port_num = port_num; + memcpy(&in.hca_vport_ctx, req, sizeof(*req)); + err = xsc_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); +err: + return err; +} +EXPORT_SYMBOL_GPL(xsc_modify_hca_vport_context); + +/** + * xsc_eswitch_get_total_vports - Get total vports of the eswitch + * + * @dev: Pointer to core device + * + * xsc_eswitch_get_total_vports returns total number of vports for + * the eswitch. + */ +u16 xsc_eswitch_get_total_vports(const struct xsc_core_device *dev) +{ + return XSC_SPECIAL_VPORTS(dev) + xsc_core_max_vfs(dev); +} +EXPORT_SYMBOL(xsc_eswitch_get_total_vports); diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/wq.c b/drivers/net/ethernet/yunsilicon/xsc/pci/wq.c new file mode 100644 index 0000000000000000000000000000000000000000..7de6aacf0b6f6478c9145fe6e8a914a6d1efa54d --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/wq.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include "wq.h" + +u32 xsc_wq_cyc_get_size(struct xsc_wq_cyc *wq) +{ + return (u32)wq->fbc.sz_m1 + 1; +} +EXPORT_SYMBOL_GPL(xsc_wq_cyc_get_size); + +static u32 wq_get_byte_sz(u8 log_sz, u8 log_stride) +{ + return ((u32)1 << log_sz) << log_stride; +} + +int xsc_eth_cqwq_create(struct xsc_core_device *xdev, struct xsc_wq_param *param, + u8 q_log_size, u8 ele_log_size, struct xsc_cqwq *wq, + struct xsc_wq_ctrl *wq_ctrl) +{ + u8 log_wq_stride = ele_log_size; + u8 log_wq_sz = q_log_size; + int err; + + err = xsc_db_alloc_node(xdev, &wq_ctrl->db, param->db_numa_node); + if (err) { + xsc_core_warn(xdev, "xsc_db_alloc_node() failed, %d\n", err); + return err; + } + + err = xsc_frag_buf_alloc_node(xdev, wq_get_byte_sz(log_wq_sz, log_wq_stride), + &wq_ctrl->buf, + param->buf_numa_node); + if (err) { + xsc_core_warn(xdev, "xsc_frag_buf_alloc_node() failed, %d\n", + err); + goto err_db_free; + } + + xsc_init_fbc(wq_ctrl->buf.frags, log_wq_stride, log_wq_sz, &wq->fbc); + + wq_ctrl->xdev = xdev; + + return 0; + +err_db_free: + xsc_db_free(xdev, &wq_ctrl->db); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_eth_cqwq_create); + +int xsc_eth_wq_cyc_create(struct xsc_core_device *xdev, struct xsc_wq_param *param, + u8 q_log_size, u8 ele_log_size, struct xsc_wq_cyc *wq, + struct xsc_wq_ctrl *wq_ctrl) +{ + u8 log_wq_stride = ele_log_size; + u8 log_wq_sz = q_log_size; + struct xsc_frag_buf_ctrl *fbc = &wq->fbc; + int err; + + err = xsc_db_alloc_node(xdev, &wq_ctrl->db, param->db_numa_node); + if (err) { + xsc_core_warn(xdev, "xsc_db_alloc_node() failed, %d\n", err); + return err; + } + + err = xsc_frag_buf_alloc_node(xdev, wq_get_byte_sz(log_wq_sz, log_wq_stride), + &wq_ctrl->buf, param->buf_numa_node); + if (err) { + xsc_core_warn(xdev, "xsc_frag_buf_alloc_node() failed, %d\n", err); + goto err_db_free; + } + + xsc_init_fbc(wq_ctrl->buf.frags, log_wq_stride, log_wq_sz, fbc); + wq->sz = xsc_wq_cyc_get_size(wq); + + wq_ctrl->xdev = xdev; + + return 0; + +err_db_free: + xsc_db_free(xdev, &wq_ctrl->db); + + return err; +} +EXPORT_SYMBOL_GPL(xsc_eth_wq_cyc_create); + +void xsc_eth_wq_destroy(struct xsc_wq_ctrl *wq_ctrl) +{ + xsc_frag_buf_free(wq_ctrl->xdev, &wq_ctrl->buf); + xsc_db_free(wq_ctrl->xdev, &wq_ctrl->db); +} +EXPORT_SYMBOL_GPL(xsc_eth_wq_destroy); + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/wq.h b/drivers/net/ethernet/yunsilicon/xsc/pci/wq.h new file mode 100644 index 0000000000000000000000000000000000000000..c6fc5a73cd4ae3601d81e3d9cc8a027f73dcddbe --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/wq.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef __XSC_WQ_H__ +#define __XSC_WQ_H__ + +#include +#include + +struct xsc_wq_param { + int buf_numa_node; + int db_numa_node; +}; + +struct xsc_wq_ctrl { + struct xsc_core_device *xdev; + struct xsc_frag_buf buf; + struct xsc_db db; +}; + +struct xsc_wq_cyc { + struct xsc_frag_buf_ctrl fbc; + u16 sz; + u16 wqe_ctr; + u16 cur_sz; +}; + +struct xsc_cqwq { + struct xsc_frag_buf_ctrl fbc; + __be32 *db; + u32 cc; /* consumer counter */ +}; + +enum xsc_res_type { + XSC_RES_UND = 0, + XSC_RES_RQ, + XSC_RES_SQ, + XSC_RES_MAX, +}; + +u32 xsc_wq_cyc_get_size(struct xsc_wq_cyc *wq); + +int xsc_buf_alloc_node(struct xsc_core_device *dev, int size, + struct xsc_frag_buf *buf, int node); + +/*api for eth driver*/ +int xsc_eth_cqwq_create(struct xsc_core_device *xdev, struct xsc_wq_param *param, + u8 q_log_size, u8 ele_log_size, struct xsc_cqwq *wq, + struct xsc_wq_ctrl *wq_ctrl); + +int xsc_eth_wq_cyc_create(struct xsc_core_device *xdev, struct xsc_wq_param *param, + u8 q_log_size, u8 ele_log_size, struct xsc_wq_cyc *wq, + struct xsc_wq_ctrl *wq_ctrl); +void xsc_eth_wq_destroy(struct xsc_wq_ctrl *wq_ctrl); + +static inline void xsc_init_fbc_offset(struct xsc_buf_list *frags, + u8 log_stride, u8 log_sz, + u16 strides_offset, + struct xsc_frag_buf_ctrl *fbc) +{ + fbc->frags = frags; + fbc->log_stride = log_stride; + fbc->log_sz = log_sz; + fbc->sz_m1 = (1 << fbc->log_sz) - 1; + fbc->log_frag_strides = PAGE_SHIFT - fbc->log_stride; + fbc->frag_sz_m1 = (1 << fbc->log_frag_strides) - 1; + fbc->strides_offset = strides_offset; +} + +static inline void xsc_init_fbc(struct xsc_buf_list *frags, + u8 log_stride, u8 log_sz, + struct xsc_frag_buf_ctrl *fbc) +{ + xsc_init_fbc_offset(frags, log_stride, log_sz, 0, fbc); +} + +static inline void *xsc_frag_buf_get_wqe(struct xsc_frag_buf_ctrl *fbc, + u32 ix) +{ + unsigned int frag; + + ix += fbc->strides_offset; + frag = ix >> fbc->log_frag_strides; + + return fbc->frags[frag].buf + ((fbc->frag_sz_m1 & ix) << fbc->log_stride); +} + +static inline u32 +xsc_frag_buf_get_idx_last_contig_stride(struct xsc_frag_buf_ctrl *fbc, u32 ix) +{ + u32 last_frag_stride_idx = (ix + fbc->strides_offset) | fbc->frag_sz_m1; + + return min_t(u32, last_frag_stride_idx - fbc->strides_offset, fbc->sz_m1); +} + +static inline int xsc_wq_cyc_missing(struct xsc_wq_cyc *wq) +{ + return wq->sz - wq->cur_sz; +} + +static inline int xsc_wq_cyc_is_empty(struct xsc_wq_cyc *wq) +{ + return !wq->cur_sz; +} + +static inline void xsc_wq_cyc_push(struct xsc_wq_cyc *wq) +{ + wq->wqe_ctr++; + wq->cur_sz++; +} + +static inline void xsc_wq_cyc_push_n(struct xsc_wq_cyc *wq, u8 n) +{ + wq->wqe_ctr += n; + wq->cur_sz += n; +} + +static inline void xsc_wq_cyc_pop(struct xsc_wq_cyc *wq) +{ + wq->cur_sz--; +} + +static inline u16 xsc_wq_cyc_ctr2ix(struct xsc_wq_cyc *wq, u16 ctr) +{ + return ctr & wq->fbc.sz_m1; +} + +static inline u16 xsc_wq_cyc_get_head(struct xsc_wq_cyc *wq) +{ + return xsc_wq_cyc_ctr2ix(wq, wq->wqe_ctr); +} + +static inline u16 xsc_wq_cyc_get_tail(struct xsc_wq_cyc *wq) +{ + return xsc_wq_cyc_ctr2ix(wq, wq->wqe_ctr - wq->cur_sz); +} + +static inline void *xsc_wq_cyc_get_wqe(struct xsc_wq_cyc *wq, u16 ix) +{ + return xsc_frag_buf_get_wqe(&wq->fbc, ix); +} + +static inline u32 xsc_cqwq_ctr2ix(struct xsc_cqwq *wq, u32 ctr) +{ + return ctr & wq->fbc.sz_m1; +} + +static inline u32 xsc_cqwq_get_ci(struct xsc_cqwq *wq) +{ + return xsc_cqwq_ctr2ix(wq, wq->cc); +} + +static inline u32 xsc_cqwq_get_ctr_wrap_cnt(struct xsc_cqwq *wq, u32 ctr) +{ + return ctr >> wq->fbc.log_sz; +} + +static inline u32 xsc_cqwq_get_wrap_cnt(struct xsc_cqwq *wq) +{ + return xsc_cqwq_get_ctr_wrap_cnt(wq, wq->cc); +} + +static inline void xsc_cqwq_pop(struct xsc_cqwq *wq) +{ + wq->cc++; +} + +#endif /* __XSC_WQ_H__ */ diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_lag.c b/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_lag.c new file mode 100644 index 0000000000000000000000000000000000000000..89c5303ede3a4098e7e5cfbddcf39c5005cadd44 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_lag.c @@ -0,0 +1,1316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include "fw/xsc_tbm.h" + +//#define XSC_LAG_DEBUG + +#ifdef fib_nh_dev +#define HAVE_FIB_NH_DEV +#endif + +static DEFINE_MUTEX(lag_mutex); + +static inline u8 xsc_lag_hashtype_convert(struct xsc_core_device *xdev, struct xsc_lag *ldev) +{ + enum netdev_lag_hash hash_type = ldev->tracker.hash_type; + u8 lag_sel_mode; + + xsc_core_info(xdev, "hash_type = %d\n", hash_type); + + switch (hash_type) { + case NETDEV_LAG_HASH_L23: + lag_sel_mode = XSC_LAG_HASH_L23; + break; + + case NETDEV_LAG_HASH_L34: + lag_sel_mode = XSC_LAG_HASH_L34; + break; + + case NETDEV_LAG_HASH_E23: + lag_sel_mode = XSC_LAG_HASH_E23; + break; + + case NETDEV_LAG_HASH_E34: + lag_sel_mode = XSC_LAG_HASH_E34; + break; + + default: + lag_sel_mode = XSC_LAG_HASH_L23; + break; + } + + return lag_sel_mode; +} + +int xsc_cmd_create_lag(struct xsc_lag *ldev, u8 flags) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + struct xsc_core_device *xdev1 = ldev->pf[1].xdev; + struct net_device *netdev0 = xdev0->netdev; + struct net_device *netdev1 = xdev1->netdev; + struct lag_tracker *tracker = &ldev->tracker; + u8 remap_port1 = ldev->v2p_map[0]; + u8 remap_port2 = ldev->v2p_map[1]; + int ret = -1; + + struct xsc_create_lag_mbox_in in = {}; + struct xsc_create_lag_mbox_out out = {}; + struct xsc_lag_port_info *info_mac0 = &in.req.info_mac0; + struct xsc_lag_port_info *info_mac1 = &in.req.info_mac1; + + bool mp_lag = flags & XSC_LAG_FLAG_MULTIPATH; + bool roce_lag = flags & XSC_LAG_FLAG_ROCE; + bool sriov_lag = flags & XSC_LAG_FLAG_SRIOV; + + u16 lag_id = XSC_LAG_PORT_START; + u8 lag_num = XSC_LAG_NUM_MAX; + + if (!(flags & XSC_LAG_MODE_FLAGS)) + return -EINVAL; + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_LAG_CREATE); + + in.req.mp_lag = mp_lag; + in.req.roce_lag = roce_lag; + in.req.lag_id = cpu_to_be16(lag_id); + in.req.lag_num = lag_num; + in.req.lag_start = cpu_to_be16(lag_id); + in.req.lag_sel_mode = mp_lag ? XSC_LAG_HASH_L34 : xsc_lag_hashtype_convert(xdev0, ldev); + in.req.remap_port1 = remap_port1; + in.req.remap_port2 = remap_port2; + + xsc_core_info(xdev0, "modify lag: lag_id = %d, mp_lag=%d, roce_lag=%d, sriov_lag=%d, lag_sel_mode = %d\n", + lag_id, mp_lag, roce_lag, sriov_lag, in.req.lag_sel_mode); + + memcpy(info_mac0->netdev_addr, netdev0->dev_addr, ETH_ALEN); + memcpy(info_mac1->netdev_addr, netdev1->dev_addr, ETH_ALEN); + memcpy(info_mac0->gw_dmac, tracker->gw_dmac0, ETH_ALEN); + memcpy(info_mac1->gw_dmac, tracker->gw_dmac1, ETH_ALEN); + info_mac0->info_mac.mac_logic_port = xdev0->mac_logic_port; + info_mac0->info_mac.logic_port = xdev0->logic_port; + info_mac0->info_mac.glb_func_id = xdev0->glb_func_id; + info_mac1->info_mac.mac_logic_port = xdev1->mac_logic_port; + info_mac1->info_mac.logic_port = xdev1->logic_port; + info_mac1->info_mac.glb_func_id = xdev1->glb_func_id; + + ret = xsc_cmd_exec(xdev0, &in, sizeof(in), &out, sizeof(out)); + if (ret) { + xsc_core_err(xdev0, "%s:error\n", __func__); + return ret; + } + + ldev->lag_id = lag_id; + + return ret; +} + +int xsc_cmd_modify_lag(struct xsc_lag *ldev) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + u16 lag_id = ldev->lag_id; + bool mp_lag = ldev->flags & XSC_LAG_FLAG_MULTIPATH; + bool roce_lag = ldev->flags & XSC_LAG_FLAG_ROCE; + bool sriov_lag = ldev->flags & XSC_LAG_FLAG_SRIOV; + u8 remap_port1 = ldev->v2p_map[0]; + u8 remap_port2 = ldev->v2p_map[1]; + + int ret = -1; + + struct xsc_modify_lag_mbox_in in = {}; + struct xsc_modify_lag_mbox_out out = {}; + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_LAG_MODIFY); + + in.req.mp_lag = mp_lag; + in.req.roce_lag = roce_lag; + in.req.lag_id = cpu_to_be16(lag_id); + in.req.remap_port1 = remap_port1; + in.req.remap_port2 = remap_port2; + in.req.lag_sel_mode = mp_lag ? XSC_LAG_HASH_L34 : xsc_lag_hashtype_convert(xdev0, ldev); + + xsc_core_info(xdev0, "modify lag: lag_id = %d, mp_lag=%d, roce_lag=%d, sriov_lag=%d, lag_sel_mode = %d\n", + lag_id, mp_lag, roce_lag, sriov_lag, in.req.lag_sel_mode); + + ret = xsc_cmd_exec(xdev0, &in, sizeof(in), &out, sizeof(out)); + if (ret) { + xsc_core_err(xdev0, "%s:error\n", __func__); + return ret; + } + + return ret; +} + +int xsc_cmd_destroy_lag(struct xsc_lag *ldev) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + u8 flags = ldev->flags; + int ret = -1; + + struct xsc_destroy_lag_mbox_in in = {}; + struct xsc_destroy_lag_mbox_out out = {}; + + if (!(flags & XSC_LAG_MODE_FLAGS)) + return -EINVAL; + + xsc_core_info(xdev0, "destroy lag: lag_id = %d\n", ldev->lag_id); + + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_LAG_DESTROY); + + in.req.lag_id = ldev->lag_id; + + ret = xsc_cmd_exec(xdev0, &in, sizeof(in), &out, sizeof(out)); + if (ret) { + xsc_core_err(xdev0, "%s:error\n", __func__); + return ret; + } + + ldev->lag_id = 0xff; + + return ret; +} + +static inline bool xsc_is_roce_lag_allowed(struct xsc_lag *ldev) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + struct xsc_core_device *xdev1 = ldev->pf[1].xdev; + + return !xsc_sriov_is_enabled(xdev0) && !xsc_sriov_is_enabled(xdev1); +} + +static int xsc_lag_set_qos(struct xsc_core_device *xdev, u16 lag_id, u8 member_bitmap, u8 lag_del) +{ + struct xsc_set_lag_qos_mbox_in in; + struct xsc_set_lag_qos_mbox_out out; + struct xsc_set_lag_qos_request *req; + int ret; + + memset(&in, 0, sizeof(in)); + memset(&out, 0, sizeof(out)); + req = &in.req; + + req->lag_id = cpu_to_be16(lag_id); + req->member_bitmap = member_bitmap; + req->lag_del = lag_del; + req->pcie_no = xsc_get_pcie_no(); + req->esw_mode = xdev->priv.eswitch->mode; + in.hdr.opcode = cpu_to_be16(XSC_CMD_OP_LAG_SET_QOS); + + ret = xsc_cmd_exec(xdev, &in, sizeof(in), &out, sizeof(out)); + return ret; +} + +static bool xsc_lag_check_prereq(struct xsc_lag *ldev) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + struct xsc_core_device *xdev1 = ldev->pf[1].xdev; + + if (!xdev0 || !xdev1) + return false; + + if (xsc_is_roce_lag_allowed(ldev)) + return true; + +#ifdef CONFIG_XSC_ESWITCH + return (xdev0->priv.eswitch->mode == XSC_ESWITCH_OFFLOADS) && + (xdev1->priv.eswitch->mode == XSC_ESWITCH_OFFLOADS); +#else +#ifdef XSC_VF_ECMP_TEST + return true; +#else + return false; +#endif +#endif +} + +static void xsc_infer_tx_affinity_mapping(struct xsc_core_device *xdev0, struct xsc_lag *ldev, + u8 *port1, u8 *port2) +{ + struct lag_tracker *tracker = &ldev->tracker; + *port1 = MAC_INVALID; + *port2 = MAC_INVALID; + + if (tracker->netdev_state[0].tx_enabled && + tracker->netdev_state[0].link_up) + *port1 = MAC_0_LOGIC; + + if (tracker->netdev_state[1].tx_enabled && + tracker->netdev_state[1].link_up) + *port2 = MAC_1_LOGIC; + + xsc_core_info(xdev0, "tx_affinity_mapping: port1 = %d, port2 = %d\n", + *port1, *port2); +} + +static int xsc_create_lag(struct xsc_lag *ldev, u8 flags) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + int err; + + xsc_infer_tx_affinity_mapping(xdev0, ldev, &ldev->v2p_map[0], + &ldev->v2p_map[1]); + + xsc_core_info(xdev0, "xsc create lag 1:%d port 2:%d", + ldev->v2p_map[0], ldev->v2p_map[1]); + + err = xsc_cmd_create_lag(ldev, flags); + if (err) + xsc_core_err(xdev0, "Failed to create LAG (%d)\n", err); + + return err; +} + +void xsc_activate_lag(struct xsc_lag *ldev, u8 flags) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + u8 member_bitmap; + + if (xsc_create_lag(ldev, flags)) { + xsc_core_err(xdev0, "Failed to activate LAG\n" + "Make sure all VFs are unbound prior to LAG activation or deactivation\n"); + return; + } + + ldev->flags |= flags; + + member_bitmap = GET_LAG_MEMBER_BITMAP(ldev->v2p_map[0], ldev->v2p_map[1]); + if (xsc_lag_set_qos(xdev0, ldev->lag_id, member_bitmap, false)) + xsc_core_err(xdev0, "failed to set QoS for LAG %u\n", ldev->lag_id); +} + +void xsc_modify_lag(struct xsc_lag *ldev) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + struct lag_tracker *tracker = &ldev->tracker; + u8 v2p_port1, v2p_port2; + u8 member_bitmap; + int err; + + xsc_infer_tx_affinity_mapping(xdev0, ldev, &v2p_port1, + &v2p_port2); + + if (v2p_port1 != ldev->v2p_map[0] || + v2p_port2 != ldev->v2p_map[1] || + tracker->hash_type != tracker->old_hash_type) { + ldev->v2p_map[0] = v2p_port1; + ldev->v2p_map[1] = v2p_port2; + + xsc_core_info(xdev0, "modify lag map port 1:%d port 2:%d", + ldev->v2p_map[0], ldev->v2p_map[1]); + + err = xsc_cmd_modify_lag(ldev); + if (err) { + xsc_core_err(xdev0, "Failed to modify LAG (%d)\n", err); + return; + } + } + + member_bitmap = GET_LAG_MEMBER_BITMAP(ldev->v2p_map[0], ldev->v2p_map[1]); + if (xsc_lag_set_qos(xdev0, ldev->lag_id, member_bitmap, false)) + xsc_core_err(xdev0, "failed to set QoS for LAG %u\n", ldev->lag_id); +} + +static void xsc_deactivate_lag(struct xsc_lag *ldev) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + + if (xsc_lag_set_qos(xdev0, ldev->lag_id, 0, true)) + xsc_core_err(xdev0, "failed to set QoS for LAG %u\n", ldev->lag_id); + + if (xsc_cmd_destroy_lag(ldev)) + xsc_core_err(xdev0, "Failed to deactivate LAG; driver restart required, Make sure all VFs are unbound prior to LAG activation or deactivation\n"); + + ldev->flags &= ~XSC_LAG_MODE_FLAGS; +} + +static void xsc_lag_remove_ib_devices(struct xsc_lag *ldev) +{ + int i; + + for (i = 0; i < XSC_MAX_PORTS; i++) + if (ldev->pf[i].xdev) + xsc_remove_dev_by_protocol(ldev->pf[i].xdev, + XSC_INTERFACE_PROTOCOL_IB); +} + +static void xsc_lag_add_ib_devices(struct xsc_lag *ldev) +{ + int i; + + for (i = 0; i < XSC_MAX_PORTS; i++) + if (ldev->pf[i].xdev) + xsc_add_dev_by_protocol(ldev->pf[i].xdev, + XSC_INTERFACE_PROTOCOL_IB); +} + +static void xsc_do_bond(struct xsc_lag *ldev) +{ + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + struct xsc_core_device *xdev1 = ldev->pf[1].xdev; + struct lag_tracker tracker; + bool do_bond; + bool roce_lag; + + if (!xdev0 || !xdev1) + return; + + roce_lag = xsc_is_roce_lag_allowed(ldev); + + mutex_lock(&lag_mutex); + tracker = ldev->tracker; + mutex_unlock(&lag_mutex); + + do_bond = tracker.is_bonded && + !tracker.lag_disable && + ldev->pf[0].netdev && + ldev->pf[1].netdev && + ldev->pf[0].netdev == tracker.ndev[0] && + ldev->pf[1].netdev == tracker.ndev[1] && + xsc_lag_check_prereq(ldev); + + xsc_core_info(xdev0, "do_bond = %d, is_bonded = %d, lag_disable = %d, lag_check = %d\n", + do_bond, tracker.is_bonded, tracker.lag_disable, xsc_lag_check_prereq(ldev)); + + if (do_bond && !__xsc_lag_is_active(ldev)) { + if (roce_lag) { + xsc_lag_remove_ib_devices(ldev); + xsc_activate_lag(ldev, XSC_LAG_FLAG_ROCE); + xsc_add_dev_by_protocol(xdev0, XSC_INTERFACE_PROTOCOL_IB); + } else { + xsc_activate_lag(ldev, XSC_LAG_FLAG_SRIOV); + } + } else if (do_bond && __xsc_lag_is_active(ldev)) { + xsc_modify_lag(ldev); + } else if (!do_bond && __xsc_lag_is_active(ldev)) { + if (roce_lag) + xsc_remove_dev_by_protocol(xdev0, XSC_INTERFACE_PROTOCOL_IB); + + xsc_deactivate_lag(ldev); + + if (roce_lag) + xsc_lag_add_ib_devices(ldev); + } +} + +static void xsc_do_bond_work(struct work_struct *work) +{ + struct delayed_work *delayed_work = to_delayed_work(work); + struct xsc_lag *ldev = container_of(delayed_work, struct xsc_lag, + bond_work); + int status; + + status = mutex_trylock(&xsc_intf_mutex); + if (!status) { + /* 1 sec delay. */ + queue_delayed_work(ldev->wq, &ldev->bond_work, HZ/2); + return; + } + + xsc_do_bond(ldev); + mutex_unlock(&xsc_intf_mutex); +} + +static struct xsc_lag *xsc_lag_dev_alloc(void) +{ + struct xsc_lag *ldev; + + ldev = kzalloc(sizeof(*ldev), GFP_KERNEL); + if (!ldev) + return NULL; + + ldev->wq = create_singlethread_workqueue("xsc_lag"); + if (!ldev->wq) { + kfree(ldev); + return NULL; + } + + kref_init(&ldev->ref); + INIT_DELAYED_WORK(&ldev->bond_work, xsc_do_bond_work); + + return ldev; +} + +static void xsc_lag_dev_add_xdev(struct xsc_lag *ldev, + struct xsc_core_device *xdev) +{ + unsigned int fn = PCI_FUNC(xdev->pdev->devfn) % XSC_MAX_PORTS; + + if (fn >= XSC_MAX_PORTS) + return; + + mutex_lock(&lag_mutex); + ldev->pf[fn].xdev = xdev; + xdev->priv.lag = ldev; + mutex_unlock(&lag_mutex); +} + +int xsc_lag_dev_get_netdev_idx(struct xsc_lag *ldev, + struct net_device *ndev) +{ + int i; + + for (i = 0; i < XSC_MAX_PORTS; i++) + if (ldev->pf[i].netdev == ndev) + return i; + + return -1; +} + +enum netdev_lag_hash bond_lag_hash_type(struct bonding *bond) +{ + switch (bond->params.xmit_policy) { + case BOND_XMIT_POLICY_LAYER2: + return NETDEV_LAG_HASH_L23; + case BOND_XMIT_POLICY_LAYER34: + return NETDEV_LAG_HASH_L34; + case BOND_XMIT_POLICY_LAYER23: + return NETDEV_LAG_HASH_L23; + case BOND_XMIT_POLICY_ENCAP23: + return NETDEV_LAG_HASH_E23; + case BOND_XMIT_POLICY_ENCAP34: + return NETDEV_LAG_HASH_E34; + default: + return NETDEV_LAG_HASH_UNKNOWN; + } +} + +static bool xsc_lag_eval_bonding_conds(struct xsc_lag *ldev, + struct lag_tracker *tracker, + struct net_device *upper) +{ + int bond_status = 0, num_slaves = 0, idx; + struct net_device *ndev_tmp; + bool is_bonded; + struct xsc_core_device *xdev0 = ldev->pf[0].xdev; + + rcu_read_lock(); + for_each_netdev_in_bond_rcu(upper, ndev_tmp) { + idx = xsc_lag_dev_get_netdev_idx(ldev, ndev_tmp); + if (idx > -1) + bond_status |= (1 << idx); + + num_slaves++; + } + rcu_read_unlock(); + + xsc_core_info(xdev0, "num_slaves = %d, bond_status = %d\n", + num_slaves, bond_status); + /* None of this lagdev's netdevs are slaves of this master. */ + + if (tracker->is_bonded && + (!tracker->netdev_state[0].link_up || !tracker->netdev_state[0].tx_enabled) && + (!tracker->netdev_state[1].link_up || !tracker->netdev_state[1].tx_enabled)) { + tracker->is_bonded = false; + return true; + } + + if (!(bond_status & 0x3)) + return false; + + /* Determine bonding status: + * A device is considered bonded if both its physical ports are slaves + * of the same lag master, and only them. + * Lag mode must be activebackup or hash. + */ + if (!tracker->is_bonded) + is_bonded = (num_slaves == XSC_MAX_PORTS) && + (bond_status == 0x3) && + ((tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) || + (tracker->tx_type == NETDEV_LAG_TX_TYPE_HASH)); + + xsc_core_info(xdev0, "is_bonded = %d\n", is_bonded); + + if (tracker->is_bonded != is_bonded) { + tracker->is_bonded = is_bonded; + return true; + } + + return false; +} + +static bool xsc_handle_changeupper_event(struct xsc_lag *ldev, + struct lag_tracker *tracker, + struct net_device *ndev, + struct netdev_notifier_changeupper_info *info) +{ + enum netdev_lag_tx_type tx_type = NETDEV_LAG_TX_TYPE_UNKNOWN; + struct netdev_lag_upper_info *lag_upper_info; + struct net_device *upper = info->upper_dev; + + if (!netif_is_lag_master(upper)) + return false; + + if (info->linking) { + lag_upper_info = info->upper_info; + + if (lag_upper_info) { + tx_type = lag_upper_info->tx_type; + tracker->hash_type = lag_upper_info->hash_type; + } + } + + tracker->tx_type = tx_type; + + return xsc_lag_eval_bonding_conds(ldev, tracker, upper); +} + +static bool xsc_handle_changelowerstate_event(struct xsc_lag *ldev, + struct lag_tracker *tracker, + struct net_device *ndev, + struct netdev_notifier_changelowerstate_info *info) +{ + struct netdev_lag_lower_state_info *lag_lower_info; + int idx; + + if (!netif_is_lag_port(ndev)) + return 0; + + idx = xsc_lag_dev_get_netdev_idx(ldev, ndev); + if (idx == -1) + return 0; + + /* This information is used to determine virtual to physical + * port mapping. + */ + lag_lower_info = info->lower_state_info; + if (!lag_lower_info) + return 0; + + tracker->netdev_state[idx] = *lag_lower_info; + + return 1; +} + +static bool xsc_handle_changehash_event(struct xsc_lag *ldev, + struct lag_tracker *tracker, + struct net_device *ndev, + struct netdev_notifier_change_info *change_info) +{ + struct bonding *bond; + enum netdev_lag_hash hash_type; + + if (!netif_is_lag_master(ndev)) + return false; + + bond = netdev_priv(ndev); + + if (!bond_mode_uses_xmit_hash(bond)) + return false; + + hash_type = bond_lag_hash_type(bond); + + if (hash_type != tracker->hash_type) { + tracker->old_hash_type = tracker->hash_type; + tracker->hash_type = hash_type; + return true; + } + + return false; +} + +static int xsc_lag_netdev_event(struct notifier_block *this, + unsigned long event, void *ptr) +{ + struct net_device *ndev = netdev_notifier_info_to_dev(ptr); + struct lag_tracker tracker; + struct xsc_lag *ldev; + bool changed = 0; + + if ((event != NETDEV_CHANGE) && (event != NETDEV_CHANGEUPPER) && + (event != NETDEV_CHANGELOWERSTATE)) + return NOTIFY_DONE; + + ldev = container_of(this, struct xsc_lag, nb); + tracker = ldev->tracker; + + switch (event) { + case NETDEV_CHANGEUPPER: + changed = xsc_handle_changeupper_event(ldev, &tracker, ndev, + ptr); + break; + case NETDEV_CHANGELOWERSTATE: + changed = xsc_handle_changelowerstate_event(ldev, &tracker, + ndev, ptr); + break; + case NETDEV_CHANGE: + changed = xsc_handle_changehash_event(ldev, &tracker, ndev, + ptr); + break; + } + + mutex_lock(&lag_mutex); + ldev->tracker = tracker; + ldev->tracker.ndev[0] = ldev->pf[0].netdev; + ldev->tracker.ndev[1] = ldev->pf[1].netdev; + mutex_unlock(&lag_mutex); + + if (changed) + queue_delayed_work(ldev->wq, &ldev->bond_work, 0); + + return NOTIFY_DONE; +} + +static void xsc_lag_fib_event_flush(struct notifier_block *nb) +{ + struct lag_mp *mp = container_of(nb, struct lag_mp, fib_nb); + + flush_workqueue(mp->wq); +} + +bool xsc_esw_multipath_prereq(struct xsc_core_device *xdev0, + struct xsc_core_device *xdev1) +{ +#ifdef CONFIG_XSC_ESWITCH + return (xdev0->priv.eswitch->mode == XSC_ESWITCH_OFFLOADS && + xdev1->priv.eswitch->mode == XSC_ESWITCH_OFFLOADS); +#else + return false; +#endif + +} + +bool xsc_lag_multipath_check_prereq(struct xsc_lag *ldev) +{ + if (!ldev->pf[0].xdev || !ldev->pf[1].xdev) + return false; + +#ifdef XSC_VF_ECMP_TEST + return xsc_esw_multipath_prereq(ldev->pf[0].xdev, ldev->pf[1].xdev); +#else + return true; +#endif +} + +static void xsc_lag_set_port_affinity(struct xsc_lag *ldev, int port) +{ + struct lag_tracker *tracker = &ldev->tracker; + + if (!__xsc_lag_is_multipath(ldev)) + return; + + switch (port) { + case MAC_0_1_LOGIC: + tracker->netdev_state[0].tx_enabled = true; + tracker->netdev_state[0].link_up = true; + tracker->netdev_state[1].tx_enabled = true; + tracker->netdev_state[1].link_up = true; + break; + case MAC_0_LOGIC: + tracker->netdev_state[0].tx_enabled = true; + tracker->netdev_state[0].link_up = true; + tracker->netdev_state[1].tx_enabled = false; + tracker->netdev_state[1].link_up = false; + break; + case MAC_1_LOGIC: + tracker->netdev_state[0].tx_enabled = false; + tracker->netdev_state[0].link_up = false; + tracker->netdev_state[1].tx_enabled = true; + tracker->netdev_state[1].link_up = true; + break; + + default: + xsc_core_warn(ldev->pf[0].xdev, "Invalid affinity port %d", port); + return; + } + + xsc_modify_lag(ldev); +} + +static void xsc_lag_fib_route_event(struct xsc_lag *ldev, + unsigned long event, + struct fib_info *fi) +{ + struct lag_mp *mp = &ldev->lag_mp; + struct fib_nh *fib_nh0, *fib_nh1; + unsigned int nhs; + struct neighbour *neigh0, *neigh1; + + /* Handle add/replace event */ +#ifdef HAVE_FIB_INFO_NH + nhs = fib_info_num_path(fi); +#else + nhs = fi->fib_nhs; +#endif + /* Handle delete event */ + if ((event == FIB_EVENT_ENTRY_DEL) && __xsc_lag_is_active(ldev) && (nhs == 2)) { + /* stop track */ + if (mp->mfi == fi) + mp->mfi = NULL; + xsc_deactivate_lag(ldev); + return; + } + +#ifdef XSC_LAG_DEBUG + xsc_core_info(ldev->pf[0].xdev, "nhs=%d\n", nhs); +#endif + + if (nhs == 1 && (event != FIB_EVENT_ENTRY_DEL)) { + if (__xsc_lag_is_active(ldev)) { +#ifdef HAVE_FIB_INFO_NH + struct net_device *nh_dev = fib_info_nh(fi, 0)->fib_nh_dev; +#elif defined HAVE_FIB_NH_DEV + struct net_device *nh_dev = fi->fib_nh[0].fib_nh_dev; +#else + struct net_device *nh_dev = fi->fib_nh[0].nh_dev; +#endif + int i = xsc_lag_dev_get_netdev_idx(ldev, nh_dev); + + xsc_lag_set_port_affinity(ldev, ++i); + } + return; + } + + if (nhs != 2) + return; + + /* Verify next hops are ports of the same hca */ +#ifdef HAVE_FIB_INFO_NH + fib_nh0 = fib_info_nh(fi, 0); + fib_nh1 = fib_info_nh(fi, 1); +#else + fib_nh0 = &fi->fib_nh[0]; + fib_nh1 = &fi->fib_nh[1]; +#endif + +#ifdef HAVE_FIB_NH_DEV + if (!(fib_nh0->fib_nh_dev == ldev->pf[0].netdev && + fib_nh1->fib_nh_dev == ldev->pf[1].netdev) && + !(fib_nh0->fib_nh_dev == ldev->pf[1].netdev && + fib_nh1->fib_nh_dev == ldev->pf[0].netdev)) { +#else + if (!(fib_nh0->nh_dev == ldev->pf[0].netdev && + fib_nh1->nh_dev == ldev->pf[1].netdev) && + !(fib_nh0->nh_dev == ldev->pf[1].netdev && + fib_nh1->nh_dev == ldev->pf[0].netdev)) { +#endif + xsc_core_err(ldev->pf[0].xdev, + "Multipath offload require two ports of the same HCA\n"); + return; + } + + /* First time we see multipath route */ + if (!mp->mfi && !__xsc_lag_is_active(ldev)) { + struct lag_tracker *tracker = &ldev->tracker; +#ifdef fib_nh_gw4 + neigh0 = neigh_lookup(&arp_tbl, &fib_nh0->fib_nh_gw4, ldev->pf[0].netdev); + neigh1 = neigh_lookup(&arp_tbl, &fib_nh1->fib_nh_gw4, ldev->pf[1].netdev); +#else + neigh0 = neigh_lookup(&arp_tbl, &fib_nh0->nh_gw, ldev->pf[0].netdev); + neigh1 = neigh_lookup(&arp_tbl, &fib_nh1->nh_gw, ldev->pf[1].netdev); +#endif + if (!neigh0 || !neigh1) { + xsc_core_err(ldev->pf[0].xdev, + "Multipath offload require two ports with valid neighbor\n"); + return; + } + + if (neigh0) { + if (!(neigh0->nud_state & NUD_NOARP)) { + read_lock_bh(&neigh0->lock); + memcpy(&tracker->gw_dmac0[0], neigh0->ha, ETH_ALEN); + read_unlock_bh(&neigh0->lock); + } else { + neigh_release(neigh0); + neigh_release(neigh1); + xsc_core_err(ldev->pf[0].xdev, + "Multipath offload require two ports with valid gw\n"); + return; + } + } + + if (neigh1) { + if (!(neigh1->nud_state & NUD_NOARP)) { + read_lock_bh(&neigh1->lock); + memcpy(&tracker->gw_dmac1[0], neigh1->ha, ETH_ALEN); + read_unlock_bh(&neigh1->lock); + } else { + neigh_release(neigh0); + neigh_release(neigh1); + xsc_core_err(ldev->pf[0].xdev, + "Multipath offload require two ports with valid gw\n"); + return; + } + } + + neigh_release(neigh0); + neigh_release(neigh1); + + xsc_activate_lag(ldev, XSC_LAG_FLAG_MULTIPATH); + } + + xsc_lag_set_port_affinity(ldev, MAC_0_1_LOGIC); + mp->mfi = fi; +} + +static void xsc_lag_fib_nexthop_event(struct xsc_lag *ldev, + unsigned long event, + struct fib_nh *fib_nh, + struct fib_info *fi) +{ + struct lag_mp *mp = &ldev->lag_mp; + + /* Check the nh event is related to the route */ + if (!mp->mfi || mp->mfi != fi) + return; + + /* nh added/removed */ + if (event == FIB_EVENT_NH_DEL) { +#ifdef HAVE_FIB_NH_DEV + int i = xsc_lag_dev_get_netdev_idx(ldev, fib_nh->fib_nh_dev); +#else + int i = xsc_lag_dev_get_netdev_idx(ldev, fib_nh->nh_dev); +#endif + if (i >= 0) { + i = (i + 1) % 2 + 1; /* peer port */ + xsc_lag_set_port_affinity(ldev, i); + } + } else if (event == FIB_EVENT_NH_ADD && +#ifdef HAVE_FIB_INFO_NH + fib_info_num_path(fi) == 2) { +#else + fi->fib_nhs == 2) { +#endif + xsc_lag_set_port_affinity(ldev, MAC_0_1_LOGIC); + } +} + +static void xsc_lag_fib_update(struct work_struct *work) +{ + struct xsc_fib_event_work *fib_work = + container_of(work, struct xsc_fib_event_work, work); + struct xsc_lag *ldev = fib_work->ldev; + struct fib_nh *fib_nh; + + /* Protect internal structures from changes */ + rtnl_lock(); + switch (fib_work->event) { + case FIB_EVENT_ENTRY_REPLACE: + case FIB_EVENT_ENTRY_APPEND: + case FIB_EVENT_ENTRY_ADD: + case FIB_EVENT_ENTRY_DEL: + xsc_lag_fib_route_event(ldev, fib_work->event, + fib_work->fen_info.fi); + fib_info_put(fib_work->fen_info.fi); + break; + case FIB_EVENT_NH_ADD: + case FIB_EVENT_NH_DEL: + fib_nh = fib_work->fnh_info.fib_nh; + xsc_lag_fib_nexthop_event(ldev, + fib_work->event, + fib_work->fnh_info.fib_nh, + fib_nh->nh_parent); + fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); + break; + } + + rtnl_unlock(); + kfree(fib_work); +} + +struct xsc_fib_event_work *xsc_lag_init_fib_work(struct xsc_lag *ldev, unsigned long event) +{ + struct xsc_fib_event_work *fib_work = + kzalloc(sizeof(struct xsc_fib_event_work), GFP_ATOMIC); + if (WARN_ON(!fib_work)) + return NULL; + + INIT_WORK(&fib_work->work, xsc_lag_fib_update); + fib_work->ldev = ldev; + fib_work->event = event; + + return fib_work; +} + +int xsc_lag_fib_event(struct notifier_block *nb, + unsigned long event, + void *ptr) +{ + struct lag_mp *mp = container_of(nb, struct lag_mp, fib_nb); + struct xsc_lag *ldev = container_of(mp, struct xsc_lag, lag_mp); + struct fib_notifier_info *info = ptr; + struct xsc_fib_event_work *fib_work; + struct fib_entry_notifier_info *fen_info; + struct fib_nh_notifier_info *fnh_info; + struct fib_info *fi; + struct net_device *fib_net_dev; + + if (info->family != AF_INET) + return NOTIFY_DONE; + + if (!xsc_lag_multipath_check_prereq(ldev)) + return NOTIFY_DONE; + +#ifdef XSC_LAG_DEBUG + xsc_core_info(ldev->pf[0].xdev, "lag fib event=%ld\n", event); +#endif + + switch (event) { + case FIB_EVENT_ENTRY_REPLACE: + case FIB_EVENT_ENTRY_APPEND: + case FIB_EVENT_ENTRY_ADD: + case FIB_EVENT_ENTRY_DEL: + fen_info = container_of(info, struct fib_entry_notifier_info, info); + fi = fen_info->fi; +#ifdef HAVE_FIB_INFO_NH + fib_net_dev = fib_info_nh(fi, 0)->fib_nh_dev; +#elif defined HAVE_FIB_NH_DEV + fib_net_dev = fi->fib_nh[0].fib_nh_dev; +#else + fib_net_dev = fi->fib_nh[0].nh_dev; +#endif + if (fib_net_dev != ldev->pf[0].netdev && + fib_net_dev != ldev->pf[1].netdev) { + return NOTIFY_DONE; + } + fib_work = xsc_lag_init_fib_work(ldev, event); + if (!fib_work) + return NOTIFY_DONE; + fib_work->fen_info = *fen_info; + /* Take reference on fib_info to prevent it from being + * freed while work is queued. Release it afterwards. + */ + fib_info_hold(fib_work->fen_info.fi); + break; + case FIB_EVENT_NH_ADD: + case FIB_EVENT_NH_DEL: + fnh_info = container_of(info, struct fib_nh_notifier_info, + info); + fib_work = xsc_lag_init_fib_work(ldev, event); + if (!fib_work) + return NOTIFY_DONE; + fib_work->fnh_info = *fnh_info; + fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); + break; + default: + return NOTIFY_DONE; + } + + queue_work(mp->wq, &fib_work->work); + + return NOTIFY_DONE; +} + +int xsc_lag_mp_init(struct xsc_lag *ldev) +{ + struct lag_mp *mp = &ldev->lag_mp; + int err; + + if (mp->fib_nb.notifier_call) + return 0; + + mp->wq = create_singlethread_workqueue("xsc_lag_mp"); + if (!mp->wq) + return -ENOMEM; + + mp->fib_nb.notifier_call = xsc_lag_fib_event; + err = register_fib_notifier(&init_net, &mp->fib_nb, + xsc_lag_fib_event_flush, NULL); + if (err) { + destroy_workqueue(mp->wq); + mp->fib_nb.notifier_call = NULL; + } + + return err; +} + +static void xsc_ldev_get(struct xsc_lag *ldev) +{ + kref_get(&ldev->ref); +} + +int __xsc_lag_add_xdev(struct xsc_core_device *xdev) +{ + struct xsc_lag *ldev = NULL; + struct xsc_core_device *tmp_xdev; + int err; + + if (!xsc_core_is_pf(xdev)) + return 0; + + tmp_xdev = xsc_get_next_phys_dev(xdev); + if (tmp_xdev) + ldev = tmp_xdev->priv.lag; + + if (!ldev) { + ldev = xsc_lag_dev_alloc(); + if (!ldev) + return -EPIPE; + } else { + xsc_ldev_get(ldev); + } + + xsc_lag_dev_add_xdev(ldev, xdev); + ldev->lag_id = 0xff; + + if (!ldev->nb.notifier_call) { + ldev->nb.notifier_call = xsc_lag_netdev_event; + if (register_netdevice_notifier(&ldev->nb)) { + ldev->nb.notifier_call = NULL; + xsc_core_err(xdev, "Failed to register LAG netdev notifier\n"); + } + } + + err = xsc_lag_mp_init(ldev); + if (err) + xsc_core_err(xdev, "Failed to init multipath lag err=%d\n", err); + return err; +} + +void xsc_lag_add_xdev(struct xsc_core_device *xdev) +{ + int err; + + mutex_lock(&xsc_intf_mutex); + err = __xsc_lag_add_xdev(xdev); + if (err) + xsc_core_dbg(xdev, "xsc lag add xdev failed: err=%d\n", err); + mutex_unlock(&xsc_intf_mutex); +} + +void xsc_lag_dev_add_pf(struct xsc_lag *ldev, + struct xsc_core_device *xdev, struct net_device *netdev) +{ + unsigned int fn = PCI_FUNC(xdev->pdev->devfn) % XSC_MAX_PORTS; + + if (fn > XSC_MAX_PORTS) + return; + + mutex_lock(&lag_mutex); + ldev->pf[fn].netdev = netdev; + ldev->tracker.netdev_state[fn].link_up = 0; + ldev->tracker.netdev_state[fn].tx_enabled = 0; + mutex_unlock(&lag_mutex); +} + +void xsc_lag_update_trackers(struct xsc_lag *ldev) +{ + enum netdev_lag_tx_type tx_type = NETDEV_LAG_TX_TYPE_UNKNOWN; + struct net_device *upper = NULL, *ndev; + struct lag_tracker *tracker; + struct bonding *bond; + struct slave *slave; + int i; + + rtnl_lock(); + tracker = &ldev->tracker; + + for (i = 0; i < XSC_MAX_PORTS; i++) { + ndev = ldev->pf[i].netdev; + if (!ndev) + continue; + + if (ndev->reg_state != NETREG_REGISTERED) + continue; + + if (!netif_is_bond_slave(ndev)) + continue; + + rcu_read_lock(); + slave = bond_slave_get_rcu(ndev); + rcu_read_unlock(); + bond = bond_get_bond_by_slave(slave); + + tracker->netdev_state[i].link_up = bond_slave_is_up(slave); + tracker->netdev_state[i].tx_enabled = bond_slave_can_tx(slave); + + if (bond_mode_uses_xmit_hash(bond)) { + tx_type = NETDEV_LAG_TX_TYPE_HASH; + tracker->hash_type = bond_lag_hash_type(bond); + tracker->old_hash_type = tracker->hash_type; + } else if (BOND_MODE(bond) == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) + tx_type = NETDEV_LAG_TX_TYPE_ACTIVEBACKUP; + + upper = bond->dev; + } + + if (!upper) + goto out; + + tracker->tx_type = tx_type; + + if (xsc_lag_eval_bonding_conds(ldev, tracker, upper)) + queue_delayed_work(ldev->wq, &ldev->bond_work, 0); + +out: + rtnl_unlock(); +} + +void xsc_lag_add(struct xsc_core_device *xdev, struct net_device *netdev) +{ + struct xsc_lag *ldev = xdev->priv.lag; + + if (!ldev || !xsc_core_is_pf(xdev)) + return; + + xsc_ldev_get(ldev); + xsc_lag_dev_add_pf(ldev, xdev, netdev); + + xsc_lag_update_trackers(ldev); +} +EXPORT_SYMBOL(xsc_lag_add); + +void xsc_lag_dev_remove_pf(struct xsc_lag *ldev, struct xsc_core_device *xdev) +{ + int i; + + for (i = 0; i < XSC_MAX_PORTS; i++) + if (ldev->pf[i].xdev == xdev) + break; + + if (i == XSC_MAX_PORTS) + return; + + mutex_lock(&lag_mutex); + ldev->pf[i].netdev = NULL; + mutex_unlock(&lag_mutex); +} + +void xsc_lag_mp_cleanup(struct xsc_lag *ldev) +{ + struct lag_mp *mp = &ldev->lag_mp; + + if (!mp->fib_nb.notifier_call) + return; + unregister_fib_notifier(&init_net, &mp->fib_nb); + destroy_workqueue(mp->wq); + mp->fib_nb.notifier_call = NULL; +} + +void xsc_lag_dev_free(struct kref *ref) +{ + struct xsc_lag *ldev = container_of(ref, struct xsc_lag, ref); + + if (ldev->nb.notifier_call) + unregister_netdevice_notifier(&ldev->nb); + + xsc_lag_mp_cleanup(ldev); + cancel_delayed_work_sync(&ldev->bond_work); + destroy_workqueue(ldev->wq); + ldev->nb.notifier_call = NULL; + kfree(ldev); +} + +void xsc_lag_dev_put(struct xsc_lag *ldev) +{ + kref_put(&ldev->ref, xsc_lag_dev_free); +} + +void xsc_lag_remove(struct xsc_core_device *xdev) +{ + struct xsc_lag *ldev = xsc_lag_dev_get(xdev); + + if (!ldev) + return; + + if (__xsc_lag_is_active(ldev)) + xsc_deactivate_lag(ldev); + + xsc_lag_dev_remove_pf(ldev, xdev); + xsc_lag_dev_put(ldev); + +} +EXPORT_SYMBOL(xsc_lag_remove); + +void xsc_lag_dev_remove_xdev(struct xsc_lag *ldev, struct xsc_core_device *xdev) +{ + int i; + + for (i = 0; i < XSC_MAX_PORTS; i++) { + if (ldev->pf[i].xdev == xdev) + break; + } + + if (i == XSC_MAX_PORTS) + return; + + mutex_lock(&lag_mutex); + ldev->pf[i].xdev = NULL; + xdev->priv.lag = NULL; + mutex_unlock(&lag_mutex); +} + +void __xsc_lag_remove_xdev(struct xsc_core_device *xdev) +{ + struct xsc_lag *ldev = xsc_lag_dev_get(xdev); + + if (!ldev) + return; + + xsc_lag_dev_remove_xdev(ldev, xdev); + xsc_lag_dev_put(ldev); +} + +void xsc_lag_remove_xdev(struct xsc_core_device *xdev) +{ + mutex_lock(&xsc_intf_mutex); + __xsc_lag_remove_xdev(xdev); + mutex_unlock(&xsc_intf_mutex); +} + +void xsc_lag_disable(struct xsc_core_device *xdev) +{ + struct xsc_lag *ldev; + struct lag_tracker *tracker; + + mutex_lock(&xsc_intf_mutex); + ldev = xsc_lag_dev_get(xdev); + if (!ldev) + goto unlock; + + xsc_ldev_get(ldev); + + if (!__xsc_lag_is_active(ldev)) + goto unlock; + + mutex_lock(&lag_mutex); + tracker = &ldev->tracker; + tracker->lag_disable = 1; + mutex_unlock(&lag_mutex); + + xsc_do_bond(ldev); + +unlock: + mutex_unlock(&xsc_intf_mutex); +} + +void xsc_lag_enable(struct xsc_core_device *xdev) +{ + struct xsc_lag *ldev; + struct lag_tracker *tracker; + + mutex_lock(&xsc_intf_mutex); + ldev = xsc_lag_dev_get(xdev); + if (!ldev) + goto unlock; + + if (__xsc_lag_is_active(ldev)) + goto unlock; + + mutex_lock(&lag_mutex); + tracker = &ldev->tracker; + tracker->lag_disable = 0; + mutex_unlock(&lag_mutex); + + xsc_do_bond(ldev); + +unlock: + mutex_unlock(&xsc_intf_mutex); +} + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_pci_ctrl.c b/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_pci_ctrl.c new file mode 100644 index 0000000000000000000000000000000000000000..565abdea4b459cc2aeb2cc347f4bb4ed8cb7c8ac --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_pci_ctrl.c @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FEATURE_ONCHIP_FT_MASK (1<<4) +#define FEATURE_DMA_RW_TBL_MASK (1<<8) +#define FEATURE_PCT_EXP_MASK (1<<9) + +#define XSC_PCI_CTRL_NAME "pci_ctrl" + +static int xsc_pci_ctrl_modify_qp(struct xsc_core_device *xdev, void *in, void *out) +{ + int ret = 0, i = 0; + struct xsc_ioctl_qp_range *resp; + struct xsc_ioctl_data_tl *tl; + int insize; + struct xsc_modify_qp_mbox_in *mailin; + struct xsc_modify_qp_mbox_out mailout; + u32 qpn; + + tl = (struct xsc_ioctl_data_tl *)out; + resp = (struct xsc_ioctl_qp_range *)(tl + 1); + xsc_core_dbg(xdev, "xsc_ioctl_qp_range: qpn:%d, num:%d, opcode:%d\n", + resp->qpn, resp->num, resp->opcode); + if (resp->num == 0) { + xsc_core_dbg(xdev, "xsc_ioctl_qp_range: resp->num ==0\n"); + return 0; + } + qpn = resp->qpn; + insize = sizeof(struct xsc_modify_qp_mbox_in); + mailin = kvzalloc(insize, GFP_KERNEL); + if (!mailin) { + xsc_core_dbg(xdev, "xsc_ioctl_qp_range: enomem\n"); + return -ENOMEM; + } + if (resp->opcode == XSC_CMD_OP_RTR2RTS_QP) { + for (i = 0; i < resp->num; i++) { + mailin->hdr.opcode = cpu_to_be16(XSC_CMD_OP_RTR2RTS_QP); + mailin->qpn = cpu_to_be32(qpn + 0); + ret = xsc_cmd_exec(xdev, mailin, insize, &mailout, sizeof(mailout)); + xsc_core_dbg(xdev, "modify qp state qpn:%d\n", qpn + i); + } + } + + kvfree(mailin); + + return ret; +} + +static int xsc_pci_ctrl_get_phy(struct xsc_core_device *xdev, + void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_data_tl *tl = (struct xsc_ioctl_data_tl *)out; + struct xsc_ioctl_get_phy_info_res *resp; + struct xsc_ioctl_get_vf_info_res *vf_res; + struct xsc_vf_info vf_info; + struct xsc_lag *ldev = xsc_lag_dev_get(xdev); + u16 lag_id = U16_MAX; + + if (ldev && __xsc_lag_is_active(ldev)) + lag_id = ldev->lag_id; + + switch (tl->opmod) { + case XSC_IOCTL_OP_GET_LOCAL: + resp = (struct xsc_ioctl_get_phy_info_res *)(tl + 1); + + resp->phy_port = xdev->pcie_port; + resp->func_id = xdev->glb_func_id; + resp->logic_in_port = xdev->logic_port; + resp->mac_phy_port = xdev->mac_port; + resp->mac_logic_in_port = xdev->mac_logic_port; + resp->lag_id = lag_id; + resp->raw_qp_id_base = xdev->caps.raweth_qp_id_base; + resp->lag_port_start = XSC_LAG_PORT_START; + resp->send_seg_num = xdev->caps.send_ds_num; + resp->recv_seg_num = xdev->caps.recv_ds_num; + resp->raw_tpe_qp_num = xdev->caps.raw_tpe_qp_num; + resp->chip_version = xdev->chip_ver_l; + resp->on_chip_tbl_vld = + (xdev->feature_flag & FEATURE_ONCHIP_FT_MASK) ? 1 : 0; + resp->dma_rw_tbl_vld = + (xdev->feature_flag & FEATURE_DMA_RW_TBL_MASK) ? 1 : 0; + resp->pct_compress_vld = + (xdev->feature_flag & FEATURE_PCT_EXP_MASK) ? 1 : 0; + + xsc_core_dbg(xdev, "%d,%d,%d,%d,%d,%d\n", resp->phy_port, + resp->func_id, resp->logic_in_port, + resp->mac_phy_port, resp->mac_logic_in_port, + resp->lag_id); + resp->funcid_encode[0] = XSC_PCIE0_VF0_FUNC_ID; + resp->funcid_encode[1] = XSC_PCIE0_VF_FUNC_ID_END; + resp->funcid_encode[2] = XSC_PCIE0_PF0_FUNC_ID; + resp->funcid_encode[3] = XSC_PCIE0_PF_FUNC_ID_END; + resp->funcid_encode[4] = XSC_PCIE1_VF0_FUNC_ID; + resp->funcid_encode[5] = XSC_PCIE1_VF_FUNC_ID_END; + resp->funcid_encode[6] = XSC_PCIE1_PF0_FUNC_ID; + resp->funcid_encode[7] = XSC_PCIE1_PF_FUNC_ID_END; + break; + + case XSC_IOCTL_OP_GET_VF_INFO: + vf_res = (struct xsc_ioctl_get_vf_info_res *)(tl + 1); + memcpy(&vf_info, vf_res, sizeof(struct xsc_vf_info)); + + xsc_pci_get_vf_info(xdev, &vf_info); + + vf_res->func_id = vf_info.func_id; + vf_res->logic_port = vf_info.logic_port; + break; + + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static int xsc_pci_ctrl_get_contextinfo(struct xsc_core_device *xdev, + void *in, void *out) +{ + int ret = 0; + struct xsc_ioctl_data_tl *tl = (struct xsc_ioctl_data_tl *)out; + struct xsc_alloc_ucontext_resp *resp; + + if (tl->opmod != XSC_IOCTL_OP_GET_CONTEXT) + ret = -EINVAL; + + resp = (struct xsc_alloc_ucontext_resp *)(tl + 1); + + // resp->qp_tab_size = 1 << xdev->caps.log_max_qp; + // resp->cache_line_size = L1_CACHE_BYTES; + // resp->max_sq_desc_sz = xdev->caps.max_sq_desc_sz; + // resp->max_rq_desc_sz = xdev->caps.max_rq_desc_sz; + // resp->max_send_wqebb = xdev->caps.max_wqes; + // resp->max_recv_wr = xdev->caps.max_wqes; + + resp->max_cq = 1 << xdev->caps.log_max_cq; + resp->max_qp = 1 << xdev->caps.log_max_qp; + resp->max_rwq_indirection_table_size = xdev->caps.max_rwq_indirection_table_size; + resp->qpm_tx_db = xdev->regs.tx_db; + resp->qpm_rx_db = xdev->regs.rx_db; + resp->cqm_next_cid_reg = xdev->regs.complete_reg; + resp->cqm_armdb = xdev->regs.complete_db; + resp->send_ds_num = xdev->caps.send_ds_num; + resp->recv_ds_num = xdev->caps.recv_ds_num; + resp->send_ds_shift = xdev->caps.send_wqe_shift; + resp->recv_ds_shift = xdev->caps.recv_wqe_shift; + resp->glb_func_id = xdev->glb_func_id; + + resp->max_wqes = xdev->caps.max_wqes; + + xsc_core_dbg(xdev, "xsc_tdi_alloc_context:\n"); + xsc_core_dbg(xdev, "resp->max_cq=%u\n", resp->max_cq); + xsc_core_dbg(xdev, "resp->max_qp=%u\n", resp->max_qp); + xsc_core_dbg(xdev, "resp->qpm_tx_db=%llx\n", resp->qpm_tx_db); + xsc_core_dbg(xdev, "resp->qpm_rx_db=%llx\n", resp->qpm_rx_db); + xsc_core_dbg(xdev, "resp->cqm_next_cid_reg=%llx\n", resp->cqm_next_cid_reg); + xsc_core_dbg(xdev, "resp->cqm_armdb=%llx\n", resp->cqm_armdb); + xsc_core_dbg(xdev, "resp->send_ds_num=%u\n", resp->send_ds_num); + xsc_core_dbg(xdev, "resp->send_ds_shift=%u\n", resp->send_ds_shift); + xsc_core_dbg(xdev, "resp->:recv_ds_num=%u\n", resp->recv_ds_num); + xsc_core_dbg(xdev, "resp->recv_ds_shift=%u\n", resp->recv_ds_shift); + xsc_core_dbg(xdev, "resp->glb_func_id=%u\n", resp->glb_func_id); + + return ret; +} + +int xsc_pci_ctrl_exec_ioctl(struct xsc_core_device *xdev, void *in, int in_size, void *out, + int out_size) +{ + int opcode, ret = 0; + struct xsc_ioctl_attr *hdr; + + hdr = (struct xsc_ioctl_attr *)in; + opcode = hdr->opcode; + switch (opcode) { + case XSC_IOCTL_GET_PHY_INFO: + ret = xsc_pci_ctrl_get_phy(xdev, in, out); + break; + case XSC_IOCTL_SET_QP_STATUS: + xsc_core_dbg(xdev, "case XSC_IOCTL_SET_QP_STATUS:\n"); + ret = xsc_pci_ctrl_modify_qp(xdev, in, out); + break; + case XSC_IOCTL_GET_CONTEXT: + xsc_core_dbg(xdev, "case XSC_IOCTL_GET_CONTEXT:\n"); + ret = xsc_pci_ctrl_get_contextinfo(xdev, in, out); + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static long xsc_pci_ctrl_getinfo(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr) +{ + struct xsc_ioctl_hdr hdr; + struct xsc_ioctl_hdr *in; + int in_size; + int err; + + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + if (err) + return -EFAULT; + if (hdr.check_filed != XSC_IOCTL_CHECK_FILED) + return -EINVAL; + switch (hdr.attr.opcode) { + case XSC_IOCTL_GET_PHY_INFO: + case XSC_IOCTL_SET_QP_STATUS: + case XSC_IOCTL_GET_CONTEXT: + break; + default: + return -EINVAL; + } + in_size = sizeof(struct xsc_ioctl_hdr) + hdr.attr.length; + in = kvzalloc(in_size, GFP_KERNEL); + if (!in) + return -EFAULT; + in->attr.opcode = hdr.attr.opcode; + in->attr.length = hdr.attr.length; + err = copy_from_user(in->attr.data, user_hdr->attr.data, hdr.attr.length); + if (err) { + kvfree(in); + return -EFAULT; + } + err = xsc_pci_ctrl_exec_ioctl(xdev, &in->attr, (in_size-sizeof(u32)), in->attr.data, + hdr.attr.length); + in->attr.error = err; + if (copy_to_user((void *)user_hdr, in, in_size)) + err = -EFAULT; + kvfree(in); + return err; +} + +static int xsc_ioctl_flow_cmdq(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr) +{ + struct xsc_ioctl_mbox_in *in; + struct xsc_ioctl_mbox_out *out; + int in_size; + int out_size; + int err; + + in_size = sizeof(struct xsc_ioctl_mbox_in) + hdr->attr.length; + in = kvzalloc(in_size, GFP_KERNEL); + if (!in) + return -EFAULT; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + in->len = __cpu_to_be16(hdr->attr.length); + err = copy_from_user(in->data, user_hdr->attr.data, hdr->attr.length); + if (err) { + kvfree(in); + return -EFAULT; + } + + out_size = sizeof(struct xsc_ioctl_mbox_out) + hdr->attr.length; + out = kvzalloc(out_size, GFP_KERNEL); + if (!out) { + kvfree(in); + return -ENOMEM; + } + memcpy(out->data, in->data, hdr->attr.length); + out->len = in->len; + err = xsc_cmd_exec(xdev, in, in_size, out, out_size); + + hdr->attr.error = __be32_to_cpu(out->error); + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + err = -EFAULT; + if (copy_to_user((void *)user_hdr->attr.data, out->data, hdr->attr.length)) + err = -EFAULT; + + kvfree(in); + kvfree(out); + return err; +} + +static int xsc_ioctl_modify_raw_qp(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr, struct xsc_ioctl_hdr *hdr) +{ + struct xsc_modify_raw_qp_mbox_in *in; + struct xsc_modify_raw_qp_mbox_out *out; + int err; + + if (hdr->attr.length != sizeof(struct xsc_modify_raw_qp_request)) + return -EINVAL; + + in = kvzalloc(sizeof(struct xsc_modify_raw_qp_mbox_in), GFP_KERNEL); + if (!in) + goto err_in; + out = kvzalloc(sizeof(struct xsc_modify_raw_qp_mbox_out), GFP_KERNEL); + if (!out) + goto err_out; + + err = copy_from_user(&in->req, user_hdr->attr.data, + sizeof(struct xsc_modify_raw_qp_request)); + if (err) + goto err; + + in->hdr.opcode = __cpu_to_be16(hdr->attr.opcode); + in->pcie_no = xsc_get_pcie_no(); + + err = xsc_cmd_exec(xdev, in, sizeof(struct xsc_modify_raw_qp_mbox_in), + out, sizeof(struct xsc_modify_raw_qp_mbox_out)); + + hdr->attr.error = __be32_to_cpu(out->hdr.status); + + if (copy_to_user((void *)user_hdr, hdr, sizeof(*hdr))) + goto err; + + kvfree(in); + kvfree(out); + return 0; + +err: + kvfree(out); +err_out: + kvfree(in); +err_in: + return -EFAULT; +} + +static long xsc_pci_ctrl_cmdq(struct xsc_core_device *xdev, + struct xsc_ioctl_hdr __user *user_hdr) +{ + struct xsc_ioctl_hdr hdr; + int err; + void *in; + void *out; + + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + if (err) + return -EFAULT; + + /* check valid */ + if (hdr.check_filed != XSC_IOCTL_CHECK_FILED) + return -EINVAL; + + /* check ioctl cmd */ + switch (hdr.attr.opcode) { + case XSC_CMD_OP_IOCTL_FLOW: + return xsc_ioctl_flow_cmdq(xdev, user_hdr, &hdr); + case XSC_CMD_OP_MODIFY_RAW_QP: + return xsc_ioctl_modify_raw_qp(xdev, user_hdr, &hdr); + case XSC_CMD_OP_CREATE_QP: + break; + case XSC_CMD_OP_DESTROY_QP: + break; + case XSC_CMD_OP_2RST_QP: + break; + case XSC_CMD_OP_CREATE_CQ: + break; + case XSC_CMD_OP_DESTROY_CQ: + break; + case XSC_CMD_OP_CREATE_MULTI_QP: + break; + case XSC_CMD_OP_ALLOC_MULTI_VIRTQ_CQ: + break; + case XSC_CMD_OP_RELEASE_MULTI_VIRTQ_CQ: + break; + case XSC_CMD_OP_ALLOC_MULTI_VIRTQ: + break; + case XSC_CMD_OP_RELEASE_MULTI_VIRTQ: + break; + + default: + return -EINVAL; + } + + in = kvzalloc(hdr.attr.length, GFP_KERNEL); + if (!in) + return -ENOMEM; + out = kvzalloc(hdr.attr.length, GFP_KERNEL); + if (!out) { + kfree(in); + return -ENOMEM; + } + + err = copy_from_user(in, user_hdr->attr.data, hdr.attr.length); + if (err) { + err = -EFAULT; + goto err_exit; + } + + xsc_cmd_exec(xdev, in, hdr.attr.length, out, hdr.attr.length); + if (copy_to_user((void *)user_hdr->attr.data, out, hdr.attr.length)) + err = -EFAULT; +err_exit: + kfree(in); + kfree(out); + return err; +} + +static void xsc_pci_ctrl_reg_cb(struct xsc_core_device *xdev, unsigned int cmd, + struct xsc_ioctl_hdr __user *user_hdr, void *data) +{ + int err; + + switch (cmd) { + case XSC_IOCTL_CMDQ: + err = xsc_pci_ctrl_cmdq(xdev, user_hdr); + break; + case XSC_IOCTL_DRV_GET: + case XSC_IOCTL_DRV_SET: + err = xsc_pci_ctrl_getinfo(xdev, user_hdr); + break; + default: + err = -EFAULT; + break; + } +} + +void xsc_pci_ctrl_fini(void) +{ + xsc_port_ctrl_cb_dereg(XSC_PCI_CTRL_NAME); +} + +int xsc_pci_ctrl_init(void) +{ + int ret; + + ret = xsc_port_ctrl_cb_reg(XSC_PCI_CTRL_NAME, xsc_pci_ctrl_reg_cb, NULL); + if (ret != 0) + pr_err("failed to register port control node for %s\n", XSC_PCI_CTRL_NAME); + + return ret; +} diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_pci_ctrl.h b/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_pci_ctrl.h new file mode 100644 index 0000000000000000000000000000000000000000..473f75cff7e64abba58459d4cb68004438af2262 --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_pci_ctrl.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#ifndef XSC_PCI_CTRL_H +#define XSC_PCI_CTRL_H + +void xsc_pci_ctrl_fini(void); +int xsc_pci_ctrl_init(void); + +#endif + diff --git a/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_port_ctrl.c b/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_port_ctrl.c new file mode 100644 index 0000000000000000000000000000000000000000..8f4924b7004ce50977f2851308ad2e76da3a3dcc --- /dev/null +++ b/drivers/net/ethernet/yunsilicon/xsc/pci/xsc_port_ctrl.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd. + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fw/xsc_tbm.h" + +#define XSC_PORT_CTRL_MAX 256 +#define XSC_PORT_CTRL_NAME_PRE "yunsilicon" +#define XSC_PORT_CTRL_NAME "port_ctrl" +#define XSC_PORT_CTRL_CB_NAME_LEN 15 + +struct xsc_port_ctrl_reg { + struct list_head node; + char name[XSC_PORT_CTRL_CB_NAME_LEN + 1]; + port_ctrl_cb cb; + void *data; +}; + +static dev_t g_port_ctrl_root_dev; +static struct class *g_port_ctrl_class; +static int g_port_ctrl_dev_cnt; +static struct list_head g_port_ctrl_cbs = LIST_HEAD_INIT(g_port_ctrl_cbs); +struct mutex g_port_ctrl_cbs_lock; + +static int _port_ctrl_open(struct inode *inode, struct file *filp) +{ + struct xsc_port_ctrl *ctrl + = container_of(inode->i_cdev, struct xsc_port_ctrl, cdev); + + filp->private_data = ctrl; + + return 0; +} + +static bool is_db_ofst(struct xsc_core_device *xdev, unsigned long offset) +{ + if (offset == (xdev->regs.tx_db & PAGE_MASK)) + return true; + else if (offset == (xdev->regs.rx_db & PAGE_MASK)) + return true; + else if (offset == (xdev->regs.complete_db & PAGE_MASK)) + return true; + else if (offset == (xdev->regs.complete_reg & PAGE_MASK)) + return true; + return false; +} + +static int _port_ctrl_mmap(struct file *filp, struct vm_area_struct *vma) +{ + resource_size_t reg_base; + unsigned long start = (unsigned long)vma->vm_start; + unsigned long size = (unsigned long)(vma->vm_end - vma->vm_start); + unsigned long offset = vma->vm_pgoff << PAGE_SHIFT; + struct xsc_port_ctrl *ctrl; + struct xsc_core_device *xdev; + + ctrl = filp->private_data; + xdev = container_of(ctrl, struct xsc_core_device, port_ctrl); + + if (is_db_ofst(xdev, offset)) + reg_base = (pci_resource_start(xdev->pdev, xdev->bar_num) + (offset & PAGE_MASK)); + else + return -EINVAL; + +#if (CHIP_VERSION_H == 0x100) && (CHIP_HOTFIX_NUM >= 0x17) + reg_base = xsc_core_is_pf(xdev) ? reg_base - 0xA0000000 : reg_base; +#else + reg_base = reg_base - 0xA0000000; +#endif + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + if (remap_pfn_range(vma, start, (reg_base >> PAGE_SHIFT), size, vma->vm_page_prot)) { + pr_err("[%s:%d] remap_pfn_range err\n", __func__, __LINE__); + return -1; + } + + return 0; +} + +static long _port_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct xsc_port_ctrl_reg *p; + struct xsc_port_ctrl *ctrl; + struct xsc_core_device *xdev; + struct xsc_ioctl_hdr __user *user_hdr; + + ctrl = filp->private_data; + xdev = container_of(ctrl, struct xsc_core_device, port_ctrl); + user_hdr = (struct xsc_ioctl_hdr __user *)arg; + + list_for_each_entry(p, &g_port_ctrl_cbs, node) { + if (p->cb) + p->cb(xdev, cmd, user_hdr, p->data); + } + + return 0; +} + +static const struct file_operations g_port_ctrl_fops = { + .owner = THIS_MODULE, + .open = _port_ctrl_open, + .mmap = _port_ctrl_mmap, + .unlocked_ioctl = _port_ctrl_ioctl, + .compat_ioctl = _port_ctrl_ioctl, +}; + +static struct xsc_port_ctrl_reg *_port_ctrl_cbs_get(const char *name) +{ + struct xsc_port_ctrl_reg *p, *found; + + found = NULL; + list_for_each_entry(p, &g_port_ctrl_cbs, node) { + if (strcmp(p->name, name) == 0) { + found = p; + break; + } + } + + return found; +} + +static void _port_ctrl_data_fini(void) +{ + class_destroy(g_port_ctrl_class); + unregister_chrdev_region(g_port_ctrl_root_dev, XSC_PORT_CTRL_MAX); +} + +static int _port_ctrl_data_init(void) +{ + int ret; + int major_devid; + + ret = alloc_chrdev_region(&g_port_ctrl_root_dev, 0, XSC_PORT_CTRL_MAX, + XSC_PORT_CTRL_NAME_PRE); + if (ret < 0) { + pr_err("%s cant't get major id\n", XSC_PORT_CTRL_NAME_PRE); + return -1; + } + + major_devid = MAJOR(g_port_ctrl_root_dev); + pr_info("requested major_devid %d\n", major_devid); + + g_port_ctrl_class = class_create(THIS_MODULE, XSC_PORT_CTRL_NAME_PRE); + if (IS_ERR(g_port_ctrl_class)) { + pr_err("failed to call create class witch name %s\n", + XSC_PORT_CTRL_NAME_PRE); + unregister_chrdev_region(g_port_ctrl_root_dev, XSC_PORT_CTRL_MAX); + return -1; + } + + g_port_ctrl_dev_cnt = 0; + + return 0; +} + +static void _port_ctrl_dev_del(struct xsc_core_device *dev) +{ + struct xsc_port_ctrl *ctrl; + + ctrl = &dev->port_ctrl; + if (!ctrl) + return; + + device_destroy(g_port_ctrl_class, ctrl->devid); + cdev_del(&ctrl->cdev); +} + +static int _port_ctrl_dev_add(struct xsc_core_device *dev) +{ + struct xsc_port_ctrl *ctrl; + int ret; + + if (g_port_ctrl_dev_cnt >= XSC_PORT_CTRL_MAX) { + xsc_core_err(dev, "too many port control devices\n"); + return -ENOMEM; + } + + ctrl = &dev->port_ctrl; + ctrl->devid = g_port_ctrl_root_dev + g_port_ctrl_dev_cnt; + ctrl->cdev.owner = THIS_MODULE; + cdev_init(&ctrl->cdev, &g_port_ctrl_fops); + ret = cdev_add(&ctrl->cdev, ctrl->devid, 1); + if (ret != 0) { + xsc_core_err(dev, "failed to add cdev\n"); + kfree(ctrl); + return -ENOMEM; + } + + ctrl->device = device_create(g_port_ctrl_class, NULL, ctrl->devid, NULL, + "%s!%s_%02x:%02x.%x", XSC_PORT_CTRL_NAME_PRE, XSC_PORT_CTRL_NAME, + dev->bus_id, dev->dev_id, dev->func_id); + if (IS_ERR(ctrl->device)) { + xsc_core_err(dev, "failed to create port control device\n"); + cdev_del(&ctrl->cdev); + kfree(ctrl); + return -ENOMEM; + } + + g_port_ctrl_dev_cnt++; + + return 0; +} + +static void _port_ctrl_cb_fini(void) +{ + struct xsc_port_ctrl_reg *p, *n; + + list_for_each_entry_safe(p, n, &g_port_ctrl_cbs, node) { + mutex_lock(&g_port_ctrl_cbs_lock); + list_del(&p->node); + mutex_unlock(&g_port_ctrl_cbs_lock); + kfree(p); + } +} + +static int _port_ctrl_cb_init(void) +{ + mutex_init(&g_port_ctrl_cbs_lock); + + return 0; +} + +static void _port_ctrl_dev_flush(void) +{ +} + +void xsc_port_ctrl_fini(void) +{ + _port_ctrl_dev_flush(); + _port_ctrl_data_fini(); + _port_ctrl_cb_fini(); +} + +int xsc_port_ctrl_init(void) +{ + int ret; + + ret = _port_ctrl_data_init(); + if (ret != 0) { + pr_err("failed to initialize port ctrl data\n"); + return -1; + } + + ret = _port_ctrl_cb_init(); + if (ret != 0) { + pr_err("failed to initialize port ctrl cb\n"); + _port_ctrl_data_fini(); + return -1; + } + + return 0; +} + +void xsc_port_ctrl_remove(struct xsc_core_device *dev) +{ +#ifndef RUN_WITH_PSV + if (xsc_core_is_pf(dev)) +#else + if (1) +#endif + _port_ctrl_dev_del(dev); +} + +int xsc_port_ctrl_probe(struct xsc_core_device *dev) +{ + int ret = 0; +#ifndef RUN_WITH_PSV + if (xsc_core_is_pf(dev)) { +#else + if (1) { +#endif + ret = _port_ctrl_dev_add(dev); + if (ret != 0) + xsc_core_err(dev, "failed to add new port control device\n"); + } + + return ret; +} + +int xsc_port_ctrl_cb_reg(const char *name, port_ctrl_cb cb, void *data) +{ + struct xsc_port_ctrl_reg *reg_node; + + if (strlen(name) > XSC_PORT_CTRL_CB_NAME_LEN) { + pr_err("the name is too long to register to port control\n"); + return -1; + } + + reg_node = _port_ctrl_cbs_get(name); + if (reg_node) { + pr_err("failed to register a duplicated node\n"); + return -1; + } + + reg_node = kmalloc(sizeof(struct xsc_port_ctrl_reg), GFP_KERNEL); + if (!reg_node) + return -1; + + strlcpy(reg_node->name, name, sizeof(reg_node->name)); + reg_node->cb = cb; + reg_node->data = data; + INIT_LIST_HEAD(®_node->node); + + mutex_lock(&g_port_ctrl_cbs_lock); + list_add_tail(®_node->node, &g_port_ctrl_cbs); + mutex_unlock(&g_port_ctrl_cbs_lock); + + return 0; +} +EXPORT_SYMBOL(xsc_port_ctrl_cb_reg); + +void xsc_port_ctrl_cb_dereg(const char *name) +{ + struct xsc_port_ctrl_reg *p, *n; + + list_for_each_entry_safe(p, n, &g_port_ctrl_cbs, node) { + if (strcmp(p->name, name) == 0) { + mutex_lock(&g_port_ctrl_cbs_lock); + list_del(&p->node); + mutex_unlock(&g_port_ctrl_cbs_lock); + kfree(p); + break; + } + } +} +EXPORT_SYMBOL(xsc_port_ctrl_cb_dereg); + diff --git a/openEuler/MAINTAINERS b/openEuler/MAINTAINERS index 5c2108d55c2df288b43a9b71d30b74f58f4e8684..ea5fa2ab10f7c1600b45179f12cd4e8242e50c7a 100644 --- a/openEuler/MAINTAINERS +++ b/openEuler/MAINTAINERS @@ -249,6 +249,12 @@ S: Maintained F: drivers/vdpa F: drivers/vhost/vdpa.c +YUNSILICON XSC DRIVERS +M: Weihonggang +S: Maintained +F: drivers/infiniband/hw/xsc +F: drivers/net/ethernet/yunsilicon/xsc + THE REST M: xiexiuqi@huawei.com M: zhengzengkai@huawei.com