diff options
author | Mark Brown <broonie@kernel.org> | 2022-08-04 17:15:59 +0100 |
---|---|---|
committer | Mark Brown <broonie@kernel.org> | 2022-08-04 17:15:59 +0100 |
commit | 5f6ec3a6646c6198579a9d406a6f617f081ddc24 (patch) | |
tree | a5cf6fcf3e27c8b2c909d6db1dbe5758e8eace9c | |
parent | 995adae979d98b42508efe3309ee457add25afb4 (diff) | |
parent | e8c04e435db5fc4bd6d55dc01d121ab75e01a134 (diff) |
Merge branch 'ntb-next' of https://github.com/jonmason/ntb.git
-rw-r--r-- | Documentation/PCI/endpoint/index.rst | 2 | ||||
-rw-r--r-- | Documentation/PCI/endpoint/pci-vntb-function.rst | 126 | ||||
-rw-r--r-- | Documentation/PCI/endpoint/pci-vntb-howto.rst | 161 | ||||
-rw-r--r-- | drivers/ntb/hw/epf/ntb_hw_epf.c | 48 | ||||
-rw-r--r-- | drivers/ntb/hw/idt/ntb_hw_idt.c | 6 | ||||
-rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_gen1.c | 4 | ||||
-rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_gen4.c | 2 | ||||
-rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_intel.h | 9 | ||||
-rw-r--r-- | drivers/ntb/test/ntb_perf.c | 828 | ||||
-rw-r--r-- | drivers/pci/controller/dwc/pcie-designware-ep.c | 10 | ||||
-rw-r--r-- | drivers/pci/endpoint/functions/Kconfig | 11 | ||||
-rw-r--r-- | drivers/pci/endpoint/functions/Makefile | 1 | ||||
-rw-r--r-- | drivers/pci/endpoint/functions/pci-epf-vntb.c | 1425 |
13 files changed, 2598 insertions, 35 deletions
diff --git a/Documentation/PCI/endpoint/index.rst b/Documentation/PCI/endpoint/index.rst index 38ea1f604b6d..4d2333e7ae06 100644 --- a/Documentation/PCI/endpoint/index.rst +++ b/Documentation/PCI/endpoint/index.rst @@ -13,6 +13,8 @@ PCI Endpoint Framework pci-test-howto pci-ntb-function pci-ntb-howto + pci-vntb-function + pci-vntb-howto function/binding/pci-test function/binding/pci-ntb diff --git a/Documentation/PCI/endpoint/pci-vntb-function.rst b/Documentation/PCI/endpoint/pci-vntb-function.rst new file mode 100644 index 000000000000..cad8013e8839 --- /dev/null +++ b/Documentation/PCI/endpoint/pci-vntb-function.rst @@ -0,0 +1,126 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================= +PCI vNTB Function +================= + +:Author: Frank Li <Frank.Li@nxp.com> + +The difference between PCI NTB function and PCI vNTB function is + +PCI NTB function need at two endpoint instances and connect HOST1 +and HOST2. + +PCI vNTB function only use one host and one endpoint(EP), use NTB +connect EP and PCI host + +.. code-block:: text + + + +------------+ +---------------------------------------+ + | | | | + +------------+ | +--------------+ + | NTB | | | NTB | + | NetDev | | | NetDev | + +------------+ | +--------------+ + | NTB | | | NTB | + | Transfer | | | Transfer | + +------------+ | +--------------+ + | | | | | + | PCI NTB | | | | + | EPF | | | | + | Driver | | | PCI Virtual | + | | +---------------+ | NTB Driver | + | | | PCI EP NTB |<------>| | + | | | FN Driver | | | + +------------+ +---------------+ +--------------+ + | | | | | | + | PCI BUS | <-----> | PCI EP BUS | | Virtual PCI | + | | PCI | | | BUS | + +------------+ +---------------+--------+--------------+ + PCI RC PCI EP + +Constructs used for Implementing vNTB +===================================== + + 1) Config Region + 2) Self Scratchpad Registers + 3) Peer Scratchpad Registers + 4) Doorbell (DB) Registers + 5) Memory Window (MW) + + +Config Region: +-------------- + +It is same as PCI NTB Function driver + +Scratchpad Registers: +--------------------- + + It is appended after Config region. + + +--------------------------------------------------+ Base + | | + | | + | | + | Common Config Register | + | | + | | + | | + +-----------------------+--------------------------+ Base + span_offset + | | | + | Peer Span Space | Span Space | + | | | + | | | + +-----------------------+--------------------------+ Base + span_offset + | | | + span_count * 4 + | | | + | Span Space | Peer Span Space | + | | | + +-----------------------+--------------------------+ + Virtual PCI Pcie Endpoint + NTB Driver NTB Driver + + +Doorbell Registers: +------------------- + + Doorbell Registers are used by the hosts to interrupt each other. + +Memory Window: +-------------- + + Actual transfer of data between the two hosts will happen using the + memory window. + +Modeling Constructs: +==================== + +32-bit BARs. + +====== =============== +BAR NO CONSTRUCTS USED +====== =============== +BAR0 Config Region +BAR1 Doorbell +BAR2 Memory Window 1 +BAR3 Memory Window 2 +BAR4 Memory Window 3 +BAR5 Memory Window 4 +====== =============== + +64-bit BARs. + +====== =============================== +BAR NO CONSTRUCTS USED +====== =============================== +BAR0 Config Region + Scratchpad +BAR1 +BAR2 Doorbell +BAR3 +BAR4 Memory Window 1 +BAR5 +====== =============================== + + diff --git a/Documentation/PCI/endpoint/pci-vntb-howto.rst b/Documentation/PCI/endpoint/pci-vntb-howto.rst new file mode 100644 index 000000000000..b4a679144692 --- /dev/null +++ b/Documentation/PCI/endpoint/pci-vntb-howto.rst @@ -0,0 +1,161 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================================================================== +PCI Non-Transparent Bridge (NTB) Endpoint Function (EPF) User Guide +=================================================================== + +:Author: Frank Li <Frank.Li@nxp.com> + +This document is a guide to help users use pci-epf-vntb function driver +and ntb_hw_epf host driver for NTB functionality. The list of steps to +be followed in the host side and EP side is given below. For the hardware +configuration and internals of NTB using configurable endpoints see +Documentation/PCI/endpoint/pci-vntb-function.rst + +Endpoint Device +=============== + +Endpoint Controller Devices +--------------------------- + +To find the list of endpoint controller devices in the system:: + + # ls /sys/class/pci_epc/ + 5f010000.pcie_ep + +If PCI_ENDPOINT_CONFIGFS is enabled:: + + # ls /sys/kernel/config/pci_ep/controllers + 5f010000.pcie_ep + +Endpoint Function Drivers +------------------------- + +To find the list of endpoint function drivers in the system:: + + # ls /sys/bus/pci-epf/drivers + pci_epf_ntb pci_epf_test pci_epf_vntb + +If PCI_ENDPOINT_CONFIGFS is enabled:: + + # ls /sys/kernel/config/pci_ep/functions + pci_epf_ntb pci_epf_test pci_epf_vntb + + +Creating pci-epf-vntb Device +---------------------------- + +PCI endpoint function device can be created using the configfs. To create +pci-epf-vntb device, the following commands can be used:: + + # mount -t configfs none /sys/kernel/config + # cd /sys/kernel/config/pci_ep/ + # mkdir functions/pci_epf_vntb/func1 + +The "mkdir func1" above creates the pci-epf-ntb function device that will +be probed by pci_epf_vntb driver. + +The PCI endpoint framework populates the directory with the following +configurable fields:: + + # ls functions/pci_epf_ntb/func1 + baseclass_code deviceid msi_interrupts pci-epf-ntb.0 + progif_code secondary subsys_id vendorid + cache_line_size interrupt_pin msix_interrupts primary + revid subclass_code subsys_vendor_id + +The PCI endpoint function driver populates these entries with default values +when the device is bound to the driver. The pci-epf-vntb driver populates +vendorid with 0xffff and interrupt_pin with 0x0001:: + + # cat functions/pci_epf_vntb/func1/vendorid + 0xffff + # cat functions/pci_epf_vntb/func1/interrupt_pin + 0x0001 + + +Configuring pci-epf-vntb Device +------------------------------- + +The user can configure the pci-epf-vntb device using its configfs entry. In order +to change the vendorid and the deviceid, the following +commands can be used:: + + # echo 0x1957 > functions/pci_epf_vntb/func1/vendorid + # echo 0x0809 > functions/pci_epf_vntb/func1/deviceid + +In order to configure NTB specific attributes, a new sub-directory to func1 +should be created:: + + # mkdir functions/pci_epf_vntb/func1/pci_epf_vntb.0/ + +The NTB function driver will populate this directory with various attributes +that can be configured by the user:: + + # ls functions/pci_epf_vntb/func1/pci_epf_vntb.0/ + db_count mw1 mw2 mw3 mw4 num_mws + spad_count + +A sample configuration for NTB function is given below:: + + # echo 4 > functions/pci_epf_vntb/func1/pci_epf_vntb.0/db_count + # echo 128 > functions/pci_epf_vntb/func1/pci_epf_vntb.0/spad_count + # echo 1 > functions/pci_epf_vntb/func1/pci_epf_vntb.0/num_mws + # echo 0x100000 > functions/pci_epf_vntb/func1/pci_epf_vntb.0/mw1 + +Binding pci-epf-ntb Device to EP Controller +-------------------------------------------- + +NTB function device should be attached to PCI endpoint controllers +connected to the host. + + # ln -s controllers/5f010000.pcie_ep functions/pci-epf-ntb/func1/primary + +Once the above step is completed, the PCI endpoint controllers are ready to +establish a link with the host. + + +Start the Link +-------------- + +In order for the endpoint device to establish a link with the host, the _start_ +field should be populated with '1'. For NTB, both the PCI endpoint controllers +should establish link with the host (imx8 don't need this steps):: + + # echo 1 > controllers/5f010000.pcie_ep/start + +RootComplex Device +================== + +lspci Output at Host side +------------------------ + +Note that the devices listed here correspond to the values populated in +"Creating pci-epf-ntb Device" section above:: + + # lspci + 00:00.0 PCI bridge: Freescale Semiconductor Inc Device 0000 (rev 01) + 01:00.0 RAM memory: Freescale Semiconductor Inc Device 0809 + +Endpoint Device / Virtual PCI bus +================================= + +lspci Output at EP Side / Virtual PCI bus +----------------------------------------- + +Note that the devices listed here correspond to the values populated in +"Creating pci-epf-ntb Device" section above:: + + # lspci + 10:00.0 Unassigned class [ffff]: Dawicontrol Computersysteme GmbH Device 1234 (rev ff) + +Using ntb_hw_epf Device +----------------------- + +The host side software follows the standard NTB software architecture in Linux. +All the existing client side NTB utilities like NTB Transport Client and NTB +Netdev, NTB Ping Pong Test Client and NTB Tool Test Client can be used with NTB +function device. + +For more information on NTB see +:doc:`Non-Transparent Bridge <../../driver-api/ntb>` diff --git a/drivers/ntb/hw/epf/ntb_hw_epf.c b/drivers/ntb/hw/epf/ntb_hw_epf.c index b019755e4e21..3ece49cb18ff 100644 --- a/drivers/ntb/hw/epf/ntb_hw_epf.c +++ b/drivers/ntb/hw/epf/ntb_hw_epf.c @@ -45,7 +45,6 @@ #define NTB_EPF_MIN_DB_COUNT 3 #define NTB_EPF_MAX_DB_COUNT 31 -#define NTB_EPF_MW_OFFSET 2 #define NTB_EPF_COMMAND_TIMEOUT 1000 /* 1 Sec */ @@ -67,6 +66,7 @@ struct ntb_epf_dev { enum pci_barno ctrl_reg_bar; enum pci_barno peer_spad_reg_bar; enum pci_barno db_reg_bar; + enum pci_barno mw_bar; unsigned int mw_count; unsigned int spad_count; @@ -92,6 +92,8 @@ struct ntb_epf_data { enum pci_barno peer_spad_reg_bar; /* BAR that contains Doorbell region and Memory window '1' */ enum pci_barno db_reg_bar; + /* BAR that contains memory windows*/ + enum pci_barno mw_bar; }; static int ntb_epf_send_command(struct ntb_epf_dev *ndev, u32 command, @@ -411,7 +413,7 @@ static int ntb_epf_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, return -EINVAL; } - bar = idx + NTB_EPF_MW_OFFSET; + bar = idx + ndev->mw_bar; mw_size = pci_resource_len(ntb->pdev, bar); @@ -453,7 +455,7 @@ static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx, if (idx == 0) offset = readl(ndev->ctrl_reg + NTB_EPF_MW1_OFFSET); - bar = idx + NTB_EPF_MW_OFFSET; + bar = idx + ndev->mw_bar; if (base) *base = pci_resource_start(ndev->ntb.pdev, bar) + offset; @@ -565,6 +567,7 @@ static int ntb_epf_init_pci(struct ntb_epf_dev *ndev, struct pci_dev *pdev) { struct device *dev = ndev->dev; + size_t spad_sz, spad_off; int ret; pci_set_drvdata(pdev, ndev); @@ -599,10 +602,16 @@ static int ntb_epf_init_pci(struct ntb_epf_dev *ndev, goto err_dma_mask; } - ndev->peer_spad_reg = pci_iomap(pdev, ndev->peer_spad_reg_bar, 0); - if (!ndev->peer_spad_reg) { - ret = -EIO; - goto err_dma_mask; + if (ndev->peer_spad_reg_bar) { + ndev->peer_spad_reg = pci_iomap(pdev, ndev->peer_spad_reg_bar, 0); + if (!ndev->peer_spad_reg) { + ret = -EIO; + goto err_dma_mask; + } + } else { + spad_sz = 4 * readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); + spad_off = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); + ndev->peer_spad_reg = ndev->ctrl_reg + spad_off + spad_sz; } ndev->db_reg = pci_iomap(pdev, ndev->db_reg_bar, 0); @@ -657,6 +666,7 @@ static int ntb_epf_pci_probe(struct pci_dev *pdev, enum pci_barno peer_spad_reg_bar = BAR_1; enum pci_barno ctrl_reg_bar = BAR_0; enum pci_barno db_reg_bar = BAR_2; + enum pci_barno mw_bar = BAR_2; struct device *dev = &pdev->dev; struct ntb_epf_data *data; struct ntb_epf_dev *ndev; @@ -671,17 +681,16 @@ static int ntb_epf_pci_probe(struct pci_dev *pdev, data = (struct ntb_epf_data *)id->driver_data; if (data) { - if (data->peer_spad_reg_bar) - peer_spad_reg_bar = data->peer_spad_reg_bar; - if (data->ctrl_reg_bar) - ctrl_reg_bar = data->ctrl_reg_bar; - if (data->db_reg_bar) - db_reg_bar = data->db_reg_bar; + peer_spad_reg_bar = data->peer_spad_reg_bar; + ctrl_reg_bar = data->ctrl_reg_bar; + db_reg_bar = data->db_reg_bar; + mw_bar = data->mw_bar; } ndev->peer_spad_reg_bar = peer_spad_reg_bar; ndev->ctrl_reg_bar = ctrl_reg_bar; ndev->db_reg_bar = db_reg_bar; + ndev->mw_bar = mw_bar; ndev->dev = dev; ntb_epf_init_struct(ndev, pdev); @@ -729,6 +738,14 @@ static const struct ntb_epf_data j721e_data = { .ctrl_reg_bar = BAR_0, .peer_spad_reg_bar = BAR_1, .db_reg_bar = BAR_2, + .mw_bar = BAR_2, +}; + +static const struct ntb_epf_data mx8_data = { + .ctrl_reg_bar = BAR_0, + .peer_spad_reg_bar = BAR_0, + .db_reg_bar = BAR_2, + .mw_bar = BAR_4, }; static const struct pci_device_id ntb_epf_pci_tbl[] = { @@ -737,6 +754,11 @@ static const struct pci_device_id ntb_epf_pci_tbl[] = { .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, .driver_data = (kernel_ulong_t)&j721e_data, }, + { + PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x0809), + .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, + .driver_data = (kernel_ulong_t)&mx8_data, + }, { }, }; diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c index 733557231ed0..0ed6f809ff2e 100644 --- a/drivers/ntb/hw/idt/ntb_hw_idt.c +++ b/drivers/ntb/hw/idt/ntb_hw_idt.c @@ -2406,7 +2406,7 @@ static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf, "\t%hhu.\t", idx); else off += scnprintf(strbuf + off, size - off, - "\t%hhu-%hhu.\t", idx, idx + cnt - 1); + "\t%hhu-%d.\t", idx, idx + cnt - 1); off += scnprintf(strbuf + off, size - off, "%s BAR%hhu, ", idt_get_mw_name(data), ndev->mws[idx].bar); @@ -2435,7 +2435,7 @@ static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf, "\t%hhu.\t", idx); else off += scnprintf(strbuf + off, size - off, - "\t%hhu-%hhu.\t", idx, idx + cnt - 1); + "\t%hhu-%d.\t", idx, idx + cnt - 1); off += scnprintf(strbuf + off, size - off, "%s BAR%hhu, ", idt_get_mw_name(data), @@ -2480,7 +2480,7 @@ static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf, int src; data = idt_ntb_msg_read(&ndev->ntb, &src, idx); off += scnprintf(strbuf + off, size - off, - "\t%hhu. 0x%08x from peer %hhu (Port %hhu)\n", + "\t%hhu. 0x%08x from peer %d (Port %hhu)\n", idx, data, src, ndev->peers[src].port); } off += scnprintf(strbuf + off, size - off, "\n"); diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.c b/drivers/ntb/hw/intel/ntb_hw_gen1.c index e5f14e20a9ff..72e2027a71c4 100644 --- a/drivers/ntb/hw/intel/ntb_hw_gen1.c +++ b/drivers/ntb/hw/intel/ntb_hw_gen1.c @@ -1874,7 +1874,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev, rc = gen3_init_dev(ndev); if (rc) goto err_init_dev; - } else if (pdev_is_gen4(pdev)) { + } else if (pdev_is_gen4(pdev) || pdev_is_gen5(pdev)) { ndev->ntb.ops = &intel_ntb4_ops; rc = intel_ntb_init_pci(ndev, pdev); if (rc) @@ -2047,6 +2047,8 @@ static const struct pci_device_id intel_ntb_pci_tbl[] = { /* GEN4 */ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_ICX)}, + /* SPR has same dev id has ICX but different revision id */ + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_GNR)}, {0} }; MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl); diff --git a/drivers/ntb/hw/intel/ntb_hw_gen4.c b/drivers/ntb/hw/intel/ntb_hw_gen4.c index 4081fc538ff4..22cac7975b3c 100644 --- a/drivers/ntb/hw/intel/ntb_hw_gen4.c +++ b/drivers/ntb/hw/intel/ntb_hw_gen4.c @@ -197,7 +197,7 @@ int gen4_init_dev(struct intel_ntb_dev *ndev) ppd1 = ioread32(ndev->self_mmio + GEN4_PPD1_OFFSET); if (pdev_is_ICX(pdev)) ndev->ntb.topo = gen4_ppd_topo(ndev, ppd1); - else if (pdev_is_SPR(pdev)) + else if (pdev_is_SPR(pdev) || pdev_is_gen5(pdev)) ndev->ntb.topo = spr_ppd_topo(ndev, ppd1); dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd1, ntb_topo_string(ndev->ntb.topo)); diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h index b233d1c6ba2d..380ec0d8e0d9 100644 --- a/drivers/ntb/hw/intel/ntb_hw_intel.h +++ b/drivers/ntb/hw/intel/ntb_hw_intel.h @@ -70,6 +70,8 @@ #define PCI_DEVICE_ID_INTEL_NTB_SS_BDX 0x6F0F #define PCI_DEVICE_ID_INTEL_NTB_B2B_SKX 0x201C #define PCI_DEVICE_ID_INTEL_NTB_B2B_ICX 0x347e +#define PCI_DEVICE_ID_INTEL_NTB_B2B_SPR 0x347e +#define PCI_DEVICE_ID_INTEL_NTB_B2B_GNR 0x0db4 /* Ntb control and link status */ #define NTB_CTL_CFG_LOCK BIT(0) @@ -225,7 +227,14 @@ static inline int pdev_is_gen4(struct pci_dev *pdev) { if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_ICX) return 1; + return 0; +} +static inline int pdev_is_gen5(struct pci_dev *pdev) +{ + if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_GNR) + return 1; return 0; } + #endif diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c index 65e1e5cf1b29..af7d3de65eb3 100644 --- a/drivers/ntb/test/ntb_perf.c +++ b/drivers/ntb/test/ntb_perf.c @@ -6,6 +6,7 @@ * * Copyright(c) 2015 Intel Corporation. All rights reserved. * Copyright(c) 2017 T-Platforms. All Rights Reserved. + * Copyright(c) 2022 YADRO. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as @@ -15,6 +16,7 @@ * * Copyright(c) 2015 Intel Corporation. All rights reserved. * Copyright(c) 2017 T-Platforms. All Rights Reserved. + * Copyright(c) 2022 YADRO. All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -65,6 +67,22 @@ * * root@self# echo 0 > $DBG_DIR/run * root@self# cat $DBG_DIR/run + *----------------------------------------------------------------------------- + * Eg: start latency test with peer (index 0) poll-waiting and get the metrics + * + * Server side: + * root@self# echo 0 > $DBG_DIR/poll_latency/run_server + * Client side: + * root@self# echo 0 > $DBG_DIR/poll_latency/run_client + * root@self# cat $DBG_DIR/poll_latency/run_client + *----------------------------------------------------------------------------- + * Eg: start doorbell latency test with peer (index 0) and get the metrics + * + * Server side: + * root@self# echo 0 > $DBG_DIR/db_latency/run_server + * Client side: + * root@self# echo 0 > $DBG_DIR/db_latency/run_client + * root@self# cat $DBG_DIR/db_latency/run_client */ #include <linux/init.h> @@ -76,6 +94,7 @@ #include <linux/dmaengine.h> #include <linux/pci.h> #include <linux/ktime.h> +#include <linux/jiffies.h> #include <linux/slab.h> #include <linux/delay.h> #include <linux/sizes.h> @@ -85,7 +104,7 @@ #include <linux/ntb.h> #define DRIVER_NAME "ntb_perf" -#define DRIVER_VERSION "2.0" +#define DRIVER_VERSION "2.3" MODULE_LICENSE("Dual BSD/GPL"); MODULE_VERSION(DRIVER_VERSION); @@ -106,6 +125,9 @@ MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool"); #define PERF_BUF_LEN 1024 +#define LAT_MIN_TRIES 20 +#define RESCHEDULE_RATIO 8192 /* power of 2, to avoid actual division */ + static unsigned long max_mw_size; module_param(max_mw_size, ulong, 0644); MODULE_PARM_DESC(max_mw_size, "Upper limit of memory window size"); @@ -122,6 +144,22 @@ static bool use_dma; /* default to 0 */ module_param(use_dma, bool, 0644); MODULE_PARM_DESC(use_dma, "Use DMA engine to measure performance"); +static bool perf_latency = true; +module_param(perf_latency, bool, 0644); +MODULE_PARM_DESC(perf_latency, "Measure burst latency"); + +static unsigned long lat_time_ms = 1000; /* default 1s */ +module_param(lat_time_ms, ulong, 0644); +MODULE_PARM_DESC(lat_time_ms, "Time (in ms) to test latency"); + +static unsigned long lat_timeout_us = 500; +module_param(lat_timeout_us, ulong, 0644); +MODULE_PARM_DESC(lat_timeout_us, "Timeout (in us) to wait for server reply"); + +static unsigned long peer_timeout_s = 60; +module_param(peer_timeout_s, ulong, 0644); +MODULE_PARM_DESC(peer_timeout_s, "Timeout (in s) to wait for peer link"); + /*============================================================================== * Perf driver data definition *============================================================================== @@ -138,7 +176,21 @@ enum perf_cmd { PERF_STS_LNKUP = 6, /* link up state flag */ }; +enum run_mode { + RUN_PL_CLIENT, + RUN_PL_SERVER, + RUN_DBL_CLIENT, + RUN_DBL_SERVER, +}; + struct perf_ctx; +struct perf_ext_lat_data; + +struct perf_ext_lat_ops { + int (*init)(struct perf_ext_lat_data *data); + int (*run)(struct perf_ext_lat_data *data); + void (*clear)(struct perf_ext_lat_data *data); +}; struct perf_peer { struct perf_ctx *perf; @@ -178,12 +230,30 @@ struct perf_thread { void *src; u64 copied; ktime_t duration; + ktime_t latency; + u64 tries; int status; struct work_struct work; }; #define to_thread_work(__work) \ container_of(__work, struct perf_thread, work) +struct perf_ext_lat_data { + struct perf_ctx *perf; + ktime_t latency; + u64 tries; + int status; + struct perf_ext_lat_ops ops; + struct work_struct work; + + union { + void *src; + int db; + }; +}; +#define to_ext_lat_data(__work) \ + container_of(__work, struct perf_ext_lat_data, work) + struct perf_ctx { struct ntb_dev *ntb; @@ -192,6 +262,12 @@ struct perf_ctx { int pcnt; struct perf_peer *peers; + /* Ext latency tests interface */ + enum run_mode mode; + struct perf_ext_lat_data pldata; + struct perf_ext_lat_data dbldata; + atomic_t running; + /* Performance measuring work-threads interface */ unsigned long busy_flag; wait_queue_head_t twait; @@ -239,6 +315,8 @@ static struct dentry *perf_dbgfs_topdir; static struct workqueue_struct *perf_wq __read_mostly; +static const u8 stop_word = 0xFF; + /*============================================================================== * NTB cross-link commands execution service *============================================================================== @@ -501,6 +579,15 @@ static void perf_link_event(void *ctx) } } +static inline void perf_dbl_pong(struct perf_ctx *perf) +{ + struct perf_ext_lat_data *data = &perf->dbldata; + + ntb_db_clear(perf->ntb, BIT_ULL(data->db)); + data->tries++; + ntb_peer_db_set(perf->ntb, BIT_ULL(data->db)); +} + static void perf_db_event(void *ctx, int vec) { struct perf_ctx *perf = ctx; @@ -509,7 +596,11 @@ static void perf_db_event(void *ctx, int vec) ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb)); /* Just receive all available commands */ - (void)perf_cmd_recv(perf); + if (perf->dbldata.db >= 0 && + BIT_ULL(perf->dbldata.db) & ntb_db_read(perf->ntb)) + perf_dbl_pong(perf); + else + (void)perf_cmd_recv(perf); } static void perf_msg_event(void *ctx) @@ -664,6 +755,8 @@ static int perf_init_service(struct perf_ctx *perf) return -EINVAL; } + perf->dbldata.db = -1; + if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) { perf->cmd_send = perf_msg_cmd_send; perf->cmd_recv = perf_msg_cmd_recv; @@ -783,7 +876,7 @@ static void perf_dma_copy_callback(void *data) } static int perf_copy_chunk(struct perf_thread *pthr, - void __iomem *dst, void *src, size_t len) + void __iomem *dst, void *src, size_t len, bool _use_dma) { struct dma_async_tx_descriptor *tx; struct dmaengine_unmap_data *unmap; @@ -794,7 +887,7 @@ static int perf_copy_chunk(struct perf_thread *pthr, void __iomem *dst_vaddr; dma_addr_t dst_dma_addr; - if (!use_dma) { + if (!_use_dma) { memcpy_toio(dst, src, len); goto ret_check_tsync; } @@ -940,7 +1033,7 @@ static int perf_run_test(struct perf_thread *pthr) /* Copied field is cleared on test launch stage */ while (pthr->copied < total_size) { - ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size); + ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size, use_dma); if (ret) { dev_err(&perf->ntb->dev, "%d: Got error %d on test\n", pthr->tidx, ret); @@ -1018,6 +1111,69 @@ no_dma_notify: kfree(pthr->src); } +static int perf_run_latency(struct perf_thread *pthr) +{ + struct perf_peer *peer = pthr->perf->test_peer; + struct ntb_dev *ntb = pthr->perf->ntb; + void __iomem *flt_dst, *bnd_dst; + void *flt_src; + u64 stop_at; + u32 rem; + int ret; + + pthr->tries = 0; + pthr->latency = ktime_get(); + flt_src = pthr->src; + flt_dst = peer->outbuf; + bnd_dst = peer->outbuf + peer->outbuf_size; + + stop_at = ktime_get_real_fast_ns() + lat_time_ms * NSEC_PER_MSEC; + while (ktime_get_real_fast_ns() < stop_at) { + ret = perf_copy_chunk(pthr, flt_dst, flt_src, 1, false); + if (ret) { + dev_err(&ntb->dev, "%d: Latency testing error %d\n", + pthr->tidx, ret); + pthr->latency = ktime_set(0, 0); + return ret; + } + + pthr->tries++; + flt_dst++; + flt_src++; + + if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) { + flt_dst = peer->outbuf; + flt_src = pthr->src; + } + + /* Avoid processor soft lock-ups */ + div_u64_rem(pthr->tries, RESCHEDULE_RATIO, &rem); + if (!rem) + schedule(); + } + + /* Stop timer */ + pthr->latency = ktime_sub(ktime_get(), pthr->latency); + + if (pthr->tries < LAT_MIN_TRIES) { + dev_err(&ntb->dev, + "%d: Too few steps (%llu) to measure Latency, recommended > %d. Increase value of 'lat_time_ms' parameter\n", + pthr->tidx, pthr->tries, LAT_MIN_TRIES); + pthr->latency = ktime_set(0, 0); + return -EINVAL; + } + + dev_dbg(&ntb->dev, "%d: made %llu tries, lasted %llu usecs\n", + pthr->tidx, pthr->tries, ktime_to_us(pthr->latency)); + + pthr->latency = ns_to_ktime(ktime_divns(pthr->latency, pthr->tries)); + + dev_dbg(&ntb->dev, "%d: latency %llu us (%llu ns)\n", pthr->tidx, + ktime_to_us(pthr->latency), ktime_to_ns(pthr->latency)); + + return 0; +} + static void perf_thread_work(struct work_struct *work) { struct perf_thread *pthr = to_thread_work(work); @@ -1043,11 +1199,284 @@ static void perf_thread_work(struct work_struct *work) } pthr->status = perf_sync_test(pthr); + if (pthr->status) + goto err_clear_test; + + if (perf_latency) + pthr->status = perf_run_latency(pthr); err_clear_test: perf_clear_test(pthr); } +static int perf_init_pl(struct perf_ext_lat_data *pldata) +{ + struct perf_ctx *perf = pldata->perf; + struct perf_peer *peer = perf->test_peer; + u8 *bp; + + pldata->src = kmalloc_node(peer->outbuf_size, GFP_KERNEL, + dev_to_node(&perf->ntb->dev)); + if (!pldata->src) + return -ENOMEM; + + /* + * Prepare random data to send, guaranteed exclusion of 0x00 (unreceived) + * and 0xFF (stop_word) + */ + get_random_bytes(pldata->src, peer->outbuf_size); + for (bp = pldata->src; bp < (u8 *) pldata->src + peer->outbuf_size; bp++) + while (*bp == 0 || *bp == stop_word) + *bp = (u8)get_random_int(); + + memset(peer->inbuf, 0, peer->inbuf_size); + + return 0; +} + +static int perf_poll_peer_reply(volatile u8 *cur) +{ + u64 wait_till = ktime_get_real_fast_ns() + lat_timeout_us * NSEC_PER_USEC; + + while (ktime_get_real_fast_ns() < wait_till) { + if (*cur == stop_word) { + *cur = 0; + return 1; + } + if (*cur != 0) { + *cur = 0; + return 0; + } + } + return -EINTR; +} + +static int perf_run_pl_client(struct perf_ext_lat_data *pldata) +{ + struct perf_ctx *perf = pldata->perf; + struct perf_peer *peer = perf->test_peer; + struct ntb_dev *ntb = perf->ntb; + void *src = pldata->src; + u64 stop_at; + int ret; + + dev_dbg(&ntb->dev, "poll_lat: client started.\n"); + + pldata->tries = 0; + pldata->latency = ktime_get(); + + stop_at = ktime_get_real_fast_ns() + lat_time_ms * NSEC_PER_MSEC; + while (ktime_get_real_fast_ns() < stop_at) { + memcpy_toio(peer->outbuf, src, 1); + + /* Avoid processor soft lock-ups */ + schedule(); + + ret = perf_poll_peer_reply(peer->inbuf); + if (ret < 0) { + dev_err(&ntb->dev, "Timeout waiting for peer reply on poll latency\n"); + pldata->latency = ktime_set(0, 0); + return -EINTR; + } else if (ret == 1) { + dev_warn(&ntb->dev, "Server terminated on poll latency, stopping\n"); + break; + } else if (!atomic_read(&perf->running)) { + dev_err(&ntb->dev, "Poll latency client terminated\n"); + return -EINTR; + } + + pldata->tries++; + src++; + + if (src >= pldata->src + peer->outbuf_size) + src = pldata->src; + } + + /* Stop timer */ + pldata->latency = ktime_sub(ktime_get(), pldata->latency); + /* Send stop to peer */ + memcpy_toio(peer->outbuf, &stop_word, 1); + + if (pldata->tries < LAT_MIN_TRIES) { + dev_err(&ntb->dev, + "Too few steps (%llu) to measure Latency, recommended > %d. Increase value of 'lat_time_ms' parameter\n", + pldata->tries, LAT_MIN_TRIES); + pldata->latency = ktime_set(0, 0); + return -EINVAL; + } + + dev_dbg(&ntb->dev, "poll_lat: made %llu tries, lasted %llu usecs\n", + pldata->tries, ktime_to_us(pldata->latency)); + + pldata->latency = ns_to_ktime(ktime_divns(pldata->latency, pldata->tries)); + + dev_dbg(&ntb->dev, "poll_lat: latency %llu us (%llu ns)\n", + ktime_to_us(pldata->latency), ktime_to_ns(pldata->latency)); + + return 0; +} + +static int perf_run_pl_server(struct perf_ext_lat_data *pldata) +{ + struct perf_ctx *perf = pldata->perf; + struct perf_peer *peer = perf->test_peer; + struct ntb_dev *ntb = perf->ntb; + void *src = pldata->src; + int ret = 0; + + dev_dbg(&ntb->dev, "poll_lat: server started.\n"); + + pldata->tries = 0; + + while (ret != 1 && atomic_read(&perf->running)) { + ret = perf_poll_peer_reply(peer->inbuf); + if (!ret) { + /* Pong to client */ + memcpy_toio(peer->outbuf, src++, 1); + if (src >= pldata->src + peer->outbuf_size) + src = pldata->src; + + pldata->tries++; + } + + /* Avoid processor soft lock-ups */ + schedule(); + } + + if (pldata->tries < LAT_MIN_TRIES) + dev_warn(&ntb->dev, + "Poll latency test terminated too early. Increase client's test time\n"); + + dev_dbg(&ntb->dev, "poll_lat: server stopped, had responded %llu times\n", + pldata->tries); + + return atomic_read(&perf->running) ? -ENODATA : -EINTR; +} + +static void perf_clear_pl(struct perf_ext_lat_data *pldata) +{ + struct perf_ctx *perf = pldata->perf; + struct perf_peer *peer = perf->test_peer; + + memset(peer->inbuf, stop_word, 1); + atomic_set(&perf->running, 0); + wake_up(&perf->twait); + kfree(pldata->src); +} + +static struct perf_ext_lat_ops perf_pl_client_ops = { + .init = perf_init_pl, + .run = perf_run_pl_client, + .clear = perf_clear_pl +}; + +static struct perf_ext_lat_ops perf_pl_server_ops = { + .init = perf_init_pl, + .run = perf_run_pl_server, + .clear = perf_clear_pl +}; + +static int perf_init_dbl(struct perf_ext_lat_data *data) +{ + struct perf_ctx *perf = data->perf; + + data->db = get_bitmask_order(ntb_db_valid_mask(perf->ntb)) - 1; + dev_dbg(&perf->ntb->dev, "DB bit for latency test: %d\n", data->db); + + if (data->db <= perf->gidx) { + dev_err(&perf->ntb->dev, "No spare DoorBell found\n"); + data->db = -1; + return -ENOSPC; + } + + return ntb_db_clear_mask(perf->ntb, BIT_ULL(data->db)); +} + +static int perf_run_dbl_client(struct perf_ext_lat_data *data) +{ + struct perf_ctx *perf = data->perf; + struct ntb_dev *ntb = perf->ntb; + u64 stop_at; + + dev_dbg(&ntb->dev, "db_lat: client started.\n"); + + data->tries = 0; + data->latency = ktime_get(); + + if (ntb_peer_db_set(perf->ntb, BIT_ULL(data->db))) + return -EIO; + + stop_at = ktime_get_real_fast_ns() + lat_time_ms * NSEC_PER_MSEC; + while (ktime_get_real_fast_ns() < stop_at) { + /* Avoid processor soft lock-ups */ + schedule(); + + if (!atomic_read(&perf->running)) { + dev_err(&ntb->dev, "DoorBell latency client terminated\n"); + return -EINTR; + } + } + + /* Stop timer */ + data->latency = ktime_sub(ktime_get(), data->latency); + + if (data->tries < LAT_MIN_TRIES) { + dev_err(&ntb->dev, + "Too few steps (%llu) to measure Latency, recommended > %d. Increase value of 'lat_time_ms' parameter\n", + data->tries, LAT_MIN_TRIES); + data->latency = ktime_set(0, 0); + return -EINVAL; + } + + dev_dbg(&ntb->dev, "db_lat: made %llu tries, lasted %llu usecs\n", + data->tries, ktime_to_us(data->latency)); + + data->latency = ns_to_ktime(ktime_divns(data->latency, data->tries)); + + dev_dbg(&ntb->dev, "db_lat: latency %llu us (%llu ns)\n", + ktime_to_us(data->latency), ktime_to_ns(data->latency)); + + return 0; +} + +static void perf_clear_dbl(struct perf_ext_lat_data *data) +{ + struct perf_ctx *perf = data->perf; + + data->db = -1; + ntb_db_set_mask(perf->ntb, BIT_ULL(data->db)); + atomic_set(&perf->running, 0); + wake_up(&perf->twait); +} + +static struct perf_ext_lat_ops perf_dbl_client_ops = { + .init = perf_init_dbl, + .run = perf_run_dbl_client, + .clear = perf_clear_dbl +}; + +static void perf_ext_lat_work(struct work_struct *work) +{ + struct perf_ext_lat_data *data = to_ext_lat_data(work); + + if (!data->ops.init || !data->ops.run || !data->ops.clear) { + struct perf_ctx *perf = data->perf; + + data->status = -EFAULT; + atomic_set(&perf->running, 0); + wake_up(&perf->twait); + return; + } + + data->status = data->ops.init(data); + if (data->status) + return; + + data->status = data->ops.run(data); + + data->ops.clear(data); +} + static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt) { if (tcnt == 0 || tcnt > MAX_THREADS_CNT) @@ -1068,7 +1497,10 @@ static void perf_terminate_test(struct perf_ctx *perf) int tidx; atomic_set(&perf->tsync, -1); + atomic_set(&perf->running, 0); wake_up(&perf->twait); + cancel_work_sync(&perf->pldata.work); + cancel_work_sync(&perf->dbldata.work); for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) { wake_up(&perf->threads[tidx].dma_wait); @@ -1082,9 +1514,10 @@ static int perf_submit_test(struct perf_peer *peer) struct perf_thread *pthr; int tidx, ret; - ret = wait_for_completion_interruptible(&peer->init_comp); - if (ret < 0) - return ret; + ret = wait_for_completion_interruptible_timeout(&peer->init_comp, + msecs_to_jiffies(peer_timeout_s * 1000)); + if (ret <= 0) + return ret ? ret : -ETIMEDOUT; if (test_and_set_bit_lock(0, &perf->busy_flag)) return -EBUSY; @@ -1114,6 +1547,63 @@ static int perf_submit_test(struct perf_peer *peer) return ret; } +static int perf_submit_ext_lat(struct perf_peer *peer) +{ + struct perf_ctx *perf = peer->perf; + int ret; + + ret = wait_for_completion_interruptible_timeout(&peer->init_comp, + msecs_to_jiffies(peer_timeout_s * 1000)); + if (ret <= 0) + return ret ? ret : -ETIMEDOUT; + + if (test_and_set_bit_lock(0, &perf->busy_flag)) + return -EBUSY; + + perf->test_peer = peer; + atomic_set(&perf->running, 1); + perf->pldata.status = -ENODATA; + perf->pldata.tries = 0; + perf->pldata.latency = ktime_set(0, 0); + perf->dbldata.status = -ENODATA; + perf->dbldata.tries = 0; + perf->dbldata.latency = ktime_set(0, 0); + + switch (perf->mode) { + case RUN_PL_SERVER: + perf->pldata.ops = perf_pl_server_ops; + (void)queue_work(perf_wq, &perf->pldata.work); + break; + case RUN_PL_CLIENT: + perf->pldata.ops = perf_pl_client_ops; + (void)queue_work(perf_wq, &perf->pldata.work); + break; + case RUN_DBL_SERVER: + ret = perf_init_dbl(&perf->dbldata); + dev_dbg(&perf->ntb->dev, "db_lat: server started.\n"); + goto submit_exit; + case RUN_DBL_CLIENT: + perf->dbldata.ops = perf_dbl_client_ops; + (void)queue_work(perf_wq, &perf->dbldata.work); + break; + default: + ret = -EINVAL; + goto submit_exit; + } + + ret = wait_event_interruptible(perf->twait, + !atomic_read(&perf->running)); + if (ret == -ERESTARTSYS) { + perf_terminate_test(perf); + ret = -EINTR; + } + +submit_exit: + clear_bit_unlock(0, &perf->busy_flag); + + return ret; +} + static int perf_read_stats(struct perf_ctx *perf, char *buf, size_t size, ssize_t *pos) { @@ -1142,6 +1632,18 @@ static int perf_read_stats(struct perf_ctx *perf, char *buf, "%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n", tidx, pthr->copied, ktime_to_us(pthr->duration), div64_u64(pthr->copied, ktime_to_us(pthr->duration))); + + if (perf_latency && ktime_compare(pthr->latency, ktime_set(0, 0))) { + if (ktime_to_us(pthr->latency) < 10) { + (*pos) += scnprintf(buf + *pos, size - *pos, + "%d: latency %llu ns\n", + tidx, ktime_to_ns(pthr->latency)); + } else { + (*pos) += scnprintf(buf + *pos, size - *pos, + "%d: latency %llu us\n", + tidx, ktime_to_us(pthr->latency)); + } + } } clear_bit_unlock(0, &perf->busy_flag); @@ -1149,7 +1651,7 @@ static int perf_read_stats(struct perf_ctx *perf, char *buf, return 0; } -static void perf_init_threads(struct perf_ctx *perf) +static void perf_init_workers(struct perf_ctx *perf) { struct perf_thread *pthr; int tidx; @@ -1158,6 +1660,14 @@ static void perf_init_threads(struct perf_ctx *perf) perf->test_peer = &perf->peers[0]; init_waitqueue_head(&perf->twait); + perf->pldata.perf = perf; + INIT_WORK(&perf->pldata.work, perf_ext_lat_work); + perf->pldata.status = -ENODATA; + + perf->dbldata.perf = perf; + INIT_WORK(&perf->dbldata.work, perf_ext_lat_work); + perf->dbldata.status = -ENODATA; + for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) { pthr = &perf->threads[tidx]; @@ -1169,7 +1679,7 @@ static void perf_init_threads(struct perf_ctx *perf) } } -static void perf_clear_threads(struct perf_ctx *perf) +static void perf_clear_workers(struct perf_ctx *perf) { perf_terminate_test(perf); } @@ -1313,6 +1823,193 @@ static const struct file_operations perf_dbgfs_run = { .write = perf_dbgfs_write_run }; +static ssize_t perf_dbgfs_read_run_pl(struct file *filep, char __user *ubuf, + size_t fsize, loff_t *offp) +{ + struct perf_ctx *perf = filep->private_data; + ssize_t size = PERF_BUF_LEN; + ssize_t pos = 0; + ssize_t ret; + char *buf; + + if (test_and_set_bit_lock(0, &perf->busy_flag)) + return -EBUSY; + + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + pos += scnprintf(buf + pos, size - pos, + " Peer %d test statistics:\n", perf->test_peer->pidx); + + if (perf->pldata.status != -ENODATA) { + if (perf->pldata.status) { + pos += scnprintf(buf + pos, size - pos, + "poll latency: error status %d\n", perf->pldata.status); + } else { + if (ktime_to_us(perf->pldata.latency) < 10) { + pos += scnprintf(buf + pos, size - pos, + "poll latency %llu ns\n", + ktime_to_ns(perf->pldata.latency)); + } else { + pos += scnprintf(buf + pos, size - pos, + "poll latency %llu us\n", + ktime_to_us(perf->pldata.latency)); + } + } + } else { + pos += scnprintf(buf + pos, size - pos, "Test did not run\n"); + } + + ret = simple_read_from_buffer(ubuf, fsize, offp, buf, pos); + + kfree(buf); + + clear_bit_unlock(0, &perf->busy_flag); + + return ret; +} + +static ssize_t perf_dbgfs_write_run_ext(struct file *filep, const char __user *ubuf, + size_t size, loff_t *offp, enum run_mode mode) +{ + struct perf_ctx *perf = filep->private_data; + struct ntb_dev *ntb = perf->ntb; + struct perf_peer *peer; + int pidx, ret; + + ret = kstrtoint_from_user(ubuf, size, 0, &pidx); + if (ret) + return ret; + + if (pidx < 0) { + switch (mode) { + case RUN_PL_SERVER: + dev_dbg(&ntb->dev, "poll_lat: kill server\n"); + if (test_bit(0, &perf->busy_flag)) { + peer = perf->test_peer; + /* Send stop to client */ + memcpy_toio(peer->outbuf, &stop_word, 1); + } + perf_terminate_test(perf); + clear_bit_unlock(0, &perf->busy_flag); + return size; + case RUN_DBL_SERVER: + dev_dbg(&ntb->dev, "db_lat: kill server\n"); + perf_clear_dbl(&perf->dbldata); + clear_bit_unlock(0, &perf->busy_flag); + return size; + default: + return -EINVAL; + } + } + + if (pidx >= perf->pcnt) + return -EINVAL; + + peer = &perf->peers[pidx]; + perf->mode = mode; + + ret = perf_submit_ext_lat(peer); + + return ret ? ret : size; +} + +static ssize_t perf_dbgfs_write_run_pl_client(struct file *filep, + const char __user *ubuf, size_t size, loff_t *offp) +{ + return perf_dbgfs_write_run_ext(filep, ubuf, size, offp, RUN_PL_CLIENT); +} + +static const struct file_operations perf_dbgfs_run_pl_client = { + .open = simple_open, + .read = perf_dbgfs_read_run_pl, + .write = perf_dbgfs_write_run_pl_client +}; + +static ssize_t perf_dbgfs_write_run_pl_server(struct file *filep, + const char __user *ubuf, size_t size, loff_t *offp) +{ + return perf_dbgfs_write_run_ext(filep, ubuf, size, offp, RUN_PL_SERVER); +} + +static const struct file_operations perf_dbgfs_run_pl_server = { + .open = simple_open, + .read = perf_dbgfs_read_run_pl, + .write = perf_dbgfs_write_run_pl_server +}; + +static ssize_t perf_dbgfs_read_run_dbl(struct file *filep, char __user *ubuf, + size_t fsize, loff_t *offp) +{ + struct perf_ctx *perf = filep->private_data; + ssize_t size = PERF_BUF_LEN; + ssize_t pos = 0; + ssize_t ret; + char *buf; + + if (test_and_set_bit_lock(0, &perf->busy_flag)) + return -EBUSY; + + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + pos += scnprintf(buf + pos, size - pos, + " Peer %d test statistics:\n", perf->test_peer->pidx); + + if (perf->dbldata.status != -ENODATA) { + if (perf->dbldata.status) { + pos += scnprintf(buf + pos, size - pos, + "doorbell latency: error status %d\n", perf->dbldata.status); + } else { + if (ktime_to_us(perf->dbldata.latency) < 10) { + pos += scnprintf(buf + pos, size - pos, + "doorbell latency %llu ns\n", + ktime_to_ns(perf->dbldata.latency)); + } else { + pos += scnprintf(buf + pos, size - pos, + "doorbell latency %llu us\n", + ktime_to_us(perf->dbldata.latency)); + } + } + } else { + pos += scnprintf(buf + pos, size - pos, "Test did not run\n"); + } + + ret = simple_read_from_buffer(ubuf, fsize, offp, buf, pos); + + kfree(buf); + + clear_bit_unlock(0, &perf->busy_flag); + + return ret; +} + +static ssize_t perf_dbgfs_write_run_dbl_client(struct file *filep, + const char __user *ubuf, size_t size, loff_t *offp) +{ + return perf_dbgfs_write_run_ext(filep, ubuf, size, offp, RUN_DBL_CLIENT); +} + +static const struct file_operations perf_dbgfs_run_dbl_client = { + .open = simple_open, + .read = perf_dbgfs_read_run_dbl, + .write = perf_dbgfs_write_run_dbl_client +}; + +static ssize_t perf_dbgfs_write_run_dbl_server(struct file *filep, + const char __user *ubuf, size_t size, loff_t *offp) +{ + return perf_dbgfs_write_run_ext(filep, ubuf, size, offp, RUN_DBL_SERVER); +} + +static const struct file_operations perf_dbgfs_run_dbl_server = { + .open = simple_open, + .read = perf_dbgfs_read_run_dbl, + .write = perf_dbgfs_write_run_dbl_server +}; + static ssize_t perf_dbgfs_read_tcnt(struct file *filep, char __user *ubuf, size_t size, loff_t *offp) { @@ -1344,15 +2041,76 @@ static ssize_t perf_dbgfs_write_tcnt(struct file *filep, return size; } +static ssize_t perf_dbgfs_read_lattrs(struct file *filep, char __user *ubuf, + size_t size, loff_t *offp) +{ + size_t buf_size = min_t(size_t, size, PERF_BUF_LEN); + struct perf_ctx *perf = filep->private_data; + ssize_t pos, ret; + char *buf; + int tidx; + + buf = kmalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + pos = scnprintf(buf, buf_size, " Peer %d latency try count:\n", + perf->test_peer->pidx); + + for (tidx = 0; tidx < perf->tcnt; tidx++) { + struct perf_thread *pthr = &perf->threads[tidx]; + + pos += scnprintf(buf + pos, buf_size - pos, + "%d: made %llu tries\n", tidx, pthr->tries); + } + + ret = simple_read_from_buffer(ubuf, size, offp, buf, pos); + + kfree(buf); + + return ret; +} + +static ssize_t perf_dbgfs_read_inbuf(struct file *filep, char __user *ubuf, + size_t size, loff_t *offp) +{ + struct perf_ctx *perf = filep->private_data; + char buf[32]; + ssize_t pos; + u64 *value; + + if (!perf->test_peer || !perf->test_peer->inbuf) { + pos = scnprintf(buf, sizeof(buf), "NULL\n"); + } else { + value = perf->test_peer->inbuf; + pos = scnprintf(buf, sizeof(buf), "0x%llx\n", *value); + } + + return simple_read_from_buffer(ubuf, size, offp, buf, pos); +} + static const struct file_operations perf_dbgfs_tcnt = { .open = simple_open, .read = perf_dbgfs_read_tcnt, .write = perf_dbgfs_write_tcnt }; +static const struct file_operations perf_dbgfs_lattrs = { + .open = simple_open, + .read = perf_dbgfs_read_lattrs +}; + +static const struct file_operations perf_dbgfs_inbuf = { + .open = simple_open, + .read = perf_dbgfs_read_inbuf, +}; + static void perf_setup_dbgfs(struct perf_ctx *perf) { struct pci_dev *pdev = perf->ntb->pdev; + struct dentry *burst_lat_dir; + struct dentry *poll_lat_dir; + struct dentry *db_lat_dir; perf->dbgfs_dir = debugfs_create_dir(pci_name(pdev), perf_dbgfs_topdir); if (!perf->dbgfs_dir) { @@ -1363,11 +2121,10 @@ static void perf_setup_dbgfs(struct perf_ctx *perf) debugfs_create_file("info", 0600, perf->dbgfs_dir, perf, &perf_dbgfs_info); - debugfs_create_file("run", 0600, perf->dbgfs_dir, perf, - &perf_dbgfs_run); + debugfs_create_symlink("run", perf->dbgfs_dir, "burst_latency/run"); - debugfs_create_file("threads_count", 0600, perf->dbgfs_dir, perf, - &perf_dbgfs_tcnt); + debugfs_create_symlink("threads_count", perf->dbgfs_dir, + "burst_latency/threads_count"); /* They are made read-only for test exec safety and integrity */ debugfs_create_u8("chunk_order", 0500, perf->dbgfs_dir, &chunk_order); @@ -1375,6 +2132,45 @@ static void perf_setup_dbgfs(struct perf_ctx *perf) debugfs_create_u8("total_order", 0500, perf->dbgfs_dir, &total_order); debugfs_create_bool("use_dma", 0500, perf->dbgfs_dir, &use_dma); + + debugfs_create_file("inbuf", 0400, perf->dbgfs_dir, perf, + &perf_dbgfs_inbuf); + + /* burst_latency subdir */ + + burst_lat_dir = debugfs_create_dir("burst_latency", perf->dbgfs_dir); + + debugfs_create_file("run", 0600, burst_lat_dir, perf, &perf_dbgfs_run); + + debugfs_create_file("threads_count", 0600, burst_lat_dir, perf, + &perf_dbgfs_tcnt); + + debugfs_create_file("tries", 0400, burst_lat_dir, perf, + &perf_dbgfs_lattrs); + + /* poll_latency subdir */ + + poll_lat_dir = debugfs_create_dir("poll_latency", perf->dbgfs_dir); + + debugfs_create_file("run_client", 0600, poll_lat_dir, perf, + &perf_dbgfs_run_pl_client); + + debugfs_create_file("run_server", 0600, poll_lat_dir, perf, + &perf_dbgfs_run_pl_server); + + debugfs_create_u64("tries", 0400, poll_lat_dir, &perf->pldata.tries); + + /* db_latency subdir */ + + db_lat_dir = debugfs_create_dir("db_latency", perf->dbgfs_dir); + + debugfs_create_file("run_client", 0600, db_lat_dir, perf, + &perf_dbgfs_run_dbl_client); + + debugfs_create_file("run_server", 0600, db_lat_dir, perf, + &perf_dbgfs_run_dbl_server); + + debugfs_create_u64("tries", 0400, db_lat_dir, &perf->dbldata.tries); } static void perf_clear_dbgfs(struct perf_ctx *perf) @@ -1494,7 +2290,7 @@ static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb) if (ret) return ret; - perf_init_threads(perf); + perf_init_workers(perf); ret = perf_init_service(perf); if (ret) @@ -1517,7 +2313,7 @@ static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb) perf_disable_service(perf); - perf_clear_threads(perf); + perf_clear_workers(perf); } static struct ntb_client perf_client = { diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index cf1627679716..83ddb190292e 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -161,7 +161,11 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no, int type, u32 free_win; struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - free_win = find_first_zero_bit(ep->ib_window_map, pci->num_ib_windows); + if (!ep->bar_to_atu[bar]) + free_win = find_first_zero_bit(ep->ib_window_map, pci->num_ib_windows); + else + free_win = ep->bar_to_atu[bar]; + if (free_win >= pci->num_ib_windows) { dev_err(pci->dev, "No free inbound window\n"); return -EINVAL; @@ -218,6 +222,7 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); clear_bit(atu_index, ep->ib_window_map); ep->epf_bar[bar] = NULL; + ep->bar_to_atu[bar] = 0; } static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, @@ -245,6 +250,9 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, if (ret) return ret; + if (ep->epf_bar[bar]) + return 0; + dw_pcie_dbi_ro_wr_en(pci); dw_pcie_writel_dbi2(pci, reg, lower_32_bits(size - 1)); diff --git a/drivers/pci/endpoint/functions/Kconfig b/drivers/pci/endpoint/functions/Kconfig index 5f1242ca2f4e..362555b024e8 100644 --- a/drivers/pci/endpoint/functions/Kconfig +++ b/drivers/pci/endpoint/functions/Kconfig @@ -25,3 +25,14 @@ config PCI_EPF_NTB device tree. If in doubt, say "N" to disable Endpoint NTB driver. + +config PCI_EPF_VNTB + tristate "PCI Endpoint NTB driver" + depends on PCI_ENDPOINT + select CONFIGFS_FS + help + Select this configuration option to enable the Non-Transparent + Bridge (NTB) driver for PCI Endpoint. NTB driver implements NTB + between PCI host and PCIe Endpoint. + + If in doubt, say "N" to disable Endpoint NTB driver. diff --git a/drivers/pci/endpoint/functions/Makefile b/drivers/pci/endpoint/functions/Makefile index 96ab932a537a..5c13001deaba 100644 --- a/drivers/pci/endpoint/functions/Makefile +++ b/drivers/pci/endpoint/functions/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_PCI_EPF_TEST) += pci-epf-test.o obj-$(CONFIG_PCI_EPF_NTB) += pci-epf-ntb.o +obj-$(CONFIG_PCI_EPF_VNTB) += pci-epf-vntb.o diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c new file mode 100644 index 000000000000..111568089d45 --- /dev/null +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -0,0 +1,1425 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Endpoint Function Driver to implement Non-Transparent Bridge functionality + * Between PCI RC and EP + * + * Copyright (C) 2020 Texas Instruments + * Copyright (C) 2022 NXP + * + * Based on pci-epf-ntb.c + * Author: Frank Li <Frank.Li@nxp.com> + * Author: Kishon Vijay Abraham I <kishon@ti.com> + */ + +/** + * +------------+ +---------------------------------------+ + * | | | | + * +------------+ | +--------------+ + * | NTB | | | NTB | + * | NetDev | | | NetDev | + * +------------+ | +--------------+ + * | NTB | | | NTB | + * | Transfer | | | Transfer | + * +------------+ | +--------------+ + * | | | | | + * | PCI NTB | | | | + * | EPF | | | | + * | Driver | | | PCI Virtual | + * | | +---------------+ | NTB Driver | + * | | | PCI EP NTB |<------>| | + * | | | FN Driver | | | + * +------------+ +---------------+ +--------------+ + * | | | | | | + * | PCI BUS | <-----> | PCI EP BUS | | Virtual PCI | + * | | PCI | | | BUS | + * +------------+ +---------------+--------+--------------+ + * PCI RC PCI EP + */ + +#include <linux/delay.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/slab.h> + +#include <linux/pci-epc.h> +#include <linux/pci-epf.h> +#include <linux/ntb.h> + +static struct workqueue_struct *kpcintb_workqueue; + +#define COMMAND_CONFIGURE_DOORBELL 1 +#define COMMAND_TEARDOWN_DOORBELL 2 +#define COMMAND_CONFIGURE_MW 3 +#define COMMAND_TEARDOWN_MW 4 +#define COMMAND_LINK_UP 5 +#define COMMAND_LINK_DOWN 6 + +#define COMMAND_STATUS_OK 1 +#define COMMAND_STATUS_ERROR 2 + +#define LINK_STATUS_UP BIT(0) + +#define SPAD_COUNT 64 +#define DB_COUNT 4 +#define NTB_MW_OFFSET 2 +#define DB_COUNT_MASK GENMASK(15, 0) +#define MSIX_ENABLE BIT(16) +#define MAX_DB_COUNT 32 +#define MAX_MW 4 + +#define VNTB_VID 0x1957 +#define VNTB_PID 0x080A + +enum epf_ntb_bar { + BAR_CONFIG, + BAR_DB, + BAR_MW0, + BAR_MW1, + BAR_MW2, +}; + +/* + * +--------------------------------------------------+ Base + * | | + * | | + * | | + * | Common Control Register | + * | | + * | | + * | | + * +-----------------------+--------------------------+ Base+span_offset + * | | | + * | Peer Span Space | Span Space | + * | | | + * | | | + * +-----------------------+--------------------------+ Base+span_offset + * | | | +span_count * 4 + * | | | + * | Span Space | Peer Span Space | + * | | | + * +-----------------------+--------------------------+ + * Virtual PCI Pcie Endpoint + * NTB Driver NTB Driver + */ +struct epf_ntb_ctrl { + u32 command; + u32 argument; + u16 command_status; + u16 link_status; + u32 topology; + u64 addr; + u64 size; + u32 num_mws; + u32 reserved; + u32 spad_offset; + u32 spad_count; + u32 db_entry_size; + u32 db_data[MAX_DB_COUNT]; + u32 db_offset[MAX_DB_COUNT]; +} __packed; + +struct epf_ntb { + struct ntb_dev ntb; + struct pci_epf *epf; + struct config_group group; + + u32 num_mws; + u32 db_count; + u32 spad_count; + u64 mws_size[MAX_MW]; + u64 db; + u32 vbus_number; + + bool linkup; + u32 spad_size; + + enum pci_barno epf_ntb_bar[6]; + + struct epf_ntb_ctrl *reg; + + phys_addr_t epf_db_phy; + void __iomem *epf_db; + + phys_addr_t vpci_mw_phy[MAX_MW]; + void __iomem *vpci_mw_addr[MAX_MW]; + + struct delayed_work cmd_handler; +}; + +#define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group) +#define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb) + +static struct pci_epf_header epf_ntb_header = { + .vendorid = PCI_ANY_ID, + .deviceid = PCI_ANY_ID, + .baseclass_code = PCI_BASE_CLASS_MEMORY, + .interrupt_pin = PCI_INTERRUPT_INTA, +}; + +/** + * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host + * @ntb: NTB device that facilitates communication between HOST and VHOST + * @link_up: true or false indicating Link is UP or Down + * + * Once NTB function in HOST invoke ntb_link_enable(), + * this NTB function driver will trigger a link event to vhost. + */ +static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) +{ + if (link_up) + ntb->reg->link_status |= LINK_STATUS_UP; + else + ntb->reg->link_status &= ~LINK_STATUS_UP; + + ntb_link_event(&ntb->ntb); + return 0; +} + +/** + * epf_ntb_configure_mw() - Configure the Outbound Address Space for vhost + * to access the memory window of host + * @ntb: NTB device that facilitates communication between host and vhost + * @mw: Index of the memory window (either 0, 1, 2 or 3) + * + * EP Outbound Window + * +--------+ +-----------+ + * | | | | + * | | | | + * | | | | + * | | | | + * | | +-----------+ + * | Virtual| | Memory Win| + * | NTB | -----------> | | + * | Driver | | | + * | | +-----------+ + * | | | | + * | | | | + * +--------+ +-----------+ + * VHost PCI EP + */ +static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw) +{ + phys_addr_t phys_addr; + u8 func_no, vfunc_no; + u64 addr, size; + int ret = 0; + + phys_addr = ntb->vpci_mw_phy[mw]; + addr = ntb->reg->addr; + size = ntb->reg->size; + + func_no = ntb->epf->func_no; + vfunc_no = ntb->epf->vfunc_no; + + ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size); + if (ret) + dev_err(&ntb->epf->epc->dev, + "intf: Failed to map memory window %d address\n", mw); + return ret; +} + +/** + * epf_ntb_teardown_mw() - Teardown the configured OB ATU + * @ntb: NTB device that facilitates communication between HOST and vHOST + * @mw: Index of the memory window (either 0, 1, 2 or 3) + * + * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using + * pci_epc_unmap_addr() + */ +static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw) +{ + pci_epc_unmap_addr(ntb->epf->epc, + ntb->epf->func_no, + ntb->epf->vfunc_no, + ntb->vpci_mw_phy[mw]); +} + +/** + * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host + * @work: work_struct for the epf_ntb_epc + * + * Workqueue function that gets invoked for the two epf_ntb_epc + * periodically (once every 5ms) to see if it has received any commands + * from NTB host. The host can send commands to configure doorbell or + * configure memory window or to update link status. + */ +static void epf_ntb_cmd_handler(struct work_struct *work) +{ + struct epf_ntb_ctrl *ctrl; + u32 command, argument; + struct epf_ntb *ntb; + struct device *dev; + int ret; + int i; + + ntb = container_of(work, struct epf_ntb, cmd_handler.work); + + for (i = 1; i < ntb->db_count; i++) { + if (readl(ntb->epf_db + i * 4)) { + if (readl(ntb->epf_db + i * 4)) + ntb->db |= 1 << (i - 1); + + ntb_db_event(&ntb->ntb, i); + writel(0, ntb->epf_db + i * 4); + } + } + + ctrl = ntb->reg; + command = ctrl->command; + if (!command) + goto reset_handler; + argument = ctrl->argument; + + ctrl->command = 0; + ctrl->argument = 0; + + ctrl = ntb->reg; + dev = &ntb->epf->dev; + + switch (command) { + case COMMAND_CONFIGURE_DOORBELL: + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_TEARDOWN_DOORBELL: + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_CONFIGURE_MW: + ret = epf_ntb_configure_mw(ntb, argument); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_TEARDOWN_MW: + epf_ntb_teardown_mw(ntb, argument); + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_LINK_UP: + ntb->linkup = true; + ret = epf_ntb_link_up(ntb, true); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + goto reset_handler; + case COMMAND_LINK_DOWN: + ntb->linkup = false; + ret = epf_ntb_link_up(ntb, false); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + break; + default: + dev_err(dev, "intf UNKNOWN command: %d\n", command); + break; + } + +reset_handler: + queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler, + msecs_to_jiffies(5)); +} + +/** + * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR + * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound + * address. + * + * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and + * self scratchpad region (removes inbound ATU configuration). While BAR0 is + * the default self scratchpad BAR, an NTB could have other BARs for self + * scratchpad (because of reserved BARs). This function can get the exact BAR + * used for self scratchpad from epf_ntb_bar[BAR_CONFIG]. + * + * Please note the self scratchpad region and config region is combined to + * a single region and mapped using the same BAR. Also note HOST2's peer + * scratchpad is HOST1's self scratchpad. + */ +static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb) +{ + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + + barno = ntb->epf_ntb_bar[BAR_CONFIG]; + epf_bar = &ntb->epf->bar[barno]; + + pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar); +} + +/** + * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST and vHOST + * + * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and + * self scratchpad region. + * + * Please note the self scratchpad region and config region is combined to + * a single region and mapped using the same BAR. + */ +static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb) +{ + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + u8 func_no, vfunc_no; + struct device *dev; + int ret; + + dev = &ntb->epf->dev; + func_no = ntb->epf->func_no; + vfunc_no = ntb->epf->vfunc_no; + barno = ntb->epf_ntb_bar[BAR_CONFIG]; + epf_bar = &ntb->epf->bar[barno]; + + ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar); + if (ret) { + dev_err(dev, "inft: Config/Status/SPAD BAR set failed\n"); + return ret; + } + return 0; +} + +/** + * epf_ntb_config_spad_bar_free() - Free the physical memory associated with + * config + scratchpad region + * @ntb: NTB device that facilitates communication between HOST and vHOST + */ +static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb) +{ + enum pci_barno barno; + + barno = ntb->epf_ntb_bar[BAR_CONFIG]; + pci_epf_free_space(ntb->epf, ntb->reg, barno, 0); +} + +/** + * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad + * region + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Allocate the Local Memory mentioned in the above diagram. The size of + * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION + * is obtained from "spad-count" configfs entry. + */ +static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb) +{ + size_t align; + enum pci_barno barno; + struct epf_ntb_ctrl *ctrl; + u32 spad_size, ctrl_size; + u64 size; + struct pci_epf *epf = ntb->epf; + struct device *dev = &epf->dev; + u32 spad_count; + void *base; + int i; + const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc, + epf->func_no, + epf->vfunc_no); + barno = ntb->epf_ntb_bar[BAR_CONFIG]; + size = epc_features->bar_fixed_size[barno]; + align = epc_features->align; + + if ((!IS_ALIGNED(size, align))) + return -EINVAL; + + spad_count = ntb->spad_count; + + ctrl_size = sizeof(struct epf_ntb_ctrl); + spad_size = 2 * spad_count * 4; + + if (!align) { + ctrl_size = roundup_pow_of_two(ctrl_size); + spad_size = roundup_pow_of_two(spad_size); + } else { + ctrl_size = ALIGN(ctrl_size, align); + spad_size = ALIGN(spad_size, align); + } + + if (!size) + size = ctrl_size + spad_size; + else if (size < ctrl_size + spad_size) + return -EINVAL; + + base = pci_epf_alloc_space(epf, size, barno, align, 0); + if (!base) { + dev_err(dev, "intf: Config/Status/SPAD alloc region fail\n"); + return -ENOMEM; + } + + ntb->reg = base; + + ctrl = ntb->reg; + ctrl->spad_offset = ctrl_size; + + ctrl->spad_count = spad_count; + ctrl->num_mws = ntb->num_mws; + ntb->spad_size = spad_size; + + ctrl->db_entry_size = 4; + + for (i = 0; i < ntb->db_count; i++) { + ntb->reg->db_data[i] = 1 + i; + ntb->reg->db_offset[i] = 0; + } + + return 0; +} + +/** + * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capaiblity + * @ntb: NTB device that facilitates communication between HOST and vHOST + * + * Configure MSI/MSI-X capability for each interface with number of + * interrupts equal to "db_count" configfs entry. + */ +static int epf_ntb_configure_interrupt(struct epf_ntb *ntb) +{ + const struct pci_epc_features *epc_features; + bool msix_capable, msi_capable; + u8 func_no, vfunc_no; + struct device *dev; + u32 db_count; + int ret; + + dev = &ntb->epf->dev; + + epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no); + msix_capable = epc_features->msix_capable; + msi_capable = epc_features->msi_capable; + + if (!(msix_capable || msi_capable)) { + dev_err(dev, "MSI or MSI-X is required for doorbell\n"); + return -EINVAL; + } + + func_no = ntb->epf->func_no; + vfunc_no = ntb->epf->vfunc_no; + + db_count = ntb->db_count; + if (db_count > MAX_DB_COUNT) { + dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); + return -EINVAL; + } + + ntb->db_count = db_count; + + if (msi_capable) { + ret = pci_epc_set_msi(ntb->epf->epc, func_no, vfunc_no, 16); + if (ret) { + dev_err(dev, "intf: MSI configuration failed\n"); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_db_bar_init() - Configure Doorbell window BARs + * @ntb: NTB device that facilitates communication between HOST and vHOST + * + */ +static int epf_ntb_db_bar_init(struct epf_ntb *ntb) +{ + const struct pci_epc_features *epc_features; + u32 align; + struct device *dev = &ntb->epf->dev; + int ret; + struct pci_epf_bar *epf_bar; + void __iomem *mw_addr; + enum pci_barno barno; + size_t size = 4 * ntb->db_count; + + epc_features = pci_epc_get_features(ntb->epf->epc, + ntb->epf->func_no, + ntb->epf->vfunc_no); + align = epc_features->align; + + if (size < 128) + size = 128; + + if (align) + size = ALIGN(size, align); + else + size = roundup_pow_of_two(size); + + barno = ntb->epf_ntb_bar[BAR_DB]; + + mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, align, 0); + if (!mw_addr) { + dev_err(dev, "intf: Failed to allocate OB address\n"); + return -ENOMEM; + } + + ntb->epf_db = mw_addr; + + epf_bar = &ntb->epf->bar[barno]; + + ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar); + if (ret) { + dev_err(dev, "intf: DoorBell BAR set failed\n"); + goto err_alloc_peer_mem; + } + return ret; + +err_alloc_peer_mem: + pci_epc_mem_free_addr(ntb->epf->epc, epf_bar->phys_addr, mw_addr, epf_bar->size); + return -1; +} + +/** + * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory + * allocated in peers outbound address space + * @ntb: NTB device that facilitates communication between HOST and vHOST + */ +static void epf_ntb_db_bar_clear(struct epf_ntb *ntb) +{ + enum pci_barno barno; + + barno = ntb->epf_ntb_bar[BAR_DB]; + pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0); + pci_epc_clear_bar(ntb->epf->epc, + ntb->epf->func_no, + ntb->epf->vfunc_no, + &ntb->epf->bar[barno]); +} + +/** + * epf_ntb_mw_bar_init() - Configure Memory window BARs + * @ntb: NTB device that facilitates communication between HOST and vHOST + * + */ +static int epf_ntb_mw_bar_init(struct epf_ntb *ntb) +{ + int ret = 0; + int i; + u64 size; + enum pci_barno barno; + struct device *dev = &ntb->epf->dev; + + for (i = 0; i < ntb->num_mws; i++) { + + size = ntb->mws_size[i]; + + barno = ntb->epf_ntb_bar[BAR_MW0 + i]; + + ntb->epf->bar[barno].barno = barno; + ntb->epf->bar[barno].size = size; + ntb->epf->bar[barno].addr = 0; + ntb->epf->bar[barno].phys_addr = 0; + ntb->epf->bar[barno].flags |= upper_32_bits(size) ? + PCI_BASE_ADDRESS_MEM_TYPE_64 : + PCI_BASE_ADDRESS_MEM_TYPE_32; + + ret = pci_epc_set_bar(ntb->epf->epc, + ntb->epf->func_no, + ntb->epf->vfunc_no, + &ntb->epf->bar[barno]); + if (ret) { + dev_err(dev, "intf: MW set failed\n"); + goto err_alloc_mem; + } + + /* allocate epc outbound memory windows to vpci vntb device */ + ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc, + &ntb->vpci_mw_phy[i], + size); + if (!ntb->vpci_mw_addr[i]) { + dev_err(dev, "Failed to allocate source address\n"); + goto err_alloc_mem; + } + } + + return ret; +err_alloc_mem: + return ret; +} + +/** + * epf_ntb_mw_bar_clear() - Clear Memory window BARs + * @ntb: NTB device that facilitates communication between HOST and vHOST + * + */ +static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb) +{ + enum pci_barno barno; + int i; + + for (i = 0; i < ntb->num_mws; i++) { + barno = ntb->epf_ntb_bar[BAR_MW0 + i]; + pci_epc_clear_bar(ntb->epf->epc, + ntb->epf->func_no, + ntb->epf->vfunc_no, + &ntb->epf->bar[barno]); + + pci_epc_mem_free_addr(ntb->epf->epc, + ntb->vpci_mw_phy[i], + ntb->vpci_mw_addr[i], + ntb->mws_size[i]); + } +} + +/** + * epf_ntb_epc_destroy() - Cleanup NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST and vHOST + * + * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces + */ +static void epf_ntb_epc_destroy(struct epf_ntb *ntb) +{ + pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0); + pci_epc_put(ntb->epf->epc); +} + +/** + * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB + * constructs (scratchpad region, doorbell, memorywindow) + * @ntb: NTB device that facilitates communication between HOST and vHOST + * + */ +static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) +{ + const struct pci_epc_features *epc_features; + enum pci_barno barno; + enum epf_ntb_bar bar; + struct device *dev; + u32 num_mws; + int i; + + barno = BAR_0; + num_mws = ntb->num_mws; + dev = &ntb->epf->dev; + epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no); + + /* These are required BARs which are mandatory for NTB functionality */ + for (bar = BAR_CONFIG; bar <= BAR_MW0; bar++, barno++) { + barno = pci_epc_get_next_free_bar(epc_features, barno); + if (barno < 0) { + dev_err(dev, "intf: Fail to get NTB function BAR\n"); + return barno; + } + ntb->epf_ntb_bar[bar] = barno; + } + + /* These are optional BARs which don't impact NTB functionality */ + for (bar = BAR_MW1, i = 1; i < num_mws; bar++, barno++, i++) { + barno = pci_epc_get_next_free_bar(epc_features, barno); + if (barno < 0) { + ntb->num_mws = i; + dev_dbg(dev, "BAR not available for > MW%d\n", i + 1); + } + ntb->epf_ntb_bar[bar] = barno; + } + + return 0; +} + +/** + * epf_ntb_epc_init() - Initialize NTB interface + * @ntb: NTB device that facilitates communication between HOST and vHOST2 + * + * Wrapper to initialize a particular EPC interface and start the workqueue + * to check for commands from host. This function will write to the + * EP controller HW for configuring it. + */ +static int epf_ntb_epc_init(struct epf_ntb *ntb) +{ + u8 func_no, vfunc_no; + struct pci_epc *epc; + struct pci_epf *epf; + struct device *dev; + int ret; + + epf = ntb->epf; + dev = &epf->dev; + epc = epf->epc; + func_no = ntb->epf->func_no; + vfunc_no = ntb->epf->vfunc_no; + + ret = epf_ntb_config_sspad_bar_set(ntb); + if (ret) { + dev_err(dev, "intf: Config/self SPAD BAR init failed"); + return ret; + } + + ret = epf_ntb_configure_interrupt(ntb); + if (ret) { + dev_err(dev, "intf: Interrupt configuration failed\n"); + goto err_config_interrupt; + } + + ret = epf_ntb_db_bar_init(ntb); + if (ret) { + dev_err(dev, "intf: DB BAR init failed\n"); + goto err_db_bar_init; + } + + ret = epf_ntb_mw_bar_init(ntb); + if (ret) { + dev_err(dev, "intf: MW BAR init failed\n"); + goto err_mw_bar_init; + } + + if (vfunc_no <= 1) { + ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header); + if (ret) { + dev_err(dev, "intf: Configuration header write failed\n"); + goto err_write_header; + } + } + + INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler); + queue_work(kpcintb_workqueue, &ntb->cmd_handler.work); + + return 0; + +err_write_header: + epf_ntb_mw_bar_clear(ntb); +err_mw_bar_init: + epf_ntb_db_bar_clear(ntb); +err_db_bar_init: +err_config_interrupt: + epf_ntb_config_sspad_bar_clear(ntb); + + return ret; +} + + +/** + * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper to cleanup all NTB interfaces. + */ +static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) +{ + epf_ntb_db_bar_clear(ntb); + epf_ntb_mw_bar_clear(ntb); +} + +#define EPF_NTB_R(_name) \ +static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ + char *page) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + \ + return sprintf(page, "%d\n", ntb->_name); \ +} + +#define EPF_NTB_W(_name) \ +static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ + const char *page, size_t len) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + u32 val; \ + int ret; \ + \ + ret = kstrtou32(page, 0, &val); \ + if (ret) \ + return ret; \ + \ + ntb->_name = val; \ + \ + return len; \ +} + +#define EPF_NTB_MW_R(_name) \ +static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ + char *page) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + int win_no; \ + \ + sscanf(#_name, "mw%d", &win_no); \ + \ + return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \ +} + +#define EPF_NTB_MW_W(_name) \ +static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ + const char *page, size_t len) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + struct device *dev = &ntb->epf->dev; \ + int win_no; \ + u64 val; \ + int ret; \ + \ + ret = kstrtou64(page, 0, &val); \ + if (ret) \ + return ret; \ + \ + if (sscanf(#_name, "mw%d", &win_no) != 1) \ + return -EINVAL; \ + \ + if (ntb->num_mws < win_no) { \ + dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \ + return -EINVAL; \ + } \ + \ + ntb->mws_size[win_no - 1] = val; \ + \ + return len; \ +} + +static ssize_t epf_ntb_num_mws_store(struct config_item *item, + const char *page, size_t len) +{ + struct config_group *group = to_config_group(item); + struct epf_ntb *ntb = to_epf_ntb(group); + u32 val; + int ret; + + ret = kstrtou32(page, 0, &val); + if (ret) + return ret; + + if (val > MAX_MW) + return -EINVAL; + + ntb->num_mws = val; + + return len; +} + +EPF_NTB_R(spad_count) +EPF_NTB_W(spad_count) +EPF_NTB_R(db_count) +EPF_NTB_W(db_count) +EPF_NTB_R(num_mws) +EPF_NTB_R(vbus_number) +EPF_NTB_W(vbus_number) +EPF_NTB_MW_R(mw1) +EPF_NTB_MW_W(mw1) +EPF_NTB_MW_R(mw2) +EPF_NTB_MW_W(mw2) +EPF_NTB_MW_R(mw3) +EPF_NTB_MW_W(mw3) +EPF_NTB_MW_R(mw4) +EPF_NTB_MW_W(mw4) + +CONFIGFS_ATTR(epf_ntb_, spad_count); +CONFIGFS_ATTR(epf_ntb_, db_count); +CONFIGFS_ATTR(epf_ntb_, num_mws); +CONFIGFS_ATTR(epf_ntb_, mw1); +CONFIGFS_ATTR(epf_ntb_, mw2); +CONFIGFS_ATTR(epf_ntb_, mw3); +CONFIGFS_ATTR(epf_ntb_, mw4); +CONFIGFS_ATTR(epf_ntb_, vbus_number); + +static struct configfs_attribute *epf_ntb_attrs[] = { + &epf_ntb_attr_spad_count, + &epf_ntb_attr_db_count, + &epf_ntb_attr_num_mws, + &epf_ntb_attr_mw1, + &epf_ntb_attr_mw2, + &epf_ntb_attr_mw3, + &epf_ntb_attr_mw4, + &epf_ntb_attr_vbus_number, + NULL, +}; + +static const struct config_item_type ntb_group_type = { + .ct_attrs = epf_ntb_attrs, + .ct_owner = THIS_MODULE, +}; + +/** + * epf_ntb_add_cfs() - Add configfs directory specific to NTB + * @epf: NTB endpoint function device + * @group: A pointer to the config_group structure referencing a group of + * config_items of a specific type that belong to a specific sub-system. + * + * Add configfs directory specific to NTB. This directory will hold + * NTB specific properties like db_count, spad_count, num_mws etc., + */ +static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf, + struct config_group *group) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + struct config_group *ntb_group = &ntb->group; + struct device *dev = &epf->dev; + + config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type); + + return ntb_group; +} + +/*==== virtual PCI bus driver, which only load virutal ntb pci driver ====*/ + +#define VPCI_BUS_NUM 0x10 + +uint32_t pci_space[] = { + (VNTB_VID | (VNTB_PID << 16)), //DeviceID, Vendor ID + 0, // status, Command + 0xffffffff, // Class code, subclass, prog if, revision id + 0x40, //bist, header type, latency Timer, cache line size + 0, //bar 0 + 0, //bar 1 + 0, //bar 2 + 0, //bar 3 + 0, //bar 4 + 0, //bar 5 + 0, //cardbus cis point + 0, //Subsystem ID Subystem vendor id + 0, //ROM Base Address + 0, //Reserved, Cap. Point + 0, //Reserved, + 0, //Max Lat, Min Gnt, interrupt pin, interrupt line +}; + +static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) +{ + if (devfn == 0) { + memcpy(val, ((uint8_t *)pci_space) + where, size); + return 0; + } + return -1; +} + +static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) +{ + return 0; +} + +struct pci_ops vpci_ops = { + .read = pci_read, + .write = pci_write, +}; + +static int vpci_bus(void *sysdata) +{ + struct pci_bus *vpci_bus; + + vpci_bus = pci_scan_bus(VPCI_BUS_NUM, &vpci_ops, sysdata); + if (vpci_bus) + pr_err("create pci bus\n"); + + pci_bus_add_devices(vpci_bus); + + return 0; +} + +/*==================== Virtual PCIe NTB driver ==========================*/ + +static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx) +{ + struct epf_ntb *ndev = ntb_ndev(ntb); + + return ndev->num_mws; +} + +static int vntb_epf_spad_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->spad_count; +} + +static int vntb_epf_peer_mw_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->num_mws; +} + +static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb) +{ + return BIT_ULL(ntb_ndev(ntb)->db_count) - 1; +} + +static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) +{ + return 0; +} + +static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx, + dma_addr_t addr, resource_size_t size) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + int ret; + struct device *dev; + + dev = &ntb->ntb.dev; + barno = ntb->epf_ntb_bar[BAR_MW0 + idx]; + epf_bar = &ntb->epf->bar[barno]; + epf_bar->phys_addr = addr; + epf_bar->barno = barno; + epf_bar->size = size; + + ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar); + if (ret) { + dev_err(dev, "failure set mw trans\n"); + return ret; + } + return 0; +} + +static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx) +{ + return 0; +} + +static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx, + phys_addr_t *base, resource_size_t *size) +{ + + struct epf_ntb *ntb = ntb_ndev(ndev); + + if (base) + *base = ntb->vpci_mw_phy[idx]; + + if (size) + *size = ntb->mws_size[idx]; + + return 0; +} + +static int vntb_epf_link_enable(struct ntb_dev *ntb, + enum ntb_speed max_speed, + enum ntb_width max_width) +{ + return 0; +} + +static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * 4; + u32 val; + void __iomem *base = ntb->reg; + + val = readl(base + off + ct + idx * 4); + return val; +} + +static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + struct epf_ntb_ctrl *ctrl = ntb->reg; + int off = ctrl->spad_offset, ct = ctrl->spad_count * 4; + void __iomem *base = ntb->reg; + + writel(val, base + off + ct + idx * 4); + return 0; +} + +static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + struct epf_ntb_ctrl *ctrl = ntb->reg; + int off = ctrl->spad_offset; + void __iomem *base = ntb->reg; + u32 val; + + val = readl(base + off + idx * 4); + return val; +} + +static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + struct epf_ntb_ctrl *ctrl = ntb->reg; + int off = ctrl->spad_offset; + void __iomem *base = ntb->reg; + + writel(val, base + off + idx * 4); + return 0; +} + +static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits) +{ + u32 interrupt_num = ffs(db_bits) + 1; + struct epf_ntb *ntb = ntb_ndev(ndev); + u8 func_no, vfunc_no; + int ret; + + func_no = ntb->epf->func_no; + vfunc_no = ntb->epf->vfunc_no; + + ret = pci_epc_raise_irq(ntb->epf->epc, + func_no, + vfunc_no, + PCI_EPC_IRQ_MSI, + interrupt_num + 1); + if (ret) { + dev_err(&ntb->ntb.dev, "intf: Failed to raise IRQ\n"); + return ret; + } + + return 0; +} + +static u64 vntb_epf_db_read(struct ntb_dev *ndev) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + + return ntb->db; +} + +static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx, + resource_size_t *addr_align, + resource_size_t *size_align, + resource_size_t *size_max) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + + if (addr_align) + *addr_align = SZ_4K; + + if (size_align) + *size_align = 1; + + if (size_max) + *size_max = ntb->mws_size[idx]; + + return 0; +} + +static u64 vntb_epf_link_is_up(struct ntb_dev *ndev, + enum ntb_speed *speed, + enum ntb_width *width) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + + return ntb->reg->link_status; +} + +static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits) +{ + return 0; +} + +static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits) +{ + struct epf_ntb *ntb = ntb_ndev(ndev); + + ntb->db &= ~db_bits; + return 0; +} + +static int vntb_epf_link_disable(struct ntb_dev *ntb) +{ + return 0; +} + +static const struct ntb_dev_ops vntb_epf_ops = { + .mw_count = vntb_epf_mw_count, + .spad_count = vntb_epf_spad_count, + .peer_mw_count = vntb_epf_peer_mw_count, + .db_valid_mask = vntb_epf_db_valid_mask, + .db_set_mask = vntb_epf_db_set_mask, + .mw_set_trans = vntb_epf_mw_set_trans, + .mw_clear_trans = vntb_epf_mw_clear_trans, + .peer_mw_get_addr = vntb_epf_peer_mw_get_addr, + .link_enable = vntb_epf_link_enable, + .spad_read = vntb_epf_spad_read, + .spad_write = vntb_epf_spad_write, + .peer_spad_read = vntb_epf_peer_spad_read, + .peer_spad_write = vntb_epf_peer_spad_write, + .peer_db_set = vntb_epf_peer_db_set, + .db_read = vntb_epf_db_read, + .mw_get_align = vntb_epf_mw_get_align, + .link_is_up = vntb_epf_link_is_up, + .db_clear_mask = vntb_epf_db_clear_mask, + .db_clear = vntb_epf_db_clear, + .link_disable = vntb_epf_link_disable, +}; + +static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + int ret; + struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata; + struct device *dev = &pdev->dev; + + ndev->ntb.pdev = pdev; + ndev->ntb.topo = NTB_TOPO_NONE; + ndev->ntb.ops = &vntb_epf_ops; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "Cannot set DMA mask\n"); + return -1; + } + + ret = ntb_register_device(&ndev->ntb); + if (ret) { + dev_err(dev, "Failed to register NTB device\n"); + goto err_register_dev; + } + + dev_info(dev, "PCI Virtual NTB driver loaded\n"); + return 0; + +err_register_dev: + return -1; +} + +static const struct pci_device_id pci_vntb_table[] = { + { + PCI_DEVICE(VNTB_VID, VNTB_PID), + }, + {}, +}; + +static struct pci_driver vntb_pci_driver = { + .name = "pci-vntb", + .id_table = pci_vntb_table, + .probe = pci_vntb_probe, +}; + +/* ============ PCIe EPF Driver Bind ====================*/ + +/** + * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality + * @epf: NTB endpoint function device + * + * Initialize both the endpoint controllers associated with NTB function device. + * Invoked when a primary interface or secondary interface is bound to EPC + * device. This function will succeed only when EPC is bound to both the + * interfaces. + */ +static int epf_ntb_bind(struct pci_epf *epf) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + struct device *dev = &epf->dev; + int ret; + + if (!epf->epc) { + dev_dbg(dev, "PRIMARY EPC interface not yet bound\n"); + return 0; + } + + ret = epf_ntb_init_epc_bar(ntb); + if (ret) { + dev_err(dev, "Failed to create NTB EPC\n"); + goto err_bar_init; + } + + ret = epf_ntb_config_spad_bar_alloc(ntb); + if (ret) { + dev_err(dev, "Failed to allocate BAR memory\n"); + goto err_bar_alloc; + } + + ret = epf_ntb_epc_init(ntb); + if (ret) { + dev_err(dev, "Failed to initialize EPC\n"); + goto err_bar_alloc; + } + + epf_set_drvdata(epf, ntb); + + if (pci_register_driver(&vntb_pci_driver)) { + dev_err(dev, "failure register vntb pci driver\n"); + goto err_bar_alloc; + } + + vpci_bus(ntb); + + return 0; + +err_bar_alloc: + epf_ntb_config_spad_bar_free(ntb); + +err_bar_init: + epf_ntb_epc_destroy(ntb); + + return ret; +} + +/** + * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind() + * @epf: NTB endpoint function device + * + * Cleanup the initialization from epf_ntb_bind() + */ +static void epf_ntb_unbind(struct pci_epf *epf) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + + epf_ntb_epc_cleanup(ntb); + epf_ntb_config_spad_bar_free(ntb); + epf_ntb_epc_destroy(ntb); + + pci_unregister_driver(&vntb_pci_driver); +} + +// EPF driver probe +static struct pci_epf_ops epf_ntb_ops = { + .bind = epf_ntb_bind, + .unbind = epf_ntb_unbind, + .add_cfs = epf_ntb_add_cfs, +}; + +/** + * epf_ntb_probe() - Probe NTB function driver + * @epf: NTB endpoint function device + * + * Probe NTB function driver when endpoint function bus detects a NTB + * endpoint function. + */ +static int epf_ntb_probe(struct pci_epf *epf) +{ + struct epf_ntb *ntb; + struct device *dev; + + dev = &epf->dev; + + ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL); + if (!ntb) + return -ENOMEM; + + epf->header = &epf_ntb_header; + ntb->epf = epf; + epf_set_drvdata(epf, ntb); + + dev_info(dev, "pci-ep epf driver loaded\n"); + return 0; +} + +static const struct pci_epf_device_id epf_ntb_ids[] = { + { + .name = "pci_epf_vntb", + }, + {}, +}; + +static struct pci_epf_driver epf_ntb_driver = { + .driver.name = "pci_epf_vntb", + .probe = epf_ntb_probe, + .id_table = epf_ntb_ids, + .ops = &epf_ntb_ops, + .owner = THIS_MODULE, +}; + + +static int __init epf_ntb_init(void) +{ + int ret; + + kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | + WQ_HIGHPRI, 0); + ret = pci_epf_register_driver(&epf_ntb_driver); + if (ret) { + destroy_workqueue(kpcintb_workqueue); + pr_err("Failed to register pci epf ntb driver --> %d\n", ret); + return ret; + } + + return 0; +} +module_init(epf_ntb_init); + +static void __exit epf_ntb_exit(void) +{ + pci_epf_unregister_driver(&epf_ntb_driver); + destroy_workqueue(kpcintb_workqueue); +} +module_exit(epf_ntb_exit); + +MODULE_DESCRIPTION("PCI EPF NTB DRIVER"); +MODULE_AUTHOR("Frank Li <Frank.li@nxp.com>"); +MODULE_LICENSE("GPL v2"); |