diff options
470 files changed, 31520 insertions, 3617 deletions
diff --git a/Documentation/IRQ-domain.txt b/Documentation/IRQ-domain.txt index 8a8b82c9ca53..39cfa72732ff 100644 --- a/Documentation/IRQ-domain.txt +++ b/Documentation/IRQ-domain.txt @@ -151,3 +151,74 @@ used and no descriptor gets allocated it is very important to make sure that the driver using the simple domain call irq_create_mapping() before any irq_find_mapping() since the latter will actually work for the static IRQ assignment case. + +==== Hierarchy IRQ domain ==== +On some architectures, there may be multiple interrupt controllers +involved in delivering an interrupt from the device to the target CPU. +Let's look at a typical interrupt delivering path on x86 platforms: + +Device --> IOAPIC -> Interrupt remapping Controller -> Local APIC -> CPU + +There are three interrupt controllers involved: +1) IOAPIC controller +2) Interrupt remapping controller +3) Local APIC controller + +To support such a hardware topology and make software architecture match +hardware architecture, an irq_domain data structure is built for each +interrupt controller and those irq_domains are organized into hierarchy. +When building irq_domain hierarchy, the irq_domain near to the device is +child and the irq_domain near to CPU is parent. So a hierarchy structure +as below will be built for the example above. + CPU Vector irq_domain (root irq_domain to manage CPU vectors) + ^ + | + Interrupt Remapping irq_domain (manage irq_remapping entries) + ^ + | + IOAPIC irq_domain (manage IOAPIC delivery entries/pins) + +There are four major interfaces to use hierarchy irq_domain: +1) irq_domain_alloc_irqs(): allocate IRQ descriptors and interrupt + controller related resources to deliver these interrupts. +2) irq_domain_free_irqs(): free IRQ descriptors and interrupt controller + related resources associated with these interrupts. +3) irq_domain_activate_irq(): activate interrupt controller hardware to + deliver the interrupt. +3) irq_domain_deactivate_irq(): deactivate interrupt controller hardware + to stop delivering the interrupt. + +Following changes are needed to support hierarchy irq_domain. +1) a new field 'parent' is added to struct irq_domain; it's used to + maintain irq_domain hierarchy information. +2) a new field 'parent_data' is added to struct irq_data; it's used to + build hierarchy irq_data to match hierarchy irq_domains. The irq_data + is used to store irq_domain pointer and hardware irq number. +3) new callbacks are added to struct irq_domain_ops to support hierarchy + irq_domain operations. + +With support of hierarchy irq_domain and hierarchy irq_data ready, an +irq_domain structure is built for each interrupt controller, and an +irq_data structure is allocated for each irq_domain associated with an +IRQ. Now we could go one step further to support stacked(hierarchy) +irq_chip. That is, an irq_chip is associated with each irq_data along +the hierarchy. A child irq_chip may implement a required action by +itself or by cooperating with its parent irq_chip. + +With stacked irq_chip, interrupt controller driver only needs to deal +with the hardware managed by itself and may ask for services from its +parent irq_chip when needed. So we could achieve a much cleaner +software architecture. + +For an interrupt controller driver to support hierarchy irq_domain, it +needs to: +1) Implement irq_domain_ops.alloc and irq_domain_ops.free +2) Optionally implement irq_domain_ops.activate and + irq_domain_ops.deactivate. +3) Optionally implement an irq_chip to manage the interrupt controller + hardware. +4) No need to implement irq_domain_ops.map and irq_domain_ops.unmap, + they are unused with hierarchy irq_domain. + +Hierarchy irq_domain may also be used to support other architectures, +such as ARM, ARM64 etc. diff --git a/Documentation/arm64/legacy_instructions.txt b/Documentation/arm64/legacy_instructions.txt new file mode 100644 index 000000000000..01bf3d9fac85 --- /dev/null +++ b/Documentation/arm64/legacy_instructions.txt @@ -0,0 +1,57 @@ +The arm64 port of the Linux kernel provides infrastructure to support +emulation of instructions which have been deprecated, or obsoleted in +the architecture. The infrastructure code uses undefined instruction +hooks to support emulation. Where available it also allows turning on +the instruction execution in hardware. + +The emulation mode can be controlled by writing to sysctl nodes +(/proc/sys/abi). The following explains the different execution +behaviours and the corresponding values of the sysctl nodes - + +* Undef + Value: 0 + Generates undefined instruction abort. Default for instructions that + have been obsoleted in the architecture, e.g., SWP + +* Emulate + Value: 1 + Uses software emulation. To aid migration of software, in this mode + usage of emulated instruction is traced as well as rate limited + warnings are issued. This is the default for deprecated + instructions, .e.g., CP15 barriers + +* Hardware Execution + Value: 2 + Although marked as deprecated, some implementations may support the + enabling/disabling of hardware support for the execution of these + instructions. Using hardware execution generally provides better + performance, but at the loss of ability to gather runtime statistics + about the use of the deprecated instructions. + +The default mode depends on the status of the instruction in the +architecture. Deprecated instructions should default to emulation +while obsolete instructions must be undefined by default. + +Note: Instruction emulation may not be possible in all cases. See +individual instruction notes for further information. + +Supported legacy instructions +----------------------------- +* SWP{B} +Node: /proc/sys/abi/swp +Status: Obsolete +Default: Undef (0) + +* CP15 Barriers +Node: /proc/sys/abi/cp15_barrier +Status: Deprecated +Default: Emulate (1) + +* SETEND +Node: /proc/sys/abi/setend +Status: Deprecated +Default: Emulate (1)* +Note: All the cpus on the system must have mixed endian support at EL0 +for this feature to be enabled. If a new CPU - which doesn't support mixed +endian - is hotplugged in after this feature has been enabled, there could +be unexpected results in the application. diff --git a/Documentation/device-mapper/dm-crypt.txt b/Documentation/device-mapper/dm-crypt.txt index c81839b52c4d..692171fe9da0 100644 --- a/Documentation/device-mapper/dm-crypt.txt +++ b/Documentation/device-mapper/dm-crypt.txt @@ -5,7 +5,7 @@ Device-Mapper's "crypt" target provides transparent encryption of block devices using the kernel crypto API. For a more detailed description of supported parameters see: -http://code.google.com/p/cryptsetup/wiki/DMCrypt +https://gitlab.com/cryptsetup/cryptsetup/wikis/DMCrypt Parameters: <cipher> <key> <iv_offset> <device path> \ <offset> [<#opt_params> <opt_params>] @@ -51,7 +51,7 @@ Parameters: <cipher> <key> <iv_offset> <device path> \ Otherwise #opt_params is the number of following arguments. Example of optional parameters section: - 1 allow_discards + 3 allow_discards same_cpu_crypt submit_from_crypt_cpus allow_discards Block discard requests (a.k.a. TRIM) are passed through the crypt device. @@ -63,11 +63,24 @@ allow_discards used space etc.) if the discarded blocks can be located easily on the device later. +same_cpu_crypt + Perform encryption using the same cpu that IO was submitted on. + The default is to use an unbound workqueue so that encryption work + is automatically balanced between available CPUs. + +submit_from_crypt_cpus + Disable offloading writes to a separate thread after encryption. + There are some situations where offloading write bios from the + encryption threads to a single thread degrades performance + significantly. The default is to offload write bios to the same + thread because it benefits CFQ to have writes submitted using the + same context. + Example scripts =============== LUKS (Linux Unified Key Setup) is now the preferred way to set up disk encryption with dm-crypt using the 'cryptsetup' utility, see -http://code.google.com/p/cryptsetup/ +https://gitlab.com/cryptsetup/cryptsetup [[ #!/bin/sh diff --git a/Documentation/device-mapper/verity.txt b/Documentation/device-mapper/verity.txt index 9884681535ee..0075f70cd3f9 100644 --- a/Documentation/device-mapper/verity.txt +++ b/Documentation/device-mapper/verity.txt @@ -125,7 +125,7 @@ block boundary) are the hash blocks which are stored a depth at a time The full specification of kernel parameters and on-disk metadata format is available at the cryptsetup project's wiki page - http://code.google.com/p/cryptsetup/wiki/DMVerity + https://gitlab.com/cryptsetup/cryptsetup/wikis/DMVerity Status ====== @@ -142,7 +142,7 @@ Set up a device: A command line tool veritysetup is available to compute or verify the hash tree or activate the kernel device. This is available from -the cryptsetup upstream repository http://code.google.com/p/cryptsetup/ +the cryptsetup upstream repository https://gitlab.com/cryptsetup/cryptsetup/ (as a libcryptsetup extension). Create hash on the device: diff --git a/Documentation/devicetree/bindings/arm/arm-boards b/Documentation/devicetree/bindings/arm/arm-boards index c554ed3d44fb..10615f5053b0 100644 --- a/Documentation/devicetree/bindings/arm/arm-boards +++ b/Documentation/devicetree/bindings/arm/arm-boards @@ -92,3 +92,134 @@ Required nodes: - core-module: the root node to the Versatile platforms must have a core-module with regs and the compatible strings "arm,core-module-versatile", "syscon" + +ARM RealView Boards +------------------- +The RealView boards cover tailored evaluation boards that are used to explore +the ARM11 and Cortex A-8 and Cortex A-9 processors. + +Required properties (in root node): + /* RealView Emulation Baseboard */ + compatible = "arm,realview-eb"; + /* RealView Platform Baseboard for ARM1176JZF-S */ + compatible = "arm,realview-pb1176"; + /* RealView Platform Baseboard for ARM11 MPCore */ + compatible = "arm,realview-pb11mp"; + /* RealView Platform Baseboard for Cortex A-8 */ + compatible = "arm,realview-pba8"; + /* RealView Platform Baseboard Explore for Cortex A-9 */ + compatible = "arm,realview-pbx"; + +Required nodes: + +- soc: some node of the RealView platforms must be the SoC + node that contain the SoC-specific devices, withe the compatible + string set to one of these tuples: + "arm,realview-eb-soc", "simple-bus" + "arm,realview-pb1176-soc", "simple-bus" + "arm,realview-pb11mp-soc", "simple-bus" + "arm,realview-pba8-soc", "simple-bus" + "arm,realview-pbx-soc", "simple-bus" + +- syscon: some subnode of the RealView SoC node must be a + system controller node pointing to the control registers, + with the compatible string set to one of these tuples: + "arm,realview-eb-syscon", "syscon" + "arm,realview-pb1176-syscon", "syscon" + "arm,realview-pb11mp-syscon", "syscon" + "arm,realview-pba8-syscon", "syscon" + "arm,realview-pbx-syscon", "syscon" + + Required properties for the system controller: + - regs: the location and size of the system controller registers, + one range of 0x1000 bytes. + +Example: + +/dts-v1/; +#include <dt-bindings/interrupt-controller/irq.h> +#include "skeleton.dtsi" + +/ { + model = "ARM RealView PB1176 with device tree"; + compatible = "arm,realview-pb1176"; + + soc { + #address-cells = <1>; + #size-cells = <1>; + compatible = "arm,realview-pb1176-soc", "simple-bus"; + ranges; + + syscon: syscon@10000000 { + compatible = "arm,realview-syscon", "syscon"; + reg = <0x10000000 0x1000>; + }; + + }; +}; + +ARM Versatile Express Boards +----------------------------- +For details on the device tree bindings for ARM Versatile Express boards +please consult the vexpress.txt file in the same directory as this file. + +ARM Juno Boards +---------------- +The Juno boards are targeting development for AArch64 systems. The first +iteration, Juno r0, is a vehicle for evaluating big.LITTLE on AArch64, +with the second iteration, Juno r1, mainly aimed at development of PCIe +based systems. Juno r1 also has support for AXI masters placed on the TLX +connectors to join the coherency domain. + +Juno boards are described in a similar way to ARM Versatile Express boards, +with the motherboard part of the hardware being described in a separate file +to highlight the fact that is part of the support infrastructure for the SoC. +Juno device tree bindings also share the Versatile Express bindings as +described under the RS1 memory mapping. + +Required properties (in root node): + compatible = "arm,juno"; /* For Juno r0 board */ + compatible = "arm,juno-r1"; /* For Juno r1 board */ + +Required nodes: +The description for the board must include: + - a "psci" node describing the boot method used for the secondary CPUs. + A detailed description of the bindings used for "psci" nodes is present + in the psci.txt file. + - a "cpus" node describing the available cores and their associated + "enable-method"s. For more details see cpus.txt file. + +Example: + +/dts-v1/; +/ { + model = "ARM Juno development board (r0)"; + compatible = "arm,juno", "arm,vexpress"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + A57_0: cpu@0 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x0>; + device_type = "cpu"; + enable-method = "psci"; + }; + + ..... + + A53_0: cpu@100 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x100>; + device_type = "cpu"; + enable-method = "psci"; + }; + + ..... + }; + +}; diff --git a/Documentation/devicetree/bindings/arm/coresight.txt b/Documentation/devicetree/bindings/arm/coresight.txt new file mode 100644 index 000000000000..88602b75418e --- /dev/null +++ b/Documentation/devicetree/bindings/arm/coresight.txt @@ -0,0 +1,199 @@ +* CoreSight Components: + +CoreSight components are compliant with the ARM CoreSight architecture +specification and can be connected in various topologies to suit a particular +SoCs tracing needs. These trace components can generally be classified as +sinks, links and sources. Trace data produced by one or more sources flows +through the intermediate links connecting the source to the currently selected +sink. Each CoreSight component device should use these properties to describe +its hardware characteristcs. + +* Required properties for all components *except* non-configurable replicators: + + * compatible: These have to be supplemented with "arm,primecell" as + drivers are using the AMBA bus interface. Possible values include: + - "arm,coresight-etb10", "arm,primecell"; + - "arm,coresight-tpiu", "arm,primecell"; + - "arm,coresight-tmc", "arm,primecell"; + - "arm,coresight-funnel", "arm,primecell"; + - "arm,coresight-etm3x", "arm,primecell"; + + * reg: physical base address and length of the register + set(s) of the component. + + * clocks: the clock associated to this component. + + * clock-names: the name of the clock as referenced by the code. + Since we are using the AMBA framework, the name should be + "apb_pclk". + + * port or ports: The representation of the component's port + layout using the generic DT graph presentation found in + "bindings/graph.txt". + +* Required properties for devices that don't show up on the AMBA bus, such as + non-configurable replicators: + + * compatible: Currently supported value is (note the absence of the + AMBA markee): + - "arm,coresight-replicator" + + * port or ports: same as above. + +* Optional properties for ETM/PTMs: + + * arm,cp14: must be present if the system accesses ETM/PTM management + registers via co-processor 14. + + * cpu: the cpu phandle this ETM/PTM is affined to. When omitted the + source is considered to belong to CPU0. + +* Optional property for TMC: + + * arm,buffer-size: size of contiguous buffer space for TMC ETR + (embedded trace router) + + +Example: + +1. Sinks + etb@20010000 { + compatible = "arm,coresight-etb10", "arm,primecell"; + reg = <0 0x20010000 0 0x1000>; + + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + etb_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&replicator_out_port0>; + }; + }; + }; + + tpiu@20030000 { + compatible = "arm,coresight-tpiu", "arm,primecell"; + reg = <0 0x20030000 0 0x1000>; + + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + tpiu_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&replicator_out_port1>; + }; + }; + }; + +2. Links + replicator { + /* non-configurable replicators don't show up on the + * AMBA bus. As such no need to add "arm,primecell". + */ + compatible = "arm,coresight-replicator"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* replicator output ports */ + port@0 { + reg = <0>; + replicator_out_port0: endpoint { + remote-endpoint = <&etb_in_port>; + }; + }; + + port@1 { + reg = <1>; + replicator_out_port1: endpoint { + remote-endpoint = <&tpiu_in_port>; + }; + }; + + /* replicator input port */ + port@2 { + reg = <0>; + replicator_in_port0: endpoint { + slave-mode; + remote-endpoint = <&funnel_out_port0>; + }; + }; + }; + }; + + funnel@20040000 { + compatible = "arm,coresight-funnel", "arm,primecell"; + reg = <0 0x20040000 0 0x1000>; + + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* funnel output port */ + port@0 { + reg = <0>; + funnel_out_port0: endpoint { + remote-endpoint = + <&replicator_in_port0>; + }; + }; + + /* funnel input ports */ + port@1 { + reg = <0>; + funnel_in_port0: endpoint { + slave-mode; + remote-endpoint = <&ptm0_out_port>; + }; + }; + + port@2 { + reg = <1>; + funnel_in_port1: endpoint { + slave-mode; + remote-endpoint = <&ptm1_out_port>; + }; + }; + + port@3 { + reg = <2>; + funnel_in_port2: endpoint { + slave-mode; + remote-endpoint = <&etm0_out_port>; + }; + }; + + }; + }; + +3. Sources + ptm@2201c000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0x2201c000 0 0x1000>; + + cpu = <&cpu0>; + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + ptm0_out_port: endpoint { + remote-endpoint = <&funnel_in_port0>; + }; + }; + }; + + ptm@2201d000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0x2201d000 0 0x1000>; + + cpu = <&cpu1>; + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + ptm1_out_port: endpoint { + remote-endpoint = <&funnel_in_port1>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/arm/gic-v3.txt b/Documentation/devicetree/bindings/arm/gic-v3.txt index 33cd05e6c125..ddfade40ac59 100644 --- a/Documentation/devicetree/bindings/arm/gic-v3.txt +++ b/Documentation/devicetree/bindings/arm/gic-v3.txt @@ -49,11 +49,29 @@ Optional occupied by the redistributors. Required if more than one such region is present. +Sub-nodes: + +GICv3 has one or more Interrupt Translation Services (ITS) that are +used to route Message Signalled Interrupts (MSI) to the CPUs. + +These nodes must have the following properties: +- compatible : Should at least contain "arm,gic-v3-its". +- msi-controller : Boolean property. Identifies the node as an MSI controller +- reg: Specifies the base physical address and size of the ITS + registers. + +The main GIC node must contain the appropriate #address-cells, +#size-cells and ranges properties for the reg property of all ITS +nodes. + Examples: gic: interrupt-controller@2cf00000 { compatible = "arm,gic-v3"; #interrupt-cells = <3>; + #address-cells = <2>; + #size-cells = <2>; + ranges; interrupt-controller; reg = <0x0 0x2f000000 0 0x10000>, // GICD <0x0 0x2f100000 0 0x200000>, // GICR @@ -61,11 +79,20 @@ Examples: <0x0 0x2c010000 0 0x2000>, // GICH <0x0 0x2c020000 0 0x2000>; // GICV interrupts = <1 9 4>; + + gic-its@2c200000 { + compatible = "arm,gic-v3-its"; + msi-controller; + reg = <0x0 0x2c200000 0 0x200000>; + }; }; gic: interrupt-controller@2c010000 { compatible = "arm,gic-v3"; #interrupt-cells = <3>; + #address-cells = <2>; + #size-cells = <2>; + ranges; interrupt-controller; redistributor-stride = <0x0 0x40000>; // 256kB stride #redistributor-regions = <2>; @@ -76,4 +103,16 @@ Examples: <0x0 0x2c060000 0 0x2000>, // GICH <0x0 0x2c080000 0 0x2000>; // GICV interrupts = <1 9 4>; + + gic-its@2c200000 { + compatible = "arm,gic-v3-its"; + msi-controller; + reg = <0x0 0x2c200000 0 0x200000>; + }; + + gic-its@2c400000 { + compatible = "arm,gic-v3-its"; + msi-controller; + reg = <0x0 0x2c400000 0 0x200000>; + }; }; diff --git a/Documentation/devicetree/bindings/arm/gic.txt b/Documentation/devicetree/bindings/arm/gic.txt index c7d2fa156678..45ccc2db3436 100644 --- a/Documentation/devicetree/bindings/arm/gic.txt +++ b/Documentation/devicetree/bindings/arm/gic.txt @@ -31,12 +31,16 @@ Main node required properties: The 3rd cell is the flags, encoded as follows: bits[3:0] trigger type and level flags. 1 = low-to-high edge triggered - 2 = high-to-low edge triggered + 2 = high-to-low edge triggered (invalid for SPIs) 4 = active high level-sensitive - 8 = active low level-sensitive + 8 = active low level-sensitive (invalid for SPIs). bits[15:8] PPI interrupt cpu mask. Each bit corresponds to each of the 8 possible cpus attached to the GIC. A bit set to '1' indicated the interrupt is wired to that CPU. Only valid for PPI interrupts. + Also note that the configurability of PPI interrupts is IMPLEMENTATION + DEFINED and as such not guaranteed to be present (most SoC available + in 2014 seem to ignore the setting of this flag and use the hardware + default value). - reg : Specifies base physical address(s) and size of the GIC registers. The first region is the GIC distributor register base and size. The 2nd region is @@ -96,3 +100,56 @@ Example: <0x2c006000 0x2000>; interrupts = <1 9 0xf04>; }; + + +* GICv2m extension for MSI/MSI-x support (Optional) + +Certain revisions of GIC-400 supports MSI/MSI-x via V2M register frame(s). +This is enabled by specifying v2m sub-node(s). + +Required properties: + +- compatible : The value here should contain "arm,gic-v2m-frame". + +- msi-controller : Identifies the node as an MSI controller. + +- reg : GICv2m MSI interface register base and size + +Optional properties: + +- arm,msi-base-spi : When the MSI_TYPER register contains an incorrect + value, this property should contain the SPI base of + the MSI frame, overriding the HW value. + +- arm,msi-num-spis : When the MSI_TYPER register contains an incorrect + value, this property should contain the number of + SPIs assigned to the frame, overriding the HW value. + +Example: + + interrupt-controller@e1101000 { + compatible = "arm,gic-400"; + #interrupt-cells = <3>; + #address-cells = <2>; + #size-cells = <2>; + interrupt-controller; + interrupts = <1 8 0xf04>; + ranges = <0 0 0 0xe1100000 0 0x100000>; + reg = <0x0 0xe1110000 0 0x01000>, + <0x0 0xe112f000 0 0x02000>, + <0x0 0xe1140000 0 0x10000>, + <0x0 0xe1160000 0 0x10000>; + v2m0: v2m@0x8000 { + compatible = "arm,gic-v2m-frame"; + msi-controller; + reg = <0x0 0x80000 0 0x1000>; + }; + + .... + + v2mN: v2m@0x9000 { + compatible = "arm,gic-v2m-frame"; + msi-controller; + reg = <0x0 0x90000 0 0x1000>; + }; + }; diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,sysirq.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,sysirq.txt new file mode 100644 index 000000000000..d680b07ec6e8 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,sysirq.txt @@ -0,0 +1,28 @@ +Mediatek 65xx/81xx sysirq + +Mediatek SOCs sysirq support controllable irq inverter for each GIC SPI +interrupt. + +Required properties: +- compatible: should be one of: + "mediatek,mt8135-sysirq" + "mediatek,mt8127-sysirq" + "mediatek,mt6589-sysirq" + "mediatek,mt6582-sysirq" + "mediatek,mt6577-sysirq" +- interrupt-controller : Identifies the node as an interrupt controller +- #interrupt-cells : Use the same format as specified by GIC in + Documentation/devicetree/bindings/arm/gic.txt +- interrupt-parent: phandle of irq parent for sysirq. The parent must + use the same interrupt-cells format as GIC. +- reg: Physical base address of the intpol registers and length of memory + mapped region. + +Example: + sysirq: interrupt-controller@10200100 { + compatible = "mediatek,mt6589-sysirq", "mediatek,mt6577-sysirq"; + interrupt-controller; + #interrupt-cells = <3>; + interrupt-parent = <&gic>; + reg = <0 0x10200100 0 0x1c>; + }; diff --git a/Documentation/devicetree/bindings/mfd/mfd.txt b/Documentation/devicetree/bindings/mfd/mfd.txt new file mode 100644 index 000000000000..af9d6931a1a2 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/mfd.txt @@ -0,0 +1,41 @@ +Multi-Function Devices (MFD) + +These devices comprise a nexus for heterogeneous hardware blocks containing +more than one non-unique yet varying hardware functionality. + +A typical MFD can be: + +- A mixed signal ASIC on an external bus, sometimes a PMIC (Power Management + Integrated Circuit) that is manufactured in a lower technology node (rough + silicon) that handles analog drivers for things like audio amplifiers, LED + drivers, level shifters, PHY (physical interfaces to things like USB or + ethernet), regulators etc. + +- A range of memory registers containing "miscellaneous system registers" also + known as a system controller "syscon" or any other memory range containing a + mix of unrelated hardware devices. + +Optional properties: + +- compatible : "simple-mfd" - this signifies that the operating system should + consider all subnodes of the MFD device as separate devices akin to how + "simple-bus" inidicates when to see subnodes as children for a simple + memory-mapped bus. For more complex devices, when the nexus driver has to + probe registers to figure out what child devices exist etc, this should not + be used. In the latter case the child devices will be determined by the + operating system. + +Example: + +foo@1000 { + compatible = "syscon", "simple-mfd"; + reg = <0x01000 0x1000>; + + led@08.0 { + compatible = "register-bit-led"; + offset = <0x08>; + mask = <0x01>; + label = "myled"; + default-state = "on"; + }; +}; diff --git a/Documentation/devicetree/bindings/opp/opp.txt b/Documentation/devicetree/bindings/opp/opp.txt new file mode 100644 index 000000000000..7202b3a09e51 --- /dev/null +++ b/Documentation/devicetree/bindings/opp/opp.txt @@ -0,0 +1,507 @@ +* Generic OPP Interface + +SoCs have a standard set of tuples consisting of frequency and +voltage pairs that the device will support per voltage domain. These +are called Operating Performance Points or OPPs. + +Properties: +- operating-points: An array of 2-tuples items, and each item consists + of frequency and voltage like <freq-kHz vol-uV>. + freq: clock frequency in kHz + vol: voltage in microvolt + +Examples: + +cpu@0 { + compatible = "arm,cortex-a9"; + reg = <0>; + next-level-cache = <&L2>; + operating-points = < + /* kHz uV */ + 792000 1100000 + 396000 950000 + 198000 850000 + >; +}; + +Binding 2: operating-points-v2 +============================ + +* Property: operating-points-v2 + +Devices supporting OPPs must set their "operating-points-v2" property with +phandle to a OPP table in their DT node. The OPP core will use this phandle to +find the operating points for the device. + +If required, this can be extended for SoC vendor specfic bindings. Such bindings +should be documented as Documentation/devicetree/bindings/power/<vendor>-opp.txt +and should have a compatible description like: "operating-points-v2-<vendor>". + +* OPP Table Node + +This describes the OPPs belonging to a device. This node can have following +properties: + +Required properties: +- compatible: Allow OPPs to express their compatibility. It should be: + "operating-points-v2". + +- OPP nodes: One or more OPP nodes describing voltage-current-frequency + combinations. Their name isn't significant but their phandle can be used to + reference an OPP. + +Optional properties: +- opp-shared: Indicates that device nodes using this OPP Table Node's phandle + switch their DVFS state together, i.e. they share clock/voltage/current lines. + Missing property means devices have independent clock/voltage/current lines, + but they share OPP tables. + +- status: Marks the OPP table enabled/disabled. + + +* OPP Node + +This defines voltage-current-frequency combinations along with other related +properties. + +Required properties: +- opp-hz: Frequency in Hz, expressed as a 64-bit big-endian integer. + +Optional properties: +- opp-microvolt: voltage in micro Volts. + + A single regulator's voltage is specified with an array of size one or three. + Single entry is for target voltage and three entries are for <target min max> + voltages. + + Entries for multiple regulators must be present in the same order as + regulators are specified in device's DT node. + +- opp-microvolt-<name>: Named opp-microvolt property. This is exactly similar to + the above opp-microvolt property, but allows multiple voltage ranges to be + provided for the same OPP. At runtime, the platform can pick a <name> and + matching opp-microvolt-<name> property will be enabled for all OPPs. If the + platform doesn't pick a specific <name> or the <name> doesn't match with any + opp-microvolt-<name> properties, then opp-microvolt property shall be used, if + present. + +- opp-microamp: The maximum current drawn by the device in microamperes + considering system specific parameters (such as transients, process, aging, + maximum operating temperature range etc.) as necessary. This may be used to + set the most efficient regulator operating mode. + + Should only be set if opp-microvolt is set for the OPP. + + Entries for multiple regulators must be present in the same order as + regulators are specified in device's DT node. If this property isn't required + for few regulators, then this should be marked as zero for them. If it isn't + required for any regulator, then this property need not be present. + +- opp-microamp-<name>: Named opp-microamp property. Similar to + opp-microvolt-<name> property, but for microamp instead. + +- clock-latency-ns: Specifies the maximum possible transition latency (in + nanoseconds) for switching to this OPP from any other OPP. + +- turbo-mode: Marks the OPP to be used only for turbo modes. Turbo mode is + available on some platforms, where the device can run over its operating + frequency for a short duration of time limited by the device's power, current + and thermal limits. + +- opp-suspend: Marks the OPP to be used during device suspend. Only one OPP in + the table should have this. + +- opp-supported-hw: This enables us to select only a subset of OPPs from the + larger OPP table, based on what version of the hardware we are running on. We + still can't have multiple nodes with the same opp-hz value in OPP table. + + It's an user defined array containing a hierarchy of hardware version numbers, + supported by the OPP. For example: a platform with hierarchy of three levels + of versions (A, B and C), this field should be like <X Y Z>, where X + corresponds to Version hierarchy A, Y corresponds to version hierarchy B and Z + corresponds to version hierarchy C. + + Each level of hierarchy is represented by a 32 bit value, and so there can be + only 32 different supported version per hierarchy. i.e. 1 bit per version. A + value of 0xFFFFFFFF will enable the OPP for all versions for that hierarchy + level. And a value of 0x00000000 will disable the OPP completely, and so we + never want that to happen. + + If 32 values aren't sufficient for a version hierarchy, than that version + hierarchy can be contained in multiple 32 bit values. i.e. <X Y Z1 Z2> in the + above example, Z1 & Z2 refer to the version hierarchy Z. + +- status: Marks the node enabled/disabled. + +Example 1: Single cluster Dual-core ARM cortex A9, switch DVFS states together. + +/ { + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + compatible = "arm,cortex-a9"; + reg = <0>; + next-level-cache = <&L2>; + clocks = <&clk_controller 0>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply0>; + operating-points-v2 = <&cpu0_opp_table>; + }; + + cpu@1 { + compatible = "arm,cortex-a9"; + reg = <1>; + next-level-cache = <&L2>; + clocks = <&clk_controller 0>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply0>; + operating-points-v2 = <&cpu0_opp_table>; + }; + }; + + cpu0_opp_table: opp_table0 { + compatible = "operating-points-v2"; + opp-shared; + + opp@1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <970000 975000 985000>; + opp-microamp = <70000>; + clock-latency-ns = <300000>; + opp-suspend; + }; + opp@1100000000 { + opp-hz = /bits/ 64 <1100000000>; + opp-microvolt = <980000 1000000 1010000>; + opp-microamp = <80000>; + clock-latency-ns = <310000>; + }; + opp@1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1025000>; + clock-latency-ns = <290000>; + turbo-mode; + }; + }; +}; + +Example 2: Single cluster, Quad-core Qualcom-krait, switches DVFS states +independently. + +/ { + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + compatible = "qcom,krait"; + reg = <0>; + next-level-cache = <&L2>; + clocks = <&clk_controller 0>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply0>; + operating-points-v2 = <&cpu_opp_table>; + }; + + cpu@1 { + compatible = "qcom,krait"; + reg = <1>; + next-level-cache = <&L2>; + clocks = <&clk_controller 1>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply1>; + operating-points-v2 = <&cpu_opp_table>; + }; + + cpu@2 { + compatible = "qcom,krait"; + reg = <2>; + next-level-cache = <&L2>; + clocks = <&clk_controller 2>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply2>; + operating-points-v2 = <&cpu_opp_table>; + }; + + cpu@3 { + compatible = "qcom,krait"; + reg = <3>; + next-level-cache = <&L2>; + clocks = <&clk_controller 3>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply3>; + operating-points-v2 = <&cpu_opp_table>; + }; + }; + + cpu_opp_table: opp_table { + compatible = "operating-points-v2"; + + /* + * Missing opp-shared property means CPUs switch DVFS states + * independently. + */ + + opp@1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <970000 975000 985000>; + opp-microamp = <70000>; + clock-latency-ns = <300000>; + opp-suspend; + }; + opp@1100000000 { + opp-hz = /bits/ 64 <1100000000>; + opp-microvolt = <980000 1000000 1010000>; + opp-microamp = <80000>; + clock-latency-ns = <310000>; + }; + opp@1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1025000>; + opp-microamp = <90000; + lock-latency-ns = <290000>; + turbo-mode; + }; + }; +}; + +Example 3: Dual-cluster, Dual-core per cluster. CPUs within a cluster switch +DVFS state together. + +/ { + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + compatible = "arm,cortex-a7"; + reg = <0>; + next-level-cache = <&L2>; + clocks = <&clk_controller 0>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply0>; + operating-points-v2 = <&cluster0_opp>; + }; + + cpu@1 { + compatible = "arm,cortex-a7"; + reg = <1>; + next-level-cache = <&L2>; + clocks = <&clk_controller 0>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply0>; + operating-points-v2 = <&cluster0_opp>; + }; + + cpu@100 { + compatible = "arm,cortex-a15"; + reg = <100>; + next-level-cache = <&L2>; + clocks = <&clk_controller 1>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply1>; + operating-points-v2 = <&cluster1_opp>; + }; + + cpu@101 { + compatible = "arm,cortex-a15"; + reg = <101>; + next-level-cache = <&L2>; + clocks = <&clk_controller 1>; + clock-names = "cpu"; + cpu-supply = <&cpu_supply1>; + operating-points-v2 = <&cluster1_opp>; + }; + }; + + cluster0_opp: opp_table0 { + compatible = "operating-points-v2"; + opp-shared; + + opp@1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <970000 975000 985000>; + opp-microamp = <70000>; + clock-latency-ns = <300000>; + opp-suspend; + }; + opp@1100000000 { + opp-hz = /bits/ 64 <1100000000>; + opp-microvolt = <980000 1000000 1010000>; + opp-microamp = <80000>; + clock-latency-ns = <310000>; + }; + opp@1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1025000>; + opp-microamp = <90000>; + clock-latency-ns = <290000>; + turbo-mode; + }; + }; + + cluster1_opp: opp_table1 { + compatible = "operating-points-v2"; + opp-shared; + + opp@1300000000 { + opp-hz = /bits/ 64 <1300000000>; + opp-microvolt = <1045000 1050000 1055000>; + opp-microamp = <95000>; + clock-latency-ns = <400000>; + opp-suspend; + }; + opp@1400000000 { + opp-hz = /bits/ 64 <1400000000>; + opp-microvolt = <1075000>; + opp-microamp = <100000>; + clock-latency-ns = <400000>; + }; + opp@1500000000 { + opp-hz = /bits/ 64 <1500000000>; + opp-microvolt = <1010000 1100000 1110000>; + opp-microamp = <95000>; + clock-latency-ns = <400000>; + turbo-mode; + }; + }; +}; + +Example 4: Handling multiple regulators + +/ { + cpus { + cpu@0 { + compatible = "arm,cortex-a7"; + ... + + cpu-supply = <&cpu_supply0>, <&cpu_supply1>, <&cpu_supply2>; + operating-points-v2 = <&cpu0_opp_table>; + }; + }; + + cpu0_opp_table: opp_table0 { + compatible = "operating-points-v2"; + opp-shared; + + opp@1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <970000>, /* Supply 0 */ + <960000>, /* Supply 1 */ + <960000>; /* Supply 2 */ + opp-microamp = <70000>, /* Supply 0 */ + <70000>, /* Supply 1 */ + <70000>; /* Supply 2 */ + clock-latency-ns = <300000>; + }; + + /* OR */ + + opp@1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <970000 975000 985000>, /* Supply 0 */ + <960000 965000 975000>, /* Supply 1 */ + <960000 965000 975000>; /* Supply 2 */ + opp-microamp = <70000>, /* Supply 0 */ + <70000>, /* Supply 1 */ + <70000>; /* Supply 2 */ + clock-latency-ns = <300000>; + }; + + /* OR */ + + opp@1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <970000 975000 985000>, /* Supply 0 */ + <960000 965000 975000>, /* Supply 1 */ + <960000 965000 975000>; /* Supply 2 */ + opp-microamp = <70000>, /* Supply 0 */ + <0>, /* Supply 1 doesn't need this */ + <70000>; /* Supply 2 */ + clock-latency-ns = <300000>; + }; + }; +}; + +Example 5: opp-supported-hw +(example: three level hierarchy of versions: cuts, substrate and process) + +/ { + cpus { + cpu@0 { + compatible = "arm,cortex-a7"; + ... + + cpu-supply = <&cpu_supply> + operating-points-v2 = <&cpu0_opp_table_slow>; + }; + }; + + opp_table { + compatible = "operating-points-v2"; + status = "okay"; + opp-shared; + + opp@600000000 { + /* + * Supports all substrate and process versions for 0xF + * cuts, i.e. only first four cuts. + */ + opp-supported-hw = <0xF 0xFFFFFFFF 0xFFFFFFFF> + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000 915000 925000>; + ... + }; + + opp@800000000 { + /* + * Supports: + * - cuts: only one, 6th cut (represented by 6th bit). + * - substrate: supports 16 different substrate versions + * - process: supports 9 different process versions + */ + opp-supported-hw = <0x20 0xff0000ff 0x0000f4f0> + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <900000 915000 925000>; + ... + }; + }; +}; + +Example 6: opp-microvolt-<name>, opp-microamp-<name>: +(example: device with two possible microvolt ranges: slow and fast) + +/ { + cpus { + cpu@0 { + compatible = "arm,cortex-a7"; + ... + + operating-points-v2 = <&cpu0_opp_table>; + }; + }; + + cpu0_opp_table: opp_table0 { + compatible = "operating-points-v2"; + opp-shared; + + opp@1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt-slow = <900000 915000 925000>; + opp-microvolt-fast = <970000 975000 985000>; + opp-microamp-slow = <70000>; + opp-microamp-fast = <71000>; + }; + + opp@1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt-slow = <900000 915000 925000>, /* Supply vcc0 */ + <910000 925000 935000>; /* Supply vcc1 */ + opp-microvolt-fast = <970000 975000 985000>, /* Supply vcc0 */ + <960000 965000 975000>; /* Supply vcc1 */ + opp-microamp = <70000>; /* Will be used for both slow/fast */ + }; + }; +}; diff --git a/Documentation/devicetree/bindings/pci/arm,juno-r1-pcie.txt b/Documentation/devicetree/bindings/pci/arm,juno-r1-pcie.txt new file mode 100644 index 000000000000..f7514c170a32 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/arm,juno-r1-pcie.txt @@ -0,0 +1,10 @@ +* ARM Juno R1 PCIe interface + +This PCIe host controller is based on PLDA XpressRICH3-AXI IP +and thus inherits all the common properties defined in plda,xpressrich3-axi.txt +as well as the base properties defined in host-generic-pci.txt. + +Required properties: + - compatible: "arm,juno-r1-pcie" + - dma-coherent: The host controller bridges the AXI transactions into PCIe bus + in a manner that makes the DMA operations to appear coherent to the CPUs. diff --git a/Documentation/devicetree/bindings/pci/plda,xpressrich3-axi.txt b/Documentation/devicetree/bindings/pci/plda,xpressrich3-axi.txt new file mode 100644 index 000000000000..f3f75bfb42bc --- /dev/null +++ b/Documentation/devicetree/bindings/pci/plda,xpressrich3-axi.txt @@ -0,0 +1,12 @@ +* PLDA XpressRICH3-AXI host controller + +The PLDA XpressRICH3-AXI host controller can be configured in a manner that +makes it compliant with the SBSA[1] standard published by ARM Ltd. For those +scenarios, the host-generic-pci.txt bindings apply with the following additions +to the compatible property: + +Required properties: + - compatible: should contain "plda,xpressrich3-axi" to identify the IP used. + + +[1] http://infocenter.arm.com/help/topic/com.arm.doc.den0029a/ diff --git a/Documentation/devicetree/bindings/pci/xgene-pci-msi.txt b/Documentation/devicetree/bindings/pci/xgene-pci-msi.txt new file mode 100644 index 000000000000..36d881c8e6d4 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/xgene-pci-msi.txt @@ -0,0 +1,68 @@ +* AppliedMicro X-Gene v1 PCIe MSI controller + +Required properties: + +- compatible: should be "apm,xgene1-msi" to identify + X-Gene v1 PCIe MSI controller block. +- msi-controller: indicates that this is X-Gene v1 PCIe MSI controller node +- reg: physical base address (0x79000000) and length (0x900000) for controller + registers. These registers include the MSI termination address and data + registers as well as the MSI interrupt status registers. +- reg-names: not required +- interrupts: A list of 16 interrupt outputs of the controller, starting from + interrupt number 0x10 to 0x1f. +- interrupt-names: not required + +Each PCIe node needs to have property msi-parent that points to msi controller node + +Examples: + +SoC DTSI: + + + MSI node: + msi@79000000 { + compatible = "apm,xgene1-msi"; + msi-controller; + reg = <0x00 0x79000000 0x0 0x900000>; + interrupts = <0x0 0x10 0x4> + <0x0 0x11 0x4> + <0x0 0x12 0x4> + <0x0 0x13 0x4> + <0x0 0x14 0x4> + <0x0 0x15 0x4> + <0x0 0x16 0x4> + <0x0 0x17 0x4> + <0x0 0x18 0x4> + <0x0 0x19 0x4> + <0x0 0x1a 0x4> + <0x0 0x1b 0x4> + <0x0 0x1c 0x4> + <0x0 0x1d 0x4> + <0x0 0x1e 0x4> + <0x0 0x1f 0x4>; + }; + + + PCIe controller node with msi-parent property pointing to MSI node: + pcie0: pcie@1f2b0000 { + status = "disabled"; + device_type = "pci"; + compatible = "apm,xgene-storm-pcie", "apm,xgene-pcie"; + #interrupt-cells = <1>; + #size-cells = <2>; + #address-cells = <3>; + reg = < 0x00 0x1f2b0000 0x0 0x00010000 /* Controller registers */ + 0xe0 0xd0000000 0x0 0x00040000>; /* PCI config space */ + reg-names = "csr", "cfg"; + ranges = <0x01000000 0x00 0x00000000 0xe0 0x10000000 0x00 0x00010000 /* io */ + 0x02000000 0x00 0x80000000 0xe1 0x80000000 0x00 0x80000000>; /* mem */ + dma-ranges = <0x42000000 0x80 0x00000000 0x80 0x00000000 0x00 0x80000000 + 0x42000000 0x00 0x00000000 0x00 0x00000000 0x80 0x00000000>; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + interrupt-map = <0x0 0x0 0x0 0x1 &gic 0x0 0xc2 0x1 + 0x0 0x0 0x0 0x2 &gic 0x0 0xc3 0x1 + 0x0 0x0 0x0 0x3 &gic 0x0 0xc4 0x1 + 0x0 0x0 0x0 0x4 &gic 0x0 0xc5 0x1>; + dma-coherent; + clocks = <&pcie0clk 0>; + msi-parent= <&msi>; + }; diff --git a/Documentation/devicetree/bindings/power/opp.txt b/Documentation/devicetree/bindings/power/opp.txt deleted file mode 100644 index 74499e5033fc..000000000000 --- a/Documentation/devicetree/bindings/power/opp.txt +++ /dev/null @@ -1,25 +0,0 @@ -* Generic OPP Interface - -SoCs have a standard set of tuples consisting of frequency and -voltage pairs that the device will support per voltage domain. These -are called Operating Performance Points or OPPs. - -Properties: -- operating-points: An array of 2-tuples items, and each item consists - of frequency and voltage like <freq-kHz vol-uV>. - freq: clock frequency in kHz - vol: voltage in microvolt - -Examples: - -cpu@0 { - compatible = "arm,cortex-a9"; - reg = <0>; - next-level-cache = <&L2>; - operating-points = < - /* kHz uV */ - 792000 1100000 - 396000 950000 - 198000 850000 - >; -}; diff --git a/Documentation/devicetree/bindings/thermal/thermal.txt b/Documentation/devicetree/bindings/thermal/thermal.txt index f5db6b72a36f..99d6608c9d5f 100644 --- a/Documentation/devicetree/bindings/thermal/thermal.txt +++ b/Documentation/devicetree/bindings/thermal/thermal.txt @@ -167,6 +167,13 @@ Optional property: by means of sensor ID. Additional coefficients are interpreted as constant offset. +- sustainable-power: An estimate of the sustainable power (in mW) that the + Type: unsigned thermal zone can dissipate at the desired + Size: one cell control temperature. For reference, the + sustainable power of a 4'' phone is typically + 2000mW, while on a 10'' tablet is around + 4500mW. + Note: The delay properties are bound to the maximum dT/dt (temperature derivative over time) in two situations for a thermal zone: (i) - when passive cooling is activated (polling-delay-passive); and @@ -546,6 +553,8 @@ thermal-zones { */ coefficients = <1200 -345 890>; + sustainable-power = <2500>; + trips { /* Trips are based on resulting linear equation */ cpu-trip: cpu-trip { diff --git a/Documentation/devicetree/bindings/unittest.txt b/Documentation/devicetree/bindings/unittest.txt new file mode 100644 index 000000000000..0f92a22fddfa --- /dev/null +++ b/Documentation/devicetree/bindings/unittest.txt @@ -0,0 +1,14 @@ +* OF selftest platform device + +** selftest + +Required properties: +- compatible: must be "selftest" + +All other properties are optional. + +Example: + selftest { + compatible = "selftest"; + status = "okay"; + }; diff --git a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt index 356ffd64fc9e..fb2c96ec71ef 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.txt +++ b/Documentation/devicetree/bindings/vendor-prefixes.txt @@ -116,6 +116,7 @@ panasonic Panasonic Corporation phytec PHYTEC Messtechnik GmbH picochip Picochip Ltd plathome Plat'Home Co., Ltd. +plda PLDA pixcir PIXCIR MICROELECTRONICS Co., Ltd powervr PowerVR (deprecated, use img) qca Qualcomm Atheros, Inc. diff --git a/Documentation/devicetree/of_selftest.txt b/Documentation/devicetree/of_selftest.txt index 1e3d5c92b5e3..57a808b588bf 100644 --- a/Documentation/devicetree/of_selftest.txt +++ b/Documentation/devicetree/of_selftest.txt @@ -63,7 +63,6 @@ struct device_node { struct device_node *parent; struct device_node *child; struct device_node *sibling; - struct device_node *allnext; /* next in list of all nodes */ ... }; @@ -99,12 +98,6 @@ child11 -> sibling12 -> sibling13 -> sibling14 -> null Figure 1: Generic structure of un-flattened device tree -*allnext: it is used to link all the nodes of DT into a list. So, for the - above tree the list would be as follows: - -root->child1->child11->sibling12->sibling13->child131->sibling14->sibling2-> -child21->sibling22->sibling23->sibling3->child31->sibling32->sibling4->null - Before executing OF selftest, it is required to attach the test data to machine's device tree (if present). So, when selftest_data_add() is called, at first it reads the flattened device tree data linked into the kernel image @@ -131,11 +124,6 @@ root ('/') test-child01 null null null -allnext list: - -root->testcase-data->test-child0->test-child01->test-sibling1->test-sibling2 -->test-sibling3->null - Figure 2: Example test data tree to be attached to live tree. According to the scenario above, the live tree is already present so it isn't @@ -204,8 +192,6 @@ detached and then moving up the parent nodes are removed, and eventually the whole tree). selftest_data_remove() calls detach_node_and_children() that uses of_detach_node() to detach the nodes from the live device tree. -To detach a node, of_detach_node() first updates all_next linked list, by -attaching the previous node's allnext to current node's allnext pointer. And -then, it either updates the child pointer of given node's parent to its -sibling or attaches the previous sibling to the given node's sibling, as -appropriate. That is it :) +To detach a node, of_detach_node() either updates the child pointer of given +node's parent to its sibling or attaches the previous sibling to the given +node's sibling, as appropriate. That is it :) diff --git a/Documentation/devicetree/overlay-notes.txt b/Documentation/devicetree/overlay-notes.txt new file mode 100644 index 000000000000..30ae758e3eef --- /dev/null +++ b/Documentation/devicetree/overlay-notes.txt @@ -0,0 +1,133 @@ +Device Tree Overlay Notes +------------------------- + +This document describes the implementation of the in-kernel +device tree overlay functionality residing in drivers/of/overlay.c and is a +companion document to Documentation/devicetree/dt-object-internal.txt[1] & +Documentation/devicetree/dynamic-resolution-notes.txt[2] + +How overlays work +----------------- + +A Device Tree's overlay purpose is to modify the kernel's live tree, and +have the modification affecting the state of the the kernel in a way that +is reflecting the changes. +Since the kernel mainly deals with devices, any new device node that result +in an active device should have it created while if the device node is either +disabled or removed all together, the affected device should be deregistered. + +Lets take an example where we have a foo board with the following base tree +which is taken from [1]. + +---- foo.dts ----------------------------------------------------------------- + /* FOO platform */ + / { + compatible = "corp,foo"; + + /* shared resources */ + res: res { + }; + + /* On chip peripherals */ + ocp: ocp { + /* peripherals that are always instantiated */ + peripheral1 { ... }; + } + }; +---- foo.dts ----------------------------------------------------------------- + +The overlay bar.dts, when loaded (and resolved as described in [2]) should + +---- bar.dts ----------------------------------------------------------------- +/plugin/; /* allow undefined label references and record them */ +/ { + .... /* various properties for loader use; i.e. part id etc. */ + fragment@0 { + target = <&ocp>; + __overlay__ { + /* bar peripheral */ + bar { + compatible = "corp,bar"; + ... /* various properties and child nodes */ + } + }; + }; +}; +---- bar.dts ----------------------------------------------------------------- + +result in foo+bar.dts + +---- foo+bar.dts ------------------------------------------------------------- + /* FOO platform + bar peripheral */ + / { + compatible = "corp,foo"; + + /* shared resources */ + res: res { + }; + + /* On chip peripherals */ + ocp: ocp { + /* peripherals that are always instantiated */ + peripheral1 { ... }; + + /* bar peripheral */ + bar { + compatible = "corp,bar"; + ... /* various properties and child nodes */ + } + } + }; +---- foo+bar.dts ------------------------------------------------------------- + +As a result of the the overlay, a new device node (bar) has been created +so a bar platform device will be registered and if a matching device driver +is loaded the device will be created as expected. + +Overlay in-kernel API +-------------------------------- + +The API is quite easy to use. + +1. Call of_overlay_create() to create and apply an overlay. The return value +is a cookie identifying this overlay. + +2. Call of_overlay_destroy() to remove and cleanup the overlay previously +created via the call to of_overlay_create(). Removal of an overlay that +is stacked by another will not be permitted. + +Finally, if you need to remove all overlays in one-go, just call +of_overlay_destroy_all() which will remove every single one in the correct +order. + +Overlay DTS Format +------------------ + +The DTS of an overlay should have the following format: + +{ + /* ignored properties by the overlay */ + + fragment@0 { /* first child node */ + + target=<phandle>; /* phandle target of the overlay */ + or + target-path="/path"; /* target path of the overlay */ + + __overlay__ { + property-a; /* add property-a to the target */ + node-a { /* add to an existing, or create a node-a */ + ... + }; + }; + } + fragment@1 { /* second child node */ + ... + }; + /* more fragments follow */ +} + +Using the non-phandle based target method allows one to use a base DT which does +not contain a __symbols__ node, i.e. it was not compiled with the -@ option. +The __symbols__ node is only required for the target=<phandle> method, since it +contains the information required to map from a phandle to a tree location. diff --git a/Documentation/devicetree/todo.txt b/Documentation/devicetree/todo.txt index c3cf0659bd19..b5139d1de811 100644 --- a/Documentation/devicetree/todo.txt +++ b/Documentation/devicetree/todo.txt @@ -2,7 +2,6 @@ Todo list for devicetree: === General structure === - Switch from custom lists to (h)list_head for nodes and properties structure -- Remove of_allnodes list and iterate using list of child nodes alone === CONFIG_OF_DYNAMIC === - Switch to RCU for tree updates and get rid of global spinlock diff --git a/Documentation/kasan.txt b/Documentation/kasan.txt new file mode 100644 index 000000000000..58b72fbf450d --- /dev/null +++ b/Documentation/kasan.txt @@ -0,0 +1,172 @@ +KernelAddressSanitizer (KASAN) +============================== + +0. Overview +=========== + +KernelAddressSANitizer (KASAN) is a dynamic memory error detector. It provides +a fast and comprehensive solution for finding use-after-free and out-of-bounds +bugs. + +KASAN uses compile-time instrumentation for checking every memory access, +therefore you will need a GCC version 4.9.2 or later. GCC 5.0 or later is +required for detection of out-of-bounds accesses to stack or global variables. + +Currently KASAN is supported only for x86_64 architecture and requires the +kernel to be built with the SLUB allocator. + +1. Usage +======== + +To enable KASAN configure kernel with: + + CONFIG_KASAN = y + +and choose between CONFIG_KASAN_OUTLINE and CONFIG_KASAN_INLINE. Outline and +inline are compiler instrumentation types. The former produces smaller binary +the latter is 1.1 - 2 times faster. Inline instrumentation requires a GCC +version 5.0 or later. + +Currently KASAN works only with the SLUB memory allocator. +For better bug detection and nicer reporting, enable CONFIG_STACKTRACE. + +To disable instrumentation for specific files or directories, add a line +similar to the following to the respective kernel Makefile: + + For a single file (e.g. main.o): + KASAN_SANITIZE_main.o := n + + For all files in one directory: + KASAN_SANITIZE := n + +1.1 Error reports +================= + +A typical out of bounds access report looks like this: + +================================================================== +BUG: AddressSanitizer: out of bounds access in kmalloc_oob_right+0x65/0x75 [test_kasan] at addr ffff8800693bc5d3 +Write of size 1 by task modprobe/1689 +============================================================================= +BUG kmalloc-128 (Not tainted): kasan error +----------------------------------------------------------------------------- + +Disabling lock debugging due to kernel taint +INFO: Allocated in kmalloc_oob_right+0x3d/0x75 [test_kasan] age=0 cpu=0 pid=1689 + __slab_alloc+0x4b4/0x4f0 + kmem_cache_alloc_trace+0x10b/0x190 + kmalloc_oob_right+0x3d/0x75 [test_kasan] + init_module+0x9/0x47 [test_kasan] + do_one_initcall+0x99/0x200 + load_module+0x2cb3/0x3b20 + SyS_finit_module+0x76/0x80 + system_call_fastpath+0x12/0x17 +INFO: Slab 0xffffea0001a4ef00 objects=17 used=7 fp=0xffff8800693bd728 flags=0x100000000004080 +INFO: Object 0xffff8800693bc558 @offset=1368 fp=0xffff8800693bc720 + +Bytes b4 ffff8800693bc548: 00 00 00 00 00 00 00 00 5a 5a 5a 5a 5a 5a 5a 5a ........ZZZZZZZZ +Object ffff8800693bc558: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk +Object ffff8800693bc568: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk +Object ffff8800693bc578: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk +Object ffff8800693bc588: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk +Object ffff8800693bc598: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk +Object ffff8800693bc5a8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk +Object ffff8800693bc5b8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk +Object ffff8800693bc5c8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b a5 kkkkkkkkkkkkkkk. +Redzone ffff8800693bc5d8: cc cc cc cc cc cc cc cc ........ +Padding ffff8800693bc718: 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZ +CPU: 0 PID: 1689 Comm: modprobe Tainted: G B 3.18.0-rc1-mm1+ #98 +Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.7.5-0-ge51488c-20140602_164612-nilsson.home.kraxel.org 04/01/2014 + ffff8800693bc000 0000000000000000 ffff8800693bc558 ffff88006923bb78 + ffffffff81cc68ae 00000000000000f3 ffff88006d407600 ffff88006923bba8 + ffffffff811fd848 ffff88006d407600 ffffea0001a4ef00 ffff8800693bc558 +Call Trace: + [<ffffffff81cc68ae>] dump_stack+0x46/0x58 + [<ffffffff811fd848>] print_trailer+0xf8/0x160 + [<ffffffffa00026a7>] ? kmem_cache_oob+0xc3/0xc3 [test_kasan] + [<ffffffff811ff0f5>] object_err+0x35/0x40 + [<ffffffffa0002065>] ? kmalloc_oob_right+0x65/0x75 [test_kasan] + [<ffffffff8120b9fa>] kasan_report_error+0x38a/0x3f0 + [<ffffffff8120a79f>] ? kasan_poison_shadow+0x2f/0x40 + [<ffffffff8120b344>] ? kasan_unpoison_shadow+0x14/0x40 + [<ffffffff8120a79f>] ? kasan_poison_shadow+0x2f/0x40 + [<ffffffffa00026a7>] ? kmem_cache_oob+0xc3/0xc3 [test_kasan] + [<ffffffff8120a995>] __asan_store1+0x75/0xb0 + [<ffffffffa0002601>] ? kmem_cache_oob+0x1d/0xc3 [test_kasan] + [<ffffffffa0002065>] ? kmalloc_oob_right+0x65/0x75 [test_kasan] + [<ffffffffa0002065>] kmalloc_oob_right+0x65/0x75 [test_kasan] + [<ffffffffa00026b0>] init_module+0x9/0x47 [test_kasan] + [<ffffffff810002d9>] do_one_initcall+0x99/0x200 + [<ffffffff811e4e5c>] ? __vunmap+0xec/0x160 + [<ffffffff81114f63>] load_module+0x2cb3/0x3b20 + [<ffffffff8110fd70>] ? m_show+0x240/0x240 + [<ffffffff81115f06>] SyS_finit_module+0x76/0x80 + [<ffffffff81cd3129>] system_call_fastpath+0x12/0x17 +Memory state around the buggy address: + ffff8800693bc300: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc380: fc fc 00 00 00 00 00 00 00 00 00 00 00 00 00 fc + ffff8800693bc400: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc480: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc500: fc fc fc fc fc fc fc fc fc fc fc 00 00 00 00 00 +>ffff8800693bc580: 00 00 00 00 00 00 00 00 00 00 03 fc fc fc fc fc + ^ + ffff8800693bc600: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc680: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc700: fc fc fc fc fb fb fb fb fb fb fb fb fb fb fb fb + ffff8800693bc780: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb + ffff8800693bc800: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb +================================================================== + +The header of the report discribe what kind of bug happened and what kind of +access caused it. It's followed by the description of the accessed slub object +(see 'SLUB Debug output' section in Documentation/vm/slub.txt for details) and +the description of the accessed memory page. + +In the last section the report shows memory state around the accessed address. +Reading this part requires some understanding of how KASAN works. + +The state of each 8 aligned bytes of memory is encoded in one shadow byte. +Those 8 bytes can be accessible, partially accessible, freed or be a redzone. +We use the following encoding for each shadow byte: 0 means that all 8 bytes +of the corresponding memory region are accessible; number N (1 <= N <= 7) means +that the first N bytes are accessible, and other (8 - N) bytes are not; +any negative value indicates that the entire 8-byte word is inaccessible. +We use different negative values to distinguish between different kinds of +inaccessible memory like redzones or freed memory (see mm/kasan/kasan.h). + +In the report above the arrows point to the shadow byte 03, which means that +the accessed address is partially accessible. + + +2. Implementation details +========================= + +From a high level, our approach to memory error detection is similar to that +of kmemcheck: use shadow memory to record whether each byte of memory is safe +to access, and use compile-time instrumentation to check shadow memory on each +memory access. + +AddressSanitizer dedicates 1/8 of kernel memory to its shadow memory +(e.g. 16TB to cover 128TB on x86_64) and uses direct mapping with a scale and +offset to translate a memory address to its corresponding shadow address. + +Here is the function witch translate an address to its corresponding shadow +address: + +static inline void *kasan_mem_to_shadow(const void *addr) +{ + return ((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT) + + KASAN_SHADOW_OFFSET; +} + +where KASAN_SHADOW_SCALE_SHIFT = 3. + +Compile-time instrumentation used for checking memory accesses. Compiler inserts +function calls (__asan_load*(addr), __asan_store*(addr)) before each memory +access of size 1, 2, 4, 8 or 16. These functions check whether memory access is +valid or not by checking corresponding shadow memory. + +GCC 5.0 has possibility to perform inline instrumentation. Instead of making +function calls GCC directly inserts the code to check the shadow memory. +This option significantly enlarges kernel but it gives x1.1-x2 performance +boost over outline instrumented kernel. diff --git a/Documentation/thermal/cpu-cooling-api.txt b/Documentation/thermal/cpu-cooling-api.txt index fca24c931ec8..71653584cd03 100644 --- a/Documentation/thermal/cpu-cooling-api.txt +++ b/Documentation/thermal/cpu-cooling-api.txt @@ -3,7 +3,7 @@ CPU cooling APIs How To Written by Amit Daniel Kachhap <amit.kachhap@linaro.org> -Updated: 12 May 2012 +Updated: 6 Jan 2015 Copyright (c) 2012 Samsung Electronics Co., Ltd(http://www.samsung.com) @@ -25,8 +25,173 @@ the user. The registration APIs returns the cooling device pointer. clip_cpus: cpumask of cpus where the frequency constraints will happen. -1.1.2 void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev) +1.1.2 struct thermal_cooling_device *of_cpufreq_cooling_register( + struct device_node *np, const struct cpumask *clip_cpus) + + This interface function registers the cpufreq cooling device with + the name "thermal-cpufreq-%x" linking it with a device tree node, in + order to bind it via the thermal DT code. This api can support multiple + instances of cpufreq cooling devices. + + np: pointer to the cooling device device tree node + clip_cpus: cpumask of cpus where the frequency constraints will happen. + +1.1.3 struct thermal_cooling_device *cpufreq_power_cooling_register( + const struct cpumask *clip_cpus, u32 capacitance, + get_static_t plat_static_func) + +Similar to cpufreq_cooling_register, this function registers a cpufreq +cooling device. Using this function, the cooling device will +implement the power extensions by using a simple cpu power model. The +cpus must have registered their OPPs using the OPP library. + +The additional parameters are needed for the power model (See 2. Power +models). "capacitance" is the dynamic power coefficient (See 2.1 +Dynamic power). "plat_static_func" is a function to calculate the +static power consumed by these cpus (See 2.2 Static power). + +1.1.4 struct thermal_cooling_device *of_cpufreq_power_cooling_register( + struct device_node *np, const struct cpumask *clip_cpus, u32 capacitance, + get_static_t plat_static_func) + +Similar to cpufreq_power_cooling_register, this function register a +cpufreq cooling device with power extensions using the device tree +information supplied by the np parameter. + +1.1.5 void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev) This interface function unregisters the "thermal-cpufreq-%x" cooling device. cdev: Cooling device pointer which has to be unregistered. + +2. Power models + +The power API registration functions provide a simple power model for +CPUs. The current power is calculated as dynamic + (optionally) +static power. This power model requires that the operating-points of +the CPUs are registered using the kernel's opp library and the +`cpufreq_frequency_table` is assigned to the `struct device` of the +cpu. If you are using CONFIG_CPUFREQ_DT then the +`cpufreq_frequency_table` should already be assigned to the cpu +device. + +The `plat_static_func` parameter of `cpufreq_power_cooling_register()` +and `of_cpufreq_power_cooling_register()` is optional. If you don't +provide it, only dynamic power will be considered. + +2.1 Dynamic power + +The dynamic power consumption of a processor depends on many factors. +For a given processor implementation the primary factors are: + +- The time the processor spends running, consuming dynamic power, as + compared to the time in idle states where dynamic consumption is + negligible. Herein we refer to this as 'utilisation'. +- The voltage and frequency levels as a result of DVFS. The DVFS + level is a dominant factor governing power consumption. +- In running time the 'execution' behaviour (instruction types, memory + access patterns and so forth) causes, in most cases, a second order + variation. In pathological cases this variation can be significant, + but typically it is of a much lesser impact than the factors above. + +A high level dynamic power consumption model may then be represented as: + +Pdyn = f(run) * Voltage^2 * Frequency * Utilisation + +f(run) here represents the described execution behaviour and its +result has a units of Watts/Hz/Volt^2 (this often expressed in +mW/MHz/uVolt^2) + +The detailed behaviour for f(run) could be modelled on-line. However, +in practice, such an on-line model has dependencies on a number of +implementation specific processor support and characterisation +factors. Therefore, in initial implementation that contribution is +represented as a constant coefficient. This is a simplification +consistent with the relative contribution to overall power variation. + +In this simplified representation our model becomes: + +Pdyn = Capacitance * Voltage^2 * Frequency * Utilisation + +Where `capacitance` is a constant that represents an indicative +running time dynamic power coefficient in fundamental units of +mW/MHz/uVolt^2. Typical values for mobile CPUs might lie in range +from 100 to 500. For reference, the approximate values for the SoC in +ARM's Juno Development Platform are 530 for the Cortex-A57 cluster and +140 for the Cortex-A53 cluster. + + +2.2 Static power + +Static leakage power consumption depends on a number of factors. For a +given circuit implementation the primary factors are: + +- Time the circuit spends in each 'power state' +- Temperature +- Operating voltage +- Process grade + +The time the circuit spends in each 'power state' for a given +evaluation period at first order means OFF or ON. However, +'retention' states can also be supported that reduce power during +inactive periods without loss of context. + +Note: The visibility of state entries to the OS can vary, according to +platform specifics, and this can then impact the accuracy of a model +based on OS state information alone. It might be possible in some +cases to extract more accurate information from system resources. + +The temperature, operating voltage and process 'grade' (slow to fast) +of the circuit are all significant factors in static leakage power +consumption. All of these have complex relationships to static power. + +Circuit implementation specific factors include the chosen silicon +process as well as the type, number and size of transistors in both +the logic gates and any RAM elements included. + +The static power consumption modelling must take into account the +power managed regions that are implemented. Taking the example of an +ARM processor cluster, the modelling would take into account whether +each CPU can be powered OFF separately or if only a single power +region is implemented for the complete cluster. + +In one view, there are others, a static power consumption model can +then start from a set of reference values for each power managed +region (e.g. CPU, Cluster/L2) in each state (e.g. ON, OFF) at an +arbitrary process grade, voltage and temperature point. These values +are then scaled for all of the following: the time in each state, the +process grade, the current temperature and the operating voltage. +However, since both implementation specific and complex relationships +dominate the estimate, the appropriate interface to the model from the +cpu cooling device is to provide a function callback that calculates +the static power in this platform. When registering the cpu cooling +device pass a function pointer that follows the `get_static_t` +prototype: + + int plat_get_static(cpumask_t *cpumask, int interval, + unsigned long voltage, u32 &power); + +`cpumask` is the cpumask of the cpus involved in the calculation. +`voltage` is the voltage at which they are operating. The function +should calculate the average static power for the last `interval` +milliseconds. It returns 0 on success, -E* on error. If it +succeeds, it should store the static power in `power`. Reading the +temperature of the cpus described by `cpumask` is left for +plat_get_static() to do as the platform knows best which thermal +sensor is closest to the cpu. + +If `plat_static_func` is NULL, static power is considered to be +negligible for this platform and only dynamic power is considered. + +The platform specific callback can then use any combination of tables +and/or equations to permute the estimated value. Process grade +information is not passed to the model since access to such data, from +on-chip measurement capability or manufacture time data, is platform +specific. + +Note: the significance of static power for CPUs in comparison to +dynamic power is highly dependent on implementation. Given the +potential complexity in implementation, the importance and accuracy of +its inclusion when using cpu cooling devices should be assessed on a +case by case basis. + diff --git a/Documentation/thermal/power_allocator.txt b/Documentation/thermal/power_allocator.txt new file mode 100644 index 000000000000..c3797b529991 --- /dev/null +++ b/Documentation/thermal/power_allocator.txt @@ -0,0 +1,247 @@ +Power allocator governor tunables +================================= + +Trip points +----------- + +The governor requires the following two passive trip points: + +1. "switch on" trip point: temperature above which the governor + control loop starts operating. This is the first passive trip + point of the thermal zone. + +2. "desired temperature" trip point: it should be higher than the + "switch on" trip point. This the target temperature the governor + is controlling for. This is the last passive trip point of the + thermal zone. + +PID Controller +-------------- + +The power allocator governor implements a +Proportional-Integral-Derivative controller (PID controller) with +temperature as the control input and power as the controlled output: + + P_max = k_p * e + k_i * err_integral + k_d * diff_err + sustainable_power + +where + e = desired_temperature - current_temperature + err_integral is the sum of previous errors + diff_err = e - previous_error + +It is similar to the one depicted below: + + k_d + | +current_temp | + | v + | +----------+ +---+ + | +----->| diff_err |-->| X |------+ + | | +----------+ +---+ | + | | | tdp actor + | | k_i | | get_requested_power() + | | | | | | | + | | | | | | | ... + v | v v v v v + +---+ | +-------+ +---+ +---+ +---+ +----------+ + | S |-------+----->| sum e |----->| X |--->| S |-->| S |-->|power | + +---+ | +-------+ +---+ +---+ +---+ |allocation| + ^ | ^ +----------+ + | | | | | + | | +---+ | | | + | +------->| X |-------------------+ v v + | +---+ granted performance +desired_temperature ^ + | + | + k_po/k_pu + +Sustainable power +----------------- + +An estimate of the sustainable dissipatable power (in mW) should be +provided while registering the thermal zone. This estimates the +sustained power that can be dissipated at the desired control +temperature. This is the maximum sustained power for allocation at +the desired maximum temperature. The actual sustained power can vary +for a number of reasons. The closed loop controller will take care of +variations such as environmental conditions, and some factors related +to the speed-grade of the silicon. `sustainable_power` is therefore +simply an estimate, and may be tuned to affect the aggressiveness of +the thermal ramp. For reference, the sustainable power of a 4" phone +is typically 2000mW, while on a 10" tablet is around 4500mW (may vary +depending on screen size). + +If you are using device tree, do add it as a property of the +thermal-zone. For example: + + thermal-zones { + soc_thermal { + polling-delay = <1000>; + polling-delay-passive = <100>; + sustainable-power = <2500>; + ... + +Instead, if the thermal zone is registered from the platform code, pass a +`thermal_zone_params` that has a `sustainable_power`. If no +`thermal_zone_params` were being passed, then something like below +will suffice: + + static const struct thermal_zone_params tz_params = { + .sustainable_power = 3500, + }; + +and then pass `tz_params` as the 5th parameter to +`thermal_zone_device_register()` + +k_po and k_pu +------------- + +The implementation of the PID controller in the power allocator +thermal governor allows the configuration of two proportional term +constants: `k_po` and `k_pu`. `k_po` is the proportional term +constant during temperature overshoot periods (current temperature is +above "desired temperature" trip point). Conversely, `k_pu` is the +proportional term constant during temperature undershoot periods +(current temperature below "desired temperature" trip point). + +These controls are intended as the primary mechanism for configuring +the permitted thermal "ramp" of the system. For instance, a lower +`k_pu` value will provide a slower ramp, at the cost of capping +available capacity at a low temperature. On the other hand, a high +value of `k_pu` will result in the governor granting very high power +whilst temperature is low, and may lead to temperature overshooting. + +The default value for `k_pu` is: + + 2 * sustainable_power / (desired_temperature - switch_on_temp) + +This means that at `switch_on_temp` the output of the controller's +proportional term will be 2 * `sustainable_power`. The default value +for `k_po` is: + + sustainable_power / (desired_temperature - switch_on_temp) + +Focusing on the proportional and feed forward values of the PID +controller equation we have: + + P_max = k_p * e + sustainable_power + +The proportional term is proportional to the difference between the +desired temperature and the current one. When the current temperature +is the desired one, then the proportional component is zero and +`P_max` = `sustainable_power`. That is, the system should operate in +thermal equilibrium under constant load. `sustainable_power` is only +an estimate, which is the reason for closed-loop control such as this. + +Expanding `k_pu` we get: + P_max = 2 * sustainable_power * (T_set - T) / (T_set - T_on) + + sustainable_power + +where + T_set is the desired temperature + T is the current temperature + T_on is the switch on temperature + +When the current temperature is the switch_on temperature, the above +formula becomes: + + P_max = 2 * sustainable_power * (T_set - T_on) / (T_set - T_on) + + sustainable_power = 2 * sustainable_power + sustainable_power = + 3 * sustainable_power + +Therefore, the proportional term alone linearly decreases power from +3 * `sustainable_power` to `sustainable_power` as the temperature +rises from the switch on temperature to the desired temperature. + +k_i and integral_cutoff +----------------------- + +`k_i` configures the PID loop's integral term constant. This term +allows the PID controller to compensate for long term drift and for +the quantized nature of the output control: cooling devices can't set +the exact power that the governor requests. When the temperature +error is below `integral_cutoff`, errors are accumulated in the +integral term. This term is then multiplied by `k_i` and the result +added to the output of the controller. Typically `k_i` is set low (1 +or 2) and `integral_cutoff` is 0. + +k_d +--- + +`k_d` configures the PID loop's derivative term constant. It's +recommended to leave it as the default: 0. + +Cooling device power API +======================== + +Cooling devices controlled by this governor must supply the additional +"power" API in their `cooling_device_ops`. It consists on three ops: + +1. int get_requested_power(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, u32 *power); +@cdev: The `struct thermal_cooling_device` pointer +@tz: thermal zone in which we are currently operating +@power: pointer in which to store the calculated power + +`get_requested_power()` calculates the power requested by the device +in milliwatts and stores it in @power . It should return 0 on +success, -E* on failure. This is currently used by the power +allocator governor to calculate how much power to give to each cooling +device. + +2. int state2power(struct thermal_cooling_device *cdev, struct + thermal_zone_device *tz, unsigned long state, u32 *power); +@cdev: The `struct thermal_cooling_device` pointer +@tz: thermal zone in which we are currently operating +@state: A cooling device state +@power: pointer in which to store the equivalent power + +Convert cooling device state @state into power consumption in +milliwatts and store it in @power. It should return 0 on success, -E* +on failure. This is currently used by thermal core to calculate the +maximum power that an actor can consume. + +3. int power2state(struct thermal_cooling_device *cdev, u32 power, + unsigned long *state); +@cdev: The `struct thermal_cooling_device` pointer +@power: power in milliwatts +@state: pointer in which to store the resulting state + +Calculate a cooling device state that would make the device consume at +most @power mW and store it in @state. It should return 0 on success, +-E* on failure. This is currently used by the thermal core to convert +a given power set by the power allocator governor to a state that the +cooling device can set. It is a function because this conversion may +depend on external factors that may change so this function should the +best conversion given "current circumstances". + +Cooling device weights +---------------------- + +Weights are a mechanism to bias the allocation among cooling +devices. They express the relative power efficiency of different +cooling devices. Higher weight can be used to express higher power +efficiency. Weighting is relative such that if each cooling device +has a weight of one they are considered equal. This is particularly +useful in heterogeneous systems where two cooling devices may perform +the same kind of compute, but with different efficiency. For example, +a system with two different types of processors. + +If the thermal zone is registered using +`thermal_zone_device_register()` (i.e., platform code), then weights +are passed as part of the thermal zone's `thermal_bind_parameters`. +If the platform is registered using device tree, then they are passed +as the `contribution` property of each map in the `cooling-maps` node. + +Limitations of the power allocator governor +=========================================== + +The power allocator governor's PID controller works best if there is a +periodic tick. If you have a driver that calls +`thermal_zone_device_update()` (or anything that ends up calling the +governor's `throttle()` function) repetitively, the governor response +won't be very good. Note that this is not particular to this +governor, step-wise will also misbehave if you call its throttle() +faster than the normal thermal framework tick (due to interrupts for +example) as it will overreact. diff --git a/Documentation/thermal/sysfs-api.txt b/Documentation/thermal/sysfs-api.txt index 87519cb379ee..c1f6864a8c5d 100644 --- a/Documentation/thermal/sysfs-api.txt +++ b/Documentation/thermal/sysfs-api.txt @@ -95,7 +95,7 @@ temperature) and throttle appropriate devices. 1.3 interface for binding a thermal zone device with a thermal cooling device 1.3.1 int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz, int trip, struct thermal_cooling_device *cdev, - unsigned long upper, unsigned long lower); + unsigned long upper, unsigned long lower, unsigned int weight); This interface function bind a thermal cooling device to the certain trip point of a thermal zone device. @@ -110,6 +110,8 @@ temperature) and throttle appropriate devices. lower:the Minimum cooling state can be used for this trip point. THERMAL_NO_LIMIT means no lower limit, and the cooling device can be in cooling state 0. + weight: the influence of this cooling device in this thermal + zone. See 1.4.1 below for more information. 1.3.2 int thermal_zone_unbind_cooling_device(struct thermal_zone_device *tz, int trip, struct thermal_cooling_device *cdev); @@ -127,9 +129,15 @@ temperature) and throttle appropriate devices. This structure defines the following parameters that are used to bind a zone with a cooling device for a particular trip point. .cdev: The cooling device pointer - .weight: The 'influence' of a particular cooling device on this zone. - This is on a percentage scale. The sum of all these weights - (for a particular zone) cannot exceed 100. + .weight: The 'influence' of a particular cooling device on this + zone. This is relative to the rest of the cooling + devices. For example, if all cooling devices have a + weight of 1, then they all contribute the same. You can + use percentages if you want, but it's not mandatory. A + weight of 0 means that this cooling device doesn't + contribute to the cooling of this zone unless all cooling + devices have a weight of 0. If all weights are 0, then + they all contribute the same. .trip_mask:This is a bit mask that gives the binding relation between this thermal zone and cdev, for a particular trip point. If nth bit is set, then the cdev and thermal zone are bound @@ -176,6 +184,14 @@ Thermal zone device sys I/F, created once it's registered: |---trip_point_[0-*]_type: Trip point type |---trip_point_[0-*]_hyst: Hysteresis value for this trip point |---emul_temp: Emulated temperature set node + |---sustainable_power: Sustainable dissipatable power + |---k_po: Proportional term during temperature overshoot + |---k_pu: Proportional term during temperature undershoot + |---k_i: PID's integral term in the power allocator gov + |---k_d: PID's derivative term in the power allocator + |---integral_cutoff: Offset above which errors are accumulated + |---slope: Slope constant applied as linear extrapolation + |---offset: Offset constant applied as linear extrapolation Thermal cooling device sys I/F, created once it's registered: /sys/class/thermal/cooling_device[0-*]: @@ -192,6 +208,8 @@ thermal_zone_bind_cooling_device/thermal_zone_unbind_cooling_device. /sys/class/thermal/thermal_zone[0-*]: |---cdev[0-*]: [0-*]th cooling device in current thermal zone |---cdev[0-*]_trip_point: Trip point that cdev[0-*] is associated with + |---cdev[0-*]_weight: Influence of the cooling device in + this thermal zone Besides the thermal zone device sysfs I/F and cooling device sysfs I/F, the generic thermal driver also creates a hwmon sysfs I/F for each _type_ @@ -265,6 +283,14 @@ cdev[0-*]_trip_point point. RO, Optional +cdev[0-*]_weight + The influence of cdev[0-*] in this thermal zone. This value + is relative to the rest of cooling devices in the thermal + zone. For example, if a cooling device has a weight double + than that of other, it's twice as effective in cooling the + thermal zone. + RW, Optional + passive Attribute is only present for zones in which the passive cooling policy is not supported by native thermal driver. Default is zero @@ -289,6 +315,66 @@ emul_temp because userland can easily disable the thermal policy by simply flooding this sysfs node with low temperature values. +sustainable_power + An estimate of the sustained power that can be dissipated by + the thermal zone. Used by the power allocator governor. For + more information see Documentation/thermal/power_allocator.txt + Unit: milliwatts + RW, Optional + +k_po + The proportional term of the power allocator governor's PID + controller during temperature overshoot. Temperature overshoot + is when the current temperature is above the "desired + temperature" trip point. For more information see + Documentation/thermal/power_allocator.txt + RW, Optional + +k_pu + The proportional term of the power allocator governor's PID + controller during temperature undershoot. Temperature undershoot + is when the current temperature is below the "desired + temperature" trip point. For more information see + Documentation/thermal/power_allocator.txt + RW, Optional + +k_i + The integral term of the power allocator governor's PID + controller. This term allows the PID controller to compensate + for long term drift. For more information see + Documentation/thermal/power_allocator.txt + RW, Optional + +k_d + The derivative term of the power allocator governor's PID + controller. For more information see + Documentation/thermal/power_allocator.txt + RW, Optional + +integral_cutoff + Temperature offset from the desired temperature trip point + above which the integral term of the power allocator + governor's PID controller starts accumulating errors. For + example, if integral_cutoff is 0, then the integral term only + accumulates error when temperature is above the desired + temperature trip point. For more information see + Documentation/thermal/power_allocator.txt + RW, Optional + +slope + The slope constant used in a linear extrapolation model + to determine a hotspot temperature based off the sensor's + raw readings. It is up to the device driver to determine + the usage of these values. + RW, Optional + +offset + The offset constant used in a linear extrapolation model + to determine a hotspot temperature based off the sensor's + raw readings. It is up to the device driver to determine + the usage of these values. + RW, Optional + ***************************** * Cooling device attributes * ***************************** @@ -318,7 +404,8 @@ passive, active. If an ACPI thermal zone supports critical, passive, active[0] and active[1] at the same time, it may register itself as a thermal_zone_device (thermal_zone1) with 4 trip points in all. It has one processor and one fan, which are both registered as -thermal_cooling_device. +thermal_cooling_device. Both are considered to have the same +effectiveness in cooling the thermal zone. If the processor is listed in _PSL method, and the fan is listed in _AL0 method, the sys I/F structure will be built like this: @@ -340,8 +427,10 @@ method, the sys I/F structure will be built like this: |---trip_point_3_type: active1 |---cdev0: --->/sys/class/thermal/cooling_device0 |---cdev0_trip_point: 1 /* cdev0 can be used for passive */ + |---cdev0_weight: 1024 |---cdev1: --->/sys/class/thermal/cooling_device3 |---cdev1_trip_point: 2 /* cdev1 can be used for active[0]*/ + |---cdev1_weight: 1024 |cooling_device0: |---type: Processor diff --git a/Documentation/trace/coresight.txt b/Documentation/trace/coresight.txt new file mode 100644 index 000000000000..77d14d51a670 --- /dev/null +++ b/Documentation/trace/coresight.txt @@ -0,0 +1,299 @@ + Coresight - HW Assisted Tracing on ARM + ====================================== + + Author: Mathieu Poirier <mathieu.poirier@linaro.org> + Date: September 11th, 2014 + +Introduction +------------ + +Coresight is an umbrella of technologies allowing for the debugging of ARM +based SoC. It includes solutions for JTAG and HW assisted tracing. This +document is concerned with the latter. + +HW assisted tracing is becoming increasingly useful when dealing with systems +that have many SoCs and other components like GPU and DMA engines. ARM has +developed a HW assisted tracing solution by means of different components, each +being added to a design at synthesis time to cater to specific tracing needs. +Compoments are generally categorised as source, link and sinks and are +(usually) discovered using the AMBA bus. + +"Sources" generate a compressed stream representing the processor instruction +path based on tracing scenarios as configured by users. From there the stream +flows through the coresight system (via ATB bus) using links that are connecting +the emanating source to a sink(s). Sinks serve as endpoints to the coresight +implementation, either storing the compressed stream in a memory buffer or +creating an interface to the outside world where data can be transferred to a +host without fear of filling up the onboard coresight memory buffer. + +At typical coresight system would look like this: + + ***************************************************************** + **************************** AMBA AXI ****************************===|| + ***************************************************************** || + ^ ^ | || + | | * ** + 0000000 ::::: 0000000 ::::: ::::: @@@@@@@ |||||||||||| + 0 CPU 0<-->: C : 0 CPU 0<-->: C : : C : @ STM @ || System || + |->0000000 : T : |->0000000 : T : : T :<--->@@@@@ || Memory || + | #######<-->: I : | #######<-->: I : : I : @@@<-| |||||||||||| + | # ETM # ::::: | # PTM # ::::: ::::: @ | + | ##### ^ ^ | ##### ^ ! ^ ! . | ||||||||| + | |->### | ! | |->### | ! | ! . | || DAP || + | | # | ! | | # | ! | ! . | ||||||||| + | | . | ! | | . | ! | ! . | | | + | | . | ! | | . | ! | ! . | | * + | | . | ! | | . | ! | ! . | | SWD/ + | | . | ! | | . | ! | ! . | | JTAG + *****************************************************************<-| + *************************** AMBA Debug APB ************************ + ***************************************************************** + | . ! . ! ! . | + | . * . * * . | + ***************************************************************** + ******************** Cross Trigger Matrix (CTM) ******************* + ***************************************************************** + | . ^ . . | + | * ! * * | + ***************************************************************** + ****************** AMBA Advanced Trace Bus (ATB) ****************** + ***************************************************************** + | ! =============== | + | * ===== F =====<---------| + | ::::::::: ==== U ==== + |-->:: CTI ::<!! === N === + | ::::::::: ! == N == + | ^ * == E == + | ! &&&&&&&&& IIIIIII == L == + |------>&& ETB &&<......II I ======= + | ! &&&&&&&&& II I . + | ! I I . + | ! I REP I<.......... + | ! I I + | !!>&&&&&&&&& II I *Source: ARM ltd. + |------>& TPIU &<......II I DAP = Debug Access Port + &&&&&&&&& IIIIIII ETM = Embedded Trace Macrocell + ; PTM = Program Trace Macrocell + ; CTI = Cross Trigger Interface + * ETB = Embedded Trace Buffer + To trace port TPIU= Trace Port Interface Unit + SWD = Serial Wire Debug + +While on target configuration of the components is done via the APB bus, +all trace data are carried out-of-band on the ATB bus. The CTM provides +a way to aggregate and distribute signals between CoreSight components. + +The coresight framework provides a central point to represent, configure and +manage coresight devices on a platform. This first implementation centers on +the basic tracing functionality, enabling components such ETM/PTM, funnel, +replicator, TMC, TPIU and ETB. Future work will enable more +intricate IP blocks such as STM and CTI. + + +Acronyms and Classification +--------------------------- + +Acronyms: + +PTM: Program Trace Macrocell +ETM: Embedded Trace Macrocell +STM: System trace Macrocell +ETB: Embedded Trace Buffer +ITM: Instrumentation Trace Macrocell +TPIU: Trace Port Interface Unit +TMC-ETR: Trace Memory Controller, configured as Embedded Trace Router +TMC-ETF: Trace Memory Controller, configured as Embedded Trace FIFO +CTI: Cross Trigger Interface + +Classification: + +Source: + ETMv3.x ETMv4, PTMv1.0, PTMv1.1, STM, STM500, ITM +Link: + Funnel, replicator (intelligent or not), TMC-ETR +Sinks: + ETBv1.0, ETB1.1, TPIU, TMC-ETF +Misc: + CTI + + +Device Tree Bindings +---------------------- + +See Documentation/devicetree/bindings/arm/coresight.txt for details. + +As of this writing drivers for ITM, STMs and CTIs are not provided but are +expected to be added as the solution matures. + + +Framework and implementation +---------------------------- + +The coresight framework provides a central point to represent, configure and +manage coresight devices on a platform. Any coresight compliant device can +register with the framework for as long as they use the right APIs: + +struct coresight_device *coresight_register(struct coresight_desc *desc); +void coresight_unregister(struct coresight_device *csdev); + +The registering function is taking a "struct coresight_device *csdev" and +register the device with the core framework. The unregister function takes +a reference to a "strut coresight_device", obtained at registration time. + +If everything goes well during the registration process the new devices will +show up under /sys/bus/coresight/devices, as showns here for a TC2 platform: + +root:~# ls /sys/bus/coresight/devices/ +replicator 20030000.tpiu 2201c000.ptm 2203c000.etm 2203e000.etm +20010000.etb 20040000.funnel 2201d000.ptm 2203d000.etm +root:~# + +The functions take a "struct coresight_device", which looks like this: + +struct coresight_desc { + enum coresight_dev_type type; + struct coresight_dev_subtype subtype; + const struct coresight_ops *ops; + struct coresight_platform_data *pdata; + struct device *dev; + const struct attribute_group **groups; +}; + + +The "coresight_dev_type" identifies what the device is, i.e, source link or +sink while the "coresight_dev_subtype" will characterise that type further. + +The "struct coresight_ops" is mandatory and will tell the framework how to +perform base operations related to the components, each component having +a different set of requirement. For that "struct coresight_ops_sink", +"struct coresight_ops_link" and "struct coresight_ops_source" have been +provided. + +The next field, "struct coresight_platform_data *pdata" is acquired by calling +"of_get_coresight_platform_data()", as part of the driver's _probe routine and +"struct device *dev" gets the device reference embedded in the "amba_device": + +static int etm_probe(struct amba_device *adev, const struct amba_id *id) +{ + ... + ... + drvdata->dev = &adev->dev; + ... +} + +Specific class of device (source, link, or sink) have generic operations +that can be performed on them (see "struct coresight_ops"). The +"**groups" is a list of sysfs entries pertaining to operations +specific to that component only. "Implementation defined" customisations are +expected to be accessed and controlled using those entries. + +Last but not least, "struct module *owner" is expected to be set to reflect +the information carried in "THIS_MODULE". + +How to use +---------- + +Before trace collection can start, a coresight sink needs to be identify. +There is no limit on the amount of sinks (nor sources) that can be enabled at +any given moment. As a generic operation, all device pertaining to the sink +class will have an "active" entry in sysfs: + +root:/sys/bus/coresight/devices# ls +replicator 20030000.tpiu 2201c000.ptm 2203c000.etm 2203e000.etm +20010000.etb 20040000.funnel 2201d000.ptm 2203d000.etm +root:/sys/bus/coresight/devices# ls 20010000.etb +enable_sink status trigger_cntr +root:/sys/bus/coresight/devices# echo 1 > 20010000.etb/enable_sink +root:/sys/bus/coresight/devices# cat 20010000.etb/enable_sink +1 +root:/sys/bus/coresight/devices# + +At boot time the current etm3x driver will configure the first address +comparator with "_stext" and "_etext", essentially tracing any instruction +that falls within that range. As such "enabling" a source will immediately +trigger a trace capture: + +root:/sys/bus/coresight/devices# echo 1 > 2201c000.ptm/enable_source +root:/sys/bus/coresight/devices# cat 2201c000.ptm/enable_source +1 +root:/sys/bus/coresight/devices# cat 20010000.etb/status +Depth: 0x2000 +Status: 0x1 +RAM read ptr: 0x0 +RAM wrt ptr: 0x19d3 <----- The write pointer is moving +Trigger cnt: 0x0 +Control: 0x1 +Flush status: 0x0 +Flush ctrl: 0x2001 +root:/sys/bus/coresight/devices# + +Trace collection is stopped the same way: + +root:/sys/bus/coresight/devices# echo 0 > 2201c000.ptm/enable_source +root:/sys/bus/coresight/devices# + +The content of the ETB buffer can be harvested directly from /dev: + +root:/sys/bus/coresight/devices# dd if=/dev/20010000.etb \ +of=~/cstrace.bin + +64+0 records in +64+0 records out +32768 bytes (33 kB) copied, 0.00125258 s, 26.2 MB/s +root:/sys/bus/coresight/devices# + +The file cstrace.bin can be decompressed using "ptm2human", DS-5 or Trace32. + +Following is a DS-5 output of an experimental loop that increments a variable up +to a certain value. The example is simple and yet provides a glimpse of the +wealth of possibilities that coresight provides. + +Info Tracing enabled +Instruction 106378866 0x8026B53C E52DE004 false PUSH {lr} +Instruction 0 0x8026B540 E24DD00C false SUB sp,sp,#0xc +Instruction 0 0x8026B544 E3A03000 false MOV r3,#0 +Instruction 0 0x8026B548 E58D3004 false STR r3,[sp,#4] +Instruction 0 0x8026B54C E59D3004 false LDR r3,[sp,#4] +Instruction 0 0x8026B550 E3530004 false CMP r3,#4 +Instruction 0 0x8026B554 E2833001 false ADD r3,r3,#1 +Instruction 0 0x8026B558 E58D3004 false STR r3,[sp,#4] +Instruction 0 0x8026B55C DAFFFFFA true BLE {pc}-0x10 ; 0x8026b54c +Timestamp Timestamp: 17106715833 +Instruction 319 0x8026B54C E59D3004 false LDR r3,[sp,#4] +Instruction 0 0x8026B550 E3530004 false CMP r3,#4 +Instruction 0 0x8026B554 E2833001 false ADD r3,r3,#1 +Instruction 0 0x8026B558 E58D3004 false STR r3,[sp,#4] +Instruction 0 0x8026B55C DAFFFFFA true BLE {pc}-0x10 ; 0x8026b54c +Instruction 9 0x8026B54C E59D3004 false LDR r3,[sp,#4] +Instruction 0 0x8026B550 E3530004 false CMP r3,#4 +Instruction 0 0x8026B554 E2833001 false ADD r3,r3,#1 +Instruction 0 0x8026B558 E58D3004 false STR r3,[sp,#4] +Instruction 0 0x8026B55C DAFFFFFA true BLE {pc}-0x10 ; 0x8026b54c +Instruction 7 0x8026B54C E59D3004 false LDR r3,[sp,#4] +Instruction 0 0x8026B550 E3530004 false CMP r3,#4 +Instruction 0 0x8026B554 E2833001 false ADD r3,r3,#1 +Instruction 0 0x8026B558 E58D3004 false STR r3,[sp,#4] +Instruction 0 0x8026B55C DAFFFFFA true BLE {pc}-0x10 ; 0x8026b54c +Instruction 7 0x8026B54C E59D3004 false LDR r3,[sp,#4] +Instruction 0 0x8026B550 E3530004 false CMP r3,#4 +Instruction 0 0x8026B554 E2833001 false ADD r3,r3,#1 +Instruction 0 0x8026B558 E58D3004 false STR r3,[sp,#4] +Instruction 0 0x8026B55C DAFFFFFA true BLE {pc}-0x10 ; 0x8026b54c +Instruction 10 0x8026B54C E59D3004 false LDR r3,[sp,#4] +Instruction 0 0x8026B550 E3530004 false CMP r3,#4 +Instruction 0 0x8026B554 E2833001 false ADD r3,r3,#1 +Instruction 0 0x8026B558 E58D3004 false STR r3,[sp,#4] +Instruction 0 0x8026B55C DAFFFFFA true BLE {pc}-0x10 ; 0x8026b54c +Instruction 6 0x8026B560 EE1D3F30 false MRC p15,#0x0,r3,c13,c0,#1 +Instruction 0 0x8026B564 E1A0100D false MOV r1,sp +Instruction 0 0x8026B568 E3C12D7F false BIC r2,r1,#0x1fc0 +Instruction 0 0x8026B56C E3C2203F false BIC r2,r2,#0x3f +Instruction 0 0x8026B570 E59D1004 false LDR r1,[sp,#4] +Instruction 0 0x8026B574 E59F0010 false LDR r0,[pc,#16] ; [0x8026B58C] = 0x80550368 +Instruction 0 0x8026B578 E592200C false LDR r2,[r2,#0xc] +Instruction 0 0x8026B57C E59221D0 false LDR r2,[r2,#0x1d0] +Instruction 0 0x8026B580 EB07A4CF true BL {pc}+0x1e9344 ; 0x804548c4 +Info Tracing enabled +Instruction 13570831 0x8026B584 E28DD00C false ADD sp,sp,#0xc +Instruction 0 0x8026B588 E8BD8000 true LDM sp!,{pc} +Timestamp Timestamp: 17107041535 diff --git a/Documentation/x86/x86_64/mm.txt b/Documentation/x86/x86_64/mm.txt index 052ee643a32e..05712ac83e38 100644 --- a/Documentation/x86/x86_64/mm.txt +++ b/Documentation/x86/x86_64/mm.txt @@ -12,6 +12,8 @@ ffffc90000000000 - ffffe8ffffffffff (=45 bits) vmalloc/ioremap space ffffe90000000000 - ffffe9ffffffffff (=40 bits) hole ffffea0000000000 - ffffeaffffffffff (=40 bits) virtual memory map (1TB) ... unused hole ... +ffffec0000000000 - fffffc0000000000 (=44 bits) kasan shadow memory (16TB) +... unused hole ... ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks ... unused hole ... ffffffff80000000 - ffffffffa0000000 (=512 MB) kernel text mapping, from phys 0 diff --git a/MAINTAINERS b/MAINTAINERS index bcfe54f2aa13..0b00192244bf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -918,6 +918,14 @@ M: Hubert Feurstein <hubert.feurstein@contec.at> S: Maintained F: arch/arm/mach-ep93xx/micro9.c +ARM/CORESIGHT FRAMEWORK AND DRIVERS +M: Mathieu Poirier <mathieu.poirier@linaro.org> +L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) +S: Maintained +F: drivers/hwtracing/coresight/* +F: Documentation/trace/coresight.txt +F: Documentation/devicetree/bindings/arm/coresight.txt + ARM/CORGI MACHINE SUPPORT M: Richard Purdie <rpurdie@rpsys.net> S: Maintained @@ -7133,6 +7141,14 @@ L: linux-pci@vger.kernel.org S: Maintained F: drivers/pci/host/*spear* +PCI MSI DRIVER FOR APPLIEDMICRO XGENE +M: Duc Dang <dhdang@apm.com> +L: linux-pci@vger.kernel.org +L: linux-arm-kernel@lists.infradead.org +S: Maintained +F: Documentation/devicetree/bindings/pci/xgene-pci-msi.txt +F: drivers/pci/host/pci-xgene-msi.c + PCMCIA SUBSYSTEM P: Linux PCMCIA Team L: linux-pcmcia@lists.infradead.org @@ -9314,6 +9330,7 @@ Q: https://patchwork.kernel.org/project/linux-pm/list/ S: Supported F: drivers/thermal/ F: include/linux/thermal.h +F: include/uapi/linux/thermal.h F: include/linux/cpu_cooling.h F: Documentation/devicetree/bindings/thermal/ @@ -422,7 +422,7 @@ export MAKE AWK GENKSYMS INSTALLKERNEL PERL PYTHON UTS_MACHINE export HOSTCXX HOSTCXXFLAGS LDFLAGS_MODULE CHECK CHECKFLAGS export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS LDFLAGS -export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE CFLAGS_GCOV +export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE CFLAGS_GCOV CFLAGS_KASAN export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL @@ -785,6 +785,7 @@ ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC)), y) KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO endif +include $(srctree)/scripts/Makefile.kasan include $(srctree)/scripts/Makefile.extrawarn # Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments diff --git a/arch/alpha/kernel/pci.c b/arch/alpha/kernel/pci.c index 076c35cd6cde..f249000a65ac 100644 --- a/arch/alpha/kernel/pci.c +++ b/arch/alpha/kernel/pci.c @@ -334,6 +334,8 @@ common_init_pci(void) bus = pci_scan_root_bus(NULL, next_busno, alpha_mv.pci_ops, hose, &resources); + if (!bus) + continue; hose->bus = bus; hose->need_domain_info = need_domain_info; next_busno = bus->busn_res.end + 1; @@ -349,6 +351,11 @@ common_init_pci(void) pci_assign_unassigned_resources(); pci_fixup_irqs(alpha_mv.pci_swizzle, alpha_mv.pci_map_irq); + for (hose = hose_head; hose; hose = hose->next) { + bus = hose->bus; + if (bus) + pci_bus_add_devices(bus); + } } diff --git a/arch/alpha/kernel/sys_nautilus.c b/arch/alpha/kernel/sys_nautilus.c index 837c0fa58317..700686d04869 100644 --- a/arch/alpha/kernel/sys_nautilus.c +++ b/arch/alpha/kernel/sys_nautilus.c @@ -207,6 +207,9 @@ nautilus_init_pci(void) /* Scan our single hose. */ bus = pci_scan_bus(0, alpha_mv.pci_ops, hose); + if (!bus) + return; + hose->bus = bus; pcibios_claim_one_bus(bus); @@ -253,6 +256,7 @@ nautilus_init_pci(void) for the root bus, so just clear it. */ bus->self = NULL; pci_fixup_irqs(alpha_mv.pci_swizzle, alpha_mv.pci_map_irq); + pci_bus_add_devices(bus); } /* diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index d8f6a2ec3d4e..21c031fe76d8 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -1299,7 +1299,7 @@ config EARLY_PRINTK config OC_ETM bool "On-chip ETM and ETB" - depends on ARM_AMBA + depends on ARM_AMBA && !CORESIGHT help Enables the on-chip embedded trace macrocell and embedded trace buffer driver that will allow you to collect traces of the @@ -1331,4 +1331,6 @@ config DEBUG_SET_MODULE_RONX against certain classes of kernel exploits. If in doubt, say "N". +source "drivers/hwtracing/coresight/Kconfig" + endmenu diff --git a/arch/arm/Makefile b/arch/arm/Makefile index b5d79884b2af..93a30a285ad2 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -320,8 +320,12 @@ $(INSTALL_TARGETS): $(Q)$(MAKE) $(build)=$(boot)/dts MACHINE=$(MACHINE) $(boot)/dts/$@ PHONY += dtbs dtbs_install -dtbs dtbs_install: prepare scripts - $(Q)$(MAKE) $(build)=$(boot)/dts MACHINE=$(MACHINE) $@ + +dtbs: prepare scripts + $(Q)$(MAKE) $(build)=$(boot)/dts MACHINE=$(MACHINE) + +dtbs_install: + $(Q)$(MAKE) $(dtbinst)=$(boot)/dts MACHINE=$(MACHINE) # We use MRPROPER_FILES and CLEAN_FILES now archclean: diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 38c89cafa1ab..6e784fac5798 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -517,15 +517,7 @@ dtb-$(CONFIG_MACH_DOVE) += dove-cm-a510.dtb \ dove-dove-db.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt6589-aquaris5.dtb -targets += dtbs dtbs_install -targets += $(dtb-y) endif -# *.dtb used to be generated in the directory above. Clean out the -# old build results so people don't accidentally use them. -dtbs: $(addprefix $(obj)/, $(dtb-y)) - $(Q)rm -f $(obj)/../*.dtb - -clean-files := *.dtb - -dtbs_install: $(addsuffix _dtbinst_, $(dtb-y)) +always := $(dtb-y) +clean-files := *.dtb diff --git a/arch/arm/boot/dts/hip04.dtsi b/arch/arm/boot/dts/hip04.dtsi index 93b6c909e991..44044f275115 100644 --- a/arch/arm/boot/dts/hip04.dtsi +++ b/arch/arm/boot/dts/hip04.dtsi @@ -190,6 +190,12 @@ clock-frequency = <168000000>; }; + clk_375m: clk_375m { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <375000000>; + }; + soc { /* It's a 32-bit SoC. */ #address-cells = <1>; @@ -264,4 +270,714 @@ }; }; + + etb@0,e3c42000 { + compatible = "arm,coresight-etb10", "arm,primecell"; + reg = <0 0xe3c42000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + port { + etb0_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&replicator0_out_port0>; + }; + }; + }; + + etb@0,e3c82000 { + compatible = "arm,coresight-etb10", "arm,primecell"; + reg = <0 0xe3c82000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + port { + etb1_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&replicator1_out_port0>; + }; + }; + }; + + etb@0,e3cc2000 { + compatible = "arm,coresight-etb10", "arm,primecell"; + reg = <0 0xe3cc2000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + port { + etb2_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&replicator2_out_port0>; + }; + }; + }; + + etb@0,e3d02000 { + compatible = "arm,coresight-etb10", "arm,primecell"; + reg = <0 0xe3d02000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + port { + etb3_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&replicator3_out_port0>; + }; + }; + }; + + tpiu@0,e3c05000 { + compatible = "arm,coresight-tpiu", "arm,primecell"; + reg = <0 0xe3c05000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + port { + tpiu_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&funnel4_out_port0>; + }; + }; + }; + + replicator0 { + /* non-configurable replicators don't show up on the + * AMBA bus. As such no need to add "arm,primecell". + */ + compatible = "arm,coresight-replicator"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* replicator output ports */ + port@0 { + reg = <0>; + replicator0_out_port0: endpoint { + remote-endpoint = <&etb0_in_port>; + }; + }; + + port@1 { + reg = <1>; + replicator0_out_port1: endpoint { + remote-endpoint = <&funnel4_in_port0>; + }; + }; + + /* replicator input port */ + port@2 { + reg = <0>; + replicator0_in_port0: endpoint { + slave-mode; + remote-endpoint = <&funnel0_out_port0>; + }; + }; + }; + }; + + replicator1 { + /* non-configurable replicators don't show up on the + * AMBA bus. As such no need to add "arm,primecell". + */ + compatible = "arm,coresight-replicator"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* replicator output ports */ + port@0 { + reg = <0>; + replicator1_out_port0: endpoint { + remote-endpoint = <&etb1_in_port>; + }; + }; + + port@1 { + reg = <1>; + replicator1_out_port1: endpoint { + remote-endpoint = <&funnel4_in_port1>; + }; + }; + + /* replicator input port */ + port@2 { + reg = <0>; + replicator1_in_port0: endpoint { + slave-mode; + remote-endpoint = <&funnel1_out_port0>; + }; + }; + }; + }; + + replicator2 { + /* non-configurable replicators don't show up on the + * AMBA bus. As such no need to add "arm,primecell". + */ + compatible = "arm,coresight-replicator"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* replicator output ports */ + port@0 { + reg = <0>; + replicator2_out_port0: endpoint { + remote-endpoint = <&etb2_in_port>; + }; + }; + + port@1 { + reg = <1>; + replicator2_out_port1: endpoint { + remote-endpoint = <&funnel4_in_port2>; + }; + }; + + /* replicator input port */ + port@2 { + reg = <0>; + replicator2_in_port0: endpoint { + slave-mode; + remote-endpoint = <&funnel2_out_port0>; + }; + }; + }; + }; + + replicator3 { + /* non-configurable replicators don't show up on the + * AMBA bus. As such no need to add "arm,primecell". + */ + compatible = "arm,coresight-replicator"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* replicator output ports */ + port@0 { + reg = <0>; + replicator3_out_port0: endpoint { + remote-endpoint = <&etb3_in_port>; + }; + }; + + port@1 { + reg = <1>; + replicator3_out_port1: endpoint { + remote-endpoint = <&funnel4_in_port3>; + }; + }; + + /* replicator input port */ + port@2 { + reg = <0>; + replicator3_in_port0: endpoint { + slave-mode; + remote-endpoint = <&funnel3_out_port0>; + }; + }; + }; + }; + + funnel@0,e3c41000 { + compatible = "arm,coresight-funnel", "arm,primecell"; + reg = <0 0xe3c41000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* funnel output port */ + port@0 { + reg = <0>; + funnel0_out_port0: endpoint { + remote-endpoint = + <&replicator0_in_port0>; + }; + }; + + /* funnel input ports */ + port@1 { + reg = <0>; + funnel0_in_port0: endpoint { + slave-mode; + remote-endpoint = <&ptm0_out_port>; + }; + }; + + port@2 { + reg = <1>; + funnel0_in_port1: endpoint { + slave-mode; + remote-endpoint = <&ptm1_out_port>; + }; + }; + + port@3 { + reg = <2>; + funnel0_in_port2: endpoint { + slave-mode; + remote-endpoint = <&ptm2_out_port>; + }; + }; + + port@4 { + reg = <3>; + funnel0_in_port3: endpoint { + slave-mode; + remote-endpoint = <&ptm3_out_port>; + }; + }; + }; + }; + + funnel@0,e3c81000 { + compatible = "arm,coresight-funnel", "arm,primecell"; + reg = <0 0xe3c81000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* funnel output port */ + port@0 { + reg = <0>; + funnel1_out_port0: endpoint { + remote-endpoint = + <&replicator1_in_port0>; + }; + }; + + /* funnel input ports */ + port@1 { + reg = <0>; + funnel1_in_port0: endpoint { + slave-mode; + remote-endpoint = <&ptm4_out_port>; + }; + }; + + port@2 { + reg = <1>; + funnel1_in_port1: endpoint { + slave-mode; + remote-endpoint = <&ptm5_out_port>; + }; + }; + + port@3 { + reg = <2>; + funnel1_in_port2: endpoint { + slave-mode; + remote-endpoint = <&ptm6_out_port>; + }; + }; + + port@4 { + reg = <3>; + funnel1_in_port3: endpoint { + slave-mode; + remote-endpoint = <&ptm7_out_port>; + }; + }; + }; + }; + + funnel@0,e3cc1000 { + compatible = "arm,coresight-funnel", "arm,primecell"; + reg = <0 0xe3cc1000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* funnel output port */ + port@0 { + reg = <0>; + funnel2_out_port0: endpoint { + remote-endpoint = + <&replicator2_in_port0>; + }; + }; + + /* funnel input ports */ + port@1 { + reg = <0>; + funnel2_in_port0: endpoint { + slave-mode; + remote-endpoint = <&ptm8_out_port>; + }; + }; + + port@2 { + reg = <1>; + funnel2_in_port1: endpoint { + slave-mode; + remote-endpoint = <&ptm9_out_port>; + }; + }; + + port@3 { + reg = <2>; + funnel2_in_port2: endpoint { + slave-mode; + remote-endpoint = <&ptm10_out_port>; + }; + }; + + port@4 { + reg = <3>; + funnel2_in_port3: endpoint { + slave-mode; + remote-endpoint = <&ptm11_out_port>; + }; + }; + }; + }; + + funnel@0,e3d01000 { + compatible = "arm,coresight-funnel", "arm,primecell"; + reg = <0 0xe3d01000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* funnel output port */ + port@0 { + reg = <0>; + funnel3_out_port0: endpoint { + remote-endpoint = + <&replicator3_in_port0>; + }; + }; + + /* funnel input ports */ + port@1 { + reg = <0>; + funnel3_in_port0: endpoint { + slave-mode; + remote-endpoint = <&ptm12_out_port>; + }; + }; + + port@2 { + reg = <1>; + funnel3_in_port1: endpoint { + slave-mode; + remote-endpoint = <&ptm13_out_port>; + }; + }; + + port@3 { + reg = <2>; + funnel3_in_port2: endpoint { + slave-mode; + remote-endpoint = <&ptm14_out_port>; + }; + }; + + port@4 { + reg = <3>; + funnel3_in_port3: endpoint { + slave-mode; + remote-endpoint = <&ptm15_out_port>; + }; + }; + }; + }; + + funnel@0,e3c04000 { + compatible = "arm,coresight-funnel", "arm,primecell"; + reg = <0 0xe3c04000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* funnel output port */ + port@0 { + reg = <0>; + funnel4_out_port0: endpoint { + remote-endpoint = <&tpiu_in_port>; + }; + }; + + /* funnel input ports */ + port@1 { + reg = <0>; + funnel4_in_port0: endpoint { + slave-mode; + remote-endpoint = + <&replicator0_out_port1>; + }; + }; + + port@2 { + reg = <1>; + funnel4_in_port1: endpoint { + slave-mode; + remote-endpoint = + <&replicator1_out_port1>; + }; + }; + + port@3 { + reg = <2>; + funnel4_in_port2: endpoint { + slave-mode; + remote-endpoint = + <&replicator2_out_port1>; + }; + }; + + port@4 { + reg = <3>; + funnel4_in_port3: endpoint { + slave-mode; + remote-endpoint = + <&replicator3_out_port1>; + }; + }; + }; + }; + + ptm@0,e3c7c000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3c7c000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU0>; + port { + ptm0_out_port: endpoint { + remote-endpoint = <&funnel0_in_port0>; + }; + }; + }; + + ptm@0,e3c7d000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3c7d000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU1>; + port { + ptm1_out_port: endpoint { + remote-endpoint = <&funnel0_in_port1>; + }; + }; + }; + + ptm@0,e3c7e000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3c7e000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU2>; + port { + ptm2_out_port: endpoint { + remote-endpoint = <&funnel0_in_port2>; + }; + }; + }; + + ptm@0,e3c7f000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3c7f000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU3>; + port { + ptm3_out_port: endpoint { + remote-endpoint = <&funnel0_in_port3>; + }; + }; + }; + + ptm@0,e3cbc000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3cbc000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU4>; + port { + ptm4_out_port: endpoint { + remote-endpoint = <&funnel1_in_port0>; + }; + }; + }; + + ptm@0,e3cbd000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3cbd000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU5>; + port { + ptm5_out_port: endpoint { + remote-endpoint = <&funnel1_in_port1>; + }; + }; + }; + + ptm@0,e3cbe000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3cbe000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU6>; + port { + ptm6_out_port: endpoint { + remote-endpoint = <&funnel1_in_port2>; + }; + }; + }; + + ptm@0,e3cbf000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3cbf000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU7>; + port { + ptm7_out_port: endpoint { + remote-endpoint = <&funnel1_in_port3>; + }; + }; + }; + + ptm@0,e3cfc000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3cfc000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU8>; + port { + ptm8_out_port: endpoint { + remote-endpoint = <&funnel2_in_port0>; + }; + }; + }; + + ptm@0,e3cfd000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3cfd000 0 0x1000>; + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU9>; + port { + ptm9_out_port: endpoint { + remote-endpoint = <&funnel2_in_port1>; + }; + }; + }; + + ptm@0,e3cfe000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3cfe000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU10>; + port { + ptm10_out_port: endpoint { + remote-endpoint = <&funnel2_in_port2>; + }; + }; + }; + + ptm@0,e3cff000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3cff000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU11>; + port { + ptm11_out_port: endpoint { + remote-endpoint = <&funnel2_in_port3>; + }; + }; + }; + + ptm@0,e3d3c000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3d3c000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU12>; + port { + ptm12_out_port: endpoint { + remote-endpoint = <&funnel3_in_port0>; + }; + }; + }; + + ptm@0,e3d3d000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3d3d000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU13>; + port { + ptm13_out_port: endpoint { + remote-endpoint = <&funnel3_in_port1>; + }; + }; + }; + + ptm@0,e3d3e000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3d3e000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU14>; + port { + ptm14_out_port: endpoint { + remote-endpoint = <&funnel3_in_port2>; + }; + }; + }; + + ptm@0,e3d3f000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0xe3d3f000 0 0x1000>; + + clocks = <&clk_375m>; + clock-names = "apb_pclk"; + cpu = <&CPU15>; + port { + ptm15_out_port: endpoint { + remote-endpoint = <&funnel3_in_port3>; + }; + }; + }; }; diff --git a/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi index d3c0bf5c84e3..b5756c21ea1d 100644 --- a/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi +++ b/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi @@ -282,7 +282,6 @@ }; &ssi1 { - fsl,mode = "i2s-slave"; status = "okay"; }; diff --git a/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi index cade1bdc97e9..86f03c1b147c 100644 --- a/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi +++ b/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi @@ -287,7 +287,6 @@ }; &ssi1 { - fsl,mode = "i2s-slave"; status = "okay"; }; diff --git a/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi index cf13239a1619..4a8d97f47759 100644 --- a/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi +++ b/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi @@ -376,12 +376,10 @@ }; &ssi1 { - fsl,mode = "i2s-slave"; status = "okay"; }; &ssi2 { - fsl,mode = "i2s-slave"; status = "okay"; }; diff --git a/arch/arm/boot/dts/imx6qdl-phytec-pbab01.dtsi b/arch/arm/boot/dts/imx6qdl-phytec-pbab01.dtsi index 584721264121..585b4f6986c1 100644 --- a/arch/arm/boot/dts/imx6qdl-phytec-pbab01.dtsi +++ b/arch/arm/boot/dts/imx6qdl-phytec-pbab01.dtsi @@ -9,17 +9,103 @@ * http://www.gnu.org/copyleft/gpl.html */ +#include <dt-bindings/sound/fsl-imx-audmux.h> + / { chosen { linux,stdout-path = &uart4; }; + + regulators { + sound_1v8: regulator@2 { + compatible = "regulator-fixed"; + reg = <2>; + regulator-name = "i2s-audio-1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + sound_3v3: regulator@3 { + compatible = "regulator-fixed"; + reg = <3>; + regulator-name = "i2s-audio-3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + }; + + tlv320_mclk: oscillator { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <19200000>; + clock-output-names = "tlv320-mclk"; + }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "OnboardTLV320AIC3007"; + simple-audio-card,format = "i2s"; + simple-audio-card,bitclock-master = <&dailink_master>; + simple-audio-card,frame-master = <&dailink_master>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Line", "Line In", + "Line", "Line Out", + "Speaker", "Speaker", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Line Out", "LLOUT", + "Line Out", "RLOUT", + "Speaker", "SPOP", + "Speaker", "SPOM", + "Headphone Jack", "HPLOUT", + "Headphone Jack", "HPROUT", + "MIC3L", "Mic Jack", + "MIC3R", "Mic Jack", + "Mic Jack", "Mic Bias", + "LINE1L", "Line In", + "LINE1R", "Line In"; + + simple-audio-card,cpu { + sound-dai = <&ssi2>; + }; + + dailink_master: simple-audio-card,codec { + sound-dai = <&codec>; + clocks = <&tlv320_mclk>; + }; + }; + }; -&fec { +&audmux { status = "okay"; + + ssi2 { + fsl,audmux-port = <1>; + fsl,port-config = < + (IMX_AUDMUX_V2_PTCR_TFSDIR | + IMX_AUDMUX_V2_PTCR_TFSEL(4) | + IMX_AUDMUX_V2_PTCR_TCLKDIR | + IMX_AUDMUX_V2_PTCR_TCSEL(4)) + IMX_AUDMUX_V2_PDCR_RXDSEL(4) + >; + }; + + pins5 { + fsl,audmux-port = <4>; + fsl,port-config = < + 0x00000000 + IMX_AUDMUX_V2_PDCR_RXDSEL(1) + >; + }; }; -&gpmi { +&can1 { + status = "okay"; +}; + +&fec { status = "okay"; }; @@ -28,14 +114,18 @@ }; &i2c2 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_i2c2>; - clock-frequency = <100000>; status = "okay"; - tlv320@18 { - compatible = "ti,tlv320aic3x"; + codec: tlv320@18 { + compatible = "ti,tlv320aic3007"; + #sound-dai-cells = <0>; reg = <0x18>; + ai3x-micbias-vg = <2>; + + AVDD-supply = <&sound_3v3>; + IOVDD-supply = <&sound_3v3>; + DRVDD-supply = <&sound_3v3>; + DVDD-supply = <&sound_1v8>; }; stmpe@41 { @@ -55,9 +145,14 @@ }; &i2c3 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_i2c3>; - clock-frequency = <100000>; + status = "okay"; +}; + +&pcie { + status = "okay"; +}; + +&ssi2 { status = "okay"; }; @@ -84,19 +179,3 @@ &usdhc3 { status = "okay"; }; - -&iomuxc { - pinctrl_i2c2: i2c2grp { - fsl,pins = < - MX6QDL_PAD_EIM_EB2__I2C2_SCL 0x4001b8b1 - MX6QDL_PAD_EIM_D16__I2C2_SDA 0x4001b8b1 - >; - }; - - pinctrl_i2c3: i2c3grp { - fsl,pins = < - MX6QDL_PAD_EIM_D17__I2C3_SCL 0x4001b8b1 - MX6QDL_PAD_EIM_D18__I2C3_SDA 0x4001b8b1 - >; - }; -}; diff --git a/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi b/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi index 413569752422..1ce6133b67f5 100644 --- a/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi +++ b/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi @@ -60,6 +60,18 @@ }; }; +&audmux { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_audmux>; + status = "disabled"; +}; + +&can1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flexcan1>; + status = "disabled"; +}; + &ecspi3 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ecspi3>; @@ -74,6 +86,22 @@ }; }; +&fec { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_enet>; + phy-mode = "rgmii"; + phy-reset-gpios = <&gpio3 23 GPIO_ACTIVE_LOW>; + phy-supply = <&vdd_eth_io_reg>; + status = "disabled"; +}; + +&gpmi { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpmi_nand>; + nand-on-flash-bbt; + status = "okay"; +}; + &i2c1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; @@ -87,8 +115,8 @@ pmic@58 { compatible = "dlg,da9063"; reg = <0x58>; - interrupt-parent = <&gpio4>; - interrupts = <17 0x8>; /* active-low GPIO4_17 */ + interrupt-parent = <&gpio2>; + interrupts = <9 0x8>; /* active-low GPIO2_9 */ regulators { vddcore_reg: bcore1 { @@ -164,6 +192,18 @@ }; }; +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c2>; + clock-frequency = <100000>; +}; + +&i2c3 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c3>; + clock-frequency = <100000>; +}; + &iomuxc { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_hog>; @@ -173,7 +213,7 @@ fsl,pins = < MX6QDL_PAD_EIM_D23__GPIO3_IO23 0x80000000 MX6QDL_PAD_DISP0_DAT3__GPIO4_IO24 0x80000000 /* SPI NOR chipselect */ - MX6QDL_PAD_DI0_PIN15__GPIO4_IO17 0x80000000 /* PMIC interrupt */ + MX6QDL_PAD_SD4_DAT1__GPIO2_IO09 0x80000000 /* PMIC interrupt */ MX6QDL_PAD_ENET_TXD0__GPIO1_IO30 0x80000000 /* Green LED */ MX6QDL_PAD_EIM_EB3__GPIO2_IO31 0x80000000 /* Red LED */ >; @@ -208,6 +248,13 @@ >; }; + pinctrl_flexcan1: flexcan1grp { + fsl,pins = < + MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX 0x1b0b0 + MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX 0x1b0b0 + >; + }; + pinctrl_gpmi_nand: gpminandgrp { fsl,pins = < MX6QDL_PAD_NANDF_CLE__NAND_CLE 0xb0b1 @@ -237,6 +284,24 @@ >; }; + pinctrl_i2c2: i2c2grp { + fsl,pins = < + MX6QDL_PAD_EIM_EB2__I2C2_SCL 0x4001b8b1 + MX6QDL_PAD_EIM_D16__I2C2_SDA 0x4001b8b1 + >; + }; + + pinctrl_i2c3: i2c3grp { + fsl,pins = < + MX6QDL_PAD_EIM_D17__I2C3_SCL 0x4001b8b1 + MX6QDL_PAD_EIM_D18__I2C3_SDA 0x4001b8b1 + >; + }; + + pinctrl_pcie: pciegrp { + fsl,pins = <MX6QDL_PAD_DI0_PIN15__GPIO4_IO17 0x80000000>; + }; + pinctrl_uart3: uart3grp { fsl,pins = < MX6QDL_PAD_EIM_D24__UART3_TX_DATA 0x1b0b1 @@ -295,22 +360,22 @@ MX6QDL_PAD_ENET_TXD1__GPIO1_IO29 0x80000000 >; }; - }; -}; -&fec { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_enet>; - phy-mode = "rgmii"; - phy-reset-gpios = <&gpio3 23 GPIO_ACTIVE_LOW>; - phy-supply = <&vdd_eth_io_reg>; - status = "disabled"; + pinctrl_audmux: audmuxgrp { + fsl,pins = < + MX6QDL_PAD_DISP0_DAT16__AUD5_TXC 0x130b0 + MX6QDL_PAD_DISP0_DAT17__AUD5_TXD 0x110b0 + MX6QDL_PAD_DISP0_DAT18__AUD5_TXFS 0x130b0 + MX6QDL_PAD_DISP0_DAT19__AUD5_RXD 0x130b0 + >; + }; + }; }; -&gpmi { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_gpmi_nand>; - nand-on-flash-bbt; +&pcie { + pinctrl-name = "default"; + pinctrl-0 = <&pinctrl_pcie>; + reset-gpio = <&gpio4 17 0>; status = "disabled"; }; diff --git a/arch/arm/boot/dts/imx6qdl-rex.dtsi b/arch/arm/boot/dts/imx6qdl-rex.dtsi index 4b5e8c87e53f..394a4ace351a 100644 --- a/arch/arm/boot/dts/imx6qdl-rex.dtsi +++ b/arch/arm/boot/dts/imx6qdl-rex.dtsi @@ -306,7 +306,6 @@ }; &ssi1 { - fsl,mode = "i2s-slave"; status = "okay"; }; diff --git a/arch/arm/boot/dts/imx6qdl-sabresd.dtsi b/arch/arm/boot/dts/imx6qdl-sabresd.dtsi index b57e554dba4e..0565921877ab 100644 --- a/arch/arm/boot/dts/imx6qdl-sabresd.dtsi +++ b/arch/arm/boot/dts/imx6qdl-sabresd.dtsi @@ -109,10 +109,8 @@ "Headphone Jack", "HPOUTR", "Ext Spk", "SPKOUTL", "Ext Spk", "SPKOUTR", - "MICBIAS", "AMIC", - "IN3R", "MICBIAS", - "DMIC", "MICBIAS", - "DMICDAT", "DMIC"; + "AMIC", "MICBIAS", + "IN3R", "AMIC"; mux-int-port = <2>; mux-ext-port = <3>; }; @@ -181,7 +179,7 @@ codec: wm8962@1a { compatible = "wlf,wm8962"; reg = <0x1a>; - clocks = <&clks 201>; + clocks = <&clks IMX6QDL_CLK_CKO>; DCVDD-supply = <®_audio>; DBVDD-supply = <®_audio>; AVDD-supply = <®_audio>; diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi index 9596ed5867e6..39abef10a7bc 100644 --- a/arch/arm/boot/dts/imx6qdl.dtsi +++ b/arch/arm/boot/dts/imx6qdl.dtsi @@ -53,6 +53,7 @@ interrupt-controller; reg = <0x00a01000 0x1000>, <0x00a00100 0x100>; + interrupt-parent = <&intc>; }; clocks { @@ -82,7 +83,7 @@ #address-cells = <1>; #size-cells = <1>; compatible = "simple-bus"; - interrupt-parent = <&intc>; + interrupt-parent = <&gpc>; ranges; dma_apbh: dma-apbh@00110000 { @@ -122,6 +123,7 @@ compatible = "arm,cortex-a9-twd-timer"; reg = <0x00a00600 0x20>; interrupts = <1 13 0xf01>; + interrupt-parent = <&intc>; clocks = <&clks IMX6QDL_CLK_TWD>; }; @@ -151,10 +153,10 @@ interrupt-names = "msi"; #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 2 &intc GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 3 &intc GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 4 &intc GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map = <0 0 0 1 &gpc GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &gpc GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &gpc GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &gpc GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>; clocks = <&clks IMX6QDL_CLK_PCIE_AXI>, <&clks IMX6QDL_CLK_LVDS1_GATE>, <&clks IMX6QDL_CLK_PCIE_REF_125M>; @@ -680,8 +682,11 @@ gpc: gpc@020dc000 { compatible = "fsl,imx6q-gpc"; reg = <0x020dc000 0x4000>; + interrupt-controller; + #interrupt-cells = <3>; interrupts = <0 89 IRQ_TYPE_LEVEL_HIGH>, <0 90 IRQ_TYPE_LEVEL_HIGH>; + interrupt-parent = <&intc>; }; gpr: iomuxc-gpr@020e0000 { diff --git a/arch/arm/boot/dts/imx6sl.dtsi b/arch/arm/boot/dts/imx6sl.dtsi index dfd83e6d8087..960a66930cc5 100644 --- a/arch/arm/boot/dts/imx6sl.dtsi +++ b/arch/arm/boot/dts/imx6sl.dtsi @@ -72,6 +72,7 @@ interrupt-controller; reg = <0x00a01000 0x1000>, <0x00a00100 0x100>; + interrupt-parent = <&intc>; }; clocks { @@ -95,7 +96,7 @@ #address-cells = <1>; #size-cells = <1>; compatible = "simple-bus"; - interrupt-parent = <&intc>; + interrupt-parent = <&gpc>; ranges; ocram: sram@00900000 { @@ -597,7 +598,10 @@ gpc: gpc@020dc000 { compatible = "fsl,imx6sl-gpc", "fsl,imx6q-gpc"; reg = <0x020dc000 0x4000>; + interrupt-controller; + #interrupt-cells = <3>; interrupts = <0 89 IRQ_TYPE_LEVEL_HIGH>; + interrupt-parent = <&intc>; }; gpr: iomuxc-gpr@020e0000 { diff --git a/arch/arm/boot/dts/imx6sx.dtsi b/arch/arm/boot/dts/imx6sx.dtsi index f3e88c03b1e4..22c9fc0fbc9e 100644 --- a/arch/arm/boot/dts/imx6sx.dtsi +++ b/arch/arm/boot/dts/imx6sx.dtsi @@ -88,6 +88,7 @@ interrupt-controller; reg = <0x00a01000 0x1000>, <0x00a00100 0x100>; + interrupt-parent = <&intc>; }; clocks { @@ -131,7 +132,7 @@ #address-cells = <1>; #size-cells = <1>; compatible = "simple-bus"; - interrupt-parent = <&intc>; + interrupt-parent = <&gpc>; ranges; pmu { @@ -694,7 +695,10 @@ gpc: gpc@020dc000 { compatible = "fsl,imx6sx-gpc", "fsl,imx6q-gpc"; reg = <0x020dc000 0x4000>; + interrupt-controller; + #interrupt-cells = <3>; interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>; + interrupt-parent = <&intc>; }; iomuxc: iomuxc@020e0000 { diff --git a/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts b/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts index 322fd1519b09..7a2aeacd62c0 100644 --- a/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts +++ b/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts @@ -358,6 +358,204 @@ }; }; + etb@0,20010000 { + compatible = "arm,coresight-etb10", "arm,primecell"; + reg = <0 0x20010000 0 0x1000>; + + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + etb_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&replicator_out_port0>; + }; + }; + }; + + tpiu@0,20030000 { + compatible = "arm,coresight-tpiu", "arm,primecell"; + reg = <0 0x20030000 0 0x1000>; + + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + tpiu_in_port: endpoint@0 { + slave-mode; + remote-endpoint = <&replicator_out_port1>; + }; + }; + }; + + replicator { + /* non-configurable replicators don't show up on the + * AMBA bus. As such no need to add "arm,primecell". + */ + compatible = "arm,coresight-replicator"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* replicator output ports */ + port@0 { + reg = <0>; + replicator_out_port0: endpoint { + remote-endpoint = <&etb_in_port>; + }; + }; + + port@1 { + reg = <1>; + replicator_out_port1: endpoint { + remote-endpoint = <&tpiu_in_port>; + }; + }; + + /* replicator input port */ + port@2 { + reg = <0>; + replicator_in_port0: endpoint { + slave-mode; + remote-endpoint = <&funnel_out_port0>; + }; + }; + }; + }; + + funnel@0,20040000 { + compatible = "arm,coresight-funnel", "arm,primecell"; + reg = <0 0x20040000 0 0x1000>; + + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* funnel output port */ + port@0 { + reg = <0>; + funnel_out_port0: endpoint { + remote-endpoint = + <&replicator_in_port0>; + }; + }; + + /* funnel input ports */ + port@1 { + reg = <0>; + funnel_in_port0: endpoint { + slave-mode; + remote-endpoint = <&ptm0_out_port>; + }; + }; + + port@2 { + reg = <1>; + funnel_in_port1: endpoint { + slave-mode; + remote-endpoint = <&ptm1_out_port>; + }; + }; + + port@3 { + reg = <2>; + funnel_in_port2: endpoint { + slave-mode; + remote-endpoint = <&etm0_out_port>; + }; + }; + + /* Input port #3 is for ITM, not supported here */ + + port@4 { + reg = <4>; + funnel_in_port4: endpoint { + slave-mode; + remote-endpoint = <&etm1_out_port>; + }; + }; + + port@5 { + reg = <5>; + funnel_in_port5: endpoint { + slave-mode; + remote-endpoint = <&etm2_out_port>; + }; + }; + }; + }; + + ptm@0,2201c000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0x2201c000 0 0x1000>; + + cpu = <&cpu0>; + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + ptm0_out_port: endpoint { + remote-endpoint = <&funnel_in_port0>; + }; + }; + }; + + ptm@0,2201d000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0x2201d000 0 0x1000>; + + cpu = <&cpu1>; + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + ptm1_out_port: endpoint { + remote-endpoint = <&funnel_in_port1>; + }; + }; + }; + + etm@0,2203c000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0x2203c000 0 0x1000>; + + cpu = <&cpu2>; + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + etm0_out_port: endpoint { + remote-endpoint = <&funnel_in_port2>; + }; + }; + }; + + etm@0,2203d000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0x2203d000 0 0x1000>; + + cpu = <&cpu3>; + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + etm1_out_port: endpoint { + remote-endpoint = <&funnel_in_port4>; + }; + }; + }; + + etm@0,2203e000 { + compatible = "arm,coresight-etm3x", "arm,primecell"; + reg = <0 0x2203e000 0 0x1000>; + + cpu = <&cpu4>; + clocks = <&oscclk6a>; + clock-names = "apb_pclk"; + port { + etm2_out_port: endpoint { + remote-endpoint = <&funnel_in_port5>; + }; + }; + }; + smb { compatible = "simple-bus"; diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 260308cd9b83..e7bef0ab9793 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -121,12 +121,14 @@ static inline unsigned long dma_max_pfn(struct device *dev) } #define dma_max_pfn(dev) dma_max_pfn(dev) -static inline int set_arch_dma_coherent_ops(struct device *dev) +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, + u64 size, struct iommu_ops *iommu, + bool coherent) { - set_dma_ops(dev, &arm_coherent_dma_ops); - return 0; + if (coherent) + set_dma_ops(dev, &arm_coherent_dma_ops); } -#define set_arch_dma_coherent_ops(dev) set_arch_dma_coherent_ops(dev) +#define arch_setup_dma_ops arch_setup_dma_ops static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) { diff --git a/arch/arm/include/asm/hardware/cp14.h b/arch/arm/include/asm/hardware/cp14.h new file mode 100644 index 000000000000..61576dc58ede --- /dev/null +++ b/arch/arm/include/asm/hardware/cp14.h @@ -0,0 +1,542 @@ +/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __ASM_HARDWARE_CP14_H +#define __ASM_HARDWARE_CP14_H + +#include <linux/types.h> + +/* Accessors for CP14 registers */ +#define dbg_read(reg) RCP14_##reg() +#define dbg_write(val, reg) WCP14_##reg(val) +#define etm_read(reg) RCP14_##reg() +#define etm_write(val, reg) WCP14_##reg(val) + +/* MRC14 and MCR14 */ +#define MRC14(op1, crn, crm, op2) \ +({ \ +u32 val; \ +asm volatile("mrc p14, "#op1", %0, "#crn", "#crm", "#op2 : "=r" (val)); \ +val; \ +}) + +#define MCR14(val, op1, crn, crm, op2) \ +({ \ +asm volatile("mcr p14, "#op1", %0, "#crn", "#crm", "#op2 : : "r" (val));\ +}) + +/* + * Debug Registers + * + * Available only in DBGv7 + * DBGECR, DBGDSCCR, DBGDSMCR, DBGDRCR + * + * Available only in DBGv7.1 + * DBGBXVRm, DBGOSDLR, DBGDEVID2, DBGDEVID1 + * + * Read only + * DBGDIDR, DBGDSCRint, DBGDTRRXint, DBGDRAR, DBGOSLSR, DBGOSSRR, DBGPRSR, + * DBGPRSR, DBGDSAR, DBGAUTHSTATUS, DBGDEVID2, DBGDEVID1, DBGDEVID + * + * Write only + * DBGDTRTXint, DBGOSLAR + */ +#define RCP14_DBGDIDR() MRC14(0, c0, c0, 0) +#define RCP14_DBGDSCRint() MRC14(0, c0, c1, 0) +#define RCP14_DBGDTRRXint() MRC14(0, c0, c5, 0) +#define RCP14_DBGWFAR() MRC14(0, c0, c6, 0) +#define RCP14_DBGVCR() MRC14(0, c0, c7, 0) +#define RCP14_DBGECR() MRC14(0, c0, c9, 0) +#define RCP14_DBGDSCCR() MRC14(0, c0, c10, 0) +#define RCP14_DBGDSMCR() MRC14(0, c0, c11, 0) +#define RCP14_DBGDTRRXext() MRC14(0, c0, c0, 2) +#define RCP14_DBGDSCRext() MRC14(0, c0, c2, 2) +#define RCP14_DBGDTRTXext() MRC14(0, c0, c3, 2) +#define RCP14_DBGDRCR() MRC14(0, c0, c4, 2) +#define RCP14_DBGBVR0() MRC14(0, c0, c0, 4) +#define RCP14_DBGBVR1() MRC14(0, c0, c1, 4) +#define RCP14_DBGBVR2() MRC14(0, c0, c2, 4) +#define RCP14_DBGBVR3() MRC14(0, c0, c3, 4) +#define RCP14_DBGBVR4() MRC14(0, c0, c4, 4) +#define RCP14_DBGBVR5() MRC14(0, c0, c5, 4) +#define RCP14_DBGBVR6() MRC14(0, c0, c6, 4) +#define RCP14_DBGBVR7() MRC14(0, c0, c7, 4) +#define RCP14_DBGBVR8() MRC14(0, c0, c8, 4) +#define RCP14_DBGBVR9() MRC14(0, c0, c9, 4) +#define RCP14_DBGBVR10() MRC14(0, c0, c10, 4) +#define RCP14_DBGBVR11() MRC14(0, c0, c11, 4) +#define RCP14_DBGBVR12() MRC14(0, c0, c12, 4) +#define RCP14_DBGBVR13() MRC14(0, c0, c13, 4) +#define RCP14_DBGBVR14() MRC14(0, c0, c14, 4) +#define RCP14_DBGBVR15() MRC14(0, c0, c15, 4) +#define RCP14_DBGBCR0() MRC14(0, c0, c0, 5) +#define RCP14_DBGBCR1() MRC14(0, c0, c1, 5) +#define RCP14_DBGBCR2() MRC14(0, c0, c2, 5) +#define RCP14_DBGBCR3() MRC14(0, c0, c3, 5) +#define RCP14_DBGBCR4() MRC14(0, c0, c4, 5) +#define RCP14_DBGBCR5() MRC14(0, c0, c5, 5) +#define RCP14_DBGBCR6() MRC14(0, c0, c6, 5) +#define RCP14_DBGBCR7() MRC14(0, c0, c7, 5) +#define RCP14_DBGBCR8() MRC14(0, c0, c8, 5) +#define RCP14_DBGBCR9() MRC14(0, c0, c9, 5) +#define RCP14_DBGBCR10() MRC14(0, c0, c10, 5) +#define RCP14_DBGBCR11() MRC14(0, c0, c11, 5) +#define RCP14_DBGBCR12() MRC14(0, c0, c12, 5) +#define RCP14_DBGBCR13() MRC14(0, c0, c13, 5) +#define RCP14_DBGBCR14() MRC14(0, c0, c14, 5) +#define RCP14_DBGBCR15() MRC14(0, c0, c15, 5) +#define RCP14_DBGWVR0() MRC14(0, c0, c0, 6) +#define RCP14_DBGWVR1() MRC14(0, c0, c1, 6) +#define RCP14_DBGWVR2() MRC14(0, c0, c2, 6) +#define RCP14_DBGWVR3() MRC14(0, c0, c3, 6) +#define RCP14_DBGWVR4() MRC14(0, c0, c4, 6) +#define RCP14_DBGWVR5() MRC14(0, c0, c5, 6) +#define RCP14_DBGWVR6() MRC14(0, c0, c6, 6) +#define RCP14_DBGWVR7() MRC14(0, c0, c7, 6) +#define RCP14_DBGWVR8() MRC14(0, c0, c8, 6) +#define RCP14_DBGWVR9() MRC14(0, c0, c9, 6) +#define RCP14_DBGWVR10() MRC14(0, c0, c10, 6) +#define RCP14_DBGWVR11() MRC14(0, c0, c11, 6) +#define RCP14_DBGWVR12() MRC14(0, c0, c12, 6) +#define RCP14_DBGWVR13() MRC14(0, c0, c13, 6) +#define RCP14_DBGWVR14() MRC14(0, c0, c14, 6) +#define RCP14_DBGWVR15() MRC14(0, c0, c15, 6) +#define RCP14_DBGWCR0() MRC14(0, c0, c0, 7) +#define RCP14_DBGWCR1() MRC14(0, c0, c1, 7) +#define RCP14_DBGWCR2() MRC14(0, c0, c2, 7) +#define RCP14_DBGWCR3() MRC14(0, c0, c3, 7) +#define RCP14_DBGWCR4() MRC14(0, c0, c4, 7) +#define RCP14_DBGWCR5() MRC14(0, c0, c5, 7) +#define RCP14_DBGWCR6() MRC14(0, c0, c6, 7) +#define RCP14_DBGWCR7() MRC14(0, c0, c7, 7) +#define RCP14_DBGWCR8() MRC14(0, c0, c8, 7) +#define RCP14_DBGWCR9() MRC14(0, c0, c9, 7) +#define RCP14_DBGWCR10() MRC14(0, c0, c10, 7) +#define RCP14_DBGWCR11() MRC14(0, c0, c11, 7) +#define RCP14_DBGWCR12() MRC14(0, c0, c12, 7) +#define RCP14_DBGWCR13() MRC14(0, c0, c13, 7) +#define RCP14_DBGWCR14() MRC14(0, c0, c14, 7) +#define RCP14_DBGWCR15() MRC14(0, c0, c15, 7) +#define RCP14_DBGDRAR() MRC14(0, c1, c0, 0) +#define RCP14_DBGBXVR0() MRC14(0, c1, c0, 1) +#define RCP14_DBGBXVR1() MRC14(0, c1, c1, 1) +#define RCP14_DBGBXVR2() MRC14(0, c1, c2, 1) +#define RCP14_DBGBXVR3() MRC14(0, c1, c3, 1) +#define RCP14_DBGBXVR4() MRC14(0, c1, c4, 1) +#define RCP14_DBGBXVR5() MRC14(0, c1, c5, 1) +#define RCP14_DBGBXVR6() MRC14(0, c1, c6, 1) +#define RCP14_DBGBXVR7() MRC14(0, c1, c7, 1) +#define RCP14_DBGBXVR8() MRC14(0, c1, c8, 1) +#define RCP14_DBGBXVR9() MRC14(0, c1, c9, 1) +#define RCP14_DBGBXVR10() MRC14(0, c1, c10, 1) +#define RCP14_DBGBXVR11() MRC14(0, c1, c11, 1) +#define RCP14_DBGBXVR12() MRC14(0, c1, c12, 1) +#define RCP14_DBGBXVR13() MRC14(0, c1, c13, 1) +#define RCP14_DBGBXVR14() MRC14(0, c1, c14, 1) +#define RCP14_DBGBXVR15() MRC14(0, c1, c15, 1) +#define RCP14_DBGOSLSR() MRC14(0, c1, c1, 4) +#define RCP14_DBGOSSRR() MRC14(0, c1, c2, 4) +#define RCP14_DBGOSDLR() MRC14(0, c1, c3, 4) +#define RCP14_DBGPRCR() MRC14(0, c1, c4, 4) +#define RCP14_DBGPRSR() MRC14(0, c1, c5, 4) +#define RCP14_DBGDSAR() MRC14(0, c2, c0, 0) +#define RCP14_DBGITCTRL() MRC14(0, c7, c0, 4) +#define RCP14_DBGCLAIMSET() MRC14(0, c7, c8, 6) +#define RCP14_DBGCLAIMCLR() MRC14(0, c7, c9, 6) +#define RCP14_DBGAUTHSTATUS() MRC14(0, c7, c14, 6) +#define RCP14_DBGDEVID2() MRC14(0, c7, c0, 7) +#define RCP14_DBGDEVID1() MRC14(0, c7, c1, 7) +#define RCP14_DBGDEVID() MRC14(0, c7, c2, 7) + +#define WCP14_DBGDTRTXint(val) MCR14(val, 0, c0, c5, 0) +#define WCP14_DBGWFAR(val) MCR14(val, 0, c0, c6, 0) +#define WCP14_DBGVCR(val) MCR14(val, 0, c0, c7, 0) +#define WCP14_DBGECR(val) MCR14(val, 0, c0, c9, 0) +#define WCP14_DBGDSCCR(val) MCR14(val, 0, c0, c10, 0) +#define WCP14_DBGDSMCR(val) MCR14(val, 0, c0, c11, 0) +#define WCP14_DBGDTRRXext(val) MCR14(val, 0, c0, c0, 2) +#define WCP14_DBGDSCRext(val) MCR14(val, 0, c0, c2, 2) +#define WCP14_DBGDTRTXext(val) MCR14(val, 0, c0, c3, 2) +#define WCP14_DBGDRCR(val) MCR14(val, 0, c0, c4, 2) +#define WCP14_DBGBVR0(val) MCR14(val, 0, c0, c0, 4) +#define WCP14_DBGBVR1(val) MCR14(val, 0, c0, c1, 4) +#define WCP14_DBGBVR2(val) MCR14(val, 0, c0, c2, 4) +#define WCP14_DBGBVR3(val) MCR14(val, 0, c0, c3, 4) +#define WCP14_DBGBVR4(val) MCR14(val, 0, c0, c4, 4) +#define WCP14_DBGBVR5(val) MCR14(val, 0, c0, c5, 4) +#define WCP14_DBGBVR6(val) MCR14(val, 0, c0, c6, 4) +#define WCP14_DBGBVR7(val) MCR14(val, 0, c0, c7, 4) +#define WCP14_DBGBVR8(val) MCR14(val, 0, c0, c8, 4) +#define WCP14_DBGBVR9(val) MCR14(val, 0, c0, c9, 4) +#define WCP14_DBGBVR10(val) MCR14(val, 0, c0, c10, 4) +#define WCP14_DBGBVR11(val) MCR14(val, 0, c0, c11, 4) +#define WCP14_DBGBVR12(val) MCR14(val, 0, c0, c12, 4) +#define WCP14_DBGBVR13(val) MCR14(val, 0, c0, c13, 4) +#define WCP14_DBGBVR14(val) MCR14(val, 0, c0, c14, 4) +#define WCP14_DBGBVR15(val) MCR14(val, 0, c0, c15, 4) +#define WCP14_DBGBCR0(val) MCR14(val, 0, c0, c0, 5) +#define WCP14_DBGBCR1(val) MCR14(val, 0, c0, c1, 5) +#define WCP14_DBGBCR2(val) MCR14(val, 0, c0, c2, 5) +#define WCP14_DBGBCR3(val) MCR14(val, 0, c0, c3, 5) +#define WCP14_DBGBCR4(val) MCR14(val, 0, c0, c4, 5) +#define WCP14_DBGBCR5(val) MCR14(val, 0, c0, c5, 5) +#define WCP14_DBGBCR6(val) MCR14(val, 0, c0, c6, 5) +#define WCP14_DBGBCR7(val) MCR14(val, 0, c0, c7, 5) +#define WCP14_DBGBCR8(val) MCR14(val, 0, c0, c8, 5) +#define WCP14_DBGBCR9(val) MCR14(val, 0, c0, c9, 5) +#define WCP14_DBGBCR10(val) MCR14(val, 0, c0, c10, 5) +#define WCP14_DBGBCR11(val) MCR14(val, 0, c0, c11, 5) +#define WCP14_DBGBCR12(val) MCR14(val, 0, c0, c12, 5) +#define WCP14_DBGBCR13(val) MCR14(val, 0, c0, c13, 5) +#define WCP14_DBGBCR14(val) MCR14(val, 0, c0, c14, 5) +#define WCP14_DBGBCR15(val) MCR14(val, 0, c0, c15, 5) +#define WCP14_DBGWVR0(val) MCR14(val, 0, c0, c0, 6) +#define WCP14_DBGWVR1(val) MCR14(val, 0, c0, c1, 6) +#define WCP14_DBGWVR2(val) MCR14(val, 0, c0, c2, 6) +#define WCP14_DBGWVR3(val) MCR14(val, 0, c0, c3, 6) +#define WCP14_DBGWVR4(val) MCR14(val, 0, c0, c4, 6) +#define WCP14_DBGWVR5(val) MCR14(val, 0, c0, c5, 6) +#define WCP14_DBGWVR6(val) MCR14(val, 0, c0, c6, 6) +#define WCP14_DBGWVR7(val) MCR14(val, 0, c0, c7, 6) +#define WCP14_DBGWVR8(val) MCR14(val, 0, c0, c8, 6) +#define WCP14_DBGWVR9(val) MCR14(val, 0, c0, c9, 6) +#define WCP14_DBGWVR10(val) MCR14(val, 0, c0, c10, 6) +#define WCP14_DBGWVR11(val) MCR14(val, 0, c0, c11, 6) +#define WCP14_DBGWVR12(val) MCR14(val, 0, c0, c12, 6) +#define WCP14_DBGWVR13(val) MCR14(val, 0, c0, c13, 6) +#define WCP14_DBGWVR14(val) MCR14(val, 0, c0, c14, 6) +#define WCP14_DBGWVR15(val) MCR14(val, 0, c0, c15, 6) +#define WCP14_DBGWCR0(val) MCR14(val, 0, c0, c0, 7) +#define WCP14_DBGWCR1(val) MCR14(val, 0, c0, c1, 7) +#define WCP14_DBGWCR2(val) MCR14(val, 0, c0, c2, 7) +#define WCP14_DBGWCR3(val) MCR14(val, 0, c0, c3, 7) +#define WCP14_DBGWCR4(val) MCR14(val, 0, c0, c4, 7) +#define WCP14_DBGWCR5(val) MCR14(val, 0, c0, c5, 7) +#define WCP14_DBGWCR6(val) MCR14(val, 0, c0, c6, 7) +#define WCP14_DBGWCR7(val) MCR14(val, 0, c0, c7, 7) +#define WCP14_DBGWCR8(val) MCR14(val, 0, c0, c8, 7) +#define WCP14_DBGWCR9(val) MCR14(val, 0, c0, c9, 7) +#define WCP14_DBGWCR10(val) MCR14(val, 0, c0, c10, 7) +#define WCP14_DBGWCR11(val) MCR14(val, 0, c0, c11, 7) +#define WCP14_DBGWCR12(val) MCR14(val, 0, c0, c12, 7) +#define WCP14_DBGWCR13(val) MCR14(val, 0, c0, c13, 7) +#define WCP14_DBGWCR14(val) MCR14(val, 0, c0, c14, 7) +#define WCP14_DBGWCR15(val) MCR14(val, 0, c0, c15, 7) +#define WCP14_DBGBXVR0(val) MCR14(val, 0, c1, c0, 1) +#define WCP14_DBGBXVR1(val) MCR14(val, 0, c1, c1, 1) +#define WCP14_DBGBXVR2(val) MCR14(val, 0, c1, c2, 1) +#define WCP14_DBGBXVR3(val) MCR14(val, 0, c1, c3, 1) +#define WCP14_DBGBXVR4(val) MCR14(val, 0, c1, c4, 1) +#define WCP14_DBGBXVR5(val) MCR14(val, 0, c1, c5, 1) +#define WCP14_DBGBXVR6(val) MCR14(val, 0, c1, c6, 1) +#define WCP14_DBGBXVR7(val) MCR14(val, 0, c1, c7, 1) +#define WCP14_DBGBXVR8(val) MCR14(val, 0, c1, c8, 1) +#define WCP14_DBGBXVR9(val) MCR14(val, 0, c1, c9, 1) +#define WCP14_DBGBXVR10(val) MCR14(val, 0, c1, c10, 1) +#define WCP14_DBGBXVR11(val) MCR14(val, 0, c1, c11, 1) +#define WCP14_DBGBXVR12(val) MCR14(val, 0, c1, c12, 1) +#define WCP14_DBGBXVR13(val) MCR14(val, 0, c1, c13, 1) +#define WCP14_DBGBXVR14(val) MCR14(val, 0, c1, c14, 1) +#define WCP14_DBGBXVR15(val) MCR14(val, 0, c1, c15, 1) +#define WCP14_DBGOSLAR(val) MCR14(val, 0, c1, c0, 4) +#define WCP14_DBGOSSRR(val) MCR14(val, 0, c1, c2, 4) +#define WCP14_DBGOSDLR(val) MCR14(val, 0, c1, c3, 4) +#define WCP14_DBGPRCR(val) MCR14(val, 0, c1, c4, 4) +#define WCP14_DBGITCTRL(val) MCR14(val, 0, c7, c0, 4) +#define WCP14_DBGCLAIMSET(val) MCR14(val, 0, c7, c8, 6) +#define WCP14_DBGCLAIMCLR(val) MCR14(val, 0, c7, c9, 6) + +/* + * ETM Registers + * + * Available only in ETMv3.3, 3.4, 3.5 + * ETMASICCR, ETMTECR2, ETMFFRR, ETMVDEVR, ETMVDCR1, ETMVDCR2, ETMVDCR3, + * ETMDCVRn, ETMDCMRn + * + * Available only in ETMv3.5 as read only + * ETMIDR2 + * + * Available only in ETMv3.5, PFTv1.0, 1.1 + * ETMTSEVR, ETMVMIDCVR, ETMPDCR + * + * Read only + * ETMCCR, ETMSCR, ETMIDR, ETMCCER, ETMOSLSR + * ETMLSR, ETMAUTHSTATUS, ETMDEVID, ETMDEVTYPE, ETMPIDR4, ETMPIDR5, ETMPIDR6, + * ETMPIDR7, ETMPIDR0, ETMPIDR1, ETMPIDR2, ETMPIDR2, ETMPIDR3, ETMCIDR0, + * ETMCIDR1, ETMCIDR2, ETMCIDR3 + * + * Write only + * ETMOSLAR, ETMLAR + * Note: ETMCCER[11] controls WO nature of certain regs. Refer ETM arch spec. + */ +#define RCP14_ETMCR() MRC14(1, c0, c0, 0) +#define RCP14_ETMCCR() MRC14(1, c0, c1, 0) +#define RCP14_ETMTRIGGER() MRC14(1, c0, c2, 0) +#define RCP14_ETMASICCR() MRC14(1, c0, c3, 0) +#define RCP14_ETMSR() MRC14(1, c0, c4, 0) +#define RCP14_ETMSCR() MRC14(1, c0, c5, 0) +#define RCP14_ETMTSSCR() MRC14(1, c0, c6, 0) +#define RCP14_ETMTECR2() MRC14(1, c0, c7, 0) +#define RCP14_ETMTEEVR() MRC14(1, c0, c8, 0) +#define RCP14_ETMTECR1() MRC14(1, c0, c9, 0) +#define RCP14_ETMFFRR() MRC14(1, c0, c10, 0) +#define RCP14_ETMFFLR() MRC14(1, c0, c11, 0) +#define RCP14_ETMVDEVR() MRC14(1, c0, c12, 0) +#define RCP14_ETMVDCR1() MRC14(1, c0, c13, 0) +#define RCP14_ETMVDCR2() MRC14(1, c0, c14, 0) +#define RCP14_ETMVDCR3() MRC14(1, c0, c15, 0) +#define RCP14_ETMACVR0() MRC14(1, c0, c0, 1) +#define RCP14_ETMACVR1() MRC14(1, c0, c1, 1) +#define RCP14_ETMACVR2() MRC14(1, c0, c2, 1) +#define RCP14_ETMACVR3() MRC14(1, c0, c3, 1) +#define RCP14_ETMACVR4() MRC14(1, c0, c4, 1) +#define RCP14_ETMACVR5() MRC14(1, c0, c5, 1) +#define RCP14_ETMACVR6() MRC14(1, c0, c6, 1) +#define RCP14_ETMACVR7() MRC14(1, c0, c7, 1) +#define RCP14_ETMACVR8() MRC14(1, c0, c8, 1) +#define RCP14_ETMACVR9() MRC14(1, c0, c9, 1) +#define RCP14_ETMACVR10() MRC14(1, c0, c10, 1) +#define RCP14_ETMACVR11() MRC14(1, c0, c11, 1) +#define RCP14_ETMACVR12() MRC14(1, c0, c12, 1) +#define RCP14_ETMACVR13() MRC14(1, c0, c13, 1) +#define RCP14_ETMACVR14() MRC14(1, c0, c14, 1) +#define RCP14_ETMACVR15() MRC14(1, c0, c15, 1) +#define RCP14_ETMACTR0() MRC14(1, c0, c0, 2) +#define RCP14_ETMACTR1() MRC14(1, c0, c1, 2) +#define RCP14_ETMACTR2() MRC14(1, c0, c2, 2) +#define RCP14_ETMACTR3() MRC14(1, c0, c3, 2) +#define RCP14_ETMACTR4() MRC14(1, c0, c4, 2) +#define RCP14_ETMACTR5() MRC14(1, c0, c5, 2) +#define RCP14_ETMACTR6() MRC14(1, c0, c6, 2) +#define RCP14_ETMACTR7() MRC14(1, c0, c7, 2) +#define RCP14_ETMACTR8() MRC14(1, c0, c8, 2) +#define RCP14_ETMACTR9() MRC14(1, c0, c9, 2) +#define RCP14_ETMACTR10() MRC14(1, c0, c10, 2) +#define RCP14_ETMACTR11() MRC14(1, c0, c11, 2) +#define RCP14_ETMACTR12() MRC14(1, c0, c12, 2) +#define RCP14_ETMACTR13() MRC14(1, c0, c13, 2) +#define RCP14_ETMACTR14() MRC14(1, c0, c14, 2) +#define RCP14_ETMACTR15() MRC14(1, c0, c15, 2) +#define RCP14_ETMDCVR0() MRC14(1, c0, c0, 3) +#define RCP14_ETMDCVR2() MRC14(1, c0, c2, 3) +#define RCP14_ETMDCVR4() MRC14(1, c0, c4, 3) +#define RCP14_ETMDCVR6() MRC14(1, c0, c6, 3) +#define RCP14_ETMDCVR8() MRC14(1, c0, c8, 3) +#define RCP14_ETMDCVR10() MRC14(1, c0, c10, 3) +#define RCP14_ETMDCVR12() MRC14(1, c0, c12, 3) +#define RCP14_ETMDCVR14() MRC14(1, c0, c14, 3) +#define RCP14_ETMDCMR0() MRC14(1, c0, c0, 4) +#define RCP14_ETMDCMR2() MRC14(1, c0, c2, 4) +#define RCP14_ETMDCMR4() MRC14(1, c0, c4, 4) +#define RCP14_ETMDCMR6() MRC14(1, c0, c6, 4) +#define RCP14_ETMDCMR8() MRC14(1, c0, c8, 4) +#define RCP14_ETMDCMR10() MRC14(1, c0, c10, 4) +#define RCP14_ETMDCMR12() MRC14(1, c0, c12, 4) +#define RCP14_ETMDCMR14() MRC14(1, c0, c14, 4) +#define RCP14_ETMCNTRLDVR0() MRC14(1, c0, c0, 5) +#define RCP14_ETMCNTRLDVR1() MRC14(1, c0, c1, 5) +#define RCP14_ETMCNTRLDVR2() MRC14(1, c0, c2, 5) +#define RCP14_ETMCNTRLDVR3() MRC14(1, c0, c3, 5) +#define RCP14_ETMCNTENR0() MRC14(1, c0, c4, 5) +#define RCP14_ETMCNTENR1() MRC14(1, c0, c5, 5) +#define RCP14_ETMCNTENR2() MRC14(1, c0, c6, 5) +#define RCP14_ETMCNTENR3() MRC14(1, c0, c7, 5) +#define RCP14_ETMCNTRLDEVR0() MRC14(1, c0, c8, 5) +#define RCP14_ETMCNTRLDEVR1() MRC14(1, c0, c9, 5) +#define RCP14_ETMCNTRLDEVR2() MRC14(1, c0, c10, 5) +#define RCP14_ETMCNTRLDEVR3() MRC14(1, c0, c11, 5) +#define RCP14_ETMCNTVR0() MRC14(1, c0, c12, 5) +#define RCP14_ETMCNTVR1() MRC14(1, c0, c13, 5) +#define RCP14_ETMCNTVR2() MRC14(1, c0, c14, 5) +#define RCP14_ETMCNTVR3() MRC14(1, c0, c15, 5) +#define RCP14_ETMSQ12EVR() MRC14(1, c0, c0, 6) +#define RCP14_ETMSQ21EVR() MRC14(1, c0, c1, 6) +#define RCP14_ETMSQ23EVR() MRC14(1, c0, c2, 6) +#define RCP14_ETMSQ31EVR() MRC14(1, c0, c3, 6) +#define RCP14_ETMSQ32EVR() MRC14(1, c0, c4, 6) +#define RCP14_ETMSQ13EVR() MRC14(1, c0, c5, 6) +#define RCP14_ETMSQR() MRC14(1, c0, c7, 6) +#define RCP14_ETMEXTOUTEVR0() MRC14(1, c0, c8, 6) +#define RCP14_ETMEXTOUTEVR1() MRC14(1, c0, c9, 6) +#define RCP14_ETMEXTOUTEVR2() MRC14(1, c0, c10, 6) +#define RCP14_ETMEXTOUTEVR3() MRC14(1, c0, c11, 6) +#define RCP14_ETMCIDCVR0() MRC14(1, c0, c12, 6) +#define RCP14_ETMCIDCVR1() MRC14(1, c0, c13, 6) +#define RCP14_ETMCIDCVR2() MRC14(1, c0, c14, 6) +#define RCP14_ETMCIDCMR() MRC14(1, c0, c15, 6) +#define RCP14_ETMIMPSPEC0() MRC14(1, c0, c0, 7) +#define RCP14_ETMIMPSPEC1() MRC14(1, c0, c1, 7) +#define RCP14_ETMIMPSPEC2() MRC14(1, c0, c2, 7) +#define RCP14_ETMIMPSPEC3() MRC14(1, c0, c3, 7) +#define RCP14_ETMIMPSPEC4() MRC14(1, c0, c4, 7) +#define RCP14_ETMIMPSPEC5() MRC14(1, c0, c5, 7) +#define RCP14_ETMIMPSPEC6() MRC14(1, c0, c6, 7) +#define RCP14_ETMIMPSPEC7() MRC14(1, c0, c7, 7) +#define RCP14_ETMSYNCFR() MRC14(1, c0, c8, 7) +#define RCP14_ETMIDR() MRC14(1, c0, c9, 7) +#define RCP14_ETMCCER() MRC14(1, c0, c10, 7) +#define RCP14_ETMEXTINSELR() MRC14(1, c0, c11, 7) +#define RCP14_ETMTESSEICR() MRC14(1, c0, c12, 7) +#define RCP14_ETMEIBCR() MRC14(1, c0, c13, 7) +#define RCP14_ETMTSEVR() MRC14(1, c0, c14, 7) +#define RCP14_ETMAUXCR() MRC14(1, c0, c15, 7) +#define RCP14_ETMTRACEIDR() MRC14(1, c1, c0, 0) +#define RCP14_ETMIDR2() MRC14(1, c1, c2, 0) +#define RCP14_ETMVMIDCVR() MRC14(1, c1, c0, 1) +#define RCP14_ETMOSLSR() MRC14(1, c1, c1, 4) +/* Not available in PFTv1.1 */ +#define RCP14_ETMOSSRR() MRC14(1, c1, c2, 4) +#define RCP14_ETMPDCR() MRC14(1, c1, c4, 4) +#define RCP14_ETMPDSR() MRC14(1, c1, c5, 4) +#define RCP14_ETMITCTRL() MRC14(1, c7, c0, 4) +#define RCP14_ETMCLAIMSET() MRC14(1, c7, c8, 6) +#define RCP14_ETMCLAIMCLR() MRC14(1, c7, c9, 6) +#define RCP14_ETMLSR() MRC14(1, c7, c13, 6) +#define RCP14_ETMAUTHSTATUS() MRC14(1, c7, c14, 6) +#define RCP14_ETMDEVID() MRC14(1, c7, c2, 7) +#define RCP14_ETMDEVTYPE() MRC14(1, c7, c3, 7) +#define RCP14_ETMPIDR4() MRC14(1, c7, c4, 7) +#define RCP14_ETMPIDR5() MRC14(1, c7, c5, 7) +#define RCP14_ETMPIDR6() MRC14(1, c7, c6, 7) +#define RCP14_ETMPIDR7() MRC14(1, c7, c7, 7) +#define RCP14_ETMPIDR0() MRC14(1, c7, c8, 7) +#define RCP14_ETMPIDR1() MRC14(1, c7, c9, 7) +#define RCP14_ETMPIDR2() MRC14(1, c7, c10, 7) +#define RCP14_ETMPIDR3() MRC14(1, c7, c11, 7) +#define RCP14_ETMCIDR0() MRC14(1, c7, c12, 7) +#define RCP14_ETMCIDR1() MRC14(1, c7, c13, 7) +#define RCP14_ETMCIDR2() MRC14(1, c7, c14, 7) +#define RCP14_ETMCIDR3() MRC14(1, c7, c15, 7) + +#define WCP14_ETMCR(val) MCR14(val, 1, c0, c0, 0) +#define WCP14_ETMTRIGGER(val) MCR14(val, 1, c0, c2, 0) +#define WCP14_ETMASICCR(val) MCR14(val, 1, c0, c3, 0) +#define WCP14_ETMSR(val) MCR14(val, 1, c0, c4, 0) +#define WCP14_ETMTSSCR(val) MCR14(val, 1, c0, c6, 0) +#define WCP14_ETMTECR2(val) MCR14(val, 1, c0, c7, 0) +#define WCP14_ETMTEEVR(val) MCR14(val, 1, c0, c8, 0) +#define WCP14_ETMTECR1(val) MCR14(val, 1, c0, c9, 0) +#define WCP14_ETMFFRR(val) MCR14(val, 1, c0, c10, 0) +#define WCP14_ETMFFLR(val) MCR14(val, 1, c0, c11, 0) +#define WCP14_ETMVDEVR(val) MCR14(val, 1, c0, c12, 0) +#define WCP14_ETMVDCR1(val) MCR14(val, 1, c0, c13, 0) +#define WCP14_ETMVDCR2(val) MCR14(val, 1, c0, c14, 0) +#define WCP14_ETMVDCR3(val) MCR14(val, 1, c0, c15, 0) +#define WCP14_ETMACVR0(val) MCR14(val, 1, c0, c0, 1) +#define WCP14_ETMACVR1(val) MCR14(val, 1, c0, c1, 1) +#define WCP14_ETMACVR2(val) MCR14(val, 1, c0, c2, 1) +#define WCP14_ETMACVR3(val) MCR14(val, 1, c0, c3, 1) +#define WCP14_ETMACVR4(val) MCR14(val, 1, c0, c4, 1) +#define WCP14_ETMACVR5(val) MCR14(val, 1, c0, c5, 1) +#define WCP14_ETMACVR6(val) MCR14(val, 1, c0, c6, 1) +#define WCP14_ETMACVR7(val) MCR14(val, 1, c0, c7, 1) +#define WCP14_ETMACVR8(val) MCR14(val, 1, c0, c8, 1) +#define WCP14_ETMACVR9(val) MCR14(val, 1, c0, c9, 1) +#define WCP14_ETMACVR10(val) MCR14(val, 1, c0, c10, 1) +#define WCP14_ETMACVR11(val) MCR14(val, 1, c0, c11, 1) +#define WCP14_ETMACVR12(val) MCR14(val, 1, c0, c12, 1) +#define WCP14_ETMACVR13(val) MCR14(val, 1, c0, c13, 1) +#define WCP14_ETMACVR14(val) MCR14(val, 1, c0, c14, 1) +#define WCP14_ETMACVR15(val) MCR14(val, 1, c0, c15, 1) +#define WCP14_ETMACTR0(val) MCR14(val, 1, c0, c0, 2) +#define WCP14_ETMACTR1(val) MCR14(val, 1, c0, c1, 2) +#define WCP14_ETMACTR2(val) MCR14(val, 1, c0, c2, 2) +#define WCP14_ETMACTR3(val) MCR14(val, 1, c0, c3, 2) +#define WCP14_ETMACTR4(val) MCR14(val, 1, c0, c4, 2) +#define WCP14_ETMACTR5(val) MCR14(val, 1, c0, c5, 2) +#define WCP14_ETMACTR6(val) MCR14(val, 1, c0, c6, 2) +#define WCP14_ETMACTR7(val) MCR14(val, 1, c0, c7, 2) +#define WCP14_ETMACTR8(val) MCR14(val, 1, c0, c8, 2) +#define WCP14_ETMACTR9(val) MCR14(val, 1, c0, c9, 2) +#define WCP14_ETMACTR10(val) MCR14(val, 1, c0, c10, 2) +#define WCP14_ETMACTR11(val) MCR14(val, 1, c0, c11, 2) +#define WCP14_ETMACTR12(val) MCR14(val, 1, c0, c12, 2) +#define WCP14_ETMACTR13(val) MCR14(val, 1, c0, c13, 2) +#define WCP14_ETMACTR14(val) MCR14(val, 1, c0, c14, 2) +#define WCP14_ETMACTR15(val) MCR14(val, 1, c0, c15, 2) +#define WCP14_ETMDCVR0(val) MCR14(val, 1, c0, c0, 3) +#define WCP14_ETMDCVR2(val) MCR14(val, 1, c0, c2, 3) +#define WCP14_ETMDCVR4(val) MCR14(val, 1, c0, c4, 3) +#define WCP14_ETMDCVR6(val) MCR14(val, 1, c0, c6, 3) +#define WCP14_ETMDCVR8(val) MCR14(val, 1, c0, c8, 3) +#define WCP14_ETMDCVR10(val) MCR14(val, 1, c0, c10, 3) +#define WCP14_ETMDCVR12(val) MCR14(val, 1, c0, c12, 3) +#define WCP14_ETMDCVR14(val) MCR14(val, 1, c0, c14, 3) +#define WCP14_ETMDCMR0(val) MCR14(val, 1, c0, c0, 4) +#define WCP14_ETMDCMR2(val) MCR14(val, 1, c0, c2, 4) +#define WCP14_ETMDCMR4(val) MCR14(val, 1, c0, c4, 4) +#define WCP14_ETMDCMR6(val) MCR14(val, 1, c0, c6, 4) +#define WCP14_ETMDCMR8(val) MCR14(val, 1, c0, c8, 4) +#define WCP14_ETMDCMR10(val) MCR14(val, 1, c0, c10, 4) +#define WCP14_ETMDCMR12(val) MCR14(val, 1, c0, c12, 4) +#define WCP14_ETMDCMR14(val) MCR14(val, 1, c0, c14, 4) +#define WCP14_ETMCNTRLDVR0(val) MCR14(val, 1, c0, c0, 5) +#define WCP14_ETMCNTRLDVR1(val) MCR14(val, 1, c0, c1, 5) +#define WCP14_ETMCNTRLDVR2(val) MCR14(val, 1, c0, c2, 5) +#define WCP14_ETMCNTRLDVR3(val) MCR14(val, 1, c0, c3, 5) +#define WCP14_ETMCNTENR0(val) MCR14(val, 1, c0, c4, 5) +#define WCP14_ETMCNTENR1(val) MCR14(val, 1, c0, c5, 5) +#define WCP14_ETMCNTENR2(val) MCR14(val, 1, c0, c6, 5) +#define WCP14_ETMCNTENR3(val) MCR14(val, 1, c0, c7, 5) +#define WCP14_ETMCNTRLDEVR0(val) MCR14(val, 1, c0, c8, 5) +#define WCP14_ETMCNTRLDEVR1(val) MCR14(val, 1, c0, c9, 5) +#define WCP14_ETMCNTRLDEVR2(val) MCR14(val, 1, c0, c10, 5) +#define WCP14_ETMCNTRLDEVR3(val) MCR14(val, 1, c0, c11, 5) +#define WCP14_ETMCNTVR0(val) MCR14(val, 1, c0, c12, 5) +#define WCP14_ETMCNTVR1(val) MCR14(val, 1, c0, c13, 5) +#define WCP14_ETMCNTVR2(val) MCR14(val, 1, c0, c14, 5) +#define WCP14_ETMCNTVR3(val) MCR14(val, 1, c0, c15, 5) +#define WCP14_ETMSQ12EVR(val) MCR14(val, 1, c0, c0, 6) +#define WCP14_ETMSQ21EVR(val) MCR14(val, 1, c0, c1, 6) +#define WCP14_ETMSQ23EVR(val) MCR14(val, 1, c0, c2, 6) +#define WCP14_ETMSQ31EVR(val) MCR14(val, 1, c0, c3, 6) +#define WCP14_ETMSQ32EVR(val) MCR14(val, 1, c0, c4, 6) +#define WCP14_ETMSQ13EVR(val) MCR14(val, 1, c0, c5, 6) +#define WCP14_ETMSQR(val) MCR14(val, 1, c0, c7, 6) +#define WCP14_ETMEXTOUTEVR0(val) MCR14(val, 1, c0, c8, 6) +#define WCP14_ETMEXTOUTEVR1(val) MCR14(val, 1, c0, c9, 6) +#define WCP14_ETMEXTOUTEVR2(val) MCR14(val, 1, c0, c10, 6) +#define WCP14_ETMEXTOUTEVR3(val) MCR14(val, 1, c0, c11, 6) +#define WCP14_ETMCIDCVR0(val) MCR14(val, 1, c0, c12, 6) +#define WCP14_ETMCIDCVR1(val) MCR14(val, 1, c0, c13, 6) +#define WCP14_ETMCIDCVR2(val) MCR14(val, 1, c0, c14, 6) +#define WCP14_ETMCIDCMR(val) MCR14(val, 1, c0, c15, 6) +#define WCP14_ETMIMPSPEC0(val) MCR14(val, 1, c0, c0, 7) +#define WCP14_ETMIMPSPEC1(val) MCR14(val, 1, c0, c1, 7) +#define WCP14_ETMIMPSPEC2(val) MCR14(val, 1, c0, c2, 7) +#define WCP14_ETMIMPSPEC3(val) MCR14(val, 1, c0, c3, 7) +#define WCP14_ETMIMPSPEC4(val) MCR14(val, 1, c0, c4, 7) +#define WCP14_ETMIMPSPEC5(val) MCR14(val, 1, c0, c5, 7) +#define WCP14_ETMIMPSPEC6(val) MCR14(val, 1, c0, c6, 7) +#define WCP14_ETMIMPSPEC7(val) MCR14(val, 1, c0, c7, 7) +/* Can be read only in ETMv3.4, ETMv3.5 */ +#define WCP14_ETMSYNCFR(val) MCR14(val, 1, c0, c8, 7) +#define WCP14_ETMEXTINSELR(val) MCR14(val, 1, c0, c11, 7) +#define WCP14_ETMTESSEICR(val) MCR14(val, 1, c0, c12, 7) +#define WCP14_ETMEIBCR(val) MCR14(val, 1, c0, c13, 7) +#define WCP14_ETMTSEVR(val) MCR14(val, 1, c0, c14, 7) +#define WCP14_ETMAUXCR(val) MCR14(val, 1, c0, c15, 7) +#define WCP14_ETMTRACEIDR(val) MCR14(val, 1, c1, c0, 0) +#define WCP14_ETMIDR2(val) MCR14(val, 1, c1, c2, 0) +#define WCP14_ETMVMIDCVR(val) MCR14(val, 1, c1, c0, 1) +#define WCP14_ETMOSLAR(val) MCR14(val, 1, c1, c0, 4) +/* Not available in PFTv1.1 */ +#define WCP14_ETMOSSRR(val) MCR14(val, 1, c1, c2, 4) +#define WCP14_ETMPDCR(val) MCR14(val, 1, c1, c4, 4) +#define WCP14_ETMPDSR(val) MCR14(val, 1, c1, c5, 4) +#define WCP14_ETMITCTRL(val) MCR14(val, 1, c7, c0, 4) +#define WCP14_ETMCLAIMSET(val) MCR14(val, 1, c7, c8, 6) +#define WCP14_ETMCLAIMCLR(val) MCR14(val, 1, c7, c9, 6) +/* Writes to this from CP14 interface are ignored */ +#define WCP14_ETMLAR(val) MCR14(val, 1, c7, c12, 6) + +#endif diff --git a/arch/arm/include/asm/mach/pci.h b/arch/arm/include/asm/mach/pci.h index 7fc42784becb..8292b5f81e23 100644 --- a/arch/arm/include/asm/mach/pci.h +++ b/arch/arm/include/asm/mach/pci.h @@ -22,6 +22,9 @@ struct hw_pci { #ifdef CONFIG_PCI_DOMAINS int domain; #endif +#ifdef CONFIG_PCI_MSI + struct msi_controller *msi_ctrl; +#endif struct pci_ops *ops; int nr_controllers; void **private_data; @@ -36,8 +39,6 @@ struct hw_pci { resource_size_t start, resource_size_t size, resource_size_t align); - void (*add_bus)(struct pci_bus *bus); - void (*remove_bus)(struct pci_bus *bus); }; /* @@ -47,6 +48,9 @@ struct pci_sys_data { #ifdef CONFIG_PCI_DOMAINS int domain; #endif +#ifdef CONFIG_PCI_MSI + struct msi_controller *msi_ctrl; +#endif struct list_head node; int busnr; /* primary bus number */ u64 mem_offset; /* bus->cpu memory mapping offset */ @@ -65,8 +69,6 @@ struct pci_sys_data { resource_size_t start, resource_size_t size, resource_size_t align); - void (*add_bus)(struct pci_bus *bus); - void (*remove_bus)(struct pci_bus *bus); void *private_data; /* platform controller private data */ }; diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c index 17a26c17f7f5..daaff73bc776 100644 --- a/arch/arm/kernel/bios32.c +++ b/arch/arm/kernel/bios32.c @@ -18,6 +18,15 @@ static int debug_pci; +#ifdef CONFIG_PCI_MSI +struct msi_controller *pcibios_msi_controller(struct pci_dev *dev) +{ + struct pci_sys_data *sysdata = dev->bus->sysdata; + + return sysdata->msi_ctrl; +} +#endif + /* * We can't use pci_get_device() here since we are * called from interrupt context. @@ -360,20 +369,6 @@ void pcibios_fixup_bus(struct pci_bus *bus) } EXPORT_SYMBOL(pcibios_fixup_bus); -void pcibios_add_bus(struct pci_bus *bus) -{ - struct pci_sys_data *sys = bus->sysdata; - if (sys->add_bus) - sys->add_bus(bus); -} - -void pcibios_remove_bus(struct pci_bus *bus) -{ - struct pci_sys_data *sys = bus->sysdata; - if (sys->remove_bus) - sys->remove_bus(bus); -} - /* * Swizzle the device pin each time we cross a bridge. If a platform does * not provide a swizzle function, we perform the standard PCI swizzling. @@ -471,12 +466,13 @@ static void pcibios_init_hw(struct device *parent, struct hw_pci *hw, #ifdef CONFIG_PCI_DOMAINS sys->domain = hw->domain; #endif +#ifdef CONFIG_PCI_MSI + sys->msi_ctrl = hw->msi_ctrl; +#endif sys->busnr = busnr; sys->swizzle = hw->swizzle; sys->map_irq = hw->map_irq; sys->align_resource = hw->align_resource; - sys->add_bus = hw->add_bus; - sys->remove_bus = hw->remove_bus; INIT_LIST_HEAD(&sys->resources); if (hw->private_data) diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index 6a4dffefd357..a7a3b154cd3d 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -41,7 +41,7 @@ void *module_alloc(unsigned long size) { return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE, + GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, __builtin_return_address(0)); } #endif diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig index 11b2957f792b..5d4017351b7b 100644 --- a/arch/arm/mach-imx/Kconfig +++ b/arch/arm/mach-imx/Kconfig @@ -50,6 +50,7 @@ config HAVE_IMX_ANATOP config HAVE_IMX_GPC bool + select PM_GENERIC_DOMAINS if PM config HAVE_IMX_MMDC bool diff --git a/arch/arm/mach-imx/Makefile b/arch/arm/mach-imx/Makefile index 6e4fcd8339cd..e5fa5eeeea52 100644 --- a/arch/arm/mach-imx/Makefile +++ b/arch/arm/mach-imx/Makefile @@ -32,8 +32,7 @@ ifeq ($(CONFIG_CPU_IDLE),y) obj-$(CONFIG_SOC_IMX5) += cpuidle-imx5.o obj-$(CONFIG_SOC_IMX6Q) += cpuidle-imx6q.o obj-$(CONFIG_SOC_IMX6SL) += cpuidle-imx6sl.o -# i.MX6SX reuses i.MX6Q cpuidle driver -obj-$(CONFIG_SOC_IMX6SX) += cpuidle-imx6q.o +obj-$(CONFIG_SOC_IMX6SX) += cpuidle-imx6sx.o endif ifdef CONFIG_SND_IMX_SOC diff --git a/arch/arm/mach-imx/clk-pllv3.c b/arch/arm/mach-imx/clk-pllv3.c index 57de74da0acf..0ad6e5442fd8 100644 --- a/arch/arm/mach-imx/clk-pllv3.c +++ b/arch/arm/mach-imx/clk-pllv3.c @@ -69,7 +69,6 @@ static int clk_pllv3_prepare(struct clk_hw *hw) { struct clk_pllv3 *pll = to_clk_pllv3(hw); u32 val; - int ret; val = readl_relaxed(pll->base); if (pll->powerup_set) @@ -78,11 +77,7 @@ static int clk_pllv3_prepare(struct clk_hw *hw) val &= ~BM_PLL_POWER; writel_relaxed(val, pll->base); - ret = clk_pllv3_wait_lock(pll); - if (ret) - return ret; - - return 0; + return clk_pllv3_wait_lock(pll); } static void clk_pllv3_unprepare(struct clk_hw *hw) diff --git a/arch/arm/mach-imx/common.h b/arch/arm/mach-imx/common.h index 1dabf435c592..f71339522156 100644 --- a/arch/arm/mach-imx/common.h +++ b/arch/arm/mach-imx/common.h @@ -61,7 +61,6 @@ struct platform_device *mxc_register_gpio(char *name, int id, void mxc_set_cpu_type(unsigned int type); void mxc_restart(enum reboot_mode, const char *); void mxc_arch_reset_init(void __iomem *); -void mxc_arch_reset_init_dt(void); int mx51_revision(void); int mx53_revision(void); void imx_set_aips(void __iomem *); @@ -71,6 +70,11 @@ void imx_set_soc_revision(unsigned int rev); unsigned int imx_get_soc_revision(void); void imx_init_revision_from_anatop(void); struct device *imx_soc_device_init(void); +void imx6_enable_rbc(bool enable); +void imx_gpc_check_dt(void); +void imx_gpc_set_arm_power_in_lpm(bool power_off); +void imx_gpc_set_arm_power_up_timing(u32 sw2iso, u32 sw); +void imx_gpc_set_arm_power_down_timing(u32 sw2iso, u32 sw); enum mxc_cpu_pwr_mode { WAIT_CLOCKED, /* wfi only */ @@ -103,13 +107,12 @@ static inline void imx_scu_map_io(void) {} static inline void imx_smp_prepare(void) {} #endif void imx_src_init(void); -void imx_gpc_init(void); void imx_gpc_pre_suspend(bool arm_power_off); void imx_gpc_post_resume(void); void imx_gpc_mask_all(void); void imx_gpc_restore_all(void); -void imx_gpc_irq_mask(struct irq_data *d); -void imx_gpc_irq_unmask(struct irq_data *d); +void imx_gpc_hwirq_mask(unsigned int hwirq); +void imx_gpc_hwirq_unmask(unsigned int hwirq); void imx_anatop_init(void); void imx_anatop_pre_suspend(void); void imx_anatop_post_resume(void); diff --git a/arch/arm/mach-imx/cpuidle-imx6sx.c b/arch/arm/mach-imx/cpuidle-imx6sx.c new file mode 100644 index 000000000000..d8a9f219e028 --- /dev/null +++ b/arch/arm/mach-imx/cpuidle-imx6sx.c @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2014 Freescale Semiconductor, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/cpuidle.h> +#include <linux/cpu_pm.h> +#include <linux/module.h> +#include <asm/cpuidle.h> +#include <asm/proc-fns.h> +#include <asm/suspend.h> + +#include "common.h" +#include "cpuidle.h" + +static int imx6sx_idle_finish(unsigned long val) +{ + cpu_do_idle(); + + return 0; +} + +static int imx6sx_enter_wait(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) +{ + imx6q_set_lpm(WAIT_UNCLOCKED); + + switch (index) { + case 1: + cpu_do_idle(); + break; + case 2: + imx6_enable_rbc(true); + imx_gpc_set_arm_power_in_lpm(true); + imx_set_cpu_jump(0, v7_cpu_resume); + /* Need to notify there is a cpu pm operation. */ + cpu_pm_enter(); + cpu_cluster_pm_enter(); + + cpu_suspend(0, imx6sx_idle_finish); + + cpu_cluster_pm_exit(); + cpu_pm_exit(); + imx_gpc_set_arm_power_in_lpm(false); + imx6_enable_rbc(false); + break; + default: + break; + } + + imx6q_set_lpm(WAIT_CLOCKED); + + return index; +} + +static struct cpuidle_driver imx6sx_cpuidle_driver = { + .name = "imx6sx_cpuidle", + .owner = THIS_MODULE, + .states = { + /* WFI */ + ARM_CPUIDLE_WFI_STATE, + /* WAIT */ + { + .exit_latency = 50, + .target_residency = 75, + .flags = CPUIDLE_FLAG_TIME_VALID | + CPUIDLE_FLAG_TIMER_STOP, + .enter = imx6sx_enter_wait, + .name = "WAIT", + .desc = "Clock off", + }, + /* WAIT + ARM power off */ + { + /* + * ARM gating 31us * 5 + RBC clear 65us + * and some margin for SW execution, here set it + * to 300us. + */ + .exit_latency = 300, + .target_residency = 500, + .flags = CPUIDLE_FLAG_TIME_VALID, + .enter = imx6sx_enter_wait, + .name = "LOW-POWER-IDLE", + .desc = "ARM power off", + }, + }, + .state_count = 3, + .safe_state_index = 0, +}; + +int __init imx6sx_cpuidle_init(void) +{ + imx6_enable_rbc(false); + /* + * set ARM power up/down timing to the fastest, + * sw2iso and sw can be set to one 32K cycle = 31us + * except for power up sw2iso which need to be + * larger than LDO ramp up time. + */ + imx_gpc_set_arm_power_up_timing(2, 1); + imx_gpc_set_arm_power_down_timing(1, 1); + + return cpuidle_register(&imx6sx_cpuidle_driver, NULL); +} diff --git a/arch/arm/mach-imx/cpuidle.h b/arch/arm/mach-imx/cpuidle.h index 24e33670417c..f9140128ba05 100644 --- a/arch/arm/mach-imx/cpuidle.h +++ b/arch/arm/mach-imx/cpuidle.h @@ -14,6 +14,7 @@ extern int imx5_cpuidle_init(void); extern int imx6q_cpuidle_init(void); extern int imx6sl_cpuidle_init(void); +extern int imx6sx_cpuidle_init(void); #else static inline int imx5_cpuidle_init(void) { @@ -27,4 +28,8 @@ static inline int imx6sl_cpuidle_init(void) { return 0; } +static inline int imx6sx_cpuidle_init(void) +{ + return 0; +} #endif diff --git a/arch/arm/mach-imx/gpc.c b/arch/arm/mach-imx/gpc.c index 82ea74e68482..bbf015056221 100644 --- a/arch/arm/mach-imx/gpc.c +++ b/arch/arm/mach-imx/gpc.c @@ -10,23 +10,67 @@ * http://www.gnu.org/copyleft/gpl.html */ +#include <linux/clk.h> +#include <linux/delay.h> #include <linux/io.h> #include <linux/irq.h> #include <linux/of.h> #include <linux/of_address.h> #include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/regulator/consumer.h> #include <linux/irqchip/arm-gic.h> #include "common.h" +#include "hardware.h" +#define GPC_CNTR 0x000 #define GPC_IMR1 0x008 +#define GPC_PGC_GPU_PDN 0x260 +#define GPC_PGC_GPU_PUPSCR 0x264 +#define GPC_PGC_GPU_PDNSCR 0x268 #define GPC_PGC_CPU_PDN 0x2a0 +#define GPC_PGC_CPU_PUPSCR 0x2a4 +#define GPC_PGC_CPU_PDNSCR 0x2a8 +#define GPC_PGC_SW2ISO_SHIFT 0x8 +#define GPC_PGC_SW_SHIFT 0x0 #define IMR_NUM 4 +#define GPC_MAX_IRQS (IMR_NUM * 32) + +#define GPU_VPU_PUP_REQ BIT(1) +#define GPU_VPU_PDN_REQ BIT(0) + +#define GPC_CLK_MAX 6 + +struct pu_domain { + struct generic_pm_domain base; + struct regulator *reg; + struct clk *clk[GPC_CLK_MAX]; + int num_clks; +}; static void __iomem *gpc_base; static u32 gpc_wake_irqs[IMR_NUM]; static u32 gpc_saved_imrs[IMR_NUM]; +void imx_gpc_set_arm_power_up_timing(u32 sw2iso, u32 sw) +{ + writel_relaxed((sw2iso << GPC_PGC_SW2ISO_SHIFT) | + (sw << GPC_PGC_SW_SHIFT), gpc_base + GPC_PGC_CPU_PUPSCR); +} + +void imx_gpc_set_arm_power_down_timing(u32 sw2iso, u32 sw) +{ + writel_relaxed((sw2iso << GPC_PGC_SW2ISO_SHIFT) | + (sw << GPC_PGC_SW_SHIFT), gpc_base + GPC_PGC_CPU_PDNSCR); +} + +void imx_gpc_set_arm_power_in_lpm(bool power_off) +{ + writel_relaxed(power_off, gpc_base + GPC_PGC_CPU_PDN); +} + void imx_gpc_pre_suspend(bool arm_power_off) { void __iomem *reg_imr1 = gpc_base + GPC_IMR1; @@ -34,7 +78,7 @@ void imx_gpc_pre_suspend(bool arm_power_off) /* Tell GPC to power off ARM core when suspend */ if (arm_power_off) - writel_relaxed(0x1, gpc_base + GPC_PGC_CPU_PDN); + imx_gpc_set_arm_power_in_lpm(arm_power_off); for (i = 0; i < IMR_NUM; i++) { gpc_saved_imrs[i] = readl_relaxed(reg_imr1 + i * 4); @@ -48,7 +92,7 @@ void imx_gpc_post_resume(void) int i; /* Keep ARM core powered on for other low-power modes */ - writel_relaxed(0x0, gpc_base + GPC_PGC_CPU_PDN); + imx_gpc_set_arm_power_in_lpm(false); for (i = 0; i < IMR_NUM; i++) writel_relaxed(gpc_saved_imrs[i], reg_imr1 + i * 4); @@ -56,17 +100,17 @@ void imx_gpc_post_resume(void) static int imx_gpc_irq_set_wake(struct irq_data *d, unsigned int on) { - unsigned int idx = d->irq / 32 - 1; + unsigned int idx = d->hwirq / 32; u32 mask; - /* Sanity check for SPI irq */ - if (d->irq < 32) - return -EINVAL; - - mask = 1 << d->irq % 32; + mask = 1 << d->hwirq % 32; gpc_wake_irqs[idx] = on ? gpc_wake_irqs[idx] | mask : gpc_wake_irqs[idx] & ~mask; + /* + * Do *not* call into the parent, as the GIC doesn't have any + * wake-up facility... + */ return 0; } @@ -91,51 +135,349 @@ void imx_gpc_restore_all(void) writel_relaxed(gpc_saved_imrs[i], reg_imr1 + i * 4); } -void imx_gpc_irq_unmask(struct irq_data *d) +void imx_gpc_hwirq_unmask(unsigned int hwirq) { void __iomem *reg; u32 val; - /* Sanity check for SPI irq */ - if (d->irq < 32) - return; - - reg = gpc_base + GPC_IMR1 + (d->irq / 32 - 1) * 4; + reg = gpc_base + GPC_IMR1 + hwirq / 32 * 4; val = readl_relaxed(reg); - val &= ~(1 << d->irq % 32); + val &= ~(1 << hwirq % 32); writel_relaxed(val, reg); } -void imx_gpc_irq_mask(struct irq_data *d) +void imx_gpc_hwirq_mask(unsigned int hwirq) { void __iomem *reg; u32 val; - /* Sanity check for SPI irq */ - if (d->irq < 32) - return; - - reg = gpc_base + GPC_IMR1 + (d->irq / 32 - 1) * 4; + reg = gpc_base + GPC_IMR1 + hwirq / 32 * 4; val = readl_relaxed(reg); - val |= 1 << (d->irq % 32); + val |= 1 << (hwirq % 32); writel_relaxed(val, reg); } -void __init imx_gpc_init(void) +static void imx_gpc_irq_unmask(struct irq_data *d) { - struct device_node *np; + imx_gpc_hwirq_unmask(d->hwirq); + irq_chip_unmask_parent(d); +} + +static void imx_gpc_irq_mask(struct irq_data *d) +{ + imx_gpc_hwirq_mask(d->hwirq); + irq_chip_mask_parent(d); +} + +static struct irq_chip imx_gpc_chip = { + .name = "GPC", + .irq_eoi = irq_chip_eoi_parent, + .irq_mask = imx_gpc_irq_mask, + .irq_unmask = imx_gpc_irq_unmask, + .irq_retrigger = irq_chip_retrigger_hierarchy, + .irq_set_wake = imx_gpc_irq_set_wake, +#ifdef CONFIG_SMP + .irq_set_affinity = irq_chip_set_affinity_parent, +#endif +}; + +static int imx_gpc_domain_xlate(struct irq_domain *domain, + struct device_node *controller, + const u32 *intspec, + unsigned int intsize, + unsigned long *out_hwirq, + unsigned int *out_type) +{ + if (domain->of_node != controller) + return -EINVAL; /* Shouldn't happen, really... */ + if (intsize != 3) + return -EINVAL; /* Not GIC compliant */ + if (intspec[0] != 0) + return -EINVAL; /* No PPI should point to this domain */ + + *out_hwirq = intspec[1]; + *out_type = intspec[2]; + return 0; +} + +static int imx_gpc_domain_alloc(struct irq_domain *domain, + unsigned int irq, + unsigned int nr_irqs, void *data) +{ + struct of_phandle_args *args = data; + struct of_phandle_args parent_args; + irq_hw_number_t hwirq; int i; - np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-gpc"); - gpc_base = of_iomap(np, 0); - WARN_ON(!gpc_base); + if (args->args_count != 3) + return -EINVAL; /* Not GIC compliant */ + if (args->args[0] != 0) + return -EINVAL; /* No PPI should point to this domain */ + + hwirq = args->args[1]; + if (hwirq >= GPC_MAX_IRQS) + return -EINVAL; /* Can't deal with this */ + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_hwirq_and_chip(domain, irq + i, hwirq + i, + &imx_gpc_chip, NULL); + + parent_args = *args; + parent_args.np = domain->parent->of_node; + return irq_domain_alloc_irqs_parent(domain, irq, nr_irqs, &parent_args); +} + +static struct irq_domain_ops imx_gpc_domain_ops = { + .xlate = imx_gpc_domain_xlate, + .alloc = imx_gpc_domain_alloc, + .free = irq_domain_free_irqs_common, +}; + +static int __init imx_gpc_init(struct device_node *node, + struct device_node *parent) +{ + struct irq_domain *parent_domain, *domain; + int i; + + if (!parent) { + pr_err("%s: no parent, giving up\n", node->full_name); + return -ENODEV; + } + + parent_domain = irq_find_host(parent); + if (!parent_domain) { + pr_err("%s: unable to obtain parent domain\n", node->full_name); + return -ENXIO; + } + + gpc_base = of_iomap(node, 0); + if (WARN_ON(!gpc_base)) + return -ENOMEM; + + domain = irq_domain_add_hierarchy(parent_domain, 0, GPC_MAX_IRQS, + node, &imx_gpc_domain_ops, + NULL); + if (!domain) { + iounmap(gpc_base); + return -ENOMEM; + } /* Initially mask all interrupts */ for (i = 0; i < IMR_NUM; i++) writel_relaxed(~0, gpc_base + GPC_IMR1 + i * 4); - /* Register GPC as the secondary interrupt controller behind GIC */ - gic_arch_extn.irq_mask = imx_gpc_irq_mask; - gic_arch_extn.irq_unmask = imx_gpc_irq_unmask; - gic_arch_extn.irq_set_wake = imx_gpc_irq_set_wake; + return 0; +} + +/* + * We cannot use the IRQCHIP_DECLARE macro that lives in + * drivers/irqchip, so we're forced to roll our own. Not very nice. + */ +OF_DECLARE_2(irqchip, imx_gpc, "fsl,imx6q-gpc", imx_gpc_init); + +void __init imx_gpc_check_dt(void) +{ + struct device_node *np; + + np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-gpc"); + if (WARN_ON(!np)) + return; + + if (WARN_ON(!of_find_property(np, "interrupt-controller", NULL))) { + pr_warn("Outdated DT detected, suspend/resume will NOT work\n"); + + /* map GPC, so that at least CPUidle and WARs keep working */ + gpc_base = of_iomap(np, 0); + } +} + +#ifdef CONFIG_PM_GENERIC_DOMAINS + +static void _imx6q_pm_pu_power_off(struct generic_pm_domain *genpd) +{ + int iso, iso2sw; + u32 val; + + /* Read ISO and ISO2SW power down delays */ + val = readl_relaxed(gpc_base + GPC_PGC_GPU_PDNSCR); + iso = val & 0x3f; + iso2sw = (val >> 8) & 0x3f; + + /* Gate off PU domain when GPU/VPU when powered down */ + writel_relaxed(0x1, gpc_base + GPC_PGC_GPU_PDN); + + /* Request GPC to power down GPU/VPU */ + val = readl_relaxed(gpc_base + GPC_CNTR); + val |= GPU_VPU_PDN_REQ; + writel_relaxed(val, gpc_base + GPC_CNTR); + + /* Wait ISO + ISO2SW IPG clock cycles */ + ndelay((iso + iso2sw) * 1000 / 66); +} + +static int imx6q_pm_pu_power_off(struct generic_pm_domain *genpd) +{ + struct pu_domain *pu = container_of(genpd, struct pu_domain, base); + + _imx6q_pm_pu_power_off(genpd); + + if (pu->reg) + regulator_disable(pu->reg); + + return 0; +} + +static int imx6q_pm_pu_power_on(struct generic_pm_domain *genpd) +{ + struct pu_domain *pu = container_of(genpd, struct pu_domain, base); + int i, ret, sw, sw2iso; + u32 val; + + if (pu->reg) + ret = regulator_enable(pu->reg); + if (pu->reg && ret) { + pr_err("%s: failed to enable regulator: %d\n", __func__, ret); + return ret; + } + + /* Enable reset clocks for all devices in the PU domain */ + for (i = 0; i < pu->num_clks; i++) + clk_prepare_enable(pu->clk[i]); + + /* Gate off PU domain when GPU/VPU when powered down */ + writel_relaxed(0x1, gpc_base + GPC_PGC_GPU_PDN); + + /* Read ISO and ISO2SW power down delays */ + val = readl_relaxed(gpc_base + GPC_PGC_GPU_PUPSCR); + sw = val & 0x3f; + sw2iso = (val >> 8) & 0x3f; + + /* Request GPC to power up GPU/VPU */ + val = readl_relaxed(gpc_base + GPC_CNTR); + val |= GPU_VPU_PUP_REQ; + writel_relaxed(val, gpc_base + GPC_CNTR); + + /* Wait ISO + ISO2SW IPG clock cycles */ + ndelay((sw + sw2iso) * 1000 / 66); + + /* Disable reset clocks for all devices in the PU domain */ + for (i = 0; i < pu->num_clks; i++) + clk_disable_unprepare(pu->clk[i]); + + return 0; +} + +static struct generic_pm_domain imx6q_arm_domain = { + .name = "ARM", +}; + +static struct pu_domain imx6q_pu_domain = { + .base = { + .name = "PU", + .power_off = imx6q_pm_pu_power_off, + .power_on = imx6q_pm_pu_power_on, + .power_off_latency_ns = 25000, + .power_on_latency_ns = 2000000, + }, +}; + +static struct generic_pm_domain imx6sl_display_domain = { + .name = "DISPLAY", +}; + +static struct generic_pm_domain *imx_gpc_domains[] = { + &imx6q_arm_domain, + &imx6q_pu_domain.base, + &imx6sl_display_domain, +}; + +static struct genpd_onecell_data imx_gpc_onecell_data = { + .domains = imx_gpc_domains, + .num_domains = ARRAY_SIZE(imx_gpc_domains), +}; + +static int imx_gpc_genpd_init(struct device *dev, struct regulator *pu_reg) +{ + struct clk *clk; + bool is_off; + int i; + + imx6q_pu_domain.reg = pu_reg; + + for (i = 0; ; i++) { + clk = of_clk_get(dev->of_node, i); + if (IS_ERR(clk)) + break; + if (i >= GPC_CLK_MAX) { + dev_err(dev, "more than %d clocks\n", GPC_CLK_MAX); + goto clk_err; + } + imx6q_pu_domain.clk[i] = clk; + } + imx6q_pu_domain.num_clks = i; + + is_off = IS_ENABLED(CONFIG_PM); + if (is_off) { + _imx6q_pm_pu_power_off(&imx6q_pu_domain.base); + } else { + /* + * Enable power if compiled without CONFIG_PM in case the + * bootloader disabled it. + */ + imx6q_pm_pu_power_on(&imx6q_pu_domain.base); + } + + pm_genpd_init(&imx6q_pu_domain.base, NULL, is_off); + return of_genpd_add_provider_onecell(dev->of_node, + &imx_gpc_onecell_data); + +clk_err: + while (i--) + clk_put(imx6q_pu_domain.clk[i]); + return -EINVAL; +} + +#else +static inline int imx_gpc_genpd_init(struct device *dev, struct regulator *reg) +{ + return 0; +} +#endif /* CONFIG_PM_GENERIC_DOMAINS */ + +static int imx_gpc_probe(struct platform_device *pdev) +{ + struct regulator *pu_reg; + int ret; + + pu_reg = devm_regulator_get_optional(&pdev->dev, "pu"); + if (PTR_ERR(pu_reg) == -ENODEV) + pu_reg = NULL; + if (IS_ERR(pu_reg)) { + ret = PTR_ERR(pu_reg); + dev_err(&pdev->dev, "failed to get pu regulator: %d\n", ret); + return ret; + } + + return imx_gpc_genpd_init(&pdev->dev, pu_reg); +} + +static const struct of_device_id imx_gpc_dt_ids[] = { + { .compatible = "fsl,imx6q-gpc" }, + { .compatible = "fsl,imx6sl-gpc" }, + { } +}; + +static struct platform_driver imx_gpc_driver = { + .driver = { + .name = "imx-gpc", + .owner = THIS_MODULE, + .of_match_table = imx_gpc_dt_ids, + }, + .probe = imx_gpc_probe, +}; + +static int __init imx_pgc_init(void) +{ + return platform_driver_register(&imx_gpc_driver); } +subsys_initcall(imx_pgc_init); diff --git a/arch/arm/mach-imx/imx25-dt.c b/arch/arm/mach-imx/imx25-dt.c index cf8032bae277..25defbdb06c4 100644 --- a/arch/arm/mach-imx/imx25-dt.c +++ b/arch/arm/mach-imx/imx25-dt.c @@ -17,13 +17,6 @@ #include "common.h" #include "mx25.h" -static void __init imx25_dt_init(void) -{ - mxc_arch_reset_init_dt(); - - of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); -} - static const char * const imx25_dt_board_compat[] __initconst = { "fsl,imx25", NULL @@ -33,7 +26,5 @@ DT_MACHINE_START(IMX25_DT, "Freescale i.MX25 (Device Tree Support)") .map_io = mx25_map_io, .init_early = imx25_init_early, .init_irq = mx25_init_irq, - .init_machine = imx25_dt_init, .dt_compat = imx25_dt_board_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/imx27-dt.c b/arch/arm/mach-imx/imx27-dt.c index dc8f1a6f45f2..bd42d1bd10af 100644 --- a/arch/arm/mach-imx/imx27-dt.c +++ b/arch/arm/mach-imx/imx27-dt.c @@ -22,8 +22,6 @@ static void __init imx27_dt_init(void) { struct platform_device_info devinfo = { .name = "cpufreq-dt", }; - mxc_arch_reset_init_dt(); - of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); platform_device_register_full(&devinfo); @@ -40,5 +38,4 @@ DT_MACHINE_START(IMX27_DT, "Freescale i.MX27 (Device Tree Support)") .init_irq = mx27_init_irq, .init_machine = imx27_dt_init, .dt_compat = imx27_dt_board_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/imx31-dt.c b/arch/arm/mach-imx/imx31-dt.c index 418dbc82adc4..32100222a017 100644 --- a/arch/arm/mach-imx/imx31-dt.c +++ b/arch/arm/mach-imx/imx31-dt.c @@ -18,13 +18,6 @@ #include "common.h" #include "mx31.h" -static void __init imx31_dt_init(void) -{ - mxc_arch_reset_init_dt(); - - of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); -} - static const char * const imx31_dt_board_compat[] __initconst = { "fsl,imx31", NULL @@ -40,7 +33,5 @@ DT_MACHINE_START(IMX31_DT, "Freescale i.MX31 (Device Tree Support)") .init_early = imx31_init_early, .init_irq = mx31_init_irq, .init_time = imx31_dt_timer_init, - .init_machine = imx31_dt_init, .dt_compat = imx31_dt_board_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/imx35-dt.c b/arch/arm/mach-imx/imx35-dt.c index 584fbe105579..e9396037235d 100644 --- a/arch/arm/mach-imx/imx35-dt.c +++ b/arch/arm/mach-imx/imx35-dt.c @@ -20,14 +20,6 @@ #include "common.h" #include "mx35.h" -static void __init imx35_dt_init(void) -{ - mxc_arch_reset_init_dt(); - - of_platform_populate(NULL, of_default_bus_match_table, - NULL, NULL); -} - static void __init imx35_irq_init(void) { imx_init_l2cache(); @@ -43,7 +35,5 @@ DT_MACHINE_START(IMX35_DT, "Freescale i.MX35 (Device Tree Support)") .map_io = mx35_map_io, .init_early = imx35_init_early, .init_irq = imx35_irq_init, - .init_machine = imx35_dt_init, .dt_compat = imx35_dt_board_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/iomux-imx31.c b/arch/arm/mach-imx/iomux-imx31.c index 1657fe64cd0f..d6a30753ca7c 100644 --- a/arch/arm/mach-imx/iomux-imx31.c +++ b/arch/arm/mach-imx/iomux-imx31.c @@ -44,9 +44,11 @@ static unsigned long mxc_pin_alloc_map[NB_PORTS * 32 / BITS_PER_LONG]; /* * set the mode for a IOMUX pin. */ -int mxc_iomux_mode(unsigned int pin_mode) +void mxc_iomux_mode(unsigned int pin_mode) { - u32 field, l, mode, ret = 0; + u32 field; + u32 l; + u32 mode; void __iomem *reg; reg = IOMUXSW_MUX_CTL + (pin_mode & IOMUX_REG_MASK); @@ -61,8 +63,6 @@ int mxc_iomux_mode(unsigned int pin_mode) __raw_writel(l, reg); spin_unlock(&gpio_mux_lock); - - return ret; } /* diff --git a/arch/arm/mach-imx/iomux-mx3.h b/arch/arm/mach-imx/iomux-mx3.h index f79f78a1c0ed..0a5adba61e0b 100644 --- a/arch/arm/mach-imx/iomux-mx3.h +++ b/arch/arm/mach-imx/iomux-mx3.h @@ -144,7 +144,7 @@ void mxc_iomux_set_gpr(enum iomux_gp_func, bool en); * It is called by the setup functions and should not be called directly anymore. * It is here visible for backward compatibility */ -int mxc_iomux_mode(unsigned int pin_mode); +void mxc_iomux_mode(unsigned int pin_mode); #define IOMUX_PADNUM_MASK 0x1ff #define IOMUX_GPIONUM_SHIFT 9 diff --git a/arch/arm/mach-imx/mach-imx50.c b/arch/arm/mach-imx/mach-imx50.c index b1e56a94a382..ecf58b9e974b 100644 --- a/arch/arm/mach-imx/mach-imx50.c +++ b/arch/arm/mach-imx/mach-imx50.c @@ -16,13 +16,6 @@ #include "common.h" -static void __init imx50_dt_init(void) -{ - mxc_arch_reset_init_dt(); - - of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); -} - static const char * const imx50_dt_board_compat[] __initconst = { "fsl,imx50", NULL @@ -30,7 +23,5 @@ static const char * const imx50_dt_board_compat[] __initconst = { DT_MACHINE_START(IMX50_DT, "Freescale i.MX50 (Device Tree Support)") .init_irq = tzic_init_irq, - .init_machine = imx50_dt_init, .dt_compat = imx50_dt_board_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/mach-imx51.c b/arch/arm/mach-imx/mach-imx51.c index 2c5fcaf8675b..b015129e4045 100644 --- a/arch/arm/mach-imx/mach-imx51.c +++ b/arch/arm/mach-imx/mach-imx51.c @@ -53,7 +53,6 @@ static void __init imx51_dt_init(void) { struct platform_device_info devinfo = { .name = "cpufreq-dt", }; - mxc_arch_reset_init_dt(); imx51_ipu_mipi_setup(); imx_src_init(); @@ -78,5 +77,4 @@ DT_MACHINE_START(IMX51_DT, "Freescale i.MX51 (Device Tree Support)") .init_machine = imx51_dt_init, .init_late = imx51_init_late, .dt_compat = imx51_dt_board_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/mach-imx53.c b/arch/arm/mach-imx/mach-imx53.c index 03dd6ea13acc..18b5c5c136db 100644 --- a/arch/arm/mach-imx/mach-imx53.c +++ b/arch/arm/mach-imx/mach-imx53.c @@ -30,7 +30,6 @@ static void __init imx53_init_early(void) static void __init imx53_dt_init(void) { - mxc_arch_reset_init_dt(); imx_src_init(); of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); @@ -54,5 +53,4 @@ DT_MACHINE_START(IMX53_DT, "Freescale i.MX53 (Device Tree Support)") .init_machine = imx53_dt_init, .init_late = imx53_init_late, .dt_compat = imx53_dt_board_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c index d51c6e99a2e9..a973970dfe10 100644 --- a/arch/arm/mach-imx/mach-imx6q.c +++ b/arch/arm/mach-imx/mach-imx6q.c @@ -268,8 +268,6 @@ static void __init imx6q_init_machine(void) imx_print_silicon_rev(cpu_is_imx6dl() ? "i.MX6DL" : "i.MX6Q", imx_get_soc_revision()); - mxc_arch_reset_init_dt(); - parent = imx_soc_device_init(); if (parent == NULL) pr_warn("failed to initialize soc device\n"); @@ -331,7 +329,7 @@ static void __init imx6q_opp_check_speed_grading(struct device *cpu_dev) if (dev_pm_opp_disable(cpu_dev, 852000000)) pr_warn("failed to disable 852 MHz OPP\n"); } - + iounmap(base); put_node: of_node_put(np); } @@ -351,7 +349,7 @@ static void __init imx6q_opp_init(void) return; } - if (of_init_opp_table(cpu_dev)) { + if (dev_pm_opp_of_add_table(cpu_dev)) { pr_warn("failed to init OPP table\n"); goto put_node; } @@ -389,10 +387,10 @@ static void __init imx6q_map_io(void) static void __init imx6q_init_irq(void) { + imx_gpc_check_dt(); imx_init_revision_from_anatop(); imx_init_l2cache(); imx_src_init(); - imx_gpc_init(); irqchip_init(); } @@ -409,5 +407,4 @@ DT_MACHINE_START(IMX6Q, "Freescale i.MX6 Quad/DualLite (Device Tree)") .init_machine = imx6q_init_machine, .init_late = imx6q_init_late, .dt_compat = imx6q_dt_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/mach-imx6sl.c b/arch/arm/mach-imx/mach-imx6sl.c index ed263a21d928..12a1b098fc6a 100644 --- a/arch/arm/mach-imx/mach-imx6sl.c +++ b/arch/arm/mach-imx/mach-imx6sl.c @@ -48,8 +48,6 @@ static void __init imx6sl_init_machine(void) { struct device *parent; - mxc_arch_reset_init_dt(); - parent = imx_soc_device_init(); if (parent == NULL) pr_warn("failed to initialize soc device\n"); @@ -63,10 +61,10 @@ static void __init imx6sl_init_machine(void) static void __init imx6sl_init_irq(void) { + imx_gpc_check_dt(); imx_init_revision_from_anatop(); imx_init_l2cache(); imx_src_init(); - imx_gpc_init(); irqchip_init(); } @@ -76,10 +74,8 @@ static const char * const imx6sl_dt_compat[] __initconst = { }; DT_MACHINE_START(IMX6SL, "Freescale i.MX6 SoloLite (Device Tree)") - .map_io = debug_ll_io_init, .init_irq = imx6sl_init_irq, .init_machine = imx6sl_init_machine, .init_late = imx6sl_init_late, .dt_compat = imx6sl_dt_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/mach-imx6sx.c b/arch/arm/mach-imx/mach-imx6sx.c index 3de3b7369aef..f17b7004c24b 100644 --- a/arch/arm/mach-imx/mach-imx6sx.c +++ b/arch/arm/mach-imx/mach-imx6sx.c @@ -8,40 +8,89 @@ #include <linux/irqchip.h> #include <linux/of_platform.h> +#include <linux/phy.h> +#include <linux/regmap.h> +#include <linux/mfd/syscon.h> +#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h> #include <asm/mach/arch.h> #include <asm/mach/map.h> #include "common.h" #include "cpuidle.h" +static int ar8031_phy_fixup(struct phy_device *dev) +{ + u16 val; + + /* Set RGMII IO voltage to 1.8V */ + phy_write(dev, 0x1d, 0x1f); + phy_write(dev, 0x1e, 0x8); + + /* introduce tx clock delay */ + phy_write(dev, 0x1d, 0x5); + val = phy_read(dev, 0x1e); + val |= 0x0100; + phy_write(dev, 0x1e, val); + + return 0; +} + +#define PHY_ID_AR8031 0x004dd074 +static void __init imx6sx_enet_phy_init(void) +{ + if (IS_BUILTIN(CONFIG_PHYLIB)) + phy_register_fixup_for_uid(PHY_ID_AR8031, 0xffffffff, + ar8031_phy_fixup); +} + +static void __init imx6sx_enet_clk_sel(void) +{ + struct regmap *gpr; + + gpr = syscon_regmap_lookup_by_compatible("fsl,imx6sx-iomuxc-gpr"); + if (!IS_ERR(gpr)) { + regmap_update_bits(gpr, IOMUXC_GPR1, + IMX6SX_GPR1_FEC_CLOCK_MUX_SEL_MASK, 0); + regmap_update_bits(gpr, IOMUXC_GPR1, + IMX6SX_GPR1_FEC_CLOCK_PAD_DIR_MASK, 0); + } else { + pr_err("failed to find fsl,imx6sx-iomux-gpr regmap\n"); + } +} + +static inline void imx6sx_enet_init(void) +{ + imx6sx_enet_phy_init(); + imx6sx_enet_clk_sel(); +} + static void __init imx6sx_init_machine(void) { struct device *parent; - mxc_arch_reset_init_dt(); - parent = imx_soc_device_init(); if (parent == NULL) pr_warn("failed to initialize soc device\n"); of_platform_populate(NULL, of_default_bus_match_table, NULL, parent); + imx6sx_enet_init(); imx_anatop_init(); imx6sx_pm_init(); } static void __init imx6sx_init_irq(void) { + imx_gpc_check_dt(); imx_init_revision_from_anatop(); imx_init_l2cache(); imx_src_init(); - imx_gpc_init(); irqchip_init(); } static void __init imx6sx_init_late(void) { - imx6q_cpuidle_init(); + imx6sx_cpuidle_init(); if (IS_ENABLED(CONFIG_ARM_IMX6Q_CPUFREQ)) platform_device_register_simple("imx6q-cpufreq", -1, NULL, 0); @@ -53,10 +102,8 @@ static const char * const imx6sx_dt_compat[] __initconst = { }; DT_MACHINE_START(IMX6SX, "Freescale i.MX6 SoloX (Device Tree)") - .map_io = debug_ll_io_init, .init_irq = imx6sx_init_irq, .init_machine = imx6sx_init_machine, .dt_compat = imx6sx_dt_compat, .init_late = imx6sx_init_late, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/mach-vf610.c b/arch/arm/mach-imx/mach-vf610.c index ee7e57b752a7..c11ab6a1dc87 100644 --- a/arch/arm/mach-imx/mach-vf610.c +++ b/arch/arm/mach-imx/mach-vf610.c @@ -12,14 +12,6 @@ #include <asm/mach/arch.h> #include <asm/hardware/cache-l2x0.h> -#include "common.h" - -static void __init vf610_init_machine(void) -{ - mxc_arch_reset_init_dt(); - of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); -} - static const char * const vf610_dt_compat[] __initconst = { "fsl,vf610", NULL, @@ -28,7 +20,5 @@ static const char * const vf610_dt_compat[] __initconst = { DT_MACHINE_START(VYBRID_VF610, "Freescale Vybrid VF610 (Device Tree)") .l2c_aux_val = 0, .l2c_aux_mask = ~0, - .init_machine = vf610_init_machine, .dt_compat = vf610_dt_compat, - .restart = mxc_restart, MACHINE_END diff --git a/arch/arm/mach-imx/pm-imx6.c b/arch/arm/mach-imx/pm-imx6.c index bdd1d9061759..7b662400813b 100644 --- a/arch/arm/mach-imx/pm-imx6.c +++ b/arch/arm/mach-imx/pm-imx6.c @@ -209,7 +209,7 @@ void imx6q_set_int_mem_clk_lpm(bool enable) writel_relaxed(val, ccm_base + CGPR); } -static void imx6q_enable_rbc(bool enable) +void imx6_enable_rbc(bool enable) { u32 val; @@ -261,7 +261,6 @@ static void imx6q_enable_wb(bool enable) int imx6q_set_lpm(enum mxc_cpu_pwr_mode mode) { - struct irq_data *iomuxc_irq_data = irq_get_irq_data(32); u32 val = readl_relaxed(ccm_base + CLPCR); val &= ~BM_CLPCR_LPM; @@ -315,10 +314,12 @@ int imx6q_set_lpm(enum mxc_cpu_pwr_mode mode) * Low-Power mode. * 3) Software should mask IRQ #32 right after CCM Low-Power mode * is set (set bits 0-1 of CCM_CLPCR). + * + * Note that IRQ #32 is GIC SPI #0. */ - imx_gpc_irq_unmask(iomuxc_irq_data); + imx_gpc_hwirq_unmask(0); writel_relaxed(val, ccm_base + CLPCR); - imx_gpc_irq_mask(iomuxc_irq_data); + imx_gpc_hwirq_mask(0); return 0; } @@ -364,7 +365,7 @@ static int imx6q_pm_enter(suspend_state_t state) * RBC setting, so we do NOT need to do that here. */ if (!imx6_suspend_in_ocram_fn) - imx6q_enable_rbc(true); + imx6_enable_rbc(true); imx_gpc_pre_suspend(true); imx_anatop_pre_suspend(); imx_set_cpu_jump(0, v7_cpu_resume); @@ -374,7 +375,7 @@ static int imx6q_pm_enter(suspend_state_t state) imx_smp_prepare(); imx_anatop_post_resume(); imx_gpc_post_resume(); - imx6q_enable_rbc(false); + imx6_enable_rbc(false); imx6q_enable_wb(false); imx6q_set_int_mem_clk_lpm(true); imx6q_set_lpm(WAIT_CLOCKED); diff --git a/arch/arm/mach-imx/system.c b/arch/arm/mach-imx/system.c index d14c33fd6b03..51c35013b673 100644 --- a/arch/arm/mach-imx/system.c +++ b/arch/arm/mach-imx/system.c @@ -89,21 +89,6 @@ void __init mxc_arch_reset_init(void __iomem *base) clk_prepare(wdog_clk); } -void __init mxc_arch_reset_init_dt(void) -{ - struct device_node *np; - - np = of_find_compatible_node(NULL, NULL, "fsl,imx21-wdt"); - wdog_base = of_iomap(np, 0); - WARN_ON(!wdog_base); - - wdog_clk = of_clk_get(np, 0); - if (IS_ERR(wdog_clk)) - pr_warn("%s: failed to get wdog clock\n", __func__); - else - clk_prepare(wdog_clk); -} - #ifdef CONFIG_CACHE_L2X0 void __init imx_init_l2cache(void) { diff --git a/arch/arm/mach-iop13xx/msi.c b/arch/arm/mach-iop13xx/msi.c index e7730cf9c15d..9f89e76dfbb9 100644 --- a/arch/arm/mach-iop13xx/msi.c +++ b/arch/arm/mach-iop13xx/msi.c @@ -126,10 +126,10 @@ static void iop13xx_msi_nop(struct irq_data *d) static struct irq_chip iop13xx_msi_chip = { .name = "PCI-MSI", .irq_ack = iop13xx_msi_nop, - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, }; int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) @@ -153,7 +153,7 @@ int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) id = iop13xx_cpu_id(); msg.data = (id << IOP13XX_MU_MIMR_CORE_SELECT) | (irq & 0x7f); - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); irq_set_chip_and_handler(irq, &iop13xx_msi_chip, handle_simple_irq); return 0; diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 00b9c4870230..1d5e13f7a298 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -13,7 +13,9 @@ config ARM64 select ARM_ARCH_TIMER select ARM_GIC select AUDIT_ARCH_COMPAT_GENERIC + select ARM_GIC_V2M if PCI_MSI select ARM_GIC_V3 + select ARM_GIC_V3_ITS if PCI_MSI select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS select COMMON_CLK @@ -36,6 +38,7 @@ config ARM64 select HARDIRQS_SW_RESEND select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_KASAN if SPARSEMEM_VMEMMAP && !(ARM64_16K_PAGES && ARM64_VA_BITS_48) select HAVE_ARCH_KGDB select HAVE_ARCH_TRACEHOOK select HAVE_BPF_JIT @@ -494,6 +497,87 @@ config FORCE_MAX_ZONEORDER default "14" if (ARM64_64K_PAGES && TRANSPARENT_HUGEPAGE) default "11" +config ARM64_PAN + bool "Enable support for Privileged Access Never (PAN)" + default y + help + Privileged Access Never (PAN; part of the ARMv8.1 Extensions) + prevents the kernel or hypervisor from accessing user-space (EL0) + memory directly. + + Choosing this option will cause any unprotected (not using + copy_to_user et al) memory access to fail with a permission fault. + + The feature is detected at runtime, and will remain as a 'nop' + instruction if the cpu does not implement the feature. + +menuconfig ARMV8_DEPRECATED + bool "Emulate deprecated/obsolete ARMv8 instructions" + depends on COMPAT + help + Legacy software support may require certain instructions + that have been deprecated or obsoleted in the architecture. + + Enable this config to enable selective emulation of these + features. + + If unsure, say Y + +if ARMV8_DEPRECATED + +config SWP_EMULATION + bool "Emulate SWP/SWPB instructions" + help + ARMv8 obsoletes the use of A32 SWP/SWPB instructions such that + they are always undefined. Say Y here to enable software + emulation of these instructions for userspace using LDXR/STXR. + + In some older versions of glibc [<=2.8] SWP is used during futex + trylock() operations with the assumption that the code will not + be preempted. This invalid assumption may be more likely to fail + with SWP emulation enabled, leading to deadlock of the user + application. + + NOTE: when accessing uncached shared regions, LDXR/STXR rely + on an external transaction monitoring block called a global + monitor to maintain update atomicity. If your system does not + implement a global monitor, this option can cause programs that + perform SWP operations to uncached memory to deadlock. + + If unsure, say Y + +config CP15_BARRIER_EMULATION + bool "Emulate CP15 Barrier instructions" + help + The CP15 barrier instructions - CP15ISB, CP15DSB, and + CP15DMB - are deprecated in ARMv8 (and ARMv7). It is + strongly recommended to use the ISB, DSB, and DMB + instructions instead. + + Say Y here to enable software emulation of these + instructions for AArch32 userspace code. When this option is + enabled, CP15 barrier usage is traced which can help + identify software that needs updating. + + If unsure, say Y + +config SETEND_EMULATION + bool "Emulate SETEND instruction" + help + The SETEND instruction alters the data-endianness of the + AArch32 EL0, and is deprecated in ARMv8. + + Say Y here to enable software emulation of the instruction + for AArch32 userspace code. + + Note: All the cpus on the system must have mixed endian support at EL0 + for this feature to be enabled. If a new CPU - which doesn't support mixed + endian - is hotplugged in after this feature has been enabled, there could + be unexpected results in the applications. + + If unsure, say Y +endif + endmenu menu "Boot options" diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 0a12933e50ed..8dd3a551c170 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -54,4 +54,6 @@ config DEBUG_SET_MODULE_RONX against certain classes of kernel exploits. If in doubt, say "N". +source "drivers/hwtracing/coresight/Kconfig" + endmenu diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 37c4fd6aeb7a..fa985ad3defc 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -15,8 +15,6 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET) OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S GZFLAGS :=-9 -LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name) - KBUILD_DEFCONFIG := defconfig KBUILD_CFLAGS += -mgeneral-regs-only @@ -48,6 +46,13 @@ else TEXT_OFFSET := 0x00080000 endif +# KASAN_SHADOW_OFFSET = VA_START + (1 << (VA_BITS - 3)) - (1 << 61) +# in 32-bit arithmetic +KASAN_SHADOW_OFFSET := $(shell printf "0x%08x00000000\n" $$(( \ + (0xffffffff & (-1 << ($(CONFIG_ARM64_VA_BITS) - 32))) \ + + (1 << ($(CONFIG_ARM64_VA_BITS) - 32 - 3)) \ + - (1 << (64 - 32 - 3)) )) ) + export TEXT_OFFSET GZFLAGS core-y += arch/arm64/kernel/ arch/arm64/mm/ @@ -56,7 +61,6 @@ core-$(CONFIG_KVM) += arch/arm64/kvm/ core-$(CONFIG_XEN) += arch/arm64/xen/ core-$(CONFIG_CRYPTO) += arch/arm64/crypto/ libs-y := arch/arm64/lib/ $(libs-y) -libs-y += $(LIBGCC) libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/ # Default target when executing plain make @@ -76,8 +80,13 @@ zinstall install: vmlinux %.dtb: scripts $(Q)$(MAKE) $(build)=$(boot)/dts $(boot)/dts/$@ -dtbs: scripts - $(Q)$(MAKE) $(build)=$(boot)/dts dtbs +PHONY += dtbs dtbs_install + +dtbs: prepare scripts + $(Q)$(MAKE) $(build)=$(boot)/dts + +dtbs_install: + $(Q)$(MAKE) $(dtbinst)=$(boot)/dts PHONY += vdso_install vdso_install: @@ -86,11 +95,13 @@ vdso_install: # We use MRPROPER_FILES and CLEAN_FILES now archclean: $(Q)$(MAKE) $(clean)=$(boot) + $(Q)$(MAKE) $(clean)=$(boot)/dts define archhelp echo '* Image.gz - Compressed kernel image (arch/$(ARCH)/boot/Image.gz)' echo ' Image - Uncompressed kernel image (arch/$(ARCH)/boot/Image)' echo '* dtbs - Build device tree blobs for enabled boards' + echo ' dtbs_install - Install dtbs to $(INSTALL_DTBS_PATH)' echo ' install - Install uncompressed kernel' echo ' zinstall - Install compressed kernel' echo ' Install using (your) ~/bin/installkernel or' diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile index f8001a62029c..7f67556f442f 100644 --- a/arch/arm64/boot/dts/Makefile +++ b/arch/arm64/boot/dts/Makefile @@ -1,10 +1,5 @@ -dtb-$(CONFIG_ARCH_THUNDER) += thunder-88xx.dtb -dtb-$(CONFIG_ARCH_VEXPRESS) += rtsm_ve-aemv8a.dtb foundation-v8.dtb -dtb-$(CONFIG_ARCH_XGENE) += apm-mustang.dtb +dts-dirs += apm +dts-dirs += arm +dts-dirs += cavium -targets += dtbs -targets += $(dtb-y) - -dtbs: $(addprefix $(obj)/, $(dtb-y)) - -clean-files := *.dtb +subdir-y := $(dts-dirs) diff --git a/arch/arm64/boot/dts/apm/Makefile b/arch/arm64/boot/dts/apm/Makefile new file mode 100644 index 000000000000..a2afabbc1717 --- /dev/null +++ b/arch/arm64/boot/dts/apm/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_XGENE) += apm-mustang.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/apm-mustang.dts b/arch/arm64/boot/dts/apm/apm-mustang.dts index 2e25de0800b9..2e25de0800b9 100644 --- a/arch/arm64/boot/dts/apm-mustang.dts +++ b/arch/arm64/boot/dts/apm/apm-mustang.dts diff --git a/arch/arm64/boot/dts/apm-storm.dtsi b/arch/arm64/boot/dts/apm/apm-storm.dtsi index f1ad9c2ab2e9..f1ad9c2ab2e9 100644 --- a/arch/arm64/boot/dts/apm-storm.dtsi +++ b/arch/arm64/boot/dts/apm/apm-storm.dtsi diff --git a/arch/arm64/boot/dts/arm/Makefile b/arch/arm64/boot/dts/arm/Makefile new file mode 100644 index 000000000000..c5c98b91514e --- /dev/null +++ b/arch/arm64/boot/dts/arm/Makefile @@ -0,0 +1,7 @@ +dtb-$(CONFIG_ARCH_VEXPRESS) += foundation-v8.dtb +dtb-$(CONFIG_ARCH_VEXPRESS) += juno.dtb juno-r1.dtb +dtb-$(CONFIG_ARCH_VEXPRESS) += rtsm_ve-aemv8a.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/foundation-v8.dts b/arch/arm64/boot/dts/arm/foundation-v8.dts index 4a060906809d..4eac8dcea423 100644 --- a/arch/arm64/boot/dts/foundation-v8.dts +++ b/arch/arm64/boot/dts/arm/foundation-v8.dts @@ -34,6 +34,7 @@ reg = <0x0 0x0>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@1 { device_type = "cpu"; @@ -41,6 +42,7 @@ reg = <0x0 0x1>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@2 { device_type = "cpu"; @@ -48,6 +50,7 @@ reg = <0x0 0x2>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@3 { device_type = "cpu"; @@ -55,6 +58,11 @@ reg = <0x0 0x3>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; + }; + + L2_0: l2-cache0 { + compatible = "cache"; }; }; @@ -78,10 +86,10 @@ timer { compatible = "arm,armv8-timer"; - interrupts = <1 13 0xff01>, - <1 14 0xff01>, - <1 11 0xff01>, - <1 10 0xff01>; + interrupts = <1 13 0xf08>, + <1 14 0xf08>, + <1 11 0xf08>, + <1 10 0xf08>; clock-frequency = <100000000>; }; diff --git a/arch/arm64/boot/dts/arm/juno-base.dtsi b/arch/arm64/boot/dts/arm/juno-base.dtsi new file mode 100644 index 000000000000..e3ee96036eca --- /dev/null +++ b/arch/arm64/boot/dts/arm/juno-base.dtsi @@ -0,0 +1,154 @@ + /* + * Devices shared by all Juno boards + */ + + memtimer: timer@2a810000 { + compatible = "arm,armv7-timer-mem"; + reg = <0x0 0x2a810000 0x0 0x10000>; + clock-frequency = <50000000>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "disabled"; + frame@2a830000 { + frame-number = <1>; + interrupts = <0 60 4>; + reg = <0x0 0x2a830000 0x0 0x10000>; + }; + }; + + gic: interrupt-controller@2c010000 { + compatible = "arm,gic-400", "arm,cortex-a15-gic"; + reg = <0x0 0x2c010000 0 0x1000>, + <0x0 0x2c02f000 0 0x2000>, + <0x0 0x2c04f000 0 0x2000>, + <0x0 0x2c06f000 0 0x2000>; + #address-cells = <2>; + #interrupt-cells = <3>; + #size-cells = <2>; + interrupt-controller; + interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_HIGH)>; + ranges = <0 0 0 0x2c1c0000 0 0x40000>; + v2m_0: v2m@0 { + compatible = "arm,gic-v2m-frame"; + msi-controller; + reg = <0 0 0 0x1000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>; + }; + + /include/ "juno-clocks.dtsi" + + dma@7ff00000 { + compatible = "arm,pl330", "arm,primecell"; + reg = <0x0 0x7ff00000 0 0x1000>; + #dma-cells = <1>; + #dma-channels = <8>; + #dma-requests = <32>; + interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_faxiclk>; + clock-names = "apb_pclk"; + }; + + soc_uart0: uart@7ff80000 { + compatible = "arm,pl011", "arm,primecell"; + reg = <0x0 0x7ff80000 0x0 0x1000>; + interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_uartclk>, <&soc_refclk100mhz>; + clock-names = "uartclk", "apb_pclk"; + }; + + i2c@7ffa0000 { + compatible = "snps,designware-i2c"; + reg = <0x0 0x7ffa0000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>; + clock-frequency = <400000>; + i2c-sda-hold-time-ns = <500>; + clocks = <&soc_smc50mhz>; + + dvi0: dvi-transmitter@70 { + compatible = "nxp,tda998x"; + reg = <0x70>; + }; + + dvi1: dvi-transmitter@71 { + compatible = "nxp,tda998x"; + reg = <0x71>; + }; + }; + + ohci@7ffb0000 { + compatible = "generic-ohci"; + reg = <0x0 0x7ffb0000 0x0 0x10000>; + interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_usb48mhz>; + }; + + ehci@7ffc0000 { + compatible = "generic-ehci"; + reg = <0x0 0x7ffc0000 0x0 0x10000>; + interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_usb48mhz>; + }; + + memory-controller@7ffd0000 { + compatible = "arm,pl354", "arm,primecell"; + reg = <0 0x7ffd0000 0 0x1000>; + interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_smc50mhz>; + clock-names = "apb_pclk"; + }; + + memory@80000000 { + device_type = "memory"; + /* last 16MB of the first memory area is reserved for secure world use by firmware */ + reg = <0x00000000 0x80000000 0x0 0x7f000000>, + <0x00000008 0x80000000 0x1 0x80000000>; + }; + + smb { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <1>; + ranges = <0 0 0 0x08000000 0x04000000>, + <1 0 0 0x14000000 0x04000000>, + <2 0 0 0x18000000 0x04000000>, + <3 0 0 0x1c000000 0x04000000>, + <4 0 0 0x0c000000 0x04000000>, + <5 0 0 0x10000000 0x04000000>; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 15>; + interrupt-map = <0 0 0 &gic 0 0 0 68 IRQ_TYPE_LEVEL_HIGH>, + <0 0 1 &gic 0 0 0 69 IRQ_TYPE_LEVEL_HIGH>, + <0 0 2 &gic 0 0 0 70 IRQ_TYPE_LEVEL_HIGH>, + <0 0 3 &gic 0 0 0 160 IRQ_TYPE_LEVEL_HIGH>, + <0 0 4 &gic 0 0 0 161 IRQ_TYPE_LEVEL_HIGH>, + <0 0 5 &gic 0 0 0 162 IRQ_TYPE_LEVEL_HIGH>, + <0 0 6 &gic 0 0 0 163 IRQ_TYPE_LEVEL_HIGH>, + <0 0 7 &gic 0 0 0 164 IRQ_TYPE_LEVEL_HIGH>, + <0 0 8 &gic 0 0 0 165 IRQ_TYPE_LEVEL_HIGH>, + <0 0 9 &gic 0 0 0 166 IRQ_TYPE_LEVEL_HIGH>, + <0 0 10 &gic 0 0 0 167 IRQ_TYPE_LEVEL_HIGH>, + <0 0 11 &gic 0 0 0 168 IRQ_TYPE_LEVEL_HIGH>, + <0 0 12 &gic 0 0 0 169 IRQ_TYPE_LEVEL_HIGH>; + + /include/ "juno-motherboard.dtsi" + }; diff --git a/arch/arm64/boot/dts/arm/juno-clocks.dtsi b/arch/arm64/boot/dts/arm/juno-clocks.dtsi new file mode 100644 index 000000000000..25352ed943e6 --- /dev/null +++ b/arch/arm64/boot/dts/arm/juno-clocks.dtsi @@ -0,0 +1,44 @@ +/* + * ARM Juno Platform clocks + * + * Copyright (c) 2013-2014 ARM Ltd + * + * This file is licensed under a dual GPLv2 or BSD license. + * + */ + + /* SoC fixed clocks */ + soc_uartclk: refclk7273800hz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <7273800>; + clock-output-names = "juno:uartclk"; + }; + + soc_usb48mhz: clk48mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <48000000>; + clock-output-names = "clk48mhz"; + }; + + soc_smc50mhz: clk50mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <50000000>; + clock-output-names = "smc_clk"; + }; + + soc_refclk100mhz: refclk100mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + clock-output-names = "apb_pclk"; + }; + + soc_faxiclk: refclk400mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <400000000>; + clock-output-names = "faxi_clk"; + }; diff --git a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi new file mode 100644 index 000000000000..c138b95a8356 --- /dev/null +++ b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi @@ -0,0 +1,129 @@ +/* + * ARM Juno Platform motherboard peripherals + * + * Copyright (c) 2013-2014 ARM Ltd + * + * This file is licensed under a dual GPLv2 or BSD license. + * + */ + + mb_clk24mhz: clk24mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "juno_mb:clk24mhz"; + }; + + mb_clk25mhz: clk25mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <25000000>; + clock-output-names = "juno_mb:clk25mhz"; + }; + + motherboard { + compatible = "arm,vexpress,v2p-p1", "simple-bus"; + #address-cells = <2>; /* SMB chipselect number and offset */ + #size-cells = <1>; + #interrupt-cells = <1>; + ranges; + model = "V2M-Juno"; + arm,hbi = <0x252>; + arm,vexpress,site = <0>; + arm,v2m-memory-map = "rs1"; + + mb_fixed_3v3: fixedregulator@0 { + compatible = "regulator-fixed"; + regulator-name = "MCC_SB_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + ethernet@2,00000000 { + compatible = "smsc,lan9118", "smsc,lan9115"; + reg = <2 0x00000000 0x10000>; + interrupts = <3>; + phy-mode = "mii"; + reg-io-width = <4>; + smsc,irq-active-high; + smsc,irq-push-pull; + clocks = <&mb_clk25mhz>; + vdd33a-supply = <&mb_fixed_3v3>; + vddvario-supply = <&mb_fixed_3v3>; + }; + + usb@5,00000000 { + compatible = "nxp,usb-isp1763"; + reg = <5 0x00000000 0x20000>; + bus-width = <16>; + interrupts = <4>; + }; + + iofpga@3,00000000 { + compatible = "arm,amba-bus", "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 3 0 0x200000>; + + mmci@050000 { + compatible = "arm,pl180", "arm,primecell"; + reg = <0x050000 0x1000>; + interrupts = <5>; + /* cd-gpios = <&v2m_mmc_gpios 0 0>; + wp-gpios = <&v2m_mmc_gpios 1 0>; */ + max-frequency = <12000000>; + vmmc-supply = <&mb_fixed_3v3>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "mclk", "apb_pclk"; + }; + + kmi@060000 { + compatible = "arm,pl050", "arm,primecell"; + reg = <0x060000 0x1000>; + interrupts = <8>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "KMIREFCLK", "apb_pclk"; + }; + + kmi@070000 { + compatible = "arm,pl050", "arm,primecell"; + reg = <0x070000 0x1000>; + interrupts = <8>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "KMIREFCLK", "apb_pclk"; + }; + + wdt@0f0000 { + compatible = "arm,sp805", "arm,primecell"; + reg = <0x0f0000 0x10000>; + interrupts = <7>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "wdogclk", "apb_pclk"; + }; + + v2m_timer01: timer@110000 { + compatible = "arm,sp804", "arm,primecell"; + reg = <0x110000 0x10000>; + interrupts = <9>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "timclken1", "apb_pclk"; + }; + + v2m_timer23: timer@120000 { + compatible = "arm,sp804", "arm,primecell"; + reg = <0x120000 0x10000>; + interrupts = <9>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "timclken1", "apb_pclk"; + }; + + rtc@170000 { + compatible = "arm,pl031", "arm,primecell"; + reg = <0x170000 0x10000>; + interrupts = <0>; + clocks = <&soc_smc50mhz>; + clock-names = "apb_pclk"; + }; + }; + }; diff --git a/arch/arm64/boot/dts/arm/juno-r1.dts b/arch/arm64/boot/dts/arm/juno-r1.dts new file mode 100644 index 000000000000..a25964d26bda --- /dev/null +++ b/arch/arm64/boot/dts/arm/juno-r1.dts @@ -0,0 +1,136 @@ +/* + * ARM Ltd. Juno Platform + * + * Copyright (c) 2015 ARM Ltd. + * + * This file is licensed under a dual GPLv2 or BSD license. + */ + +/dts-v1/; + +#include <dt-bindings/interrupt-controller/arm-gic.h> + +/ { + model = "ARM Juno development board (r1)"; + compatible = "arm,juno-r1", "arm,juno", "arm,vexpress"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + serial0 = &soc_uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + A57_0: cpu@0 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x0>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + }; + + A57_1: cpu@1 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x1>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + }; + + A53_0: cpu@100 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x100>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_1: cpu@101 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x101>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_2: cpu@102 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x102>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_3: cpu@103 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x103>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A57_L2: l2-cache0 { + compatible = "cache"; + }; + + A53_L2: l2-cache1 { + compatible = "cache"; + }; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>; + interrupt-affinity = <&A57_0>, + <&A57_1>, + <&A53_0>, + <&A53_1>, + <&A53_2>, + <&A53_3>; + }; + + #include "juno-base.dtsi" + + pcie-controller@40000000 { + compatible = "arm,juno-r1-pcie", "plda,xpressrich3-axi", "pci-host-ecam-generic"; + device_type = "pci"; + reg = <0 0x40000000 0 0x10000000>; /* ECAM config space */ + bus-range = <0 255>; + linux,pci-domain = <0>; + #address-cells = <3>; + #size-cells = <2>; + dma-coherent; + ranges = <0x01000000 0x00 0x5f800000 0x00 0x5f800000 0x0 0x00800000>, + <0x02000000 0x00 0x50000000 0x00 0x50000000 0x0 0x08000000>, + <0x42000000 0x40 0x00000000 0x40 0x00000000 0x1 0x00000000>; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &gic 0 0 0 136 4>, + <0 0 0 2 &gic 0 0 0 137 4>, + <0 0 0 3 &gic 0 0 0 138 4>, + <0 0 0 4 &gic 0 0 0 139 4>; + msi-parent = <&v2m_0>; + }; +}; + +&memtimer { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/arm/juno.dts b/arch/arm64/boot/dts/arm/juno.dts new file mode 100644 index 000000000000..d7cbdd482a61 --- /dev/null +++ b/arch/arm64/boot/dts/arm/juno.dts @@ -0,0 +1,111 @@ +/* + * ARM Ltd. Juno Platform + * + * Copyright (c) 2013-2014 ARM Ltd. + * + * This file is licensed under a dual GPLv2 or BSD license. + */ + +/dts-v1/; + +#include <dt-bindings/interrupt-controller/arm-gic.h> + +/ { + model = "ARM Juno development board (r0)"; + compatible = "arm,juno", "arm,vexpress"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + serial0 = &soc_uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + A57_0: cpu@0 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x0>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + }; + + A57_1: cpu@1 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x1>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + }; + + A53_0: cpu@100 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x100>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_1: cpu@101 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x101>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_2: cpu@102 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x102>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_3: cpu@103 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x103>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A57_L2: l2-cache0 { + compatible = "cache"; + }; + + A53_L2: l2-cache1 { + compatible = "cache"; + }; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>; + interrupt-affinity = <&A57_0>, + <&A57_1>, + <&A53_0>, + <&A53_1>, + <&A53_2>, + <&A53_3>; + }; + + #include "juno-base.dtsi" +}; diff --git a/arch/arm64/boot/dts/rtsm_ve-aemv8a.dts b/arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts index 572005ea2217..20addabbd127 100644 --- a/arch/arm64/boot/dts/rtsm_ve-aemv8a.dts +++ b/arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts @@ -37,6 +37,7 @@ reg = <0x0 0x0>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@1 { device_type = "cpu"; @@ -44,6 +45,7 @@ reg = <0x0 0x1>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@2 { device_type = "cpu"; @@ -51,6 +53,7 @@ reg = <0x0 0x2>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@3 { device_type = "cpu"; @@ -58,6 +61,11 @@ reg = <0x0 0x3>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; + }; + + L2_0: l2-cache0 { + compatible = "cache"; }; }; @@ -81,10 +89,10 @@ timer { compatible = "arm,armv8-timer"; - interrupts = <1 13 0xff01>, - <1 14 0xff01>, - <1 11 0xff01>, - <1 10 0xff01>; + interrupts = <1 13 0xf08>, + <1 14 0xf08>, + <1 11 0xf08>, + <1 10 0xf08>; clock-frequency = <100000000>; }; diff --git a/arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi b/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi index c46cbb29f3c6..c46cbb29f3c6 100644 --- a/arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi +++ b/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi diff --git a/arch/arm64/boot/dts/cavium/Makefile b/arch/arm64/boot/dts/cavium/Makefile new file mode 100644 index 000000000000..e34f89ddabb2 --- /dev/null +++ b/arch/arm64/boot/dts/cavium/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_THUNDER) += thunder-88xx.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/thunder-88xx.dts b/arch/arm64/boot/dts/cavium/thunder-88xx.dts index 800ba65991f7..800ba65991f7 100644 --- a/arch/arm64/boot/dts/thunder-88xx.dts +++ b/arch/arm64/boot/dts/cavium/thunder-88xx.dts diff --git a/arch/arm64/boot/dts/thunder-88xx.dtsi b/arch/arm64/boot/dts/cavium/thunder-88xx.dtsi index d8c0bdc51882..d8c0bdc51882 100644 --- a/arch/arm64/boot/dts/thunder-88xx.dtsi +++ b/arch/arm64/boot/dts/cavium/thunder-88xx.dtsi diff --git a/arch/arm64/boot/dts/include/dt-bindings b/arch/arm64/boot/dts/include/dt-bindings new file mode 120000 index 000000000000..08c00e4972fa --- /dev/null +++ b/arch/arm64/boot/dts/include/dt-bindings @@ -0,0 +1 @@ +../../../../../include/dt-bindings
\ No newline at end of file diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index dd301be89ecc..6738cb24c058 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -37,6 +37,7 @@ CONFIG_ARCH_VEXPRESS=y CONFIG_ARCH_XGENE=y CONFIG_PCI=y CONFIG_PCI_MSI=y +CONFIG_PCI_HOST_GENERIC=y CONFIG_PCI_XGENE=y CONFIG_SMP=y CONFIG_PREEMPT=y diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index dc770bd4f5a5..e315bd833c10 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -28,6 +28,7 @@ generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mman.h generic-y += msgbuf.h +generic-y += msi.h generic-y += mutex.h generic-y += pci.h generic-y += pci-bridge.h diff --git a/arch/arm64/include/asm/alternative-asm.h b/arch/arm64/include/asm/alternative-asm.h deleted file mode 100644 index 919a67855b63..000000000000 --- a/arch/arm64/include/asm/alternative-asm.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef __ASM_ALTERNATIVE_ASM_H -#define __ASM_ALTERNATIVE_ASM_H - -#ifdef __ASSEMBLY__ - -.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len - .word \orig_offset - . - .word \alt_offset - . - .hword \feature - .byte \orig_len - .byte \alt_len -.endm - -.macro alternative_insn insn1 insn2 cap -661: \insn1 -662: .pushsection .altinstructions, "a" - altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f - .popsection - .pushsection .altinstr_replacement, "ax" -663: \insn2 -664: .popsection - .if ((664b-663b) != (662b-661b)) - .error "Alternatives instruction length mismatch" - .endif -.endm - -#endif /* __ASSEMBLY__ */ - -#endif /* __ASM_ALTERNATIVE_ASM_H */ diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index f6d206e7f9e9..4e3d4c8b50d1 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -1,6 +1,9 @@ #ifndef __ASM_ALTERNATIVE_H #define __ASM_ALTERNATIVE_H +#ifndef __ASSEMBLY__ + +#include <linux/kconfig.h> #include <linux/types.h> #include <linux/stddef.h> #include <linux/stringify.h> @@ -23,8 +26,22 @@ void free_alternatives_memory(void); " .byte 662b-661b\n" /* source len */ \ " .byte 664f-663f\n" /* replacement len */ -/* alternative assembly primitive: */ -#define ALTERNATIVE(oldinstr, newinstr, feature) \ +/* + * alternative assembly primitive: + * + * If any of these .org directive fail, it means that insn1 and insn2 + * don't have the same length. This used to be written as + * + * .if ((664b-663b) != (662b-661b)) + * .error "Alternatives instruction length mismatch" + * .endif + * + * but most assemblers die if insn1 or insn2 have a .inst. This should + * be fixed in a binutils release posterior to 2.25.51.0.2 (anything + * containing commit 4e4d08cf7399b606 or c1baaddf8861). + */ +#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \ + ".if "__stringify(cfg_enabled)" == 1\n" \ "661:\n\t" \ oldinstr "\n" \ "662:\n" \ @@ -36,8 +53,92 @@ void free_alternatives_memory(void); newinstr "\n" \ "664:\n\t" \ ".popsection\n\t" \ - ".if ((664b-663b) != (662b-661b))\n\t" \ - " .error \"Alternatives instruction length mismatch\"\n\t"\ + ".org . - (664b-663b) + (662b-661b)\n\t" \ + ".org . - (662b-661b) + (664b-663b)\n" \ ".endif\n" +#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ + __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg)) + +#else + +.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len + .word \orig_offset - . + .word \alt_offset - . + .hword \feature + .byte \orig_len + .byte \alt_len +.endm + +.macro alternative_insn insn1, insn2, cap, enable = 1 + .if \enable +661: \insn1 +662: .pushsection .altinstructions, "a" + altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f + .popsection + .pushsection .altinstr_replacement, "ax" +663: \insn2 +664: .popsection + .org . - (664b-663b) + (662b-661b) + .org . - (662b-661b) + (664b-663b) + .endif +.endm + +/* + * Begin an alternative code sequence. + * + * The code that follows this macro will be assembled and linked as + * normal. There are no restrictions on this code. + */ +.macro alternative_if_not cap + .pushsection .altinstructions, "a" + altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f + .popsection +661: +.endm + +/* + * Provide the alternative code sequence. + * + * The code that follows this macro is assembled into a special + * section to be used for dynamic patching. Code that follows this + * macro must: + * + * 1. Be exactly the same length (in bytes) as the default code + * sequence. + * + * 2. Not contain a branch target that is used outside of the + * alternative sequence it is defined in (branches into an + * alternative sequence are not fixed up). + */ +.macro alternative_else +662: .pushsection .altinstr_replacement, "ax" +663: +.endm + +/* + * Complete an alternative code sequence. + */ +.macro alternative_endif +664: .popsection + .org . - (664b-663b) + (662b-661b) + .org . - (662b-661b) + (664b-663b) +.endm + +#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ + alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) + + +#endif /* __ASSEMBLY__ */ + +/* + * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature)); + * + * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO)); + * N.B. If CONFIG_FOO is specified, but not selected, the whole block + * will be omitted, including oldinstr. + */ +#define ALTERNATIVE(oldinstr, newinstr, ...) \ + _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1) + #endif /* __ASM_ALTERNATIVE_H */ diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 5901480bfdca..3579988b23f9 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -20,6 +20,9 @@ #error "Only include this from assembly code" #endif +#ifndef __ASM_ASSEMBLER_H +#define __ASM_ASSEMBLER_H + #include <asm/ptrace.h> #include <asm/thread_info.h> @@ -155,3 +158,64 @@ lr .req x30 // link register #endif orr \rd, \lbits, \hbits, lsl #32 .endm + +/* + * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where + * <symbol> is within the range +/- 4 GB of the PC. + */ + /* + * @dst: destination register (64 bit wide) + * @sym: name of the symbol + * @tmp: optional scratch register to be used if <dst> == sp, which + * is not allowed in an adrp instruction + */ + .macro adr_l, dst, sym, tmp= + .ifb \tmp + adrp \dst, \sym + add \dst, \dst, :lo12:\sym + .else + adrp \tmp, \sym + add \dst, \tmp, :lo12:\sym + .endif + .endm + + /* + * @dst: destination register (32 or 64 bit wide) + * @sym: name of the symbol + * @tmp: optional 64-bit scratch register to be used if <dst> is a + * 32-bit wide register, in which case it cannot be used to hold + * the address + */ + .macro ldr_l, dst, sym, tmp= + .ifb \tmp + adrp \dst, \sym + ldr \dst, [\dst, :lo12:\sym] + .else + adrp \tmp, \sym + ldr \dst, [\tmp, :lo12:\sym] + .endif + .endm + + /* + * @src: source register (32 or 64 bit wide) + * @sym: name of the symbol + * @tmp: mandatory 64-bit scratch register to calculate the address + * while <src> needs to be preserved. + */ + .macro str_l, src, sym, tmp + adrp \tmp, \sym + str \src, [\tmp, :lo12:\sym] + .endm + +/* + * Annotate a function as position independent, i.e., safe to be called before + * the kernel virtual mapping is activated. + */ +#define ENDPIPROC(x) \ + .globl __pi_##x; \ + .type __pi_##x, %function; \ + .set __pi_##x, x; \ + .size __pi_##x, . - x; \ + ENDPROC(x) + +#endif /* __ASM_ASSEMBLER_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 5fe4befda1a5..d71140b76773 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -24,11 +24,31 @@ #define ARM64_WORKAROUND_CLEAN_CACHE 0 #define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1 #define ARM64_WORKAROUND_845719 2 +#define ARM64_HAS_SYSREG_GIC_CPUIF 3 +#define ARM64_HAS_PAN 4 -#define ARM64_NCAPS 3 +#define ARM64_NCAPS 5 #ifndef __ASSEMBLY__ +struct arm64_cpu_capabilities { + const char *desc; + u16 capability; + bool (*matches)(const struct arm64_cpu_capabilities *); + void (*enable)(void); + union { + struct { /* To be used for erratum handling only */ + u32 midr_model; + u32 midr_range_min, midr_range_max; + }; + + struct { /* Feature register checking */ + int field_pos; + int min_field_value; + }; + }; +}; + extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); static inline bool cpu_have_feature(unsigned int num) @@ -52,7 +72,19 @@ static inline void cpus_set_cap(unsigned int num) __set_bit(num, cpu_hwcaps); } +static inline int __attribute_const__ cpuid_feature_extract_field(u64 features, + int field) +{ + return (s64)(features << (64 - 4 - field)) >> (64 - 4); +} + + +void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, + const char *info); void check_local_cpu_errata(void); +void check_local_cpu_features(void); +bool cpu_supports_mixed_endian_el0(void); +bool system_supports_mixed_endian_el0(void); #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 8adb986a3086..ee6403df9fe4 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -72,6 +72,15 @@ #define APM_CPU_PART_POTENZA 0x000 +#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16 +#define ID_AA64MMFR0_BIGENDEL0_MASK (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT) +#define ID_AA64MMFR0_BIGENDEL0(mmfr0) \ + (((mmfr0) & ID_AA64MMFR0_BIGENDEL0_MASK) >> ID_AA64MMFR0_BIGENDEL0_SHIFT) +#define ID_AA64MMFR0_BIGEND_SHIFT 8 +#define ID_AA64MMFR0_BIGEND_MASK (0xf << ID_AA64MMFR0_BIGEND_SHIFT) +#define ID_AA64MMFR0_BIGEND(mmfr0) \ + (((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT) + #ifndef __ASSEMBLY__ /* @@ -104,6 +113,11 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void) return read_cpuid(CTR_EL0); } +static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0) +{ + return (ID_AA64MMFR0_BIGEND(mmfr0) == 0x1) || + (ID_AA64MMFR0_BIGENDEL0(mmfr0) == 0x1); +} #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index 5f750dc96e0f..667346273d9b 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -20,10 +20,16 @@ #include <linux/futex.h> #include <linux/uaccess.h> + +#include <asm/alternative.h> +#include <asm/cpufeature.h> #include <asm/errno.h> +#include <asm/sysreg.h> #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \ asm volatile( \ + ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) \ "1: ldxr %w1, %2\n" \ insn "\n" \ "2: stlxr %w3, %w0, %2\n" \ @@ -39,6 +45,8 @@ " .align 3\n" \ " .quad 1b, 4b, 2b, 4b\n" \ " .popsection\n" \ + ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) \ : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \ : "r" (oparg), "Ir" (-EFAULT) \ : "memory") diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 56a9e63b6c33..e2ff32a93b5c 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -354,6 +354,16 @@ bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); int aarch64_insn_patch_text_nosync(void *addr, u32 insn); int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); + +bool aarch32_insn_is_wide(u32 insn); + +#define A32_RN_OFFSET 16 +#define A32_RT_OFFSET 12 +#define A32_RT2_OFFSET 0 + +u32 aarch32_insn_extract_reg_num(u32 insn, int offset); +u32 aarch32_insn_mcr_extract_opc2(u32 insn); +u32 aarch32_insn_mcr_extract_crm(u32 insn); #endif /* __ASSEMBLY__ */ #endif /* __ASM_INSN_H */ diff --git a/arch/arm64/include/asm/kasan.h b/arch/arm64/include/asm/kasan.h new file mode 100644 index 000000000000..2774fa384c47 --- /dev/null +++ b/arch/arm64/include/asm/kasan.h @@ -0,0 +1,38 @@ +#ifndef __ASM_KASAN_H +#define __ASM_KASAN_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_KASAN + +#include <linux/linkage.h> +#include <asm/memory.h> + +/* + * KASAN_SHADOW_START: beginning of the kernel virtual addresses. + * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/8 of kernel virtual addresses. + */ +#define KASAN_SHADOW_START (VA_START) +#define KASAN_SHADOW_END (KASAN_SHADOW_START + (1UL << (VA_BITS - 3))) + +/* + * This value is used to map an address to the corresponding shadow + * address by the following formula: + * shadow_addr = (address >> 3) + KASAN_SHADOW_OFFSET; + * + * (1 << 61) shadow addresses - [KASAN_SHADOW_OFFSET,KASAN_SHADOW_END] + * cover all 64-bits of virtual addresses. So KASAN_SHADOW_OFFSET + * should satisfy the following equation: + * KASAN_SHADOW_OFFSET = KASAN_SHADOW_END - (1ULL << 61) + */ +#define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << (64 - 3))) + +void kasan_init(void); +asmlinkage void kasan_early_init(void); + +#else +static inline void kasan_init(void) { } +#endif + +#endif +#endif diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index a62cd077457b..e4a2ef9cf998 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -36,12 +36,14 @@ * PAGE_OFFSET - the virtual address of the start of the kernel image (top * (VA_BITS - 1)) * VA_BITS - the maximum number of bits for virtual addresses. + * VA_START - the first kernel virtual address. * TASK_SIZE - the maximum size of a user space task. * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area. * The module space lives between the addresses given by TASK_SIZE * and PAGE_OFFSET - it must be within 128MB of the kernel text. */ #define VA_BITS (CONFIG_ARM64_VA_BITS) +#define VA_START (UL(0xffffffffffffffff) << VA_BITS) #define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1)) #define MODULES_END (PAGE_OFFSET) #define MODULES_VADDR (MODULES_END - SZ_64M) diff --git a/arch/arm64/include/asm/opcodes.h b/arch/arm64/include/asm/opcodes.h new file mode 100644 index 000000000000..4e603ea36ad3 --- /dev/null +++ b/arch/arm64/include/asm/opcodes.h @@ -0,0 +1 @@ +#include <../../arm/include/asm/opcodes.h> diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index d5bed02073d6..e838b9adc4d6 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -26,11 +26,14 @@ #define check_pgt_cache() do { } while (0) +#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) +#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) + #if CONFIG_ARM64_PGTABLE_LEVELS > 2 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); + return (pmd_t *)__get_free_page(PGALLOC_GFP); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) @@ -50,7 +53,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); + return (pud_t *)__get_free_page(PGALLOC_GFP); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) @@ -69,8 +72,6 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) extern pgd_t *pgd_alloc(struct mm_struct *mm); extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); -#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) - static inline pte_t * pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) { diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 1dcbde6036a8..239192d72a7b 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -41,7 +41,14 @@ * fixed mappings and modules */ #define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE) -#define VMALLOC_START (UL(0xffffffffffffffff) << VA_BITS) + +#ifndef CONFIG_KASAN +#define VMALLOC_START (VA_START) +#else +#include <asm/kasan.h> +#define VMALLOC_START (KASAN_SHADOW_END + SZ_64K) +#endif + #define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K) #define VMEMMAP_START (VMALLOC_END + SZ_64K) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 286b1bec547c..f3a965ed5259 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -166,4 +166,6 @@ static inline void spin_lock_prefetch(const void *x) #endif +void cpu_enable_pan(void); + #endif /* __ASM_PROCESSOR_H */ diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 41ed9e13795e..d6dd9fdbc3be 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -58,6 +58,13 @@ #define COMPAT_PSR_Z_BIT 0x40000000 #define COMPAT_PSR_N_BIT 0x80000000 #define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */ + +#ifdef CONFIG_CPU_BIG_ENDIAN +#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT +#else +#define COMPAT_PSR_ENDSTATE 0 +#endif + /* * These are 'magic' values for PTRACE_PEEKUSR that return info about where a * process is located in memory. diff --git a/arch/arm64/include/asm/string.h b/arch/arm64/include/asm/string.h index 64d2d4884a9d..2eb714c4639f 100644 --- a/arch/arm64/include/asm/string.h +++ b/arch/arm64/include/asm/string.h @@ -36,17 +36,33 @@ extern __kernel_size_t strnlen(const char *, __kernel_size_t); #define __HAVE_ARCH_MEMCPY extern void *memcpy(void *, const void *, __kernel_size_t); +extern void *__memcpy(void *, const void *, __kernel_size_t); #define __HAVE_ARCH_MEMMOVE extern void *memmove(void *, const void *, __kernel_size_t); +extern void *__memmove(void *, const void *, __kernel_size_t); #define __HAVE_ARCH_MEMCHR extern void *memchr(const void *, int, __kernel_size_t); #define __HAVE_ARCH_MEMSET extern void *memset(void *, int, __kernel_size_t); +extern void *__memset(void *, int, __kernel_size_t); #define __HAVE_ARCH_MEMCMP extern int memcmp(const void *, const void *, size_t); + +#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) + +/* + * For files that are not instrumented (e.g. mm/slub.c) we + * should use not instrumented version of mem* functions. + */ + +#define memcpy(dst, src, len) __memcpy(dst, src, len) +#define memmove(dst, src, len) __memmove(dst, src, len) +#define memset(s, c, n) __memset(s, c, n) +#endif + #endif diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 5c89df0acbcb..a7f3d4b2514d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -20,8 +20,29 @@ #ifndef __ASM_SYSREG_H #define __ASM_SYSREG_H +#include <asm/opcodes.h> + +#define SCTLR_EL1_CP15BEN (0x1 << 5) +#define SCTLR_EL1_SED (0x1 << 8) + +/* + * ARMv8 ARM reserves the following encoding for system registers: + * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview", + * C5.2, version:ARM DDI 0487A.f) + * [20-19] : Op0 + * [18-16] : Op1 + * [15-12] : CRn + * [11-8] : CRm + * [7-5] : Op2 + */ #define sys_reg(op0, op1, crn, crm, op2) \ - ((((op0)-2)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5)) + ((((op0)&3)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5)) + +#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4) +#define SCTLR_EL1_SPAN (1 << 23) + +#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\ + (!!x)<<8 | 0x1f) #ifdef __ASSEMBLY__ @@ -31,11 +52,11 @@ .equ __reg_num_xzr, 31 .macro mrs_s, rt, sreg - .inst 0xd5300000|(\sreg)|(__reg_num_\rt) + .inst 0xd5200000|(\sreg)|(__reg_num_\rt) .endm .macro msr_s, sreg, rt - .inst 0xd5100000|(\sreg)|(__reg_num_\rt) + .inst 0xd5000000|(\sreg)|(__reg_num_\rt) .endm #else @@ -47,14 +68,23 @@ asm( " .equ __reg_num_xzr, 31\n" "\n" " .macro mrs_s, rt, sreg\n" -" .inst 0xd5300000|(\\sreg)|(__reg_num_\\rt)\n" +" .inst 0xd5200000|(\\sreg)|(__reg_num_\\rt)\n" " .endm\n" "\n" " .macro msr_s, sreg, rt\n" -" .inst 0xd5100000|(\\sreg)|(__reg_num_\\rt)\n" +" .inst 0xd5000000|(\\sreg)|(__reg_num_\\rt)\n" " .endm\n" ); +static inline void config_sctlr_el1(u32 clear, u32 set) +{ + u32 val; + + asm volatile("mrs %0, sctlr_el1" : "=r" (val)); + val &= ~clear; + val |= set; + asm volatile("msr sctlr_el1, %0" : : "r" (val)); +} #endif #endif /* __ASM_SYSREG_H */ diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h index 10ca8ff93cc2..232e4ba5d314 100644 --- a/arch/arm64/include/asm/traps.h +++ b/arch/arm64/include/asm/traps.h @@ -18,6 +18,22 @@ #ifndef __ASM_TRAP_H #define __ASM_TRAP_H +#include <linux/list.h> + +struct pt_regs; + +struct undef_hook { + struct list_head node; + u32 instr_mask; + u32 instr_val; + u64 pstate_mask; + u64 pstate_val; + int (*fn)(struct pt_regs *regs, u32 instr); +}; + +void register_undef_hook(struct undef_hook *hook); +void unregister_undef_hook(struct undef_hook *hook); + static inline int in_exception_text(unsigned long ptr) { extern char __exception_text_start[]; diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 6f752728b4c1..14450f742554 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -25,7 +25,10 @@ #include <linux/string.h> #include <linux/thread_info.h> +#include <asm/alternative.h> +#include <asm/cpufeature.h> #include <asm/ptrace.h> +#include <asm/sysreg.h> #include <asm/errno.h> #include <asm/memory.h> #include <asm/compiler.h> @@ -140,6 +143,8 @@ static inline void set_fs(mm_segment_t fs) do { \ unsigned long __gu_val; \ __chk_user_ptr(ptr); \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN)); \ switch (sizeof(*(ptr))) { \ case 1: \ __get_user_asm("ldrb", "%w", __gu_val, (ptr), (err)); \ @@ -156,7 +161,9 @@ do { \ default: \ BUILD_BUG(); \ } \ - (x) = (__typeof__(*(ptr)))__gu_val; \ + (x) = (__force __typeof__(*(ptr)))__gu_val; \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN)); \ } while (0) #define __get_user(x, ptr) \ @@ -203,6 +210,8 @@ do { \ do { \ __typeof__(*(ptr)) __pu_val = (x); \ __chk_user_ptr(ptr); \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN)); \ switch (sizeof(*(ptr))) { \ case 1: \ __put_user_asm("strb", "%w", __pu_val, (ptr), (err)); \ @@ -219,6 +228,8 @@ do { \ default: \ BUILD_BUG(); \ } \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN)); \ } while (0) #define __put_user(x, ptr) \ diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h index 6913643bbe54..208db3df135a 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -44,6 +44,7 @@ #define PSR_I_BIT 0x00000080 #define PSR_A_BIT 0x00000100 #define PSR_D_BIT 0x00000200 +#define PSR_PAN_BIT 0x00400000 #define PSR_Q_BIT 0x08000000 #define PSR_V_BIT 0x10000000 #define PSR_C_BIT 0x20000000 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 33cba01d4f33..fd65134d1053 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -5,6 +5,9 @@ CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) CFLAGS_efi-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET) +CFLAGS_armv8_deprecated.o := -I$(src) + +KASAN_SANITIZE_efi-stub.o := n CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_insn.o = -pg @@ -14,11 +17,12 @@ CFLAGS_REMOVE_return_address.o = -pg arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \ - cpuinfo.o cpu_errata.o alternative.o psci-call.o + hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o \ + return_address.o cpuinfo.o cpu_errata.o cpufeature.o alternative.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ - sys_compat.o + sys_compat.o \ + ../../arm/kernel/opcodes.o arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o @@ -31,6 +35,7 @@ arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o arm64-obj-$(CONFIG_KGDB) += kgdb.o arm64-obj-$(CONFIG_EFI) += efi.o efi-stub.o efi-entry.o arm64-obj-$(CONFIG_PCI) += pci.o +arm64-obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o obj-y += $(arm64-obj-y) vdso/ obj-m += $(arm64-obj-m) diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index a85843ddbde8..3b6d8cc9dfe0 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -51,6 +51,9 @@ EXPORT_SYMBOL(strnlen); EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memcpy); EXPORT_SYMBOL(memmove); +EXPORT_SYMBOL(__memset); +EXPORT_SYMBOL(__memcpy); +EXPORT_SYMBOL(__memmove); EXPORT_SYMBOL(memchr); EXPORT_SYMBOL(memcmp); diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c new file mode 100644 index 000000000000..bcee7abac68e --- /dev/null +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -0,0 +1,659 @@ +/* + * Copyright (C) 2014 ARM Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/cpu.h> +#include <linux/init.h> +#include <linux/list.h> +#include <linux/perf_event.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/sysctl.h> + +#include <asm/alternative.h> +#include <asm/cpufeature.h> +#include <asm/insn.h> +#include <asm/opcodes.h> +#include <asm/sysreg.h> +#include <asm/system_misc.h> +#include <asm/traps.h> +#include <asm/uaccess.h> +#include <asm/cpufeature.h> + +#define CREATE_TRACE_POINTS +#include "trace-events-emulation.h" + +/* + * The runtime support for deprecated instruction support can be in one of + * following three states - + * + * 0 = undef + * 1 = emulate (software emulation) + * 2 = hw (supported in hardware) + */ +enum insn_emulation_mode { + INSN_UNDEF, + INSN_EMULATE, + INSN_HW, +}; + +enum legacy_insn_status { + INSN_DEPRECATED, + INSN_OBSOLETE, +}; + +struct insn_emulation_ops { + const char *name; + enum legacy_insn_status status; + struct undef_hook *hooks; + int (*set_hw_mode)(bool enable); +}; + +struct insn_emulation { + struct list_head node; + struct insn_emulation_ops *ops; + int current_mode; + int min; + int max; +}; + +static LIST_HEAD(insn_emulation); +static int nr_insn_emulated; +static DEFINE_RAW_SPINLOCK(insn_emulation_lock); + +static void register_emulation_hooks(struct insn_emulation_ops *ops) +{ + struct undef_hook *hook; + + BUG_ON(!ops->hooks); + + for (hook = ops->hooks; hook->instr_mask; hook++) + register_undef_hook(hook); + + pr_notice("Registered %s emulation handler\n", ops->name); +} + +static void remove_emulation_hooks(struct insn_emulation_ops *ops) +{ + struct undef_hook *hook; + + BUG_ON(!ops->hooks); + + for (hook = ops->hooks; hook->instr_mask; hook++) + unregister_undef_hook(hook); + + pr_notice("Removed %s emulation handler\n", ops->name); +} + +static void enable_insn_hw_mode(void *data) +{ + struct insn_emulation *insn = (struct insn_emulation *)data; + if (insn->ops->set_hw_mode) + insn->ops->set_hw_mode(true); +} + +static void disable_insn_hw_mode(void *data) +{ + struct insn_emulation *insn = (struct insn_emulation *)data; + if (insn->ops->set_hw_mode) + insn->ops->set_hw_mode(false); +} + +/* Run set_hw_mode(mode) on all active CPUs */ +static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable) +{ + if (!insn->ops->set_hw_mode) + return -EINVAL; + if (enable) + on_each_cpu(enable_insn_hw_mode, (void *)insn, true); + else + on_each_cpu(disable_insn_hw_mode, (void *)insn, true); + return 0; +} + +/* + * Run set_hw_mode for all insns on a starting CPU. + * Returns: + * 0 - If all the hooks ran successfully. + * -EINVAL - At least one hook is not supported by the CPU. + */ +static int run_all_insn_set_hw_mode(unsigned long cpu) +{ + int rc = 0; + unsigned long flags; + struct insn_emulation *insn; + + raw_spin_lock_irqsave(&insn_emulation_lock, flags); + list_for_each_entry(insn, &insn_emulation, node) { + bool enable = (insn->current_mode == INSN_HW); + if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) { + pr_warn("CPU[%ld] cannot support the emulation of %s", + cpu, insn->ops->name); + rc = -EINVAL; + } + } + raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); + return rc; +} + +static int update_insn_emulation_mode(struct insn_emulation *insn, + enum insn_emulation_mode prev) +{ + int ret = 0; + + switch (prev) { + case INSN_UNDEF: /* Nothing to be done */ + break; + case INSN_EMULATE: + remove_emulation_hooks(insn->ops); + break; + case INSN_HW: + if (!run_all_cpu_set_hw_mode(insn, false)) + pr_notice("Disabled %s support\n", insn->ops->name); + break; + } + + switch (insn->current_mode) { + case INSN_UNDEF: + break; + case INSN_EMULATE: + register_emulation_hooks(insn->ops); + break; + case INSN_HW: + ret = run_all_cpu_set_hw_mode(insn, true); + if (!ret) + pr_notice("Enabled %s support\n", insn->ops->name); + break; + } + + return ret; +} + +static void register_insn_emulation(struct insn_emulation_ops *ops) +{ + unsigned long flags; + struct insn_emulation *insn; + + insn = kzalloc(sizeof(*insn), GFP_KERNEL); + insn->ops = ops; + insn->min = INSN_UNDEF; + + switch (ops->status) { + case INSN_DEPRECATED: + insn->current_mode = INSN_EMULATE; + /* Disable the HW mode if it was turned on at early boot time */ + run_all_cpu_set_hw_mode(insn, false); + insn->max = INSN_HW; + break; + case INSN_OBSOLETE: + insn->current_mode = INSN_UNDEF; + insn->max = INSN_EMULATE; + break; + } + + raw_spin_lock_irqsave(&insn_emulation_lock, flags); + list_add(&insn->node, &insn_emulation); + nr_insn_emulated++; + raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); + + /* Register any handlers if required */ + update_insn_emulation_mode(insn, INSN_UNDEF); +} + +static int emulation_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret = 0; + struct insn_emulation *insn = (struct insn_emulation *) table->data; + enum insn_emulation_mode prev_mode = insn->current_mode; + + table->data = &insn->current_mode; + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (ret || !write || prev_mode == insn->current_mode) + goto ret; + + ret = update_insn_emulation_mode(insn, prev_mode); + if (ret) { + /* Mode change failed, revert to previous mode. */ + insn->current_mode = prev_mode; + update_insn_emulation_mode(insn, INSN_UNDEF); + } +ret: + table->data = insn; + return ret; +} + +static struct ctl_table ctl_abi[] = { + { + .procname = "abi", + .mode = 0555, + }, + { } +}; + +static void register_insn_emulation_sysctl(struct ctl_table *table) +{ + unsigned long flags; + int i = 0; + struct insn_emulation *insn; + struct ctl_table *insns_sysctl, *sysctl; + + insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1), + GFP_KERNEL); + + raw_spin_lock_irqsave(&insn_emulation_lock, flags); + list_for_each_entry(insn, &insn_emulation, node) { + sysctl = &insns_sysctl[i]; + + sysctl->mode = 0644; + sysctl->maxlen = sizeof(int); + + sysctl->procname = insn->ops->name; + sysctl->data = insn; + sysctl->extra1 = &insn->min; + sysctl->extra2 = &insn->max; + sysctl->proc_handler = emulation_proc_handler; + i++; + } + raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); + + table->child = insns_sysctl; + register_sysctl_table(table); +} + +/* + * Implement emulation of the SWP/SWPB instructions using load-exclusive and + * store-exclusive. + * + * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>] + * Where: Rt = destination + * Rt2 = source + * Rn = address + */ + +/* + * Error-checking SWP macros implemented using ldxr{b}/stxr{b} + */ +#define __user_swpX_asm(data, addr, res, temp, B) \ + __asm__ __volatile__( \ + ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) \ + " mov %w2, %w1\n" \ + "0: ldxr"B" %w1, [%3]\n" \ + "1: stxr"B" %w0, %w2, [%3]\n" \ + " cbz %w0, 2f\n" \ + " mov %w0, %w4\n" \ + "2:\n" \ + " .pushsection .fixup,\"ax\"\n" \ + " .align 2\n" \ + "3: mov %w0, %w5\n" \ + " b 2b\n" \ + " .popsection" \ + " .pushsection __ex_table,\"a\"\n" \ + " .align 3\n" \ + " .quad 0b, 3b\n" \ + " .quad 1b, 3b\n" \ + " .popsection\n" \ + ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) \ + : "=&r" (res), "+r" (data), "=&r" (temp) \ + : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ + : "memory") + +#define __user_swp_asm(data, addr, res, temp) \ + __user_swpX_asm(data, addr, res, temp, "") +#define __user_swpb_asm(data, addr, res, temp) \ + __user_swpX_asm(data, addr, res, temp, "b") + +/* + * Bit 22 of the instruction encoding distinguishes between + * the SWP and SWPB variants (bit set means SWPB). + */ +#define TYPE_SWPB (1 << 22) + +/* + * Set up process info to signal segmentation fault - called on access error. + */ +static void set_segfault(struct pt_regs *regs, unsigned long addr) +{ + siginfo_t info; + + down_read(¤t->mm->mmap_sem); + if (find_vma(current->mm, addr) == NULL) + info.si_code = SEGV_MAPERR; + else + info.si_code = SEGV_ACCERR; + up_read(¤t->mm->mmap_sem); + + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void *) instruction_pointer(regs); + + pr_debug("SWP{B} emulation: access caused memory abort!\n"); + arm64_notify_die("Illegal memory access", regs, &info, 0); +} + +static int emulate_swpX(unsigned int address, unsigned int *data, + unsigned int type) +{ + unsigned int res = 0; + + if ((type != TYPE_SWPB) && (address & 0x3)) { + /* SWP to unaligned address not permitted */ + pr_debug("SWP instruction on unaligned pointer!\n"); + return -EFAULT; + } + + while (1) { + unsigned long temp; + + if (type == TYPE_SWPB) + __user_swpb_asm(*data, address, res, temp); + else + __user_swp_asm(*data, address, res, temp); + + if (likely(res != -EAGAIN) || signal_pending(current)) + break; + + cond_resched(); + } + + return res; +} + +/* + * swp_handler logs the id of calling process, dissects the instruction, sanity + * checks the memory location, calls emulate_swpX for the actual operation and + * deals with fixup/error handling before returning + */ +static int swp_handler(struct pt_regs *regs, u32 instr) +{ + u32 destreg, data, type, address = 0; + int rn, rt2, res = 0; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); + + type = instr & TYPE_SWPB; + + switch (arm_check_condition(instr, regs->pstate)) { + case ARM_OPCODE_CONDTEST_PASS: + break; + case ARM_OPCODE_CONDTEST_FAIL: + /* Condition failed - return to next instruction */ + goto ret; + case ARM_OPCODE_CONDTEST_UNCOND: + /* If unconditional encoding - not a SWP, undef */ + return -EFAULT; + default: + return -EINVAL; + } + + rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET); + rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET); + + address = (u32)regs->user_regs.regs[rn]; + data = (u32)regs->user_regs.regs[rt2]; + destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET); + + pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n", + rn, address, destreg, + aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data); + + /* Check access in reasonable access range for both SWP and SWPB */ + if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { + pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n", + address); + goto fault; + } + + res = emulate_swpX(address, &data, type); + if (res == -EFAULT) + goto fault; + else if (res == 0) + regs->user_regs.regs[destreg] = data; + +ret: + if (type == TYPE_SWPB) + trace_instruction_emulation("swpb", regs->pc); + else + trace_instruction_emulation("swp", regs->pc); + + pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n", + current->comm, (unsigned long)current->pid, regs->pc); + + regs->pc += 4; + return 0; + +fault: + set_segfault(regs, address); + + return 0; +} + +/* + * Only emulate SWP/SWPB executed in ARM state/User mode. + * The kernel must be SWP free and SWP{B} does not exist in Thumb. + */ +static struct undef_hook swp_hooks[] = { + { + .instr_mask = 0x0fb00ff0, + .instr_val = 0x01000090, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = swp_handler + }, + { } +}; + +static struct insn_emulation_ops swp_ops = { + .name = "swp", + .status = INSN_OBSOLETE, + .hooks = swp_hooks, + .set_hw_mode = NULL, +}; + +static int cp15barrier_handler(struct pt_regs *regs, u32 instr) +{ + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); + + switch (arm_check_condition(instr, regs->pstate)) { + case ARM_OPCODE_CONDTEST_PASS: + break; + case ARM_OPCODE_CONDTEST_FAIL: + /* Condition failed - return to next instruction */ + goto ret; + case ARM_OPCODE_CONDTEST_UNCOND: + /* If unconditional encoding - not a barrier instruction */ + return -EFAULT; + default: + return -EINVAL; + } + + switch (aarch32_insn_mcr_extract_crm(instr)) { + case 10: + /* + * dmb - mcr p15, 0, Rt, c7, c10, 5 + * dsb - mcr p15, 0, Rt, c7, c10, 4 + */ + if (aarch32_insn_mcr_extract_opc2(instr) == 5) { + dmb(sy); + trace_instruction_emulation( + "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc); + } else { + dsb(sy); + trace_instruction_emulation( + "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc); + } + break; + case 5: + /* + * isb - mcr p15, 0, Rt, c7, c5, 4 + * + * Taking an exception or returning from one acts as an + * instruction barrier. So no explicit barrier needed here. + */ + trace_instruction_emulation( + "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc); + break; + } + +ret: + pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n", + current->comm, (unsigned long)current->pid, regs->pc); + + regs->pc += 4; + return 0; +} + +static int cp15_barrier_set_hw_mode(bool enable) +{ + if (enable) + config_sctlr_el1(0, SCTLR_EL1_CP15BEN); + else + config_sctlr_el1(SCTLR_EL1_CP15BEN, 0); + return 0; +} + +static struct undef_hook cp15_barrier_hooks[] = { + { + .instr_mask = 0x0fff0fdf, + .instr_val = 0x0e070f9a, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = cp15barrier_handler, + }, + { + .instr_mask = 0x0fff0fff, + .instr_val = 0x0e070f95, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = cp15barrier_handler, + }, + { } +}; + +static struct insn_emulation_ops cp15_barrier_ops = { + .name = "cp15_barrier", + .status = INSN_DEPRECATED, + .hooks = cp15_barrier_hooks, + .set_hw_mode = cp15_barrier_set_hw_mode, +}; + +static int setend_set_hw_mode(bool enable) +{ + if (!cpu_supports_mixed_endian_el0()) + return -EINVAL; + + if (enable) + config_sctlr_el1(SCTLR_EL1_SED, 0); + else + config_sctlr_el1(0, SCTLR_EL1_SED); + return 0; +} + +static int compat_setend_handler(struct pt_regs *regs, u32 big_endian) +{ + char *insn; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); + + if (big_endian) { + insn = "setend be"; + regs->pstate |= COMPAT_PSR_E_BIT; + } else { + insn = "setend le"; + regs->pstate &= ~COMPAT_PSR_E_BIT; + } + + trace_instruction_emulation(insn, regs->pc); + pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n", + current->comm, (unsigned long)current->pid, regs->pc); + + return 0; +} + +static int a32_setend_handler(struct pt_regs *regs, u32 instr) +{ + int rc = compat_setend_handler(regs, (instr >> 9) & 1); + regs->pc += 4; + return rc; +} + +static int t16_setend_handler(struct pt_regs *regs, u32 instr) +{ + int rc = compat_setend_handler(regs, (instr >> 3) & 1); + regs->pc += 2; + return rc; +} + +static struct undef_hook setend_hooks[] = { + { + .instr_mask = 0xfffffdff, + .instr_val = 0xf1010000, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = a32_setend_handler, + }, + { + /* Thumb mode */ + .instr_mask = 0x0000fff7, + .instr_val = 0x0000b650, + .pstate_mask = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_MASK), + .pstate_val = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_USR), + .fn = t16_setend_handler, + }, + {} +}; + +static struct insn_emulation_ops setend_ops = { + .name = "setend", + .status = INSN_DEPRECATED, + .hooks = setend_hooks, + .set_hw_mode = setend_set_hw_mode, +}; + +static int insn_cpu_hotplug_notify(struct notifier_block *b, + unsigned long action, void *hcpu) +{ + int rc = 0; + if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING) + rc = run_all_insn_set_hw_mode((unsigned long)hcpu); + + return notifier_from_errno(rc); +} + +static struct notifier_block insn_cpu_hotplug_notifier = { + .notifier_call = insn_cpu_hotplug_notify, +}; + +/* + * Invoked as late_initcall, since not needed before init spawned. + */ +static int __init armv8_deprecated_init(void) +{ + if (IS_ENABLED(CONFIG_SWP_EMULATION)) + register_insn_emulation(&swp_ops); + + if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) + register_insn_emulation(&cp15_barrier_ops); + + if (IS_ENABLED(CONFIG_SETEND_EMULATION)) { + if(system_supports_mixed_endian_el0()) + register_insn_emulation(&setend_ops); + else + pr_info("setend instruction emulation is not supported on the system"); + } + + register_cpu_notifier(&insn_cpu_hotplug_notifier); + register_insn_emulation_sysctl(ctl_abi); + + return 0; +} + +late_initcall(armv8_deprecated_init); diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index bbc710aafb37..6ffd91438560 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -16,8 +16,6 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#define pr_fmt(fmt) "alternative: " fmt - #include <linux/types.h> #include <asm/cpu.h> #include <asm/cputype.h> @@ -26,27 +24,11 @@ #define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) #define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) -/* - * Add a struct or another datatype to the union below if you need - * different means to detect an affected CPU. - */ -struct arm64_cpu_capabilities { - const char *desc; - u16 capability; - bool (*is_affected)(struct arm64_cpu_capabilities *); - union { - struct { - u32 midr_model; - u32 midr_range_min, midr_range_max; - }; - }; -}; - #define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \ MIDR_ARCHITECTURE_MASK) static bool __maybe_unused -is_affected_midr_range(struct arm64_cpu_capabilities *entry) +is_affected_midr_range(const struct arm64_cpu_capabilities *entry) { u32 midr = read_cpuid_id(); @@ -59,12 +41,12 @@ is_affected_midr_range(struct arm64_cpu_capabilities *entry) } #define MIDR_RANGE(model, min, max) \ - .is_affected = is_affected_midr_range, \ + .matches = is_affected_midr_range, \ .midr_model = model, \ .midr_range_min = min, \ .midr_range_max = max -struct arm64_cpu_capabilities arm64_errata[] = { +const struct arm64_cpu_capabilities arm64_errata[] = { #if defined(CONFIG_ARM64_ERRATUM_826319) || \ defined(CONFIG_ARM64_ERRATUM_827319) || \ defined(CONFIG_ARM64_ERRATUM_824069) @@ -106,15 +88,5 @@ struct arm64_cpu_capabilities arm64_errata[] = { void check_local_cpu_errata(void) { - struct arm64_cpu_capabilities *cpus = arm64_errata; - int i; - - for (i = 0; cpus[i].desc; i++) { - if (!cpus[i].is_affected(&cpus[i])) - continue; - - if (!cpus_have_cap(cpus[i].capability)) - pr_info("enabling workaround for %s\n", cpus[i].desc); - cpus_set_cap(cpus[i].capability); - } + check_cpu_capabilities(arm64_errata, "enabling workaround for"); } diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c new file mode 100644 index 000000000000..978fa169d3c3 --- /dev/null +++ b/arch/arm64/kernel/cpufeature.c @@ -0,0 +1,97 @@ +/* + * Contains CPU feature definitions + * + * Copyright (C) 2015 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) "alternatives: " fmt + +#include <linux/types.h> +#include <asm/cpu.h> +#include <asm/cpufeature.h> +#include <asm/processor.h> + +static bool +feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) +{ + int val = cpuid_feature_extract_field(reg, entry->field_pos); + + return val >= entry->min_field_value; +} + +static bool +has_id_aa64pfr0_feature(const struct arm64_cpu_capabilities *entry) +{ + u64 val; + + val = read_cpuid(id_aa64pfr0_el1); + return feature_matches(val, entry); +} + +static bool __maybe_unused +has_id_aa64mmfr1_feature(const struct arm64_cpu_capabilities *entry) +{ + u64 val; + + val = read_cpuid(id_aa64mmfr1_el1); + return feature_matches(val, entry); +} + +static const struct arm64_cpu_capabilities arm64_features[] = { + { + .desc = "GIC system register CPU interface", + .capability = ARM64_HAS_SYSREG_GIC_CPUIF, + .matches = has_id_aa64pfr0_feature, + .field_pos = 24, + .min_field_value = 1, + }, +#ifdef CONFIG_ARM64_PAN + { + .desc = "Privileged Access Never", + .capability = ARM64_HAS_PAN, + .matches = has_id_aa64mmfr1_feature, + .field_pos = 20, + .min_field_value = 1, + .enable = cpu_enable_pan, + }, +#endif /* CONFIG_ARM64_PAN */ + {}, +}; + +void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, + const char *info) +{ + int i; + + for (i = 0; caps[i].desc; i++) { + if (!caps[i].matches(&caps[i])) + continue; + + if (!cpus_have_cap(caps[i].capability)) + pr_info("%s %s\n", info, caps[i].desc); + cpus_set_cap(caps[i].capability); + } + + /* second pass allows enable() to consider interacting capabilities */ + for (i = 0; caps[i].desc; i++) { + if (cpus_have_cap(caps[i].capability) && caps[i].enable) + caps[i].enable(); + } +} + +void check_local_cpu_features(void) +{ + check_cpu_capabilities(arm64_features, "detected feature"); +} diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 16d6d032ecf1..faf5cadbd391 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -35,6 +35,7 @@ */ DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data); static struct cpuinfo_arm64 boot_cpu_data; +static bool mixed_endian_el0 = true; static char *icache_policy_str[] = { [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN", @@ -68,6 +69,26 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); } +bool cpu_supports_mixed_endian_el0(void) +{ + return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); +} + +bool system_supports_mixed_endian_el0(void) +{ + return mixed_endian_el0; +} + +static void update_mixed_endian_el0_support(struct cpuinfo_arm64 *info) +{ + mixed_endian_el0 &= id_aa64mmfr0_mixed_endian_el0(info->reg_id_aa64mmfr0); +} + +static void update_cpu_features(struct cpuinfo_arm64 *info) +{ + update_mixed_endian_el0_support(info); +} + static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu) { if ((boot & mask) == (cur & mask)) @@ -189,6 +210,8 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) cpuinfo_detect_icache_policy(info); check_local_cpu_errata(); + check_local_cpu_features(); + update_cpu_features(info); } void cpuinfo_store_cpu(void) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index b6bb97026fdd..075e42e74cc9 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -21,7 +21,7 @@ #include <linux/init.h> #include <linux/linkage.h> -#include <asm/alternative-asm.h> +#include <asm/alternative.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/cpufeature.h> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 5c4b8d6e8ba0..4f9a2b83dc8e 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -238,7 +238,13 @@ ENTRY(stext) mov x0, x22 bl lookup_processor_type mov x23, x0 // x23=current cpu_table - cbz x23, __error_p // invalid processor (x23=0)? + /* + * __error_p may end up out of range for cbz if text areas are + * aligned up to section sizes. + */ + cbnz x23, 1f // invalid processor (x23=0)? + b __error_p +1: bl __vet_fdt bl __create_page_tables // x25=TTBR0, x26=TTBR1 /* @@ -250,13 +256,217 @@ ENTRY(stext) */ ldr x27, __switch_data // address to jump to after // MMU has been enabled - adr lr, __enable_mmu // return (PIC) address + adrp lr, __enable_mmu // return (PIC) address + add lr, lr, #:lo12:__enable_mmu ldr x12, [x23, #CPU_INFO_SETUP] add x12, x12, x28 // __virt_to_phys br x12 // initialise processor ENDPROC(stext) /* + * Determine validity of the x21 FDT pointer. + * The dtb must be 8-byte aligned and live in the first 512M of memory. + */ +__vet_fdt: + tst x21, #0x7 + b.ne 1f + cmp x21, x24 + b.lt 1f + mov x0, #(1 << 29) + add x0, x0, x24 + cmp x21, x0 + b.ge 1f + ret +1: + mov x21, #0 + ret +ENDPROC(__vet_fdt) +/* + * Macro to create a table entry to the next page. + * + * tbl: page table address + * virt: virtual address + * shift: #imm page table shift + * ptrs: #imm pointers per table page + * + * Preserves: virt + * Corrupts: tmp1, tmp2 + * Returns: tbl -> next level table page address + */ + .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2 + lsr \tmp1, \virt, #\shift + and \tmp1, \tmp1, #\ptrs - 1 // table index + add \tmp2, \tbl, #PAGE_SIZE + orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type + str \tmp2, [\tbl, \tmp1, lsl #3] + add \tbl, \tbl, #PAGE_SIZE // next level table page + .endm + +/* + * Macro to populate the PGD (and possibily PUD) for the corresponding + * block entry in the next level (tbl) for the given virtual address. + * + * Preserves: tbl, next, virt + * Corrupts: tmp1, tmp2 + */ + .macro create_pgd_entry, tbl, virt, tmp1, tmp2 + create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2 +#if SWAPPER_PGTABLE_LEVELS == 3 + create_table_entry \tbl, \virt, TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2 +#endif + .endm + +/* + * Macro to populate block entries in the page table for the start..end + * virtual range (inclusive). + * + * Preserves: tbl, flags + * Corrupts: phys, start, end, pstate + */ + .macro create_block_map, tbl, flags, phys, start, end + lsr \phys, \phys, #BLOCK_SHIFT + lsr \start, \start, #BLOCK_SHIFT + and \start, \start, #PTRS_PER_PTE - 1 // table index + orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry + lsr \end, \end, #BLOCK_SHIFT + and \end, \end, #PTRS_PER_PTE - 1 // table end index +9999: str \phys, [\tbl, \start, lsl #3] // store the entry + add \start, \start, #1 // next entry + add \phys, \phys, #BLOCK_SIZE // next block + cmp \start, \end + b.ls 9999b + .endm + +/* + * Setup the initial page tables. We only setup the barest amount which is + * required to get the kernel running. The following sections are required: + * - identity mapping to enable the MMU (low address, TTBR0) + * - first few MB of the kernel linear mapping to jump to once the MMU has + * been enabled, including the FDT blob (TTBR1) + * - pgd entry for fixed mappings (TTBR1) + */ +__create_page_tables: + pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses + mov x27, lr + + /* + * Invalidate the idmap and swapper page tables to avoid potential + * dirty cache lines being evicted. + */ + mov x0, x25 + add x1, x26, #SWAPPER_DIR_SIZE + bl __inval_cache_range + + /* + * Clear the idmap and swapper page tables. + */ + mov x0, x25 + add x6, x26, #SWAPPER_DIR_SIZE +1: stp xzr, xzr, [x0], #16 + stp xzr, xzr, [x0], #16 + stp xzr, xzr, [x0], #16 + stp xzr, xzr, [x0], #16 + cmp x0, x6 + b.lo 1b + + ldr x7, =MM_MMUFLAGS + + /* + * Create the identity mapping. + */ + mov x0, x25 // idmap_pg_dir + ldr x3, =KERNEL_START + add x3, x3, x28 // __pa(KERNEL_START) + create_pgd_entry x0, x3, x5, x6 + ldr x6, =KERNEL_END + mov x5, x3 // __pa(KERNEL_START) + add x6, x6, x28 // __pa(KERNEL_END) + create_block_map x0, x7, x3, x5, x6 + + /* + * Map the kernel image (starting with PHYS_OFFSET). + */ + mov x0, x26 // swapper_pg_dir + mov x5, #PAGE_OFFSET + create_pgd_entry x0, x5, x3, x6 + ldr x6, =KERNEL_END + mov x3, x24 // phys offset + create_block_map x0, x7, x3, x5, x6 + + /* + * Map the FDT blob (maximum 2MB; must be within 512MB of + * PHYS_OFFSET). + */ + mov x3, x21 // FDT phys address + and x3, x3, #~((1 << 21) - 1) // 2MB aligned + mov x6, #PAGE_OFFSET + sub x5, x3, x24 // subtract PHYS_OFFSET + tst x5, #~((1 << 29) - 1) // within 512MB? + csel x21, xzr, x21, ne // zero the FDT pointer + b.ne 1f + add x5, x5, x6 // __va(FDT blob) + add x6, x5, #1 << 21 // 2MB for the FDT blob + sub x6, x6, #1 // inclusive range + create_block_map x0, x7, x3, x5, x6 +1: + /* + * Since the page tables have been populated with non-cacheable + * accesses (MMU disabled), invalidate the idmap and swapper page + * tables again to remove any speculatively loaded cache lines. + */ + mov x0, x25 + add x1, x26, #SWAPPER_DIR_SIZE + bl __inval_cache_range + + mov lr, x27 + ret +ENDPROC(__create_page_tables) + .ltorg + + .align 3 + .type __switch_data, %object +__switch_data: + .quad __mmap_switched + .quad __bss_start // x6 + .quad __bss_stop // x7 + .quad processor_id // x4 + .quad __fdt_pointer // x5 + .quad memstart_addr // x6 + .quad init_thread_union + THREAD_START_SP // sp + +/* + * The following fragment of code is executed with the MMU on in MMU mode, and + * uses absolute addresses; this is not position independent. + */ +__mmap_switched: + adr x3, __switch_data + 8 + + ldp x6, x7, [x3], #16 +1: cmp x6, x7 + b.hs 2f + str xzr, [x6], #8 // Clear BSS + b 1b +2: + ldp x4, x5, [x3], #16 + ldr x6, [x3], #8 + ldr x16, [x3] + mov sp, x16 + str x22, [x4] // Save processor ID + str x21, [x5] // Save FDT pointer + str x24, [x6] // Save PHYS_OFFSET + mov x29, #0 +#ifdef CONFIG_KASAN + bl kasan_early_init +#endif + b start_kernel +ENDPROC(__mmap_switched) + +/* + * end early head section, begin head code that is also used for + * hotplug and needs to have the same protections as the text region + */ + .section ".text","ax" +/* * If we're fortunate enough to boot at EL2, ensure that the world is * sane before dropping to EL1. * @@ -498,183 +708,6 @@ ENDPROC(__calc_phys_offset) .quad PAGE_OFFSET /* - * Macro to create a table entry to the next page. - * - * tbl: page table address - * virt: virtual address - * shift: #imm page table shift - * ptrs: #imm pointers per table page - * - * Preserves: virt - * Corrupts: tmp1, tmp2 - * Returns: tbl -> next level table page address - */ - .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2 - lsr \tmp1, \virt, #\shift - and \tmp1, \tmp1, #\ptrs - 1 // table index - add \tmp2, \tbl, #PAGE_SIZE - orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type - str \tmp2, [\tbl, \tmp1, lsl #3] - add \tbl, \tbl, #PAGE_SIZE // next level table page - .endm - -/* - * Macro to populate the PGD (and possibily PUD) for the corresponding - * block entry in the next level (tbl) for the given virtual address. - * - * Preserves: tbl, next, virt - * Corrupts: tmp1, tmp2 - */ - .macro create_pgd_entry, tbl, virt, tmp1, tmp2 - create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2 -#if SWAPPER_PGTABLE_LEVELS == 3 - create_table_entry \tbl, \virt, TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2 -#endif - .endm - -/* - * Macro to populate block entries in the page table for the start..end - * virtual range (inclusive). - * - * Preserves: tbl, flags - * Corrupts: phys, start, end, pstate - */ - .macro create_block_map, tbl, flags, phys, start, end - lsr \phys, \phys, #BLOCK_SHIFT - lsr \start, \start, #BLOCK_SHIFT - and \start, \start, #PTRS_PER_PTE - 1 // table index - orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry - lsr \end, \end, #BLOCK_SHIFT - and \end, \end, #PTRS_PER_PTE - 1 // table end index -9999: str \phys, [\tbl, \start, lsl #3] // store the entry - add \start, \start, #1 // next entry - add \phys, \phys, #BLOCK_SIZE // next block - cmp \start, \end - b.ls 9999b - .endm - -/* - * Setup the initial page tables. We only setup the barest amount which is - * required to get the kernel running. The following sections are required: - * - identity mapping to enable the MMU (low address, TTBR0) - * - first few MB of the kernel linear mapping to jump to once the MMU has - * been enabled, including the FDT blob (TTBR1) - * - pgd entry for fixed mappings (TTBR1) - */ -__create_page_tables: - pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses - mov x27, lr - - /* - * Invalidate the idmap and swapper page tables to avoid potential - * dirty cache lines being evicted. - */ - mov x0, x25 - add x1, x26, #SWAPPER_DIR_SIZE - bl __inval_cache_range - - /* - * Clear the idmap and swapper page tables. - */ - mov x0, x25 - add x6, x26, #SWAPPER_DIR_SIZE -1: stp xzr, xzr, [x0], #16 - stp xzr, xzr, [x0], #16 - stp xzr, xzr, [x0], #16 - stp xzr, xzr, [x0], #16 - cmp x0, x6 - b.lo 1b - - ldr x7, =MM_MMUFLAGS - - /* - * Create the identity mapping. - */ - mov x0, x25 // idmap_pg_dir - ldr x3, =KERNEL_START - add x3, x3, x28 // __pa(KERNEL_START) - create_pgd_entry x0, x3, x5, x6 - ldr x6, =KERNEL_END - mov x5, x3 // __pa(KERNEL_START) - add x6, x6, x28 // __pa(KERNEL_END) - create_block_map x0, x7, x3, x5, x6 - - /* - * Map the kernel image (starting with PHYS_OFFSET). - */ - mov x0, x26 // swapper_pg_dir - mov x5, #PAGE_OFFSET - create_pgd_entry x0, x5, x3, x6 - ldr x6, =KERNEL_END - mov x3, x24 // phys offset - create_block_map x0, x7, x3, x5, x6 - - /* - * Map the FDT blob (maximum 2MB; must be within 512MB of - * PHYS_OFFSET). - */ - mov x3, x21 // FDT phys address - and x3, x3, #~((1 << 21) - 1) // 2MB aligned - mov x6, #PAGE_OFFSET - sub x5, x3, x24 // subtract PHYS_OFFSET - tst x5, #~((1 << 29) - 1) // within 512MB? - csel x21, xzr, x21, ne // zero the FDT pointer - b.ne 1f - add x5, x5, x6 // __va(FDT blob) - add x6, x5, #1 << 21 // 2MB for the FDT blob - sub x6, x6, #1 // inclusive range - create_block_map x0, x7, x3, x5, x6 -1: - /* - * Since the page tables have been populated with non-cacheable - * accesses (MMU disabled), invalidate the idmap and swapper page - * tables again to remove any speculatively loaded cache lines. - */ - mov x0, x25 - add x1, x26, #SWAPPER_DIR_SIZE - bl __inval_cache_range - - mov lr, x27 - ret -ENDPROC(__create_page_tables) - .ltorg - - .align 3 - .type __switch_data, %object -__switch_data: - .quad __mmap_switched - .quad __bss_start // x6 - .quad __bss_stop // x7 - .quad processor_id // x4 - .quad __fdt_pointer // x5 - .quad memstart_addr // x6 - .quad init_thread_union + THREAD_START_SP // sp - -/* - * The following fragment of code is executed with the MMU on in MMU mode, and - * uses absolute addresses; this is not position independent. - */ -__mmap_switched: - adr x3, __switch_data + 8 - - ldp x6, x7, [x3], #16 -1: cmp x6, x7 - b.hs 2f - str xzr, [x6], #8 // Clear BSS - b 1b -2: - ldp x4, x5, [x3], #16 - ldr x6, [x3], #8 - ldr x16, [x3] - mov sp, x16 - str x22, [x4] // Save processor ID - str x21, [x5] // Save FDT pointer - str x24, [x6] // Save PHYS_OFFSET - mov x29, #0 - b start_kernel -ENDPROC(__mmap_switched) - -/* * Exception handling. Something went wrong and we can't proceed. We ought to * tell the user, but since we don't have any guarantee that we're even * running on the right architecture, we do virtually nothing. @@ -721,22 +754,3 @@ __lookup_processor_type_data: .quad . .quad cpu_table .size __lookup_processor_type_data, . - __lookup_processor_type_data - -/* - * Determine validity of the x21 FDT pointer. - * The dtb must be 8-byte aligned and live in the first 512M of memory. - */ -__vet_fdt: - tst x21, #0x7 - b.ne 1f - cmp x21, x24 - b.lt 1f - mov x0, #(1 << 29) - add x0, x0, x24 - cmp x21, x0 - b.ge 1f - ret -1: - mov x21, #0 - ret -ENDPROC(__vet_fdt) diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c index 8cd27fedc8b6..7e9327a0986d 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/kernel/insn.c @@ -960,3 +960,29 @@ u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst, return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); } + +bool aarch32_insn_is_wide(u32 insn) +{ + return insn >= 0xe800; +} + +/* + * Macros/defines for extracting register numbers from instruction. + */ +u32 aarch32_insn_extract_reg_num(u32 insn, int offset) +{ + return (insn & (0xf << offset)) >> offset; +} + +#define OPC2_MASK 0x7 +#define OPC2_OFFSET 5 +u32 aarch32_insn_mcr_extract_opc2(u32 insn) +{ + return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET; +} + +#define CRM_MASK 0xf +u32 aarch32_insn_mcr_extract_crm(u32 insn) +{ + return insn & CRM_MASK; +} diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index e366329d96d8..51128018b907 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -21,6 +21,7 @@ #include <linux/bitops.h> #include <linux/elf.h> #include <linux/gfp.h> +#include <linux/kasan.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/moduleloader.h> @@ -32,9 +33,18 @@ void *module_alloc(unsigned long size) { - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE, - __builtin_return_address(0)); + void *p; + + p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END, + GFP_KERNEL, PAGE_KERNEL_EXEC, 0, + NUMA_NO_NODE, __builtin_return_address(0)); + + if (p && (kasan_module_alloc(p, size) < 0)) { + vfree(p); + return NULL; + } + + return p; } enum aarch64_reloc_op { diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 0c641a82f78c..3843e546ab3d 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -52,6 +52,7 @@ #include <asm/cputable.h> #include <asm/cpufeature.h> #include <asm/cpu_ops.h> +#include <asm/kasan.h> #include <asm/sections.h> #include <asm/setup.h> #include <asm/smp_plat.h> @@ -316,25 +317,6 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys) } } -/* - * Limit the memory size that was specified via FDT. - */ -static int __init early_mem(char *p) -{ - phys_addr_t limit; - - if (!p) - return 1; - - limit = memparse(p, &p) & PAGE_MASK; - pr_notice("Memory limited to %lldMB\n", limit >> 20); - - memblock_enforce_memory_limit(limit); - - return 0; -} -early_param("mem", early_mem); - static void __init request_standard_resources(void) { struct memblock_region *region; @@ -363,6 +345,69 @@ static void __init request_standard_resources(void) } } +#ifdef CONFIG_BLK_DEV_INITRD +/* + * Relocate initrd if it is not completely within the linear mapping. + * This would be the case if mem= cuts out all or part of it. + */ +static void __init relocate_initrd(void) +{ + phys_addr_t orig_start = __virt_to_phys(initrd_start); + phys_addr_t orig_end = __virt_to_phys(initrd_end); + phys_addr_t ram_end = memblock_end_of_DRAM(); + phys_addr_t new_start; + unsigned long size, to_free = 0; + void *dest; + + if (orig_end <= ram_end) + return; + + /* + * Any of the original initrd which overlaps the linear map should + * be freed after relocating. + */ + if (orig_start < ram_end) + to_free = ram_end - orig_start; + + size = orig_end - orig_start; + if (!size) + return; + + /* initrd needs to be relocated completely inside linear mapping */ + new_start = memblock_find_in_range(0, PFN_PHYS(max_pfn), + size, PAGE_SIZE); + if (!new_start) + panic("Cannot relocate initrd of size %ld\n", size); + memblock_reserve(new_start, size); + + initrd_start = __phys_to_virt(new_start); + initrd_end = initrd_start + size; + + pr_info("Moving initrd from [%llx-%llx] to [%llx-%llx]\n", + orig_start, orig_start + size - 1, + new_start, new_start + size - 1); + + dest = (void *)initrd_start; + + if (to_free) { + memcpy(dest, (void *)__phys_to_virt(orig_start), to_free); + dest += to_free; + } + + copy_from_early_mem(dest, orig_start + to_free, size - to_free); + + if (to_free) { + pr_info("Freeing original RAMDISK from [%llx-%llx]\n", + orig_start, orig_start + to_free - 1); + memblock_free(orig_start, to_free); + } +} +#else +static inline void __init relocate_initrd(void) +{ +} +#endif + u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID }; void __init setup_arch(char **cmdline_p) @@ -392,6 +437,10 @@ void __init setup_arch(char **cmdline_p) arm64_memblock_init(); paging_init(); + relocate_initrd(); + + kasan_init(); + request_standard_resources(); efi_idmap_init(); diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index 15dd021b0025..b6da20fa7a48 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c @@ -457,7 +457,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, { compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler); compat_ulong_t retcode; - compat_ulong_t spsr = regs->pstate & ~PSR_f; + compat_ulong_t spsr = regs->pstate & ~(PSR_f | COMPAT_PSR_E_BIT); int thumb; /* Check if the handler is written for ARM or Thumb */ @@ -471,6 +471,9 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, /* The IT state must be cleared for both ARM and Thumb-2 */ spsr &= ~COMPAT_PSR_IT_MASK; + /* Restore the original endianness */ + spsr |= COMPAT_PSR_ENDSTATE; + if (ka->sa.sa_flags & SA_RESTORER) { retcode = ptr_to_compat(ka->sa.sa_restorer); } else { diff --git a/arch/arm64/kernel/trace-events-emulation.h b/arch/arm64/kernel/trace-events-emulation.h new file mode 100644 index 000000000000..ae1dd598ea65 --- /dev/null +++ b/arch/arm64/kernel/trace-events-emulation.h @@ -0,0 +1,35 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM emulation + +#if !defined(_TRACE_EMULATION_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_EMULATION_H + +#include <linux/tracepoint.h> + +TRACE_EVENT(instruction_emulation, + + TP_PROTO(const char *instr, u64 addr), + TP_ARGS(instr, addr), + + TP_STRUCT__entry( + __string(instr, instr) + __field(u64, addr) + ), + + TP_fast_assign( + __assign_str(instr, instr); + __entry->addr = addr; + ), + + TP_printk("instr=\"%s\" addr=0x%llx", __get_str(instr), __entry->addr) +); + +#endif /* _TRACE_EMULATION_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH . + +#define TRACE_INCLUDE_FILE trace-events-emulation +#include <trace/define_trace.h> diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 1a1436c481ef..f1f6b16fb362 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -259,6 +259,69 @@ void arm64_notify_die(const char *str, struct pt_regs *regs, } } +static LIST_HEAD(undef_hook); +static DEFINE_RAW_SPINLOCK(undef_lock); + +void register_undef_hook(struct undef_hook *hook) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&undef_lock, flags); + list_add(&hook->node, &undef_hook); + raw_spin_unlock_irqrestore(&undef_lock, flags); +} + +void unregister_undef_hook(struct undef_hook *hook) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&undef_lock, flags); + list_del(&hook->node); + raw_spin_unlock_irqrestore(&undef_lock, flags); +} + +static int call_undef_hook(struct pt_regs *regs) +{ + struct undef_hook *hook; + unsigned long flags; + u32 instr; + int (*fn)(struct pt_regs *regs, u32 instr) = NULL; + void __user *pc = (void __user *)instruction_pointer(regs); + + if (!user_mode(regs)) + return 1; + + if (compat_thumb_mode(regs)) { + /* 16-bit Thumb instruction */ + if (get_user(instr, (u16 __user *)pc)) + goto exit; + instr = le16_to_cpu(instr); + if (aarch32_insn_is_wide(instr)) { + u32 instr2; + + if (get_user(instr2, (u16 __user *)(pc + 2))) + goto exit; + instr2 = le16_to_cpu(instr2); + instr = (instr << 16) | instr2; + } + } else { + /* 32-bit ARM instruction */ + if (get_user(instr, (u32 __user *)pc)) + goto exit; + instr = le32_to_cpu(instr); + } + + raw_spin_lock_irqsave(&undef_lock, flags); + list_for_each_entry(hook, &undef_hook, node) + if ((instr & hook->instr_mask) == hook->instr_val && + (regs->pstate & hook->pstate_mask) == hook->pstate_val) + fn = hook->fn; + + raw_spin_unlock_irqrestore(&undef_lock, flags); +exit: + return fn ? fn(regs, instr) : 1; +} + asmlinkage void __exception do_undefinstr(struct pt_regs *regs) { siginfo_t info; @@ -268,6 +331,9 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs) if (!aarch32_break_handler(regs)) return; + if (call_undef_hook(regs) == 0) + return; + if (show_unhandled_signals && unhandled_signal(current, SIGILL) && printk_ratelimit()) { pr_info("%s[%d]: undefined instruction: pc=%p\n", diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S index c17967fdf5f6..a9723c71c52b 100644 --- a/arch/arm64/lib/clear_user.S +++ b/arch/arm64/lib/clear_user.S @@ -16,7 +16,11 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ #include <linux/linkage.h> + +#include <asm/alternative.h> #include <asm/assembler.h> +#include <asm/cpufeature.h> +#include <asm/sysreg.h> .text @@ -29,6 +33,8 @@ * Alignment fixed up by hardware. */ ENTRY(__clear_user) +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) mov x2, x1 // save the size for fixup return subs x1, x1, #8 b.mi 2f @@ -48,6 +54,8 @@ USER(9f, strh wzr, [x0], #2 ) b.mi 5f USER(9f, strb wzr, [x0] ) 5: mov x0, #0 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) ret ENDPROC(__clear_user) diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 5e27add9d362..1be9ef27be97 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -15,7 +15,11 @@ */ #include <linux/linkage.h> + +#include <asm/alternative.h> #include <asm/assembler.h> +#include <asm/cpufeature.h> +#include <asm/sysreg.h> /* * Copy from user space to a kernel buffer (alignment handled by the hardware) @@ -28,14 +32,21 @@ * x0 - bytes not copied */ ENTRY(__copy_from_user) - add x4, x1, x2 // upper user buffer boundary - subs x2, x2, #8 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) + add x5, x1, x2 // upper user buffer boundary + subs x2, x2, #16 + b.mi 1f +0: +USER(9f, ldp x3, x4, [x1], #16) + subs x2, x2, #16 + stp x3, x4, [x0], #16 + b.pl 0b +1: adds x2, x2, #8 b.mi 2f -1: USER(9f, ldr x3, [x1], #8 ) - subs x2, x2, #8 + sub x2, x2, #8 str x3, [x0], #8 - b.pl 1b 2: adds x2, x2, #4 b.mi 3f USER(9f, ldr w3, [x1], #4 ) @@ -51,12 +62,14 @@ USER(9f, ldrh w3, [x1], #2 ) USER(9f, ldrb w3, [x1] ) strb w3, [x0] 5: mov x0, #0 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) ret ENDPROC(__copy_from_user) .section .fixup,"ax" .align 2 -9: sub x2, x4, x1 +9: sub x2, x5, x1 mov x3, x2 10: strb wzr, [x0], #1 // zero remaining buffer space subs x3, x3, #1 diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S index 84b6c9bb9b93..1b94661e22b3 100644 --- a/arch/arm64/lib/copy_in_user.S +++ b/arch/arm64/lib/copy_in_user.S @@ -17,7 +17,11 @@ */ #include <linux/linkage.h> + +#include <asm/alternative.h> #include <asm/assembler.h> +#include <asm/cpufeature.h> +#include <asm/sysreg.h> /* * Copy from user space to user space (alignment handled by the hardware) @@ -30,14 +34,21 @@ * x0 - bytes not copied */ ENTRY(__copy_in_user) - add x4, x0, x2 // upper user buffer boundary - subs x2, x2, #8 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) + add x5, x0, x2 // upper user buffer boundary + subs x2, x2, #16 + b.mi 1f +0: +USER(9f, ldp x3, x4, [x1], #16) + subs x2, x2, #16 +USER(9f, stp x3, x4, [x0], #16) + b.pl 0b +1: adds x2, x2, #8 b.mi 2f -1: USER(9f, ldr x3, [x1], #8 ) - subs x2, x2, #8 + sub x2, x2, #8 USER(9f, str x3, [x0], #8 ) - b.pl 1b 2: adds x2, x2, #4 b.mi 3f USER(9f, ldr w3, [x1], #4 ) @@ -53,11 +64,13 @@ USER(9f, strh w3, [x0], #2 ) USER(9f, ldrb w3, [x1] ) USER(9f, strb w3, [x0] ) 5: mov x0, #0 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) ret ENDPROC(__copy_in_user) .section .fixup,"ax" .align 2 -9: sub x0, x4, x0 // bytes not copied +9: sub x0, x5, x0 // bytes not copied ret .previous diff --git a/arch/arm64/lib/copy_template.S b/arch/arm64/lib/copy_template.S new file mode 100644 index 000000000000..410fbdb8163f --- /dev/null +++ b/arch/arm64/lib/copy_template.S @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2013 Linaro. + * + * This code is based on glibc cortex strings work originally authored by Linaro + * and re-licensed under GPLv2 for the Linux kernel. The original code can + * be found @ + * + * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ + * files/head:/src/aarch64/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + + +/* + * Copy a buffer from src to dest (alignment handled by the hardware) + * + * Parameters: + * x0 - dest + * x1 - src + * x2 - n + * Returns: + * x0 - dest + */ +dstin .req x0 +src .req x1 +count .req x2 +tmp1 .req x3 +tmp1w .req w3 +tmp2 .req x4 +tmp2w .req w4 +dst .req x6 + +A_l .req x7 +A_h .req x8 +B_l .req x9 +B_h .req x10 +C_l .req x11 +C_h .req x12 +D_l .req x13 +D_h .req x14 + + mov dst, dstin + cmp count, #16 + /*When memory length is less than 16, the accessed are not aligned.*/ + b.lo .Ltiny15 + + neg tmp2, src + ands tmp2, tmp2, #15/* Bytes to reach alignment. */ + b.eq .LSrcAligned + sub count, count, tmp2 + /* + * Copy the leading memory data from src to dst in an increasing + * address order.By this way,the risk of overwritting the source + * memory data is eliminated when the distance between src and + * dst is less than 16. The memory accesses here are alignment. + */ + tbz tmp2, #0, 1f + ldrb1 tmp1w, src, #1 + strb1 tmp1w, dst, #1 +1: + tbz tmp2, #1, 2f + ldrh1 tmp1w, src, #2 + strh1 tmp1w, dst, #2 +2: + tbz tmp2, #2, 3f + ldr1 tmp1w, src, #4 + str1 tmp1w, dst, #4 +3: + tbz tmp2, #3, .LSrcAligned + ldr1 tmp1, src, #8 + str1 tmp1, dst, #8 + +.LSrcAligned: + cmp count, #64 + b.ge .Lcpy_over64 + /* + * Deal with small copies quickly by dropping straight into the + * exit block. + */ +.Ltail63: + /* + * Copy up to 48 bytes of data. At this point we only need the + * bottom 6 bits of count to be accurate. + */ + ands tmp1, count, #0x30 + b.eq .Ltiny15 + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + ldp1 A_l, A_h, src, #16 + stp1 A_l, A_h, dst, #16 +1: + ldp1 A_l, A_h, src, #16 + stp1 A_l, A_h, dst, #16 +2: + ldp1 A_l, A_h, src, #16 + stp1 A_l, A_h, dst, #16 +.Ltiny15: + /* + * Prefer to break one ldp/stp into several load/store to access + * memory in an increasing address order,rather than to load/store 16 + * bytes from (src-16) to (dst-16) and to backward the src to aligned + * address,which way is used in original cortex memcpy. If keeping + * the original memcpy process here, memmove need to satisfy the + * precondition that src address is at least 16 bytes bigger than dst + * address,otherwise some source data will be overwritten when memove + * call memcpy directly. To make memmove simpler and decouple the + * memcpy's dependency on memmove, withdrew the original process. + */ + tbz count, #3, 1f + ldr1 tmp1, src, #8 + str1 tmp1, dst, #8 +1: + tbz count, #2, 2f + ldr1 tmp1w, src, #4 + str1 tmp1w, dst, #4 +2: + tbz count, #1, 3f + ldrh1 tmp1w, src, #2 + strh1 tmp1w, dst, #2 +3: + tbz count, #0, .Lexitfunc + ldrb1 tmp1w, src, #1 + strb1 tmp1w, dst, #1 + + b .Lexitfunc + +.Lcpy_over64: + subs count, count, #128 + b.ge .Lcpy_body_large + /* + * Less than 128 bytes to copy, so handle 64 here and then jump + * to the tail. + */ + ldp1 A_l, A_h, src, #16 + stp1 A_l, A_h, dst, #16 + ldp1 B_l, B_h, src, #16 + ldp1 C_l, C_h, src, #16 + stp1 B_l, B_h, dst, #16 + stp1 C_l, C_h, dst, #16 + ldp1 D_l, D_h, src, #16 + stp1 D_l, D_h, dst, #16 + + tst count, #0x3f + b.ne .Ltail63 + b .Lexitfunc + + /* + * Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line this ensures the entire loop is in one line. + */ + .p2align L1_CACHE_SHIFT +.Lcpy_body_large: + /* pre-get 64 bytes data. */ + ldp1 A_l, A_h, src, #16 + ldp1 B_l, B_h, src, #16 + ldp1 C_l, C_h, src, #16 + ldp1 D_l, D_h, src, #16 +1: + /* + * interlace the load of next 64 bytes data block with store of the last + * loaded 64 bytes data. + */ + stp1 A_l, A_h, dst, #16 + ldp1 A_l, A_h, src, #16 + stp1 B_l, B_h, dst, #16 + ldp1 B_l, B_h, src, #16 + stp1 C_l, C_h, dst, #16 + ldp1 C_l, C_h, src, #16 + stp1 D_l, D_h, dst, #16 + ldp1 D_l, D_h, src, #16 + subs count, count, #64 + b.ge 1b + stp1 A_l, A_h, dst, #16 + stp1 B_l, B_h, dst, #16 + stp1 C_l, C_h, dst, #16 + stp1 D_l, D_h, dst, #16 + + tst count, #0x3f + b.ne .Ltail63 +.Lexitfunc: diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index a0aeeb9b7a28..a257b47e2dc4 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -15,7 +15,11 @@ */ #include <linux/linkage.h> + +#include <asm/alternative.h> #include <asm/assembler.h> +#include <asm/cpufeature.h> +#include <asm/sysreg.h> /* * Copy to user space from a kernel buffer (alignment handled by the hardware) @@ -28,14 +32,21 @@ * x0 - bytes not copied */ ENTRY(__copy_to_user) - add x4, x0, x2 // upper user buffer boundary - subs x2, x2, #8 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) + add x5, x0, x2 // upper user buffer boundary + subs x2, x2, #16 + b.mi 1f +0: + ldp x3, x4, [x1], #16 + subs x2, x2, #16 +USER(9f, stp x3, x4, [x0], #16) + b.pl 0b +1: adds x2, x2, #8 b.mi 2f -1: ldr x3, [x1], #8 - subs x2, x2, #8 + sub x2, x2, #8 USER(9f, str x3, [x0], #8 ) - b.pl 1b 2: adds x2, x2, #4 b.mi 3f ldr w3, [x1], #4 @@ -51,11 +62,13 @@ USER(9f, strh w3, [x0], #2 ) ldrb w3, [x1] USER(9f, strb w3, [x0] ) 5: mov x0, #0 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) ret ENDPROC(__copy_to_user) .section .fixup,"ax" .align 2 -9: sub x0, x4, x0 // bytes not copied +9: sub x0, x5, x0 // bytes not copied ret .previous diff --git a/arch/arm64/lib/memchr.S b/arch/arm64/lib/memchr.S index 8636b7549163..4444c1d25f4b 100644 --- a/arch/arm64/lib/memchr.S +++ b/arch/arm64/lib/memchr.S @@ -41,4 +41,4 @@ ENTRY(memchr) ret 2: mov x0, #0 ret -ENDPROC(memchr) +ENDPIPROC(memchr) diff --git a/arch/arm64/lib/memcmp.S b/arch/arm64/lib/memcmp.S index 6ea0776ba6de..ffbdec00327d 100644 --- a/arch/arm64/lib/memcmp.S +++ b/arch/arm64/lib/memcmp.S @@ -255,4 +255,4 @@ CPU_LE( rev data2, data2 ) .Lret0: mov result, #0 ret -ENDPROC(memcmp) +ENDPIPROC(memcmp) diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S index 8a9a96d3ddae..67613937711f 100644 --- a/arch/arm64/lib/memcpy.S +++ b/arch/arm64/lib/memcpy.S @@ -36,166 +36,42 @@ * Returns: * x0 - dest */ -dstin .req x0 -src .req x1 -count .req x2 -tmp1 .req x3 -tmp1w .req w3 -tmp2 .req x4 -tmp2w .req w4 -tmp3 .req x5 -tmp3w .req w5 -dst .req x6 + .macro ldrb1 ptr, regB, val + ldrb \ptr, [\regB], \val + .endm -A_l .req x7 -A_h .req x8 -B_l .req x9 -B_h .req x10 -C_l .req x11 -C_h .req x12 -D_l .req x13 -D_h .req x14 + .macro strb1 ptr, regB, val + strb \ptr, [\regB], \val + .endm -ENTRY(memcpy) - mov dst, dstin - cmp count, #16 - /*When memory length is less than 16, the accessed are not aligned.*/ - b.lo .Ltiny15 + .macro ldrh1 ptr, regB, val + ldrh \ptr, [\regB], \val + .endm - neg tmp2, src - ands tmp2, tmp2, #15/* Bytes to reach alignment. */ - b.eq .LSrcAligned - sub count, count, tmp2 - /* - * Copy the leading memory data from src to dst in an increasing - * address order.By this way,the risk of overwritting the source - * memory data is eliminated when the distance between src and - * dst is less than 16. The memory accesses here are alignment. - */ - tbz tmp2, #0, 1f - ldrb tmp1w, [src], #1 - strb tmp1w, [dst], #1 -1: - tbz tmp2, #1, 2f - ldrh tmp1w, [src], #2 - strh tmp1w, [dst], #2 -2: - tbz tmp2, #2, 3f - ldr tmp1w, [src], #4 - str tmp1w, [dst], #4 -3: - tbz tmp2, #3, .LSrcAligned - ldr tmp1, [src],#8 - str tmp1, [dst],#8 + .macro strh1 ptr, regB, val + strh \ptr, [\regB], \val + .endm -.LSrcAligned: - cmp count, #64 - b.ge .Lcpy_over64 - /* - * Deal with small copies quickly by dropping straight into the - * exit block. - */ -.Ltail63: - /* - * Copy up to 48 bytes of data. At this point we only need the - * bottom 6 bits of count to be accurate. - */ - ands tmp1, count, #0x30 - b.eq .Ltiny15 - cmp tmp1w, #0x20 - b.eq 1f - b.lt 2f - ldp A_l, A_h, [src], #16 - stp A_l, A_h, [dst], #16 -1: - ldp A_l, A_h, [src], #16 - stp A_l, A_h, [dst], #16 -2: - ldp A_l, A_h, [src], #16 - stp A_l, A_h, [dst], #16 -.Ltiny15: - /* - * Prefer to break one ldp/stp into several load/store to access - * memory in an increasing address order,rather than to load/store 16 - * bytes from (src-16) to (dst-16) and to backward the src to aligned - * address,which way is used in original cortex memcpy. If keeping - * the original memcpy process here, memmove need to satisfy the - * precondition that src address is at least 16 bytes bigger than dst - * address,otherwise some source data will be overwritten when memove - * call memcpy directly. To make memmove simpler and decouple the - * memcpy's dependency on memmove, withdrew the original process. - */ - tbz count, #3, 1f - ldr tmp1, [src], #8 - str tmp1, [dst], #8 -1: - tbz count, #2, 2f - ldr tmp1w, [src], #4 - str tmp1w, [dst], #4 -2: - tbz count, #1, 3f - ldrh tmp1w, [src], #2 - strh tmp1w, [dst], #2 -3: - tbz count, #0, .Lexitfunc - ldrb tmp1w, [src] - strb tmp1w, [dst] + .macro ldr1 ptr, regB, val + ldr \ptr, [\regB], \val + .endm -.Lexitfunc: - ret + .macro str1 ptr, regB, val + str \ptr, [\regB], \val + .endm -.Lcpy_over64: - subs count, count, #128 - b.ge .Lcpy_body_large - /* - * Less than 128 bytes to copy, so handle 64 here and then jump - * to the tail. - */ - ldp A_l, A_h, [src],#16 - stp A_l, A_h, [dst],#16 - ldp B_l, B_h, [src],#16 - ldp C_l, C_h, [src],#16 - stp B_l, B_h, [dst],#16 - stp C_l, C_h, [dst],#16 - ldp D_l, D_h, [src],#16 - stp D_l, D_h, [dst],#16 + .macro ldp1 ptr, regB, regC, val + ldp \ptr, \regB, [\regC], \val + .endm - tst count, #0x3f - b.ne .Ltail63 - ret + .macro stp1 ptr, regB, regC, val + stp \ptr, \regB, [\regC], \val + .endm - /* - * Critical loop. Start at a new cache line boundary. Assuming - * 64 bytes per line this ensures the entire loop is in one line. - */ - .p2align L1_CACHE_SHIFT -.Lcpy_body_large: - /* pre-get 64 bytes data. */ - ldp A_l, A_h, [src],#16 - ldp B_l, B_h, [src],#16 - ldp C_l, C_h, [src],#16 - ldp D_l, D_h, [src],#16 -1: - /* - * interlace the load of next 64 bytes data block with store of the last - * loaded 64 bytes data. - */ - stp A_l, A_h, [dst],#16 - ldp A_l, A_h, [src],#16 - stp B_l, B_h, [dst],#16 - ldp B_l, B_h, [src],#16 - stp C_l, C_h, [dst],#16 - ldp C_l, C_h, [src],#16 - stp D_l, D_h, [dst],#16 - ldp D_l, D_h, [src],#16 - subs count, count, #64 - b.ge 1b - stp A_l, A_h, [dst],#16 - stp B_l, B_h, [dst],#16 - stp C_l, C_h, [dst],#16 - stp D_l, D_h, [dst],#16 - - tst count, #0x3f - b.ne .Ltail63 + .weak memcpy +ENTRY(__memcpy) +ENTRY(memcpy) +#include "copy_template.S" ret -ENDPROC(memcpy) +ENDPIPROC(memcpy) +ENDPROC(__memcpy) diff --git a/arch/arm64/lib/memmove.S b/arch/arm64/lib/memmove.S index 57b19ea2dad4..a5a4459013b1 100644 --- a/arch/arm64/lib/memmove.S +++ b/arch/arm64/lib/memmove.S @@ -57,12 +57,14 @@ C_h .req x12 D_l .req x13 D_h .req x14 + .weak memmove +ENTRY(__memmove) ENTRY(memmove) cmp dstin, src - b.lo memcpy + b.lo __memcpy add tmp1, src, count cmp dstin, tmp1 - b.hs memcpy /* No overlap. */ + b.hs __memcpy /* No overlap. */ add dst, dstin, count add src, src, count @@ -194,4 +196,5 @@ ENTRY(memmove) tst count, #0x3f b.ne .Ltail63 ret -ENDPROC(memmove) +ENDPIPROC(memmove) +ENDPROC(__memmove) diff --git a/arch/arm64/lib/memset.S b/arch/arm64/lib/memset.S index 7c72dfd36b63..f2670a9f218c 100644 --- a/arch/arm64/lib/memset.S +++ b/arch/arm64/lib/memset.S @@ -54,6 +54,8 @@ dst .req x8 tmp3w .req w9 tmp3 .req x9 + .weak memset +ENTRY(__memset) ENTRY(memset) mov dst, dstin /* Preserve return value. */ and A_lw, val, #255 @@ -213,4 +215,5 @@ ENTRY(memset) ands count, count, zva_bits_x b.ne .Ltail_maybe_long ret -ENDPROC(memset) +ENDPIPROC(memset) +ENDPROC(__memset) diff --git a/arch/arm64/lib/strcmp.S b/arch/arm64/lib/strcmp.S index 42f828b06c59..471fe61760ef 100644 --- a/arch/arm64/lib/strcmp.S +++ b/arch/arm64/lib/strcmp.S @@ -231,4 +231,4 @@ CPU_BE( orr syndrome, diff, has_nul ) lsr data1, data1, #56 sub result, data1, data2, lsr #56 ret -ENDPROC(strcmp) +ENDPIPROC(strcmp) diff --git a/arch/arm64/lib/strlen.S b/arch/arm64/lib/strlen.S index 987b68b9ce44..55ccc8e24c08 100644 --- a/arch/arm64/lib/strlen.S +++ b/arch/arm64/lib/strlen.S @@ -123,4 +123,4 @@ CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ csinv data1, data1, xzr, le csel data2, data2, data2a, le b .Lrealigned -ENDPROC(strlen) +ENDPIPROC(strlen) diff --git a/arch/arm64/lib/strncmp.S b/arch/arm64/lib/strncmp.S index 0224cf5a5533..e267044761c6 100644 --- a/arch/arm64/lib/strncmp.S +++ b/arch/arm64/lib/strncmp.S @@ -307,4 +307,4 @@ CPU_BE( orr syndrome, diff, has_nul ) .Lret0: mov result, #0 ret -ENDPROC(strncmp) +ENDPIPROC(strncmp) diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index c56179ed2c09..00bc265f87e9 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -3,3 +3,6 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ ioremap.o mmap.o pgd.o mmu.o \ context.o proc.o pageattr.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o + +obj-$(CONFIG_KASAN) += kasan_init.o +KASAN_SANITIZE_kasan_init.o := n diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 8eaf18577d71..321a6ac84a94 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -21,7 +21,7 @@ #include <linux/init.h> #include <asm/assembler.h> #include <asm/cpufeature.h> -#include <asm/alternative-asm.h> +#include <asm/alternative.h> #include "proc-macros.S" @@ -167,7 +167,7 @@ ENTRY(__flush_dcache_area) b.lo 1b dsb sy ret -ENDPROC(__flush_dcache_area) +ENDPIPROC(__flush_dcache_area) /* * __inval_cache_range(start, end) @@ -200,7 +200,7 @@ __dma_inv_range: b.lo 2b dsb sy ret -ENDPROC(__inval_cache_range) +ENDPIPROC(__inval_cache_range) ENDPROC(__dma_inv_range) /* @@ -235,7 +235,7 @@ ENTRY(__dma_flush_range) b.lo 1b dsb sy ret -ENDPROC(__dma_flush_range) +ENDPIPROC(__dma_flush_range) /* * __dma_map_area(start, size, dir) @@ -248,7 +248,7 @@ ENTRY(__dma_map_area) cmp w2, #DMA_FROM_DEVICE b.eq __dma_inv_range b __dma_clean_range -ENDPROC(__dma_map_area) +ENDPIPROC(__dma_map_area) /* * __dma_unmap_area(start, size, dir) @@ -261,4 +261,4 @@ ENTRY(__dma_unmap_area) cmp w2, #DMA_TO_DEVICE b.ne __dma_inv_range ret -ENDPROC(__dma_unmap_area) +ENDPIPROC(__dma_unmap_area) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index df0f5347029f..7788e084fddb 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -30,9 +30,11 @@ #include <linux/highmem.h> #include <linux/perf_event.h> +#include <asm/cpufeature.h> #include <asm/exception.h> #include <asm/debug-monitors.h> #include <asm/esr.h> +#include <asm/sysreg.h> #include <asm/system_misc.h> #include <asm/pgtable.h> #include <asm/tlbflush.h> @@ -225,6 +227,13 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, } /* + * PAN bit set implies the fault happened in kernel space, but not + * in the arch's user access functions. + */ + if (IS_ENABLED(CONFIG_ARM64_PAN) && (regs->pstate & PSR_PAN_BIT)) + goto no_context; + + /* * As per x86, we may deadlock here. However, since the kernel only * validly references user space from well defined areas of the code, * we can bug out early if this is from code which shouldn't. @@ -534,3 +543,10 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, return 0; } + +#ifdef CONFIG_ARM64_PAN +void cpu_enable_pan(void) +{ + config_sctlr_el1(SCTLR_EL1_SPAN, 0); +} +#endif /* CONFIG_ARM64_PAN */ diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index ed8affdf8f58..bc4f9c0fc325 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -136,10 +136,29 @@ static void arm64_memory_present(void) } #endif +static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX; + +/* + * Limit the memory size that was specified via FDT. + */ +static int __init early_mem(char *p) +{ + if (!p) + return 1; + + memory_limit = memparse(p, &p) & PAGE_MASK; + pr_notice("Memory limited to %lldMB\n", memory_limit >> 20); + + return 0; +} +early_param("mem", early_mem); + void __init arm64_memblock_init(void) { phys_addr_t dma_phys_limit = 0; + memblock_enforce_memory_limit(memory_limit); + /* * Register the kernel text, kernel data, initrd, and initial * pagetables with memblock. @@ -273,6 +292,9 @@ void __init mem_init(void) #define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K) pr_notice("Virtual kernel memory layout:\n" +#ifdef CONFIG_KASAN + " kasan : 0x%16lx - 0x%16lx (%6ld GB)\n" +#endif " vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n" #ifdef CONFIG_SPARSEMEM_VMEMMAP " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n" @@ -285,6 +307,9 @@ void __init mem_init(void) " .init : 0x%p" " - 0x%p" " (%6ld KB)\n" " .text : 0x%p" " - 0x%p" " (%6ld KB)\n" " .data : 0x%p" " - 0x%p" " (%6ld KB)\n", +#ifdef CONFIG_KASAN + MLG(KASAN_SHADOW_START, KASAN_SHADOW_END), +#endif MLG(VMALLOC_START, VMALLOC_END), #ifdef CONFIG_SPARSEMEM_VMEMMAP MLG(VMEMMAP_START, diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c new file mode 100644 index 000000000000..cf038c7d9fa9 --- /dev/null +++ b/arch/arm64/mm/kasan_init.c @@ -0,0 +1,165 @@ +/* + * This file contains kasan initialization code for ARM64. + * + * Copyright (c) 2015 Samsung Electronics Co., Ltd. + * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define pr_fmt(fmt) "kasan: " fmt +#include <linux/kasan.h> +#include <linux/kernel.h> +#include <linux/memblock.h> +#include <linux/start_kernel.h> + +#include <asm/page.h> +#include <asm/pgalloc.h> +#include <asm/pgtable.h> +#include <asm/tlbflush.h> + +static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE); + +static void __init kasan_early_pte_populate(pmd_t *pmd, unsigned long addr, + unsigned long end) +{ + pte_t *pte; + unsigned long next; + + if (pmd_none(*pmd)) + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); + + pte = pte_offset_kernel(pmd, addr); + do { + next = addr + PAGE_SIZE; + set_pte(pte, pfn_pte(virt_to_pfn(kasan_zero_page), + PAGE_KERNEL)); + } while (pte++, addr = next, addr != end && pte_none(*pte)); +} + +static void __init kasan_early_pmd_populate(pud_t *pud, + unsigned long addr, + unsigned long end) +{ + pmd_t *pmd; + unsigned long next; + + if (pud_none(*pud)) + pud_populate(&init_mm, pud, kasan_zero_pmd); + + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + kasan_early_pte_populate(pmd, addr, next); + } while (pmd++, addr = next, addr != end && pmd_none(*pmd)); +} + +static void __init kasan_early_pud_populate(pgd_t *pgd, + unsigned long addr, + unsigned long end) +{ + pud_t *pud; + unsigned long next; + + if (pgd_none(*pgd)) + pgd_populate(&init_mm, pgd, kasan_zero_pud); + + pud = pud_offset(pgd, addr); + do { + next = pud_addr_end(addr, end); + kasan_early_pmd_populate(pud, addr, next); + } while (pud++, addr = next, addr != end && pud_none(*pud)); +} + +static void __init kasan_map_early_shadow(void) +{ + unsigned long addr = KASAN_SHADOW_START; + unsigned long end = KASAN_SHADOW_END; + unsigned long next; + pgd_t *pgd; + + pgd = pgd_offset_k(addr); + do { + next = pgd_addr_end(addr, end); + kasan_early_pud_populate(pgd, addr, next); + } while (pgd++, addr = next, addr != end); +} + +asmlinkage void __init kasan_early_init(void) +{ + BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_END - (1UL << 61)); + BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE)); + BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE)); + kasan_map_early_shadow(); +} + +static void __init clear_pgds(unsigned long start, + unsigned long end) +{ + /* + * Remove references to kasan page tables from + * swapper_pg_dir. pgd_clear() can't be used + * here because it's nop on 2,3-level pagetable setups + */ + for (; start < end; start += PGDIR_SIZE) + set_pgd(pgd_offset_k(start), __pgd(0)); +} + +static void __init cpu_set_ttbr1(unsigned long ttbr1) +{ + asm( + " msr ttbr1_el1, %0\n" + " isb" + : + : "r" (ttbr1)); +} + +void __init kasan_init(void) +{ + struct memblock_region *reg; + + /* + * We are going to perform proper setup of shadow memory. + * At first we should unmap early shadow (clear_pgds() call bellow). + * However, instrumented code couldn't execute without shadow memory. + * tmp_pg_dir used to keep early shadow mapped until full shadow + * setup will be finished. + */ + memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir)); + cpu_set_ttbr1(__pa(tmp_pg_dir)); + flush_tlb_all(); + + clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); + + kasan_populate_zero_shadow((void *)KASAN_SHADOW_START, + kasan_mem_to_shadow((void *)MODULES_VADDR)); + + for_each_memblock(memory, reg) { + void *start = (void *)__phys_to_virt(reg->base); + void *end = (void *)__phys_to_virt(reg->base + reg->size); + + if (start >= end) + break; + + /* + * end + 1 here is intentional. We check several shadow bytes in + * advance to slightly speed up fastpath. In some rare cases + * we could cross boundary of mapped shadow, so we just map + * some more here. + */ + vmemmap_populate((unsigned long)kasan_mem_to_shadow(start), + (unsigned long)kasan_mem_to_shadow(end) + 1, + pfn_to_nid(virt_to_pfn(start))); + } + + memset(kasan_zero_page, 0, PAGE_SIZE); + cpu_set_ttbr1(__pa(swapper_pg_dir)); + flush_tlb_all(); + + /* At this point kasan is fully initialized. Enable error messages */ + init_task.kasan_depth = 0; + pr_info("KernelAddressSanitizer initialized\n"); +} diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index 6682b361d3ac..cb3ba1b812e7 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -28,16 +28,14 @@ #include "mm.h" -#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) - static struct kmem_cache *pgd_cache; pgd_t *pgd_alloc(struct mm_struct *mm) { if (PGD_SIZE == PAGE_SIZE) - return (pgd_t *)get_zeroed_page(GFP_KERNEL); + return (pgd_t *)__get_free_page(PGALLOC_GFP); else - return kmem_cache_zalloc(pgd_cache, GFP_KERNEL); + return kmem_cache_alloc(pgd_cache, PGALLOC_GFP); } void pgd_free(struct mm_struct *mm, pgd_t *pgd) diff --git a/arch/frv/mb93090-mb00/pci-vdk.c b/arch/frv/mb93090-mb00/pci-vdk.c index efa5d65b0007..34fb53700e91 100644 --- a/arch/frv/mb93090-mb00/pci-vdk.c +++ b/arch/frv/mb93090-mb00/pci-vdk.c @@ -316,6 +316,7 @@ void pcibios_fixup_bus(struct pci_bus *bus) int __init pcibios_init(void) { + struct pci_bus *bus; struct pci_ops *dir = NULL; LIST_HEAD(resources); @@ -383,12 +384,15 @@ int __init pcibios_init(void) printk("PCI: Probing PCI hardware\n"); pci_add_resource(&resources, &pci_ioport_resource); pci_add_resource(&resources, &pci_iomem_resource); - pci_scan_root_bus(NULL, 0, pci_root_ops, NULL, &resources); + bus = pci_scan_root_bus(NULL, 0, pci_root_ops, NULL, &resources); pcibios_irq_init(); pcibios_fixup_irqs(); pcibios_resource_survey(); + if (!bus) + return 0; + pci_bus_add_devices(bus); return 0; } diff --git a/arch/ia64/kernel/msi_ia64.c b/arch/ia64/kernel/msi_ia64.c index 8c3730c3c63d..8ae36ea177d3 100644 --- a/arch/ia64/kernel/msi_ia64.c +++ b/arch/ia64/kernel/msi_ia64.c @@ -35,7 +35,7 @@ static int ia64_set_msi_irq_affinity(struct irq_data *idata, data |= MSI_DATA_VECTOR(irq_to_vector(irq)); msg.data = data; - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); cpumask_copy(idata->affinity, cpumask_of(cpu)); return 0; @@ -71,7 +71,7 @@ int ia64_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) MSI_DATA_DELIVERY_FIXED | MSI_DATA_VECTOR(vector); - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); irq_set_chip_and_handler(irq, &ia64_msi_chip, handle_edge_irq); return 0; @@ -102,8 +102,8 @@ static int ia64_msi_retrigger_irq(struct irq_data *data) */ static struct irq_chip ia64_msi_chip = { .name = "PCI-MSI", - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, .irq_ack = ia64_ack_msi_irq, #ifdef CONFIG_SMP .irq_set_affinity = ia64_set_msi_irq_affinity, diff --git a/arch/ia64/sn/kernel/io_init.c b/arch/ia64/sn/kernel/io_init.c index 0b5ce82d203d..1be65eb074ec 100644 --- a/arch/ia64/sn/kernel/io_init.c +++ b/arch/ia64/sn/kernel/io_init.c @@ -271,7 +271,9 @@ sn_pci_controller_fixup(int segment, int busnum, struct pci_bus *bus) if (bus == NULL) { kfree(res); kfree(controller); + return; } + pci_bus_add_devices(bus); } /* diff --git a/arch/ia64/sn/kernel/msi_sn.c b/arch/ia64/sn/kernel/msi_sn.c index 446e7799928c..a0eb27b66d13 100644 --- a/arch/ia64/sn/kernel/msi_sn.c +++ b/arch/ia64/sn/kernel/msi_sn.c @@ -145,7 +145,7 @@ int sn_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *entry) msg.data = 0x100 + irq; irq_set_msi_desc(irq, entry); - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); irq_set_chip_and_handler(irq, &sn_msi_chip, handle_edge_irq); return 0; @@ -205,7 +205,7 @@ static int sn_set_msi_irq_affinity(struct irq_data *data, msg.address_hi = (u32)(bus_addr >> 32); msg.address_lo = (u32)(bus_addr & 0x00000000ffffffff); - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); cpumask_copy(data->affinity, cpu_mask); return 0; @@ -228,8 +228,8 @@ static int sn_msi_retrigger_irq(struct irq_data *data) static struct irq_chip sn_msi_chip = { .name = "PCI-MSI", - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, .irq_ack = sn_ack_msi_irq, #ifdef CONFIG_SMP .irq_set_affinity = sn_set_msi_irq_affinity, diff --git a/arch/m68k/coldfire/pci.c b/arch/m68k/coldfire/pci.c index df9679238b6d..821de928dc3f 100644 --- a/arch/m68k/coldfire/pci.c +++ b/arch/m68k/coldfire/pci.c @@ -313,12 +313,16 @@ static int __init mcf_pci_init(void) schedule_timeout(msecs_to_jiffies(200)); rootbus = pci_scan_bus(0, &mcf_pci_ops, NULL); + if (!rootbus) + return -ENODEV; + rootbus->resource[0] = &mcf_pci_io; rootbus->resource[1] = &mcf_pci_mem; pci_fixup_irqs(pci_common_swizzle, mcf_pci_map_irq); pci_bus_size_bridges(rootbus); pci_bus_assign_resources(rootbus); + pci_bus_add_devices(rootbus); return 0; } diff --git a/arch/microblaze/pci/pci-common.c b/arch/microblaze/pci/pci-common.c index b30e41c0c033..a014e5b2e396 100644 --- a/arch/microblaze/pci/pci-common.c +++ b/arch/microblaze/pci/pci-common.c @@ -1371,6 +1371,10 @@ static int __init pcibios_init(void) /* Call common code to handle resource allocation */ pcibios_resource_survey(); + list_for_each_entry_safe(hose, tmp, &hose_list, list_node) { + if (hose->bus) + pci_bus_add_devices(hose->bus); + } return 0; } diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c index 2a52568dbcd6..1833f5171ccd 100644 --- a/arch/mips/kernel/module.c +++ b/arch/mips/kernel/module.c @@ -47,7 +47,7 @@ static DEFINE_SPINLOCK(dbe_lock); void *module_alloc(unsigned long size) { return __vmalloc_node_range(size, 1, MODULE_START, MODULE_END, - GFP_KERNEL, PAGE_KERNEL, NUMA_NO_NODE, + GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); } #endif diff --git a/arch/mips/pci/msi-octeon.c b/arch/mips/pci/msi-octeon.c index 63bbe07a1ccd..cffaaf4aae3c 100644 --- a/arch/mips/pci/msi-octeon.c +++ b/arch/mips/pci/msi-octeon.c @@ -178,7 +178,7 @@ msi_irq_allocated: pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); irq_set_msi_desc(irq, desc); - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); return 0; } diff --git a/arch/mips/pci/msi-xlp.c b/arch/mips/pci/msi-xlp.c index f7ac3edda1b2..6a40f24c91b4 100644 --- a/arch/mips/pci/msi-xlp.c +++ b/arch/mips/pci/msi-xlp.c @@ -217,7 +217,7 @@ static void xlp_msix_mask_ack(struct irq_data *d) msixvec = nlm_irq_msixvec(d->irq); link = nlm_irq_msixlink(msixvec); - mask_msi_irq(d); + pci_msi_mask_irq(d); md = irq_data_get_irq_handler_data(d); /* Ack MSI on bridge */ @@ -239,10 +239,10 @@ static void xlp_msix_mask_ack(struct irq_data *d) static struct irq_chip xlp_msix_chip = { .name = "XLP-MSIX", - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, .irq_mask_ack = xlp_msix_mask_ack, - .irq_unmask = unmask_msi_irq, + .irq_unmask = pci_msi_unmask_irq, }; void arch_teardown_msi_irq(unsigned int irq) @@ -345,7 +345,7 @@ static int xlp_setup_msi(uint64_t lnkbase, int node, int link, if (ret < 0) return ret; - write_msi_msg(xirq, &msg); + pci_write_msi_msg(xirq, &msg); return 0; } @@ -446,7 +446,7 @@ static int xlp_setup_msix(uint64_t lnkbase, int node, int link, if (ret < 0) return ret; - write_msi_msg(xirq, &msg); + pci_write_msi_msg(xirq, &msg); return 0; } diff --git a/arch/mips/pci/pci-xlr.c b/arch/mips/pci/pci-xlr.c index 0dde80332d3a..26d2dabef281 100644 --- a/arch/mips/pci/pci-xlr.c +++ b/arch/mips/pci/pci-xlr.c @@ -260,7 +260,7 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) if (ret < 0) return ret; - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); return 0; } #endif diff --git a/arch/mips/pci/pci.c b/arch/mips/pci/pci.c index 1bf60b127377..9eb54b557c9f 100644 --- a/arch/mips/pci/pci.c +++ b/arch/mips/pci/pci.c @@ -114,6 +114,7 @@ static void pcibios_scanbus(struct pci_controller *hose) pci_bus_size_bridges(bus); pci_bus_assign_resources(bus); } + pci_bus_add_devices(bus); } } diff --git a/arch/mn10300/unit-asb2305/pci.c b/arch/mn10300/unit-asb2305/pci.c index 6b4339f8c9c2..1d91c74b8d3f 100644 --- a/arch/mn10300/unit-asb2305/pci.c +++ b/arch/mn10300/unit-asb2305/pci.c @@ -347,6 +347,7 @@ static int __init pcibios_init(void) { resource_size_t io_offset, mem_offset; LIST_HEAD(resources); + struct pci_bus *bus; ioport_resource.start = 0xA0000000; ioport_resource.end = 0xDFFFFFFF; @@ -376,11 +377,14 @@ static int __init pcibios_init(void) pci_add_resource_offset(&resources, &pci_ioport_resource, io_offset); pci_add_resource_offset(&resources, &pci_iomem_resource, mem_offset); - pci_scan_root_bus(NULL, 0, &pci_direct_ampci, NULL, &resources); + bus = pci_scan_root_bus(NULL, 0, &pci_direct_ampci, NULL, &resources); + if (!bus) + return 0; pcibios_irq_init(); pcibios_fixup_irqs(); pcibios_resource_survey(); + pci_bus_add_devices(bus); return 0; } diff --git a/arch/parisc/kernel/module.c b/arch/parisc/kernel/module.c index 50dfafc3f2c1..0d498efddab2 100644 --- a/arch/parisc/kernel/module.c +++ b/arch/parisc/kernel/module.c @@ -219,7 +219,7 @@ void *module_alloc(unsigned long size) * init_data correctly */ return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, GFP_KERNEL | __GFP_HIGHMEM, - PAGE_KERNEL_RWX, NUMA_NO_NODE, + PAGE_KERNEL_RWX, 0, NUMA_NO_NODE, __builtin_return_address(0)); } diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index b9d1dfdbe5bb..9fe6002c1d5a 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -1711,12 +1711,11 @@ static void stage_topology_update(int core_id) static int dt_update_callback(struct notifier_block *nb, unsigned long action, void *data) { - struct of_prop_reconfig *update; + struct of_reconfig_data *update = data; int rc = NOTIFY_DONE; switch (action) { case OF_RECONFIG_UPDATE_PROPERTY: - update = (struct of_prop_reconfig *)data; if (!of_prop_cmp(update->dn->type, "cpu") && !of_prop_cmp(update->prop->name, "ibm,associativity")) { u32 core_id; diff --git a/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c b/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c index ca3a062ed1b9..11090ab4bf59 100644 --- a/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c +++ b/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c @@ -123,7 +123,8 @@ cpld_pic_cascade(unsigned int irq, struct irq_desc *desc) } static int -cpld_pic_host_match(struct irq_domain *h, struct device_node *node) +cpld_pic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { return cpld_pic_node == node; } diff --git a/arch/powerpc/platforms/cell/axon_msi.c b/arch/powerpc/platforms/cell/axon_msi.c index 862b32702d29..0883994df384 100644 --- a/arch/powerpc/platforms/cell/axon_msi.c +++ b/arch/powerpc/platforms/cell/axon_msi.c @@ -279,7 +279,7 @@ static int axon_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) irq_set_msi_desc(virq, entry); msg.data = virq; - write_msi_msg(virq, &msg); + pci_write_msi_msg(virq, &msg); } return 0; @@ -301,9 +301,9 @@ static void axon_msi_teardown_msi_irqs(struct pci_dev *dev) } static struct irq_chip msic_irq_chip = { - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, - .irq_shutdown = mask_msi_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, + .irq_shutdown = pci_msi_mask_irq, .name = "AXON-MSI", }; diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c index 28e558d3316d..109d236ca492 100644 --- a/arch/powerpc/platforms/cell/interrupt.c +++ b/arch/powerpc/platforms/cell/interrupt.c @@ -222,7 +222,8 @@ void iic_request_IPIs(void) #endif /* CONFIG_SMP */ -static int iic_host_match(struct irq_domain *h, struct device_node *node) +static int iic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { return of_device_is_compatible(node, "IBM,CBEA-Internal-Interrupt-Controller"); diff --git a/arch/powerpc/platforms/embedded6xx/flipper-pic.c b/arch/powerpc/platforms/embedded6xx/flipper-pic.c index 4cde8e7da4b8..b7866e01483d 100644 --- a/arch/powerpc/platforms/embedded6xx/flipper-pic.c +++ b/arch/powerpc/platforms/embedded6xx/flipper-pic.c @@ -108,7 +108,8 @@ static int flipper_pic_map(struct irq_domain *h, unsigned int virq, return 0; } -static int flipper_pic_match(struct irq_domain *h, struct device_node *np) +static int flipper_pic_match(struct irq_domain *h, struct device_node *np, + enum irq_domain_bus_token bus_token) { return 1; } diff --git a/arch/powerpc/platforms/powermac/pic.c b/arch/powerpc/platforms/powermac/pic.c index 4c24bf60d39d..246cab46bffe 100644 --- a/arch/powerpc/platforms/powermac/pic.c +++ b/arch/powerpc/platforms/powermac/pic.c @@ -268,7 +268,8 @@ static struct irqaction gatwick_cascade_action = { .name = "cascade", }; -static int pmac_pic_host_match(struct irq_domain *h, struct device_node *node) +static int pmac_pic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* We match all, we don't always have a node anyway */ return 1; diff --git a/arch/powerpc/platforms/powernv/opal-irqchip.c b/arch/powerpc/platforms/powernv/opal-irqchip.c new file mode 100644 index 000000000000..2c91ee7800b9 --- /dev/null +++ b/arch/powerpc/platforms/powernv/opal-irqchip.c @@ -0,0 +1,254 @@ +/* + * This file implements an irqchip for OPAL events. Whenever there is + * an interrupt that is handled by OPAL we get passed a list of events + * that Linux needs to do something about. These basically look like + * interrupts to Linux so we implement an irqchip to handle them. + * + * Copyright Alistair Popple, IBM Corporation 2014. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ +#include <linux/bitops.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqdomain.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/kthread.h> +#include <linux/delay.h> +#include <linux/slab.h> +#include <linux/irq_work.h> + +#include <asm/machdep.h> +#include <asm/opal.h> + +#include "powernv.h" + +/* Maximum number of events supported by OPAL firmware */ +#define MAX_NUM_EVENTS 64 + +struct opal_event_irqchip { + struct irq_chip irqchip; + struct irq_domain *domain; + unsigned long mask; +}; +static struct opal_event_irqchip opal_event_irqchip; + +static unsigned int opal_irq_count; +static unsigned int *opal_irqs; + +static void opal_handle_irq_work(struct irq_work *work); +static __be64 last_outstanding_events; +static struct irq_work opal_event_irq_work = { + .func = opal_handle_irq_work, +}; + +static void opal_event_mask(struct irq_data *d) +{ + clear_bit(d->hwirq, &opal_event_irqchip.mask); +} + +static void opal_event_unmask(struct irq_data *d) +{ + set_bit(d->hwirq, &opal_event_irqchip.mask); + + opal_poll_events(&last_outstanding_events); + if (last_outstanding_events & opal_event_irqchip.mask) + /* Need to retrigger the interrupt */ + irq_work_queue(&opal_event_irq_work); +} + +static int opal_event_set_type(struct irq_data *d, unsigned int flow_type) +{ + /* + * For now we only support level triggered events. The irq + * handler will be called continuously until the event has + * been cleared in OPAL. + */ + if (flow_type != IRQ_TYPE_LEVEL_HIGH) + return -EINVAL; + + return 0; +} + +static struct opal_event_irqchip opal_event_irqchip = { + .irqchip = { + .name = "OPAL EVT", + .irq_mask = opal_event_mask, + .irq_unmask = opal_event_unmask, + .irq_set_type = opal_event_set_type, + }, + .mask = 0, +}; + +static int opal_event_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_data(irq, &opal_event_irqchip); + irq_set_chip_and_handler(irq, &opal_event_irqchip.irqchip, + handle_level_irq); + + return 0; +} + +void opal_handle_events(uint64_t events) +{ + int virq, hwirq = 0; + u64 mask = opal_event_irqchip.mask; + + if (!in_irq() && (events & mask)) { + last_outstanding_events = events; + irq_work_queue(&opal_event_irq_work); + return; + } + + while (events & mask) { + hwirq = fls64(events) - 1; + if (BIT_ULL(hwirq) & mask) { + virq = irq_find_mapping(opal_event_irqchip.domain, + hwirq); + if (virq) + generic_handle_irq(virq); + } + events &= ~BIT_ULL(hwirq); + } +} + +static irqreturn_t opal_interrupt(int irq, void *data) +{ + __be64 events; + + opal_handle_interrupt(virq_to_hw(irq), &events); + opal_handle_events(be64_to_cpu(events)); + + return IRQ_HANDLED; +} + +static void opal_handle_irq_work(struct irq_work *work) +{ + opal_handle_events(be64_to_cpu(last_outstanding_events)); +} + +static int opal_event_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) +{ + return h->of_node == node; +} + +static int opal_event_xlate(struct irq_domain *h, struct device_node *np, + const u32 *intspec, unsigned int intsize, + irq_hw_number_t *out_hwirq, unsigned int *out_flags) +{ + *out_hwirq = intspec[0]; + *out_flags = IRQ_TYPE_LEVEL_HIGH; + + return 0; +} + +static const struct irq_domain_ops opal_event_domain_ops = { + .match = opal_event_match, + .map = opal_event_map, + .xlate = opal_event_xlate, +}; + +void opal_event_shutdown(void) +{ + unsigned int i; + + /* First free interrupts, which will also mask them */ + for (i = 0; i < opal_irq_count; i++) { + if (opal_irqs[i]) + free_irq(opal_irqs[i], NULL); + opal_irqs[i] = 0; + } +} + +int __init opal_event_init(void) +{ + struct device_node *dn, *opal_node; + const __be32 *irqs; + int i, irqlen, rc = 0; + + opal_node = of_find_node_by_path("/ibm,opal"); + if (!opal_node) { + pr_warn("opal: Node not found\n"); + return -ENODEV; + } + + /* If dn is NULL it means the domain won't be linked to a DT + * node so therefore irq_of_parse_and_map(...) wont work. But + * that shouldn't be problem because if we're running a + * version of skiboot that doesn't have the dn then the + * devices won't have the correct properties and will have to + * fall back to the legacy method (opal_event_request(...)) + * anyway. */ + dn = of_find_compatible_node(NULL, NULL, "ibm,opal-event"); + opal_event_irqchip.domain = irq_domain_add_linear(dn, MAX_NUM_EVENTS, + &opal_event_domain_ops, &opal_event_irqchip); + of_node_put(dn); + if (!opal_event_irqchip.domain) { + pr_warn("opal: Unable to create irq domain\n"); + rc = -ENOMEM; + goto out; + } + + /* Get interrupt property */ + irqs = of_get_property(opal_node, "opal-interrupts", &irqlen); + opal_irq_count = irqs ? (irqlen / 4) : 0; + pr_debug("Found %d interrupts reserved for OPAL\n", opal_irq_count); + + /* Install interrupt handlers */ + opal_irqs = kcalloc(opal_irq_count, sizeof(*opal_irqs), GFP_KERNEL); + for (i = 0; irqs && i < opal_irq_count; i++, irqs++) { + unsigned int irq, virq; + + /* Get hardware and virtual IRQ */ + irq = be32_to_cpup(irqs); + virq = irq_create_mapping(NULL, irq); + if (virq == NO_IRQ) { + pr_warn("Failed to map irq 0x%x\n", irq); + continue; + } + + /* Install interrupt handler */ + rc = request_irq(virq, opal_interrupt, 0, "opal", NULL); + if (rc) { + irq_dispose_mapping(virq); + pr_warn("Error %d requesting irq %d (0x%x)\n", + rc, virq, irq); + continue; + } + + /* Cache IRQ */ + opal_irqs[i] = virq; + } + +out: + of_node_put(opal_node); + return rc; +} +machine_arch_initcall(powernv, opal_event_init); + +/** + * opal_event_request(unsigned int opal_event_nr) - Request an event + * @opal_event_nr: the opal event number to request + * + * This routine can be used to find the linux virq number which can + * then be passed to request_irq to assign a handler for a particular + * opal event. This should only be used by legacy devices which don't + * have proper device tree bindings. Most devices should use + * irq_of_parse_and_map() instead. + */ +int opal_event_request(unsigned int opal_event_nr) +{ + if (WARN_ON_ONCE(!opal_event_irqchip.domain)) + return NO_IRQ; + + return irq_create_mapping(opal_event_irqchip.domain, opal_event_nr); +} +EXPORT_SYMBOL(opal_event_request); diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index 9ff55d59ac76..019991d574a0 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -90,7 +90,7 @@ static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) return rc; } irq_set_msi_desc(virq, entry); - write_msi_msg(virq, &msg); + pci_write_msi_msg(virq, &msg); } return 0; } diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c index 5f3b23220b8e..df0c086d8197 100644 --- a/arch/powerpc/platforms/ps3/interrupt.c +++ b/arch/powerpc/platforms/ps3/interrupt.c @@ -678,7 +678,8 @@ static int ps3_host_map(struct irq_domain *h, unsigned int virq, return 0; } -static int ps3_host_match(struct irq_domain *h, struct device_node *np) +static int ps3_host_match(struct irq_domain *h, struct device_node *np, + enum irq_domain_bus_token bus_token) { /* Match all */ return 1; diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c index 5c375f93c669..f30cf4d136a4 100644 --- a/arch/powerpc/platforms/pseries/hotplug-cpu.c +++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c @@ -340,16 +340,17 @@ static void pseries_remove_processor(struct device_node *np) } static int pseries_smp_notifier(struct notifier_block *nb, - unsigned long action, void *node) + unsigned long action, void *data) { + struct of_reconfig_data *rd = data; int err = 0; switch (action) { case OF_RECONFIG_ATTACH_NODE: - err = pseries_add_processor(node); + err = pseries_add_processor(rd->dn); break; case OF_RECONFIG_DETACH_NODE: - pseries_remove_processor(node); + pseries_remove_processor(rd->dn); break; } return notifier_from_errno(err); diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c index 3c4c0dcd90d3..1bbb78fab530 100644 --- a/arch/powerpc/platforms/pseries/hotplug-memory.c +++ b/arch/powerpc/platforms/pseries/hotplug-memory.c @@ -183,7 +183,7 @@ static int pseries_add_mem_node(struct device_node *np) return (ret < 0) ? -EINVAL : 0; } -static int pseries_update_drconf_memory(struct of_prop_reconfig *pr) +static int pseries_update_drconf_memory(struct of_reconfig_data *pr) { struct of_drconf_cell *new_drmem, *old_drmem; unsigned long memblock_size; @@ -232,22 +232,21 @@ static int pseries_update_drconf_memory(struct of_prop_reconfig *pr) } static int pseries_memory_notifier(struct notifier_block *nb, - unsigned long action, void *node) + unsigned long action, void *data) { - struct of_prop_reconfig *pr; + struct of_reconfig_data *rd = data; int err = 0; switch (action) { case OF_RECONFIG_ATTACH_NODE: - err = pseries_add_mem_node(node); + err = pseries_add_mem_node(rd->dn); break; case OF_RECONFIG_DETACH_NODE: - err = pseries_remove_mem_node(node); + err = pseries_remove_mem_node(rd->dn); break; case OF_RECONFIG_UPDATE_PROPERTY: - pr = (struct of_prop_reconfig *)node; - if (!strcmp(pr->prop->name, "ibm,dynamic-memory")) - err = pseries_update_drconf_memory(pr); + if (!strcmp(rd->prop->name, "ibm,dynamic-memory")) + err = pseries_update_drconf_memory(rd); break; } return notifier_from_errno(err); diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index 05a2c9eefc08..1e1fa545eb69 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -1251,10 +1251,11 @@ static struct notifier_block iommu_mem_nb = { .notifier_call = iommu_mem_notifier, }; -static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node) +static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data) { int err = NOTIFY_OK; - struct device_node *np = node; + struct of_reconfig_data *rd = data; + struct device_node *np = rd->dn; struct pci_dn *pci = PCI_DN(np); struct direct_window *window; diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index 8b909e94fd9a..691a154c286d 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -476,7 +476,7 @@ again: irq_set_msi_desc(virq, entry); /* Read config space back so we can restore after reset */ - __read_msi_msg(entry, &msg); + __pci_read_msi_msg(entry, &msg); entry->msg = msg; } diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index 125c589eeef5..ed8a90022a3d 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -251,9 +251,10 @@ static void __init pseries_discover_pic(void) " interrupt-controller\n"); } -static int pci_dn_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node) +static int pci_dn_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data) { - struct device_node *np = node; + struct of_reconfig_data *rd = data; + struct device_node *np = rd->dn; struct pci_dn *pci = NULL; int err = NOTIFY_OK; diff --git a/arch/powerpc/sysdev/ehv_pic.c b/arch/powerpc/sysdev/ehv_pic.c index 2d20f10a4203..eca0b00794fa 100644 --- a/arch/powerpc/sysdev/ehv_pic.c +++ b/arch/powerpc/sysdev/ehv_pic.c @@ -177,7 +177,8 @@ unsigned int ehv_pic_get_irq(void) return irq_linear_revmap(global_ehv_pic->irqhost, irq); } -static int ehv_pic_host_match(struct irq_domain *h, struct device_node *node) +static int ehv_pic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* Exact match, unless ehv_pic node is NULL */ return h->of_node == NULL || h->of_node == node; diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index ea6b3a1c79d8..f13282ca3ee9 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -82,8 +82,8 @@ static void fsl_msi_print_chip(struct irq_data *irqd, struct seq_file *p) static struct irq_chip fsl_msi_chip = { - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, .irq_ack = fsl_msi_end_irq, .irq_print_chip = fsl_msi_print_chip, }; @@ -243,7 +243,7 @@ static int fsl_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) irq_set_msi_desc(virq, entry); fsl_compose_msi_msg(pdev, hwirq, &msg, msi_data); - write_msi_msg(virq, &msg); + pci_write_msi_msg(virq, &msg); } return 0; diff --git a/arch/powerpc/sysdev/i8259.c b/arch/powerpc/sysdev/i8259.c index 45598da0b321..8c3756cbc4f9 100644 --- a/arch/powerpc/sysdev/i8259.c +++ b/arch/powerpc/sysdev/i8259.c @@ -162,7 +162,8 @@ static struct resource pic_edgectrl_iores = { .flags = IORESOURCE_BUSY, }; -static int i8259_host_match(struct irq_domain *h, struct device_node *node) +static int i8259_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { return h->of_node == NULL || h->of_node == node; } diff --git a/arch/powerpc/sysdev/ipic.c b/arch/powerpc/sysdev/ipic.c index b32900f8fbf3..23d42d796512 100644 --- a/arch/powerpc/sysdev/ipic.c +++ b/arch/powerpc/sysdev/ipic.c @@ -672,7 +672,8 @@ static struct irq_chip ipic_edge_irq_chip = { .irq_set_type = ipic_set_irq_type, }; -static int ipic_host_match(struct irq_domain *h, struct device_node *node) +static int ipic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* Exact match, unless ipic node is NULL */ return h->of_node == NULL || h->of_node == node; diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index 89cec0ed6a58..bf6f77e274b2 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -1009,7 +1009,8 @@ static struct irq_chip mpic_irq_ht_chip = { #endif /* CONFIG_MPIC_U3_HT_IRQS */ -static int mpic_host_match(struct irq_domain *h, struct device_node *node) +static int mpic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* Exact match, unless mpic node is NULL */ return h->of_node == NULL || h->of_node == node; diff --git a/arch/powerpc/sysdev/mpic_pasemi_msi.c b/arch/powerpc/sysdev/mpic_pasemi_msi.c index a6add4ae6c5a..5a4c4741cf14 100644 --- a/arch/powerpc/sysdev/mpic_pasemi_msi.c +++ b/arch/powerpc/sysdev/mpic_pasemi_msi.c @@ -42,7 +42,7 @@ static struct mpic *msi_mpic; static void mpic_pasemi_msi_mask_irq(struct irq_data *data) { pr_debug("mpic_pasemi_msi_mask_irq %d\n", data->irq); - mask_msi_irq(data); + pci_msi_mask_irq(data); mpic_mask_irq(data); } @@ -50,7 +50,7 @@ static void mpic_pasemi_msi_unmask_irq(struct irq_data *data) { pr_debug("mpic_pasemi_msi_unmask_irq %d\n", data->irq); mpic_unmask_irq(data); - unmask_msi_irq(data); + pci_msi_unmask_irq(data); } static struct irq_chip mpic_pasemi_msi_chip = { @@ -138,7 +138,7 @@ static int pasemi_msi_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) * register to generate MSI [512...1023] */ msg.data = hwirq-0x200; - write_msi_msg(virq, &msg); + pci_write_msi_msg(virq, &msg); } return 0; diff --git a/arch/powerpc/sysdev/mpic_u3msi.c b/arch/powerpc/sysdev/mpic_u3msi.c index db35a4073127..65880ccd3d36 100644 --- a/arch/powerpc/sysdev/mpic_u3msi.c +++ b/arch/powerpc/sysdev/mpic_u3msi.c @@ -25,14 +25,14 @@ static struct mpic *msi_mpic; static void mpic_u3msi_mask_irq(struct irq_data *data) { - mask_msi_irq(data); + pci_msi_mask_irq(data); mpic_mask_irq(data); } static void mpic_u3msi_unmask_irq(struct irq_data *data) { mpic_unmask_irq(data); - unmask_msi_irq(data); + pci_msi_unmask_irq(data); } static struct irq_chip mpic_u3msi_chip = { @@ -172,7 +172,7 @@ static int u3msi_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) printk("u3msi: allocated virq 0x%x (hw 0x%x) addr 0x%lx\n", virq, hwirq, (unsigned long)addr); msg.data = hwirq; - write_msi_msg(virq, &msg); + pci_write_msi_msg(virq, &msg); hwirq++; } diff --git a/arch/powerpc/sysdev/ppc4xx_hsta_msi.c b/arch/powerpc/sysdev/ppc4xx_hsta_msi.c index a6a4dbda9078..908105f835d1 100644 --- a/arch/powerpc/sysdev/ppc4xx_hsta_msi.c +++ b/arch/powerpc/sysdev/ppc4xx_hsta_msi.c @@ -85,7 +85,7 @@ static int hsta_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) msi_bitmap_free_hwirqs(&ppc4xx_hsta_msi.bmp, irq, 1); return -EINVAL; } - write_msi_msg(hwirq, &msg); + pci_write_msi_msg(hwirq, &msg); } return 0; diff --git a/arch/powerpc/sysdev/ppc4xx_msi.c b/arch/powerpc/sysdev/ppc4xx_msi.c index 85d9c1852d19..c6df3e205eab 100644 --- a/arch/powerpc/sysdev/ppc4xx_msi.c +++ b/arch/powerpc/sysdev/ppc4xx_msi.c @@ -116,7 +116,7 @@ static int ppc4xx_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) irq_set_msi_desc(virq, entry); msg.data = int_no; - write_msi_msg(virq, &msg); + pci_write_msi_msg(virq, &msg); } return 0; } diff --git a/arch/powerpc/sysdev/qe_lib/qe_ic.c b/arch/powerpc/sysdev/qe_lib/qe_ic.c index b2b87c30e266..a433b3d40d18 100644 --- a/arch/powerpc/sysdev/qe_lib/qe_ic.c +++ b/arch/powerpc/sysdev/qe_lib/qe_ic.c @@ -245,7 +245,8 @@ static struct irq_chip qe_ic_irq_chip = { .irq_mask_ack = qe_ic_mask_irq, }; -static int qe_ic_host_match(struct irq_domain *h, struct device_node *node) +static int qe_ic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* Exact match, unless qe_ic node is NULL */ return h->of_node == NULL || h->of_node == node; diff --git a/arch/powerpc/sysdev/xics/ics-opal.c b/arch/powerpc/sysdev/xics/ics-opal.c index 3c6ee1b64e5d..4ba554ec8eaf 100644 --- a/arch/powerpc/sysdev/xics/ics-opal.c +++ b/arch/powerpc/sysdev/xics/ics-opal.c @@ -73,7 +73,7 @@ static unsigned int ics_opal_startup(struct irq_data *d) * at that level, so we do it here by hand. */ if (d->msi_desc) - unmask_msi_irq(d); + pci_msi_unmask_irq(d); #endif /* unmask it */ diff --git a/arch/powerpc/sysdev/xics/ics-rtas.c b/arch/powerpc/sysdev/xics/ics-rtas.c index 936575d99c5c..bc81335b2cbc 100644 --- a/arch/powerpc/sysdev/xics/ics-rtas.c +++ b/arch/powerpc/sysdev/xics/ics-rtas.c @@ -76,7 +76,7 @@ static unsigned int ics_rtas_startup(struct irq_data *d) * at that level, so we do it here by hand. */ if (d->msi_desc) - unmask_msi_irq(d); + pci_msi_unmask_irq(d); #endif /* unmask it */ ics_rtas_unmask_irq(d); diff --git a/arch/powerpc/sysdev/xics/xics-common.c b/arch/powerpc/sysdev/xics/xics-common.c index fe0cca477164..13ab71690923 100644 --- a/arch/powerpc/sysdev/xics/xics-common.c +++ b/arch/powerpc/sysdev/xics/xics-common.c @@ -300,7 +300,8 @@ int xics_get_irq_server(unsigned int virq, const struct cpumask *cpumask, } #endif /* CONFIG_SMP */ -static int xics_host_match(struct irq_domain *h, struct device_node *node) +static int xics_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { struct ics *ics; diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index b89b59158b95..411a7eea81a1 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -50,7 +50,7 @@ void *module_alloc(unsigned long size) if (PAGE_ALIGN(size) > MODULES_LEN) return NULL; return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, NUMA_NO_NODE, + GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); } #endif diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 944818617718..b6b76785f879 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -50,8 +50,8 @@ static DEFINE_SPINLOCK(zpci_list_lock); static struct irq_chip zpci_irq_chip = { .name = "zPCI", - .irq_unmask = unmask_msi_irq, - .irq_mask = mask_msi_irq, + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, }; static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); @@ -408,7 +408,7 @@ int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) msg.data = hwirq; msg.address_lo = zdev->msi_addr & 0xffffffff; msg.address_hi = zdev->msi_addr >> 32; - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); airq_iv_set_data(zdev->aibv, hwirq, irq); hwirq++; } @@ -453,9 +453,9 @@ void arch_teardown_msi_irqs(struct pci_dev *pdev) /* Release MSI interrupts */ list_for_each_entry(msi, &pdev->msi_list, list) { if (msi->msi_attrib.is_msix) - default_msix_mask_irq(msi, 1); + __pci_msix_desc_mask_irq(msi, 1); else - default_msi_mask_irq(msi, 1, 1); + __pci_msi_desc_mask_irq(msi, 1, 1); irq_set_msi_desc(msi->irq, NULL); irq_free_desc(msi->irq); msi->msg.address_lo = 0; @@ -760,8 +760,8 @@ static int zpci_scan_bus(struct zpci_dev *zdev) zpci_cleanup_bus_resources(zdev); return -EIO; } - zdev->bus->max_bus_speed = zdev->max_bus_speed; + pci_bus_add_devices(zdev->bus); return 0; } diff --git a/arch/sh/drivers/pci/pci.c b/arch/sh/drivers/pci/pci.c index 1bc09ee7948f..efc10519916a 100644 --- a/arch/sh/drivers/pci/pci.c +++ b/arch/sh/drivers/pci/pci.c @@ -69,6 +69,7 @@ static void pcibios_scanbus(struct pci_channel *hose) pci_bus_size_bridges(bus); pci_bus_assign_resources(bus); + pci_bus_add_devices(bus); } else { pci_free_resource_list(&resources); } diff --git a/arch/sparc/kernel/leon_pci.c b/arch/sparc/kernel/leon_pci.c index 899b7203a4e4..297107679fdf 100644 --- a/arch/sparc/kernel/leon_pci.c +++ b/arch/sparc/kernel/leon_pci.c @@ -40,6 +40,7 @@ void leon_pci_init(struct platform_device *ofdev, struct leon_pci_info *info) /* Assign devices with resources */ pci_assign_unassigned_resources(); + pci_bus_add_devices(root_bus); } else { pci_free_resource_list(&resources); } diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c index 97655e0fd243..192a617a32f3 100644 --- a/arch/sparc/kernel/module.c +++ b/arch/sparc/kernel/module.c @@ -29,7 +29,7 @@ static void *module_map(unsigned long size) if (PAGE_ALIGN(size) > MODULES_LEN) return NULL; return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, NUMA_NO_NODE, + GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); } #else diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c index 580651af73f2..84e16d81a6d8 100644 --- a/arch/sparc/kernel/pci_msi.c +++ b/arch/sparc/kernel/pci_msi.c @@ -111,10 +111,10 @@ static void free_msi(struct pci_pbm_info *pbm, int msi_num) static struct irq_chip msi_irq = { .name = "PCI-MSI", - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, /* XXX affinity XXX */ }; @@ -161,7 +161,7 @@ static int sparc64_setup_msi_irq(unsigned int *irq_p, msg.data = msi; irq_set_msi_desc(*irq_p, entry); - write_msi_msg(*irq_p, &msg); + pci_write_msi_msg(*irq_p, &msg); return 0; diff --git a/arch/sparc/kernel/pcic.c b/arch/sparc/kernel/pcic.c index 6cc78c213c01..24384e1dc33d 100644 --- a/arch/sparc/kernel/pcic.c +++ b/arch/sparc/kernel/pcic.c @@ -391,12 +391,16 @@ static void __init pcic_pbm_scan_bus(struct linux_pcic *pcic) struct linux_pbm_info *pbm = &pcic->pbm; pbm->pci_bus = pci_scan_bus(pbm->pci_first_busno, &pcic_ops, pbm); + if (!pbm->pci_bus) + return; + #if 0 /* deadwood transplanted from sparc64 */ pci_fill_in_pbm_cookies(pbm->pci_bus, pbm, pbm->prom_node); pci_record_assignments(pbm, pbm->pci_bus); pci_assign_unassigned(pbm, pbm->pci_bus); pci_fixup_irq(pbm, pbm->pci_bus); #endif + pci_bus_add_devices(pbm->pci_bus); } /* diff --git a/arch/tile/kernel/pci.c b/arch/tile/kernel/pci.c index 1f80a88c75a6..9503957f5d9e 100644 --- a/arch/tile/kernel/pci.c +++ b/arch/tile/kernel/pci.c @@ -339,6 +339,8 @@ int __init pcibios_init(void) struct pci_bus *next_bus; struct pci_dev *dev; + pci_bus_add_devices(root_bus); + list_for_each_entry(dev, &root_bus->devices, bus_list) { /* * Find the PCI host controller, ie. the 1st diff --git a/arch/tile/kernel/pci_gx.c b/arch/tile/kernel/pci_gx.c index e39f9c542807..be79d2894e73 100644 --- a/arch/tile/kernel/pci_gx.c +++ b/arch/tile/kernel/pci_gx.c @@ -1042,6 +1042,8 @@ int __init pcibios_init(void) alloc_mem_map_failed: break; } + + pci_bus_add_devices(root_bus); } return 0; @@ -1453,7 +1455,7 @@ static struct pci_ops tile_cfg_ops = { static unsigned int tilegx_msi_startup(struct irq_data *d) { if (d->msi_desc) - unmask_msi_irq(d); + pci_msi_unmask_irq(d); return 0; } @@ -1465,14 +1467,14 @@ static void tilegx_msi_ack(struct irq_data *d) static void tilegx_msi_mask(struct irq_data *d) { - mask_msi_irq(d); + pci_msi_mask_irq(d); __insn_mtspr(SPR_IPI_MASK_SET_K, 1UL << d->irq); } static void tilegx_msi_unmask(struct irq_data *d) { __insn_mtspr(SPR_IPI_MASK_RESET_K, 1UL << d->irq); - unmask_msi_irq(d); + pci_msi_unmask_irq(d); } static struct irq_chip tilegx_msi_chip = { @@ -1590,7 +1592,7 @@ int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) msg.address_hi = msi_addr >> 32; msg.address_lo = msi_addr & 0xffffffff; - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); irq_set_chip_and_handler(irq, &tilegx_msi_chip, handle_level_irq); irq_set_handler_data(irq, controller); diff --git a/arch/unicore32/kernel/module.c b/arch/unicore32/kernel/module.c index dc41f6dfedb6..e191b3448bd3 100644 --- a/arch/unicore32/kernel/module.c +++ b/arch/unicore32/kernel/module.c @@ -25,7 +25,7 @@ void *module_alloc(unsigned long size) { return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE, + GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, __builtin_return_address(0)); } diff --git a/arch/unicore32/kernel/pci.c b/arch/unicore32/kernel/pci.c index 374a055a8e6b..d45fa5f3e9c4 100644 --- a/arch/unicore32/kernel/pci.c +++ b/arch/unicore32/kernel/pci.c @@ -266,17 +266,10 @@ static int __init pci_common_init(void) pci_fixup_irqs(pci_common_swizzle, pci_puv3_map_irq); if (!pci_has_flag(PCI_PROBE_ONLY)) { - /* - * Size the bridge windows. - */ pci_bus_size_bridges(puv3_bus); - - /* - * Assign resources. - */ pci_bus_assign_resources(puv3_bus); } - + pci_bus_add_devices(puv3_bus); return 0; } subsys_initcall(pci_common_init); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index a5d100cf8ed6..c50d0116714d 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -83,6 +83,7 @@ config X86 select HAVE_CMPXCHG_LOCAL select HAVE_CMPXCHG_DOUBLE select HAVE_ARCH_KMEMCHECK + select HAVE_ARCH_KASAN if X86_64 && SPARSEMEM_VMEMMAP select HAVE_USER_RETURN_NOTIFIER select ARCH_BINFMT_ELF_RANDOMIZE_PIE select HAVE_ARCH_JUMP_LABEL diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index 32342a6034df..c45301c272d0 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -9,6 +9,8 @@ # Changed by many, many contributors over the years. # +KASAN_SANITIZE := n + # If you want to preset the SVGA mode, uncomment the next line and # set SVGA_MODE to whatever number you want. # Set it to -DSVGA_MODE=NORMAL_VGA if you just want the EGA/VGA mode. diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 30c0acf4ea6c..2959cca0b90c 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -4,6 +4,8 @@ # create a compressed vmlinux image from the original vmlinux # +KASAN_SANITIZE := n + targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c index acdf06bc418b..2da9cef8ee43 100644 --- a/arch/x86/boot/compressed/eboot.c +++ b/arch/x86/boot/compressed/eboot.c @@ -13,8 +13,7 @@ #include <asm/setup.h> #include <asm/desc.h> -#undef memcpy /* Use memcpy from misc.c */ - +#include "../string.h" #include "eboot.h" static efi_system_table_t *sys_table; diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 124312be129b..1ea2f195f41e 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -9,6 +9,10 @@ */ #undef CONFIG_PARAVIRT #undef CONFIG_PARAVIRT_SPINLOCKS +#undef CONFIG_KASAN +#ifdef CONFIG_X86_32 +#define _ASM_X86_DESC_H 1 +#endif #include <linux/linkage.h> #include <linux/screen_info.h> diff --git a/arch/x86/include/asm/kasan.h b/arch/x86/include/asm/kasan.h new file mode 100644 index 000000000000..491e4fd7754e --- /dev/null +++ b/arch/x86/include/asm/kasan.h @@ -0,0 +1,34 @@ +#ifndef _ASM_X86_KASAN_H +#define _ASM_X86_KASAN_H + +#include <linux/const.h> +#define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL) + +/* + * Compiler uses shadow offset assuming that addresses start + * from 0. Kernel addresses don't start from 0, so shadow + * for kernel really starts from compiler's shadow offset + + * 'kernel address space start' >> KASAN_SHADOW_SCALE_SHIFT + */ +#define KASAN_SHADOW_START (KASAN_SHADOW_OFFSET + \ + (0xffff800000000000ULL >> 3)) +/* 47 bits for kernel address -> (47 - 3) bits for shadow */ +#define KASAN_SHADOW_END (KASAN_SHADOW_START + (1ULL << (47 - 3))) + +#ifndef __ASSEMBLY__ + +extern pte_t kasan_zero_pte[]; +extern pte_t kasan_zero_pmd[]; +extern pte_t kasan_zero_pud[]; + +#ifdef CONFIG_KASAN +void __init kasan_map_early_shadow(pgd_t *pgd); +void __init kasan_init(void); +#else +static inline void kasan_map_early_shadow(pgd_t *pgd) { } +static inline void kasan_init(void) { } +#endif + +#endif + +#endif diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h index 75450b2c7be4..4edd53b79a81 100644 --- a/arch/x86/include/asm/page_64_types.h +++ b/arch/x86/include/asm/page_64_types.h @@ -1,17 +1,23 @@ #ifndef _ASM_X86_PAGE_64_DEFS_H #define _ASM_X86_PAGE_64_DEFS_H -#define THREAD_SIZE_ORDER 2 +#ifdef CONFIG_KASAN +#define KASAN_STACK_ORDER 1 +#else +#define KASAN_STACK_ORDER 0 +#endif + +#define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER) #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #define CURRENT_MASK (~(THREAD_SIZE - 1)) -#define EXCEPTION_STACK_ORDER 0 +#define EXCEPTION_STACK_ORDER (0 + KASAN_STACK_ORDER) #define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER) #define DEBUG_STACK_ORDER (EXCEPTION_STACK_ORDER + 1) #define DEBUG_STKSZ (PAGE_SIZE << DEBUG_STACK_ORDER) -#define IRQ_STACK_ORDER 2 +#define IRQ_STACK_ORDER (2 + KASAN_STACK_ORDER) #define IRQ_STACK_SIZE (PAGE_SIZE << IRQ_STACK_ORDER) #define DOUBLEFAULT_STACK 1 diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 19e2c468fc2c..e4661196994e 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -27,11 +27,12 @@ static __always_inline void *__inline_memcpy(void *to, const void *from, size_t function. */ #define __HAVE_ARCH_MEMCPY 1 +extern void *__memcpy(void *to, const void *from, size_t len); + #ifndef CONFIG_KMEMCHECK #if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4 extern void *memcpy(void *to, const void *from, size_t len); #else -extern void *__memcpy(void *to, const void *from, size_t len); #define memcpy(dst, src, len) \ ({ \ size_t __len = (len); \ @@ -53,9 +54,11 @@ extern void *__memcpy(void *to, const void *from, size_t len); #define __HAVE_ARCH_MEMSET void *memset(void *s, int c, size_t n); +void *__memset(void *s, int c, size_t n); #define __HAVE_ARCH_MEMMOVE void *memmove(void *dest, const void *src, size_t count); +void *__memmove(void *dest, const void *src, size_t count); int memcmp(const void *cs, const void *ct, size_t count); size_t strlen(const char *s); @@ -63,6 +66,19 @@ char *strcpy(char *dest, const char *src); char *strcat(char *dest, const char *src); int strcmp(const char *cs, const char *ct); +#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) + +/* + * For files that not instrumented (e.g. mm/slub.c) we + * should use not instrumented version of mem* functions. + */ + +#undef memcpy +#define memcpy(dst, src, len) __memcpy(dst, src, len) +#define memmove(dst, src, len) __memmove(dst, src, len) +#define memset(s, c, n) __memset(s, c, n) +#endif + #endif /* __KERNEL__ */ #endif /* _ASM_X86_STRING_64_H */ diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index e45e4da96bf1..f58a9c7a3c86 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -172,7 +172,6 @@ struct x86_platform_ops { struct pci_dev; struct msi_msg; -struct msi_desc; struct x86_msi_ops { int (*setup_msi_irqs)(struct pci_dev *dev, int nvec, int type); @@ -183,8 +182,6 @@ struct x86_msi_ops { void (*teardown_msi_irqs)(struct pci_dev *dev); void (*restore_msi_irqs)(struct pci_dev *dev); int (*setup_hpet_msi)(unsigned int irq, unsigned int id); - u32 (*msi_mask_irq)(struct msi_desc *desc, u32 mask, u32 flag); - u32 (*msix_mask_irq)(struct msi_desc *desc, u32 flag); }; struct IO_APIC_route_entry; diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 8f1e77440b2b..2e7f86e8a4a6 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -16,6 +16,10 @@ CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_early_printk.o = -pg endif +KASAN_SANITIZE_head$(BITS).o := n +KASAN_SANITIZE_dumpstack.o := n +KASAN_SANITIZE_dumpstack_$(BITS).o := n + CFLAGS_irq.o := -I$(src)/../include/asm/trace obj-y := process_$(BITS).o signal.o entry_$(BITS).o diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 1183d545da1e..7ffe0a2b870f 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -3158,7 +3158,7 @@ msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK; msg.address_lo |= MSI_ADDR_DEST_ID(dest); - __write_msi_msg(data->msi_desc, &msg); + __pci_write_msi_msg(data->msi_desc, &msg); return IRQ_SET_MASK_OK_NOCOPY; } @@ -3169,8 +3169,8 @@ msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) */ static struct irq_chip msi_chip = { .name = "PCI-MSI", - .irq_unmask = unmask_msi_irq, - .irq_mask = mask_msi_irq, + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, .irq_ack = ack_apic_edge, .irq_set_affinity = msi_set_affinity, .irq_retrigger = ioapic_retrigger_irq, @@ -3196,7 +3196,7 @@ int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, * MSI message denotes a contiguous group of IRQs, written for 0th IRQ. */ if (!irq_offset) - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); setup_remapped_irq(irq, irq_cfg(irq), chip); diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index b74ebc7c4402..cf3df1d8d039 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -265,7 +265,10 @@ int __die(const char *str, struct pt_regs *regs, long err) printk("SMP "); #endif #ifdef CONFIG_DEBUG_PAGEALLOC - printk("DEBUG_PAGEALLOC"); + printk("DEBUG_PAGEALLOC "); +#endif +#ifdef CONFIG_KASAN + printk("KASAN"); #endif printk("\n"); if (notify_die(DIE_OOPS, str, regs, err, diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 957fdb5eb2e3..b111ab5c4509 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -27,6 +27,7 @@ #include <asm/bios_ebda.h> #include <asm/bootparam_utils.h> #include <asm/microcode.h> +#include <asm/kasan.h> /* * Manage page tables very early on. @@ -46,7 +47,7 @@ static void __init reset_early_page_tables(void) next_early_pgt = 0; - write_cr3(__pa(early_level4_pgt)); + write_cr3(__pa_nodebug(early_level4_pgt)); } /* Create a new PMD entry */ @@ -59,7 +60,7 @@ int __init early_make_pgtable(unsigned long address) pmdval_t pmd, *pmd_p; /* Invalid address or early pgt is done ? */ - if (physaddr >= MAXMEM || read_cr3() != __pa(early_level4_pgt)) + if (physaddr >= MAXMEM || read_cr3() != __pa_nodebug(early_level4_pgt)) return -1; again: @@ -160,6 +161,8 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) /* Kill off the identity-map trampoline */ reset_early_page_tables(); + kasan_map_early_shadow(early_level4_pgt); + /* clear bss before set_intr_gate with early_idt_handler */ clear_bss(); @@ -181,6 +184,8 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) /* set init_level4_pgt kernel high mapping*/ init_level4_pgt[511] = early_level4_pgt[511]; + kasan_map_early_shadow(init_level4_pgt); + x86_64_start_reservations(real_mode_data); } diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index a2dc0add72ed..f8a8406033c3 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -516,8 +516,38 @@ ENTRY(phys_base) /* This must match the first entry in level2_kernel_pgt */ .quad 0x0000000000000000 +#ifdef CONFIG_KASAN +#define FILL(VAL, COUNT) \ + .rept (COUNT) ; \ + .quad (VAL) ; \ + .endr + +NEXT_PAGE(kasan_zero_pte) + FILL(kasan_zero_page - __START_KERNEL_map + _KERNPG_TABLE, 512) +NEXT_PAGE(kasan_zero_pmd) + FILL(kasan_zero_pte - __START_KERNEL_map + _KERNPG_TABLE, 512) +NEXT_PAGE(kasan_zero_pud) + FILL(kasan_zero_pmd - __START_KERNEL_map + _KERNPG_TABLE, 512) + +#undef FILL +#endif + + #include "../../x86/xen/xen-head.S" __PAGE_ALIGNED_BSS NEXT_PAGE(empty_zero_page) .skip PAGE_SIZE + +#ifdef CONFIG_KASAN +/* + * This page used as early shadow. We don't use empty_zero_page + * at early stages, stack instrumentation could write some garbage + * to this page. + * Latter we reuse it as zero shadow for large ranges of memory + * that allowed to access, but not instrumented by kasan + * (vmalloc/vmemmap ...). + */ +NEXT_PAGE(kasan_zero_page) + .skip PAGE_SIZE +#endif diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index 53ce44fe8f21..0bc26d71a9dc 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -24,6 +24,7 @@ #include <linux/fs.h> #include <linux/string.h> #include <linux/kernel.h> +#include <linux/kasan.h> #include <linux/bug.h> #include <linux/mm.h> #include <linux/gfp.h> @@ -83,13 +84,22 @@ static unsigned long int get_module_load_offset(void) void *module_alloc(unsigned long size) { + void *p; + if (PAGE_ALIGN(size) > MODULES_LEN) return NULL; - return __vmalloc_node_range(size, 1, + + p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR + get_module_load_offset(), MODULES_END, GFP_KERNEL | __GFP_HIGHMEM, - PAGE_KERNEL_EXEC, NUMA_NO_NODE, + PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, __builtin_return_address(0)); + if (p && (kasan_module_alloc(p, size) < 0)) { + vfree(p); + return NULL; + } + + return p; } #ifdef CONFIG_X86_32 diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 87b6496a32f1..28a850625672 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -89,6 +89,7 @@ #include <asm/cacheflush.h> #include <asm/processor.h> #include <asm/bugs.h> +#include <asm/kasan.h> #include <asm/vsyscall.h> #include <asm/cpu.h> @@ -1174,6 +1175,8 @@ void __init setup_arch(char **cmdline_p) x86_init.paging.pagetable_init(); + kasan_init(); + if (boot_cpu_data.cpuid_level >= 0) { /* A CPU has %cr4 if and only if it has CPUID */ mmu_cr4_features = __read_cr4(); diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c index 040681928e9d..37d8fa4438f0 100644 --- a/arch/x86/kernel/x8664_ksyms_64.c +++ b/arch/x86/kernel/x8664_ksyms_64.c @@ -50,13 +50,19 @@ EXPORT_SYMBOL(csum_partial); #undef memset #undef memmove +extern void *__memset(void *, int, __kernel_size_t); +extern void *__memcpy(void *, const void *, __kernel_size_t); +extern void *__memmove(void *, const void *, __kernel_size_t); extern void *memset(void *, int, __kernel_size_t); extern void *memcpy(void *, const void *, __kernel_size_t); -extern void *__memcpy(void *, const void *, __kernel_size_t); +extern void *memmove(void *, const void *, __kernel_size_t); + +EXPORT_SYMBOL(__memset); +EXPORT_SYMBOL(__memcpy); +EXPORT_SYMBOL(__memmove); EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memcpy); -EXPORT_SYMBOL(__memcpy); EXPORT_SYMBOL(memmove); #ifndef CONFIG_DEBUG_VIRTUAL diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index e48b674639cc..234b0722de53 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -116,8 +116,6 @@ struct x86_msi_ops x86_msi = { .teardown_msi_irqs = default_teardown_msi_irqs, .restore_msi_irqs = default_restore_msi_irqs, .setup_hpet_msi = default_setup_hpet_msi, - .msi_mask_irq = default_msi_mask_irq, - .msix_mask_irq = default_msix_mask_irq, }; /* MSI arch specific hooks */ @@ -140,14 +138,6 @@ void arch_restore_msi_irqs(struct pci_dev *dev) { x86_msi.restore_msi_irqs(dev); } -u32 arch_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) -{ - return x86_msi.msi_mask_irq(desc, mask, flag); -} -u32 arch_msix_mask_irq(struct msi_desc *desc, u32 flag) -{ - return x86_msi.msix_mask_irq(desc, flag); -} #endif struct x86_io_apic_ops x86_io_apic_ops = { diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index 56313a326188..89b53c9968e7 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S @@ -53,6 +53,8 @@ .Lmemcpy_e_e: .previous +.weak memcpy + ENTRY(__memcpy) ENTRY(memcpy) CFI_STARTPROC @@ -199,8 +201,8 @@ ENDPROC(__memcpy) * only outcome... */ .section .altinstructions, "a" - altinstruction_entry memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\ + altinstruction_entry __memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\ .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c - altinstruction_entry memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \ + altinstruction_entry __memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \ .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e .previous diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S index 65268a6104f4..9c4b530575da 100644 --- a/arch/x86/lib/memmove_64.S +++ b/arch/x86/lib/memmove_64.S @@ -24,7 +24,10 @@ * Output: * rax: dest */ +.weak memmove + ENTRY(memmove) +ENTRY(__memmove) CFI_STARTPROC /* Handle more 32 bytes in loop */ @@ -220,4 +223,5 @@ ENTRY(memmove) .Lmemmove_end_forward-.Lmemmove_begin_forward, \ .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs .previous +ENDPROC(__memmove) ENDPROC(memmove) diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S index 2dcb3808cbda..6f44935c6a60 100644 --- a/arch/x86/lib/memset_64.S +++ b/arch/x86/lib/memset_64.S @@ -56,6 +56,8 @@ .Lmemset_e_e: .previous +.weak memset + ENTRY(memset) ENTRY(__memset) CFI_STARTPROC @@ -147,8 +149,8 @@ ENDPROC(__memset) * feature to implement the right patch order. */ .section .altinstructions,"a" - altinstruction_entry memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\ - .Lfinal-memset,.Lmemset_e-.Lmemset_c - altinstruction_entry memset,.Lmemset_c_e,X86_FEATURE_ERMS, \ - .Lfinal-memset,.Lmemset_e_e-.Lmemset_c_e + altinstruction_entry __memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\ + .Lfinal-__memset,.Lmemset_e-.Lmemset_c + altinstruction_entry __memset,.Lmemset_c_e,X86_FEATURE_ERMS, \ + .Lfinal-__memset,.Lmemset_e_e-.Lmemset_c_e .previous diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 6a19ad9f370d..9648838b78fa 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -20,6 +20,9 @@ obj-$(CONFIG_HIGHMEM) += highmem_32.o obj-$(CONFIG_KMEMCHECK) += kmemcheck/ +KASAN_SANITIZE_kasan_init_$(BITS).o := n +obj-$(CONFIG_KASAN) += kasan_init_$(BITS).o + obj-$(CONFIG_MMIOTRACE) += mmiotrace.o mmiotrace-y := kmmio.o pf_in.o mmio-mod.o obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c new file mode 100644 index 000000000000..23dc7673e110 --- /dev/null +++ b/arch/x86/mm/kasan_init_64.c @@ -0,0 +1,209 @@ +#include <linux/bootmem.h> +#include <linux/kasan.h> +#include <linux/kdebug.h> +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/vmalloc.h> + +#include <asm/tlbflush.h> +#include <asm/sections.h> + +extern pgd_t early_level4_pgt[PTRS_PER_PGD]; +extern struct range pfn_mapped[E820_X_MAX]; + +extern unsigned char kasan_zero_page[PAGE_SIZE]; + +static int __init map_range(struct range *range) +{ + unsigned long start; + unsigned long end; + + start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start)); + end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end)); + + /* + * end + 1 here is intentional. We check several shadow bytes in advance + * to slightly speed up fastpath. In some rare cases we could cross + * boundary of mapped shadow, so we just map some more here. + */ + return vmemmap_populate(start, end + 1, NUMA_NO_NODE); +} + +static void __init clear_pgds(unsigned long start, + unsigned long end) +{ + for (; start < end; start += PGDIR_SIZE) + pgd_clear(pgd_offset_k(start)); +} + +void __init kasan_map_early_shadow(pgd_t *pgd) +{ + int i; + unsigned long start = KASAN_SHADOW_START; + unsigned long end = KASAN_SHADOW_END; + + for (i = pgd_index(start); start < end; i++) { + pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud) + | _KERNPG_TABLE); + start += PGDIR_SIZE; + } +} + +static int __init zero_pte_populate(pmd_t *pmd, unsigned long addr, + unsigned long end) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); + + while (addr + PAGE_SIZE <= end) { + WARN_ON(!pte_none(*pte)); + set_pte(pte, __pte(__pa_nodebug(kasan_zero_page) + | __PAGE_KERNEL_RO)); + addr += PAGE_SIZE; + pte = pte_offset_kernel(pmd, addr); + } + return 0; +} + +static int __init zero_pmd_populate(pud_t *pud, unsigned long addr, + unsigned long end) +{ + int ret = 0; + pmd_t *pmd = pmd_offset(pud, addr); + + while (IS_ALIGNED(addr, PMD_SIZE) && addr + PMD_SIZE <= end) { + WARN_ON(!pmd_none(*pmd)); + set_pmd(pmd, __pmd(__pa_nodebug(kasan_zero_pte) + | __PAGE_KERNEL_RO)); + addr += PMD_SIZE; + pmd = pmd_offset(pud, addr); + } + if (addr < end) { + if (pmd_none(*pmd)) { + void *p = vmemmap_alloc_block(PAGE_SIZE, NUMA_NO_NODE); + if (!p) + return -ENOMEM; + set_pmd(pmd, __pmd(__pa_nodebug(p) | _KERNPG_TABLE)); + } + ret = zero_pte_populate(pmd, addr, end); + } + return ret; +} + + +static int __init zero_pud_populate(pgd_t *pgd, unsigned long addr, + unsigned long end) +{ + int ret = 0; + pud_t *pud = pud_offset(pgd, addr); + + while (IS_ALIGNED(addr, PUD_SIZE) && addr + PUD_SIZE <= end) { + WARN_ON(!pud_none(*pud)); + set_pud(pud, __pud(__pa_nodebug(kasan_zero_pmd) + | __PAGE_KERNEL_RO)); + addr += PUD_SIZE; + pud = pud_offset(pgd, addr); + } + + if (addr < end) { + if (pud_none(*pud)) { + void *p = vmemmap_alloc_block(PAGE_SIZE, NUMA_NO_NODE); + if (!p) + return -ENOMEM; + set_pud(pud, __pud(__pa_nodebug(p) | _KERNPG_TABLE)); + } + ret = zero_pmd_populate(pud, addr, end); + } + return ret; +} + +static int __init zero_pgd_populate(unsigned long addr, unsigned long end) +{ + int ret = 0; + pgd_t *pgd = pgd_offset_k(addr); + + while (IS_ALIGNED(addr, PGDIR_SIZE) && addr + PGDIR_SIZE <= end) { + WARN_ON(!pgd_none(*pgd)); + set_pgd(pgd, __pgd(__pa_nodebug(kasan_zero_pud) + | __PAGE_KERNEL_RO)); + addr += PGDIR_SIZE; + pgd = pgd_offset_k(addr); + } + + if (addr < end) { + if (pgd_none(*pgd)) { + void *p = vmemmap_alloc_block(PAGE_SIZE, NUMA_NO_NODE); + if (!p) + return -ENOMEM; + set_pgd(pgd, __pgd(__pa_nodebug(p) | _KERNPG_TABLE)); + } + ret = zero_pud_populate(pgd, addr, end); + } + return ret; +} + + +static void __init populate_zero_shadow(const void *start, const void *end) +{ + if (zero_pgd_populate((unsigned long)start, (unsigned long)end)) + panic("kasan: unable to map zero shadow!"); +} + + +#ifdef CONFIG_KASAN_INLINE +static int kasan_die_handler(struct notifier_block *self, + unsigned long val, + void *data) +{ + if (val == DIE_GPF) { + pr_emerg("CONFIG_KASAN_INLINE enabled"); + pr_emerg("GPF could be caused by NULL-ptr deref or user memory access"); + } + return NOTIFY_OK; +} + +static struct notifier_block kasan_die_notifier = { + .notifier_call = kasan_die_handler, +}; +#endif + +void __init kasan_init(void) +{ + int i; + +#ifdef CONFIG_KASAN_INLINE + register_die_notifier(&kasan_die_notifier); +#endif + + memcpy(early_level4_pgt, init_level4_pgt, sizeof(early_level4_pgt)); + load_cr3(early_level4_pgt); + + clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); + + kasan_populate_zero_shadow((void *)KASAN_SHADOW_START, + kasan_mem_to_shadow((void *)PAGE_OFFSET)); + + for (i = 0; i < E820_X_MAX; i++) { + if (pfn_mapped[i].end == 0) + break; + + if (map_range(&pfn_mapped[i])) + panic("kasan: unable to allocate shadow!"); + } + kasan_populate_zero_shadow( + kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), + kasan_mem_to_shadow((void *)__START_KERNEL_map)); + + vmemmap_populate((unsigned long)kasan_mem_to_shadow(_stext), + (unsigned long)kasan_mem_to_shadow(_end), + NUMA_NO_NODE); + + kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), + (void *)KASAN_SHADOW_END); + + memset(kasan_zero_page, 0, PAGE_SIZE); + + load_cr3(init_level4_pgt); + init_task.kasan_depth = 0; + + pr_info("KernelAddressSanitizer initialized\n"); +} diff --git a/arch/x86/pci/common.c b/arch/x86/pci/common.c index 2fb384724ebb..8fd6f44aee83 100644 --- a/arch/x86/pci/common.c +++ b/arch/x86/pci/common.c @@ -490,7 +490,9 @@ void pcibios_scan_root(int busnum) if (!bus) { pci_free_resource_list(&resources); kfree(sd); + return; } + pci_bus_add_devices(bus); } void __init pcibios_set_cache_line_size(void) diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 41c15dc4241c..df8101fa5bd6 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -229,7 +229,7 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return 1; list_for_each_entry(msidesc, &dev->msi_list, list) { - __read_msi_msg(msidesc, &msg); + __pci_read_msi_msg(msidesc, &msg); pirq = MSI_ADDR_EXT_DEST_ID(msg.address_hi) | ((msg.address_lo >> MSI_ADDR_DEST_ID_SHIFT) & 0xff); if (msg.data != XEN_PIRQ_MSI_DATA || @@ -240,7 +240,7 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) goto error; } xen_msi_compose_msg(dev, pirq, &msg); - __write_msi_msg(msidesc, &msg); + __pci_write_msi_msg(msidesc, &msg); dev_dbg(&dev->dev, "xen: msi bound to pirq=%d\n", pirq); } else { dev_dbg(&dev->dev, @@ -394,14 +394,7 @@ static void xen_teardown_msi_irq(unsigned int irq) { xen_destroy_irq(irq); } -static u32 xen_nop_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) -{ - return 0; -} -static u32 xen_nop_msix_mask_irq(struct msi_desc *desc, u32 flag) -{ - return 0; -} + #endif int __init pci_xen_init(void) @@ -425,8 +418,7 @@ int __init pci_xen_init(void) x86_msi.setup_msi_irqs = xen_setup_msi_irqs; x86_msi.teardown_msi_irq = xen_teardown_msi_irq; x86_msi.teardown_msi_irqs = xen_teardown_msi_irqs; - x86_msi.msi_mask_irq = xen_nop_msi_mask_irq; - x86_msi.msix_mask_irq = xen_nop_msix_mask_irq; + pci_msi_ignore_mask = 1; #endif return 0; } @@ -461,8 +453,7 @@ int __init pci_xen_initial_domain(void) x86_msi.setup_msi_irqs = xen_initdom_setup_msi_irqs; x86_msi.teardown_msi_irq = xen_teardown_msi_irq; x86_msi.restore_msi_irqs = xen_initdom_restore_msi_irqs; - x86_msi.msi_mask_irq = xen_nop_msi_mask_irq; - x86_msi.msix_mask_irq = xen_nop_msix_mask_irq; + pci_msi_ignore_mask = 1; #endif __acpi_register_gsi = acpi_register_gsi_xen; __acpi_unregister_gsi = NULL; diff --git a/arch/x86/realmode/Makefile b/arch/x86/realmode/Makefile index 94f7fbe97b08..e02c2c6c56a5 100644 --- a/arch/x86/realmode/Makefile +++ b/arch/x86/realmode/Makefile @@ -6,7 +6,7 @@ # for more details. # # - +KASAN_SANITIZE := n subdir- := rm obj-y += init.o diff --git a/arch/x86/realmode/rm/Makefile b/arch/x86/realmode/rm/Makefile index 7c0d7be176a5..2730d775ef9a 100644 --- a/arch/x86/realmode/rm/Makefile +++ b/arch/x86/realmode/rm/Makefile @@ -6,6 +6,7 @@ # for more details. # # +KASAN_SANITIZE := n always := realmode.bin realmode.relocs diff --git a/arch/x86/vdso/Makefile b/arch/x86/vdso/Makefile index 5a4affe025e8..2aacd7c63c7b 100644 --- a/arch/x86/vdso/Makefile +++ b/arch/x86/vdso/Makefile @@ -3,6 +3,7 @@ # KBUILD_CFLAGS += $(DISABLE_LTO) +KASAN_SANITIZE := n VDSO64-$(CONFIG_X86_64) := y VDSOX32-$(CONFIG_X86_X32_ABI) := y diff --git a/arch/xtensa/kernel/pci.c b/arch/xtensa/kernel/pci.c index 5b3403388d7f..b848cc3dc913 100644 --- a/arch/xtensa/kernel/pci.c +++ b/arch/xtensa/kernel/pci.c @@ -174,7 +174,7 @@ static int __init pcibios_init(void) struct pci_controller *pci_ctrl; struct list_head resources; struct pci_bus *bus; - int next_busno = 0; + int next_busno = 0, ret; printk("PCI: Probing PCI hardware\n"); @@ -185,14 +185,25 @@ static int __init pcibios_init(void) pci_controller_apertures(pci_ctrl, &resources); bus = pci_scan_root_bus(NULL, pci_ctrl->first_busno, pci_ctrl->ops, pci_ctrl, &resources); + if (!bus) + continue; + pci_ctrl->bus = bus; pci_ctrl->last_busno = bus->busn_res.end; if (next_busno <= pci_ctrl->last_busno) next_busno = pci_ctrl->last_busno+1; } pci_bus_count = next_busno; + ret = platform_pcibios_fixup(); + if (ret) + return ret; - return platform_pcibios_fixup(); + for (pci_ctrl = pci_ctrl_head; pci_ctrl; pci_ctrl = pci_ctrl->next) { + if (pci_ctrl->bus) + pci_bus_add_devices(pci_ctrl->bus); + } + + return 0; } subsys_initcall(pcibios_init); diff --git a/drivers/Makefile b/drivers/Makefile index 76304657163b..5db034ea8175 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -162,3 +162,4 @@ obj-$(CONFIG_POWERCAP) += powercap/ obj-$(CONFIG_MCB) += mcb/ obj-$(CONFIG_RAS) += ras/ obj-$(CONFIG_THUNDERBOLT) += thunderbolt/ +obj-$(CONFIG_CORESIGHT) += hwtracing/coresight/ diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index d24fa1964eb8..6d4e44ea74ac 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -800,7 +800,8 @@ static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, result = thermal_zone_bind_cooling_device (thermal, trip, cdev, - THERMAL_NO_LIMIT, THERMAL_NO_LIMIT); + THERMAL_NO_LIMIT, THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT); else result = thermal_zone_unbind_cooling_device @@ -824,7 +825,8 @@ static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, if (bind) result = thermal_zone_bind_cooling_device (thermal, trip, cdev, - THERMAL_NO_LIMIT, THERMAL_NO_LIMIT); + THERMAL_NO_LIMIT, THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT); else result = thermal_zone_unbind_cooling_device (thermal, trip, cdev); @@ -841,7 +843,8 @@ static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, result = thermal_zone_bind_cooling_device (thermal, THERMAL_TRIPS_NONE, cdev, THERMAL_NO_LIMIT, - THERMAL_NO_LIMIT); + THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT); else result = thermal_zone_unbind_cooling_device (thermal, THERMAL_TRIPS_NONE, diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index 47bbdc1b5be3..a4ac490dd784 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -336,7 +336,7 @@ int amba_device_add(struct amba_device *dev, struct resource *parent) amba_put_disable_pclk(dev); - if (cid == AMBA_CID) + if (cid == AMBA_CID || cid == CORESIGHT_CID) dev->periphid = pid; if (!dev->periphid) diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c index 50e8bd00e431..fe95d6ca047f 100644 --- a/drivers/base/firmware_class.c +++ b/drivers/base/firmware_class.c @@ -866,7 +866,6 @@ fw_create_instance(struct firmware *firmware, const char *fw_name, fw_priv = kzalloc(sizeof(*fw_priv), GFP_KERNEL); if (!fw_priv) { - dev_err(device, "%s: kmalloc failed\n", __func__); fw_priv = ERR_PTR(-ENOMEM); goto exit; } diff --git a/drivers/base/platform.c b/drivers/base/platform.c index a33ae377728f..ead6ea0b1a71 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -1019,6 +1019,7 @@ int __init platform_bus_init(void) error = bus_register(&platform_bus_type); if (error) device_unregister(&platform_bus); + of_platform_register_reconfig_notifier(); return error; } diff --git a/drivers/base/power/Makefile b/drivers/base/power/Makefile index 1cb8544598d5..49fddd1964cc 100644 --- a/drivers/base/power/Makefile +++ b/drivers/base/power/Makefile @@ -1,7 +1,7 @@ obj-$(CONFIG_PM) += sysfs.o generic_ops.o common.o qos.o runtime.o obj-$(CONFIG_PM_SLEEP) += main.o wakeup.o obj-$(CONFIG_PM_TRACE_RTC) += trace.o -obj-$(CONFIG_PM_OPP) += opp.o +obj-$(CONFIG_PM_OPP) += opp/ obj-$(CONFIG_PM_GENERIC_DOMAINS) += domain.o domain_governor.o obj-$(CONFIG_HAVE_CLK) += clock_ops.o diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c index 78369305e069..308e9ffa4abf 100644 --- a/drivers/base/power/clock_ops.c +++ b/drivers/base/power/clock_ops.c @@ -80,10 +80,8 @@ int pm_clk_add(struct device *dev, const char *con_id) return -EINVAL; ce = kzalloc(sizeof(*ce), GFP_KERNEL); - if (!ce) { - dev_err(dev, "Not enough memory for clock entry.\n"); + if (!ce) return -ENOMEM; - } if (con_id) { ce->con_id = kstrdup(con_id, GFP_KERNEL); diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c index b0f138806bbc..f32b802b98f4 100644 --- a/drivers/base/power/common.c +++ b/drivers/base/power/common.c @@ -19,8 +19,8 @@ * @dev: Device to handle. * * If power.subsys_data is NULL, point it to a new object, otherwise increment - * its reference counter. Return 1 if a new object has been created, otherwise - * return 0 or error code. + * its reference counter. Return 0 if new object has been created or refcount + * increased, otherwise negative error code. */ int dev_pm_get_subsys_data(struct device *dev) { @@ -56,13 +56,11 @@ EXPORT_SYMBOL_GPL(dev_pm_get_subsys_data); * @dev: Device to handle. * * If the reference counter of power.subsys_data is zero after dropping the - * reference, power.subsys_data is removed. Return 1 if that happens or 0 - * otherwise. + * reference, power.subsys_data is removed. */ -int dev_pm_put_subsys_data(struct device *dev) +void dev_pm_put_subsys_data(struct device *dev) { struct pm_subsys_data *psd; - int ret = 1; spin_lock_irq(&dev->power.lock); @@ -70,18 +68,14 @@ int dev_pm_put_subsys_data(struct device *dev) if (!psd) goto out; - if (--psd->refcount == 0) { + if (--psd->refcount == 0) dev->power.subsys_data = NULL; - } else { + else psd = NULL; - ret = 0; - } out: spin_unlock_irq(&dev->power.lock); kfree(psd); - - return ret; } EXPORT_SYMBOL_GPL(dev_pm_put_subsys_data); diff --git a/drivers/base/power/opp.c b/drivers/base/power/opp.c deleted file mode 100644 index 89ced955fafa..000000000000 --- a/drivers/base/power/opp.c +++ /dev/null @@ -1,678 +0,0 @@ -/* - * Generic OPP Interface - * - * Copyright (C) 2009-2010 Texas Instruments Incorporated. - * Nishanth Menon - * Romit Dasgupta - * Kevin Hilman - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/kernel.h> -#include <linux/errno.h> -#include <linux/err.h> -#include <linux/slab.h> -#include <linux/device.h> -#include <linux/list.h> -#include <linux/rculist.h> -#include <linux/rcupdate.h> -#include <linux/pm_opp.h> -#include <linux/of.h> -#include <linux/export.h> - -/* - * Internal data structure organization with the OPP layer library is as - * follows: - * dev_opp_list (root) - * |- device 1 (represents voltage domain 1) - * | |- opp 1 (availability, freq, voltage) - * | |- opp 2 .. - * ... ... - * | `- opp n .. - * |- device 2 (represents the next voltage domain) - * ... - * `- device m (represents mth voltage domain) - * device 1, 2.. are represented by dev_opp structure while each opp - * is represented by the opp structure. - */ - -/** - * struct dev_pm_opp - Generic OPP description structure - * @node: opp list node. The nodes are maintained throughout the lifetime - * of boot. It is expected only an optimal set of OPPs are - * added to the library by the SoC framework. - * RCU usage: opp list is traversed with RCU locks. node - * modification is possible realtime, hence the modifications - * are protected by the dev_opp_list_lock for integrity. - * IMPORTANT: the opp nodes should be maintained in increasing - * order. - * @available: true/false - marks if this OPP as available or not - * @rate: Frequency in hertz - * @u_volt: Nominal voltage in microvolts corresponding to this OPP - * @dev_opp: points back to the device_opp struct this opp belongs to - * @head: RCU callback head used for deferred freeing - * - * This structure stores the OPP information for a given device. - */ -struct dev_pm_opp { - struct list_head node; - - bool available; - unsigned long rate; - unsigned long u_volt; - - struct device_opp *dev_opp; - struct rcu_head head; -}; - -/** - * struct device_opp - Device opp structure - * @node: list node - contains the devices with OPPs that - * have been registered. Nodes once added are not modified in this - * list. - * RCU usage: nodes are not modified in the list of device_opp, - * however addition is possible and is secured by dev_opp_list_lock - * @dev: device pointer - * @head: notifier head to notify the OPP availability changes. - * @opp_list: list of opps - * - * This is an internal data structure maintaining the link to opps attached to - * a device. This structure is not meant to be shared to users as it is - * meant for book keeping and private to OPP library - */ -struct device_opp { - struct list_head node; - - struct device *dev; - struct srcu_notifier_head head; - struct list_head opp_list; -}; - -/* - * The root of the list of all devices. All device_opp structures branch off - * from here, with each device_opp containing the list of opp it supports in - * various states of availability. - */ -static LIST_HEAD(dev_opp_list); -/* Lock to allow exclusive modification to the device and opp lists */ -static DEFINE_MUTEX(dev_opp_list_lock); - -/** - * find_device_opp() - find device_opp struct using device pointer - * @dev: device pointer used to lookup device OPPs - * - * Search list of device OPPs for one containing matching device. Does a RCU - * reader operation to grab the pointer needed. - * - * Returns pointer to 'struct device_opp' if found, otherwise -ENODEV or - * -EINVAL based on type of error. - * - * Locking: This function must be called under rcu_read_lock(). device_opp - * is a RCU protected pointer. This means that device_opp is valid as long - * as we are under RCU lock. - */ -static struct device_opp *find_device_opp(struct device *dev) -{ - struct device_opp *tmp_dev_opp, *dev_opp = ERR_PTR(-ENODEV); - - if (unlikely(IS_ERR_OR_NULL(dev))) { - pr_err("%s: Invalid parameters\n", __func__); - return ERR_PTR(-EINVAL); - } - - list_for_each_entry_rcu(tmp_dev_opp, &dev_opp_list, node) { - if (tmp_dev_opp->dev == dev) { - dev_opp = tmp_dev_opp; - break; - } - } - - return dev_opp; -} - -/** - * dev_pm_opp_get_voltage() - Gets the voltage corresponding to an available opp - * @opp: opp for which voltage has to be returned for - * - * Return voltage in micro volt corresponding to the opp, else - * return 0 - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. This means that opp which could have been fetched by - * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are - * under RCU lock. The pointer returned by the opp_find_freq family must be - * used in the same section as the usage of this function with the pointer - * prior to unlocking with rcu_read_unlock() to maintain the integrity of the - * pointer. - */ -unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp) -{ - struct dev_pm_opp *tmp_opp; - unsigned long v = 0; - - tmp_opp = rcu_dereference(opp); - if (unlikely(IS_ERR_OR_NULL(tmp_opp)) || !tmp_opp->available) - pr_err("%s: Invalid parameters\n", __func__); - else - v = tmp_opp->u_volt; - - return v; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage); - -/** - * dev_pm_opp_get_freq() - Gets the frequency corresponding to an available opp - * @opp: opp for which frequency has to be returned for - * - * Return frequency in hertz corresponding to the opp, else - * return 0 - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. This means that opp which could have been fetched by - * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are - * under RCU lock. The pointer returned by the opp_find_freq family must be - * used in the same section as the usage of this function with the pointer - * prior to unlocking with rcu_read_unlock() to maintain the integrity of the - * pointer. - */ -unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp) -{ - struct dev_pm_opp *tmp_opp; - unsigned long f = 0; - - tmp_opp = rcu_dereference(opp); - if (unlikely(IS_ERR_OR_NULL(tmp_opp)) || !tmp_opp->available) - pr_err("%s: Invalid parameters\n", __func__); - else - f = tmp_opp->rate; - - return f; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq); - -/** - * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list - * @dev: device for which we do this operation - * - * This function returns the number of available opps if there are any, - * else returns 0 if none or the corresponding error value. - * - * Locking: This function must be called under rcu_read_lock(). This function - * internally references two RCU protected structures: device_opp and opp which - * are safe as long as we are under a common RCU locked section. - */ -int dev_pm_opp_get_opp_count(struct device *dev) -{ - struct device_opp *dev_opp; - struct dev_pm_opp *temp_opp; - int count = 0; - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) { - int r = PTR_ERR(dev_opp); - dev_err(dev, "%s: device OPP not found (%d)\n", __func__, r); - return r; - } - - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { - if (temp_opp->available) - count++; - } - - return count; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count); - -/** - * dev_pm_opp_find_freq_exact() - search for an exact frequency - * @dev: device for which we do this operation - * @freq: frequency to search for - * @available: true/false - match for available opp - * - * Searches for exact match in the opp list and returns pointer to the matching - * opp if found, else returns ERR_PTR in case of error and should be handled - * using IS_ERR. Error return values can be: - * EINVAL: for bad pointer - * ERANGE: no match found for search - * ENODEV: if device not found in list of registered devices - * - * Note: available is a modifier for the search. if available=true, then the - * match is for exact matching frequency and is available in the stored OPP - * table. if false, the match is for exact frequency which is not available. - * - * This provides a mechanism to enable an opp which is not available currently - * or the opposite as well. - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. The reason for the same is that the opp pointer which is - * returned will remain valid for use with opp_get_{voltage, freq} only while - * under the locked area. The pointer returned must be used prior to unlocking - * with rcu_read_unlock() to maintain the integrity of the pointer. - */ -struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev, - unsigned long freq, - bool available) -{ - struct device_opp *dev_opp; - struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) { - int r = PTR_ERR(dev_opp); - dev_err(dev, "%s: device OPP not found (%d)\n", __func__, r); - return ERR_PTR(r); - } - - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { - if (temp_opp->available == available && - temp_opp->rate == freq) { - opp = temp_opp; - break; - } - } - - return opp; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact); - -/** - * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq - * @dev: device for which we do this operation - * @freq: Start frequency - * - * Search for the matching ceil *available* OPP from a starting freq - * for a device. - * - * Returns matching *opp and refreshes *freq accordingly, else returns - * ERR_PTR in case of error and should be handled using IS_ERR. Error return - * values can be: - * EINVAL: for bad pointer - * ERANGE: no match found for search - * ENODEV: if device not found in list of registered devices - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. The reason for the same is that the opp pointer which is - * returned will remain valid for use with opp_get_{voltage, freq} only while - * under the locked area. The pointer returned must be used prior to unlocking - * with rcu_read_unlock() to maintain the integrity of the pointer. - */ -struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev, - unsigned long *freq) -{ - struct device_opp *dev_opp; - struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); - - if (!dev || !freq) { - dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq); - return ERR_PTR(-EINVAL); - } - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); - - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { - if (temp_opp->available && temp_opp->rate >= *freq) { - opp = temp_opp; - *freq = opp->rate; - break; - } - } - - return opp; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil); - -/** - * dev_pm_opp_find_freq_floor() - Search for a rounded floor freq - * @dev: device for which we do this operation - * @freq: Start frequency - * - * Search for the matching floor *available* OPP from a starting freq - * for a device. - * - * Returns matching *opp and refreshes *freq accordingly, else returns - * ERR_PTR in case of error and should be handled using IS_ERR. Error return - * values can be: - * EINVAL: for bad pointer - * ERANGE: no match found for search - * ENODEV: if device not found in list of registered devices - * - * Locking: This function must be called under rcu_read_lock(). opp is a rcu - * protected pointer. The reason for the same is that the opp pointer which is - * returned will remain valid for use with opp_get_{voltage, freq} only while - * under the locked area. The pointer returned must be used prior to unlocking - * with rcu_read_unlock() to maintain the integrity of the pointer. - */ -struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, - unsigned long *freq) -{ - struct device_opp *dev_opp; - struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); - - if (!dev || !freq) { - dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq); - return ERR_PTR(-EINVAL); - } - - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); - - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { - if (temp_opp->available) { - /* go to the next node, before choosing prev */ - if (temp_opp->rate > *freq) - break; - else - opp = temp_opp; - } - } - if (!IS_ERR(opp)) - *freq = opp->rate; - - return opp; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor); - -/** - * dev_pm_opp_add() - Add an OPP table from a table definitions - * @dev: device for which we do this operation - * @freq: Frequency in Hz for this OPP - * @u_volt: Voltage in uVolts for this OPP - * - * This function adds an opp definition to the opp list and returns status. - * The opp is made available by default and it can be controlled using - * dev_pm_opp_enable/disable functions. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function internally uses RCU updater strategy with mutex locks - * to keep the integrity of the internal data structures. Callers should ensure - * that this function is *NOT* called under RCU protection or in contexts where - * mutex cannot be locked. - * - * Return: - * 0: On success OR - * Duplicate OPPs (both freq and volt are same) and opp->available - * -EEXIST: Freq are same and volt are different OR - * Duplicate OPPs (both freq and volt are same) and !opp->available - * -ENOMEM: Memory allocation failure - */ -int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt) -{ - struct device_opp *dev_opp = NULL; - struct dev_pm_opp *opp, *new_opp; - struct list_head *head; - - /* allocate new OPP node */ - new_opp = kzalloc(sizeof(*new_opp), GFP_KERNEL); - if (!new_opp) { - dev_warn(dev, "%s: Unable to create new OPP node\n", __func__); - return -ENOMEM; - } - - /* Hold our list modification lock here */ - mutex_lock(&dev_opp_list_lock); - - /* Check for existing list for 'dev' */ - dev_opp = find_device_opp(dev); - if (IS_ERR(dev_opp)) { - /* - * Allocate a new device OPP table. In the infrequent case - * where a new device is needed to be added, we pay this - * penalty. - */ - dev_opp = kzalloc(sizeof(struct device_opp), GFP_KERNEL); - if (!dev_opp) { - mutex_unlock(&dev_opp_list_lock); - kfree(new_opp); - dev_warn(dev, - "%s: Unable to create device OPP structure\n", - __func__); - return -ENOMEM; - } - - dev_opp->dev = dev; - srcu_init_notifier_head(&dev_opp->head); - INIT_LIST_HEAD(&dev_opp->opp_list); - - /* Secure the device list modification */ - list_add_rcu(&dev_opp->node, &dev_opp_list); - } - - /* populate the opp table */ - new_opp->dev_opp = dev_opp; - new_opp->rate = freq; - new_opp->u_volt = u_volt; - new_opp->available = true; - - /* - * Insert new OPP in order of increasing frequency - * and discard if already present - */ - head = &dev_opp->opp_list; - list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) { - if (new_opp->rate <= opp->rate) - break; - else - head = &opp->node; - } - - /* Duplicate OPPs ? */ - if (new_opp->rate == opp->rate) { - int ret = opp->available && new_opp->u_volt == opp->u_volt ? - 0 : -EEXIST; - - dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n", - __func__, opp->rate, opp->u_volt, opp->available, - new_opp->rate, new_opp->u_volt, new_opp->available); - mutex_unlock(&dev_opp_list_lock); - kfree(new_opp); - return ret; - } - - list_add_rcu(&new_opp->node, head); - mutex_unlock(&dev_opp_list_lock); - - /* - * Notify the changes in the availability of the operable - * frequency/voltage list. - */ - srcu_notifier_call_chain(&dev_opp->head, OPP_EVENT_ADD, new_opp); - return 0; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_add); - -/** - * opp_set_availability() - helper to set the availability of an opp - * @dev: device for which we do this operation - * @freq: OPP frequency to modify availability - * @availability_req: availability status requested for this opp - * - * Set the availability of an OPP with an RCU operation, opp_{enable,disable} - * share a common logic which is isolated here. - * - * Returns -EINVAL for bad pointers, -ENOMEM if no memory available for the - * copy operation, returns 0 if no modifcation was done OR modification was - * successful. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function internally uses RCU updater strategy with mutex locks to - * keep the integrity of the internal data structures. Callers should ensure - * that this function is *NOT* called under RCU protection or in contexts where - * mutex locking or synchronize_rcu() blocking calls cannot be used. - */ -static int opp_set_availability(struct device *dev, unsigned long freq, - bool availability_req) -{ - struct device_opp *tmp_dev_opp, *dev_opp = ERR_PTR(-ENODEV); - struct dev_pm_opp *new_opp, *tmp_opp, *opp = ERR_PTR(-ENODEV); - int r = 0; - - /* keep the node allocated */ - new_opp = kmalloc(sizeof(*new_opp), GFP_KERNEL); - if (!new_opp) { - dev_warn(dev, "%s: Unable to create OPP\n", __func__); - return -ENOMEM; - } - - mutex_lock(&dev_opp_list_lock); - - /* Find the device_opp */ - list_for_each_entry(tmp_dev_opp, &dev_opp_list, node) { - if (dev == tmp_dev_opp->dev) { - dev_opp = tmp_dev_opp; - break; - } - } - if (IS_ERR(dev_opp)) { - r = PTR_ERR(dev_opp); - dev_warn(dev, "%s: Device OPP not found (%d)\n", __func__, r); - goto unlock; - } - - /* Do we have the frequency? */ - list_for_each_entry(tmp_opp, &dev_opp->opp_list, node) { - if (tmp_opp->rate == freq) { - opp = tmp_opp; - break; - } - } - if (IS_ERR(opp)) { - r = PTR_ERR(opp); - goto unlock; - } - - /* Is update really needed? */ - if (opp->available == availability_req) - goto unlock; - /* copy the old data over */ - *new_opp = *opp; - - /* plug in new node */ - new_opp->available = availability_req; - - list_replace_rcu(&opp->node, &new_opp->node); - mutex_unlock(&dev_opp_list_lock); - kfree_rcu(opp, head); - - /* Notify the change of the OPP availability */ - if (availability_req) - srcu_notifier_call_chain(&dev_opp->head, OPP_EVENT_ENABLE, - new_opp); - else - srcu_notifier_call_chain(&dev_opp->head, OPP_EVENT_DISABLE, - new_opp); - - return 0; - -unlock: - mutex_unlock(&dev_opp_list_lock); - kfree(new_opp); - return r; -} - -/** - * dev_pm_opp_enable() - Enable a specific OPP - * @dev: device for which we do this operation - * @freq: OPP frequency to enable - * - * Enables a provided opp. If the operation is valid, this returns 0, else the - * corresponding error value. It is meant to be used for users an OPP available - * after being temporarily made unavailable with dev_pm_opp_disable. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function indirectly uses RCU and mutex locks to keep the - * integrity of the internal data structures. Callers should ensure that - * this function is *NOT* called under RCU protection or in contexts where - * mutex locking or synchronize_rcu() blocking calls cannot be used. - */ -int dev_pm_opp_enable(struct device *dev, unsigned long freq) -{ - return opp_set_availability(dev, freq, true); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_enable); - -/** - * dev_pm_opp_disable() - Disable a specific OPP - * @dev: device for which we do this operation - * @freq: OPP frequency to disable - * - * Disables a provided opp. If the operation is valid, this returns - * 0, else the corresponding error value. It is meant to be a temporary - * control by users to make this OPP not available until the circumstances are - * right to make it available again (with a call to dev_pm_opp_enable). - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function indirectly uses RCU and mutex locks to keep the - * integrity of the internal data structures. Callers should ensure that - * this function is *NOT* called under RCU protection or in contexts where - * mutex locking or synchronize_rcu() blocking calls cannot be used. - */ -int dev_pm_opp_disable(struct device *dev, unsigned long freq) -{ - return opp_set_availability(dev, freq, false); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_disable); - -/** - * dev_pm_opp_get_notifier() - find notifier_head of the device with opp - * @dev: device pointer used to lookup device OPPs. - */ -struct srcu_notifier_head *dev_pm_opp_get_notifier(struct device *dev) -{ - struct device_opp *dev_opp = find_device_opp(dev); - - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); /* matching type */ - - return &dev_opp->head; -} - -#ifdef CONFIG_OF -/** - * of_init_opp_table() - Initialize opp table from device tree - * @dev: device pointer used to lookup device OPPs. - * - * Register the initial OPP table with the OPP library for given device. - */ -int of_init_opp_table(struct device *dev) -{ - const struct property *prop; - const __be32 *val; - int nr; - - prop = of_find_property(dev->of_node, "operating-points", NULL); - if (!prop) - return -ENODEV; - if (!prop->value) - return -ENODATA; - - /* - * Each OPP is a set of tuples consisting of frequency and - * voltage like <freq-kHz vol-uV>. - */ - nr = prop->length / sizeof(u32); - if (nr % 2) { - dev_err(dev, "%s: Invalid OPP list\n", __func__); - return -EINVAL; - } - - val = prop->value; - while (nr) { - unsigned long freq = be32_to_cpup(val++) * 1000; - unsigned long volt = be32_to_cpup(val++); - - if (dev_pm_opp_add(dev, freq, volt)) - dev_warn(dev, "%s: Failed to add OPP %ld\n", - __func__, freq); - nr -= 2; - } - - return 0; -} -EXPORT_SYMBOL_GPL(of_init_opp_table); -#endif diff --git a/drivers/base/power/opp/Makefile b/drivers/base/power/opp/Makefile new file mode 100644 index 000000000000..33c1e18c41a4 --- /dev/null +++ b/drivers/base/power/opp/Makefile @@ -0,0 +1,2 @@ +ccflags-$(CONFIG_DEBUG_DRIVER) := -DDEBUG +obj-y += core.o cpu.o diff --git a/drivers/base/power/opp/core.c b/drivers/base/power/opp/core.c new file mode 100644 index 000000000000..ed19c41d782e --- /dev/null +++ b/drivers/base/power/opp/core.c @@ -0,0 +1,2008 @@ +/* + * Generic OPP Interface + * + * Copyright (C) 2009-2010 Texas Instruments Incorporated. + * Nishanth Menon + * Romit Dasgupta + * Kevin Hilman + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/clk.h> +#include <linux/errno.h> +#include <linux/err.h> +#include <linux/slab.h> +#include <linux/device.h> +#include <linux/of.h> +#include <linux/export.h> +#include <linux/regulator/consumer.h> + +#include "opp.h" + +/* + * The root of the list of all devices. All device_opp structures branch off + * from here, with each device_opp containing the list of opp it supports in + * various states of availability. + */ +static LIST_HEAD(dev_opp_list); +/* Lock to allow exclusive modification to the device and opp lists */ +DEFINE_MUTEX(dev_opp_list_lock); + +#define opp_rcu_lockdep_assert() \ +do { \ + rcu_lockdep_assert(rcu_read_lock_held() || \ + lockdep_is_held(&dev_opp_list_lock), \ + "Missing rcu_read_lock() or " \ + "dev_opp_list_lock protection"); \ +} while (0) + +static struct device_list_opp *_find_list_dev(const struct device *dev, + struct device_opp *dev_opp) +{ + struct device_list_opp *list_dev; + + list_for_each_entry(list_dev, &dev_opp->dev_list, node) + if (list_dev->dev == dev) + return list_dev; + + return NULL; +} + +static struct device_opp *_managed_opp(const struct device_node *np) +{ + struct device_opp *dev_opp; + + list_for_each_entry_rcu(dev_opp, &dev_opp_list, node) { + if (dev_opp->np == np) { + /* + * Multiple devices can point to the same OPP table and + * so will have same node-pointer, np. + * + * But the OPPs will be considered as shared only if the + * OPP table contains a "opp-shared" property. + */ + return dev_opp->shared_opp ? dev_opp : NULL; + } + } + + return NULL; +} + +/** + * _find_device_opp() - find device_opp struct using device pointer + * @dev: device pointer used to lookup device OPPs + * + * Search list of device OPPs for one containing matching device. Does a RCU + * reader operation to grab the pointer needed. + * + * Return: pointer to 'struct device_opp' if found, otherwise -ENODEV or + * -EINVAL based on type of error. + * + * Locking: For readers, this function must be called under rcu_read_lock(). + * device_opp is a RCU protected pointer, which means that device_opp is valid + * as long as we are under RCU lock. + * + * For Writers, this function must be called with dev_opp_list_lock held. + */ +struct device_opp *_find_device_opp(struct device *dev) +{ + struct device_opp *dev_opp; + + opp_rcu_lockdep_assert(); + + if (IS_ERR_OR_NULL(dev)) { + pr_err("%s: Invalid parameters\n", __func__); + return ERR_PTR(-EINVAL); + } + + list_for_each_entry_rcu(dev_opp, &dev_opp_list, node) + if (_find_list_dev(dev, dev_opp)) + return dev_opp; + + return ERR_PTR(-ENODEV); +} + +/** + * dev_pm_opp_get_voltage() - Gets the voltage corresponding to an available opp + * @opp: opp for which voltage has to be returned for + * + * Return: voltage in micro volt corresponding to the opp, else + * return 0 + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. This means that opp which could have been fetched by + * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are + * under RCU lock. The pointer returned by the opp_find_freq family must be + * used in the same section as the usage of this function with the pointer + * prior to unlocking with rcu_read_unlock() to maintain the integrity of the + * pointer. + */ +unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp) +{ + struct dev_pm_opp *tmp_opp; + unsigned long v = 0; + + opp_rcu_lockdep_assert(); + + tmp_opp = rcu_dereference(opp); + if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available) + pr_err("%s: Invalid parameters\n", __func__); + else + v = tmp_opp->u_volt; + + return v; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage); + +/** + * dev_pm_opp_get_freq() - Gets the frequency corresponding to an available opp + * @opp: opp for which frequency has to be returned for + * + * Return: frequency in hertz corresponding to the opp, else + * return 0 + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. This means that opp which could have been fetched by + * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are + * under RCU lock. The pointer returned by the opp_find_freq family must be + * used in the same section as the usage of this function with the pointer + * prior to unlocking with rcu_read_unlock() to maintain the integrity of the + * pointer. + */ +unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp) +{ + struct dev_pm_opp *tmp_opp; + unsigned long f = 0; + + opp_rcu_lockdep_assert(); + + tmp_opp = rcu_dereference(opp); + if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available) + pr_err("%s: Invalid parameters\n", __func__); + else + f = tmp_opp->rate; + + return f; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq); + +/** + * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not + * @opp: opp for which turbo mode is being verified + * + * Turbo OPPs are not for normal use, and can be enabled (under certain + * conditions) for short duration of times to finish high throughput work + * quickly. Running on them for longer times may overheat the chip. + * + * Return: true if opp is turbo opp, else false. + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. This means that opp which could have been fetched by + * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are + * under RCU lock. The pointer returned by the opp_find_freq family must be + * used in the same section as the usage of this function with the pointer + * prior to unlocking with rcu_read_unlock() to maintain the integrity of the + * pointer. + */ +bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp) +{ + struct dev_pm_opp *tmp_opp; + + opp_rcu_lockdep_assert(); + + tmp_opp = rcu_dereference(opp); + if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available) { + pr_err("%s: Invalid parameters\n", __func__); + return false; + } + + return tmp_opp->turbo; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo); + +/** + * dev_pm_opp_get_max_clock_latency() - Get max clock latency in nanoseconds + * @dev: device for which we do this operation + * + * Return: This function returns the max clock latency in nanoseconds. + * + * Locking: This function takes rcu_read_lock(). + */ +unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev) +{ + struct device_opp *dev_opp; + unsigned long clock_latency_ns; + + rcu_read_lock(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) + clock_latency_ns = 0; + else + clock_latency_ns = dev_opp->clock_latency_ns_max; + + rcu_read_unlock(); + return clock_latency_ns; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency); + +/** + * dev_pm_opp_get_max_volt_latency() - Get max voltage latency in nanoseconds + * @dev: device for which we do this operation + * + * Return: This function returns the max voltage latency in nanoseconds. + * + * Locking: This function takes rcu_read_lock(). + */ +unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *opp; + struct regulator *reg; + unsigned long latency_ns = 0; + unsigned long min_uV = ~0, max_uV = 0; + int ret; + + rcu_read_lock(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + rcu_read_unlock(); + return 0; + } + + reg = dev_opp->regulator; + if (IS_ERR(reg)) { + /* Regulator may not be required for device */ + if (reg) + dev_err(dev, "%s: Invalid regulator (%ld)\n", __func__, + PTR_ERR(reg)); + rcu_read_unlock(); + return 0; + } + + list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) { + if (!opp->available) + continue; + + if (opp->u_volt_min < min_uV) + min_uV = opp->u_volt_min; + if (opp->u_volt_max > max_uV) + max_uV = opp->u_volt_max; + } + + rcu_read_unlock(); + + /* + * The caller needs to ensure that dev_opp (and hence the regulator) + * isn't freed, while we are executing this routine. + */ + ret = regulator_set_voltage_time(reg, min_uV, max_uV); + if (ret > 0) + latency_ns = ret * 1000; + + return latency_ns; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_volt_latency); + +/** + * dev_pm_opp_get_max_transition_latency() - Get max transition latency in + * nanoseconds + * @dev: device for which we do this operation + * + * Return: This function returns the max transition latency, in nanoseconds, to + * switch from one OPP to other. + * + * Locking: This function takes rcu_read_lock(). + */ +unsigned long dev_pm_opp_get_max_transition_latency(struct device *dev) +{ + return dev_pm_opp_get_max_volt_latency(dev) + + dev_pm_opp_get_max_clock_latency(dev); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_transition_latency); + +/** + * dev_pm_opp_get_suspend_opp() - Get suspend opp + * @dev: device for which we do this operation + * + * Return: This function returns pointer to the suspend opp if it is + * defined and available, otherwise it returns NULL. + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. The reason for the same is that the opp pointer which is + * returned will remain valid for use with opp_get_{voltage, freq} only while + * under the locked area. The pointer returned must be used prior to unlocking + * with rcu_read_unlock() to maintain the integrity of the pointer. + */ +struct dev_pm_opp *dev_pm_opp_get_suspend_opp(struct device *dev) +{ + struct device_opp *dev_opp; + + opp_rcu_lockdep_assert(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp) || !dev_opp->suspend_opp || + !dev_opp->suspend_opp->available) + return NULL; + + return dev_opp->suspend_opp; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp); + +/** + * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list + * @dev: device for which we do this operation + * + * Return: This function returns the number of available opps if there are any, + * else returns 0 if none or the corresponding error value. + * + * Locking: This function takes rcu_read_lock(). + */ +int dev_pm_opp_get_opp_count(struct device *dev) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *temp_opp; + int count = 0; + + rcu_read_lock(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + count = PTR_ERR(dev_opp); + dev_err(dev, "%s: device OPP not found (%d)\n", + __func__, count); + goto out_unlock; + } + + list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { + if (temp_opp->available) + count++; + } + +out_unlock: + rcu_read_unlock(); + return count; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count); + +/** + * dev_pm_opp_find_freq_exact() - search for an exact frequency + * @dev: device for which we do this operation + * @freq: frequency to search for + * @available: true/false - match for available opp + * + * Return: Searches for exact match in the opp list and returns pointer to the + * matching opp if found, else returns ERR_PTR in case of error and should + * be handled using IS_ERR. Error return values can be: + * EINVAL: for bad pointer + * ERANGE: no match found for search + * ENODEV: if device not found in list of registered devices + * + * Note: available is a modifier for the search. if available=true, then the + * match is for exact matching frequency and is available in the stored OPP + * table. if false, the match is for exact frequency which is not available. + * + * This provides a mechanism to enable an opp which is not available currently + * or the opposite as well. + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. The reason for the same is that the opp pointer which is + * returned will remain valid for use with opp_get_{voltage, freq} only while + * under the locked area. The pointer returned must be used prior to unlocking + * with rcu_read_unlock() to maintain the integrity of the pointer. + */ +struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev, + unsigned long freq, + bool available) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); + + opp_rcu_lockdep_assert(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + int r = PTR_ERR(dev_opp); + dev_err(dev, "%s: device OPP not found (%d)\n", __func__, r); + return ERR_PTR(r); + } + + list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { + if (temp_opp->available == available && + temp_opp->rate == freq) { + opp = temp_opp; + break; + } + } + + return opp; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact); + +/** + * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq + * @dev: device for which we do this operation + * @freq: Start frequency + * + * Search for the matching ceil *available* OPP from a starting freq + * for a device. + * + * Return: matching *opp and refreshes *freq accordingly, else returns + * ERR_PTR in case of error and should be handled using IS_ERR. Error return + * values can be: + * EINVAL: for bad pointer + * ERANGE: no match found for search + * ENODEV: if device not found in list of registered devices + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. The reason for the same is that the opp pointer which is + * returned will remain valid for use with opp_get_{voltage, freq} only while + * under the locked area. The pointer returned must be used prior to unlocking + * with rcu_read_unlock() to maintain the integrity of the pointer. + */ +struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev, + unsigned long *freq) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); + + opp_rcu_lockdep_assert(); + + if (!dev || !freq) { + dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq); + return ERR_PTR(-EINVAL); + } + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) + return ERR_CAST(dev_opp); + + list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { + if (temp_opp->available && temp_opp->rate >= *freq) { + opp = temp_opp; + *freq = opp->rate; + break; + } + } + + return opp; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil); + +/** + * dev_pm_opp_find_freq_floor() - Search for a rounded floor freq + * @dev: device for which we do this operation + * @freq: Start frequency + * + * Search for the matching floor *available* OPP from a starting freq + * for a device. + * + * Return: matching *opp and refreshes *freq accordingly, else returns + * ERR_PTR in case of error and should be handled using IS_ERR. Error return + * values can be: + * EINVAL: for bad pointer + * ERANGE: no match found for search + * ENODEV: if device not found in list of registered devices + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. The reason for the same is that the opp pointer which is + * returned will remain valid for use with opp_get_{voltage, freq} only while + * under the locked area. The pointer returned must be used prior to unlocking + * with rcu_read_unlock() to maintain the integrity of the pointer. + */ +struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, + unsigned long *freq) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); + + opp_rcu_lockdep_assert(); + + if (!dev || !freq) { + dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq); + return ERR_PTR(-EINVAL); + } + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) + return ERR_CAST(dev_opp); + + list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { + if (temp_opp->available) { + /* go to the next node, before choosing prev */ + if (temp_opp->rate > *freq) + break; + else + opp = temp_opp; + } + } + if (!IS_ERR(opp)) + *freq = opp->rate; + + return opp; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor); + +/* + * The caller needs to ensure that device_opp (and hence the clk) isn't freed, + * while clk returned here is used. + */ +static struct clk *_get_opp_clk(struct device *dev) +{ + struct device_opp *dev_opp; + struct clk *clk; + + rcu_read_lock(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + dev_err(dev, "%s: device opp doesn't exist\n", __func__); + clk = ERR_CAST(dev_opp); + goto unlock; + } + + clk = dev_opp->clk; + if (IS_ERR(clk)) + dev_err(dev, "%s: No clock available for the device\n", + __func__); + +unlock: + rcu_read_unlock(); + return clk; +} + +static int _set_opp_voltage(struct device *dev, struct regulator *reg, + unsigned long u_volt, unsigned long u_volt_min, + unsigned long u_volt_max) +{ + int ret; + + /* Regulator not available for device */ + if (IS_ERR(reg)) { + dev_dbg(dev, "%s: regulator not available: %ld\n", __func__, + PTR_ERR(reg)); + return 0; + } + + dev_dbg(dev, "%s: voltages (mV): %lu %lu %lu\n", __func__, u_volt_min, + u_volt, u_volt_max); + + ret = regulator_set_voltage_triplet(reg, u_volt_min, u_volt, + u_volt_max); + if (ret) + dev_err(dev, "%s: failed to set voltage (%lu %lu %lu mV): %d\n", + __func__, u_volt_min, u_volt, u_volt_max, ret); + + return ret; +} + +/** + * dev_pm_opp_set_rate() - Configure new OPP based on frequency + * @dev: device for which we do this operation + * @target_freq: frequency to achieve + * + * This configures the power-supplies and clock source to the levels specified + * by the OPP corresponding to the target_freq. + * + * Locking: This function takes rcu_read_lock(). + */ +int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *old_opp, *opp; + struct regulator *reg; + struct clk *clk; + unsigned long freq, old_freq; + unsigned long u_volt, u_volt_min, u_volt_max; + unsigned long ou_volt, ou_volt_min, ou_volt_max; + int ret; + + if (unlikely(!target_freq)) { + dev_err(dev, "%s: Invalid target frequency %lu\n", __func__, + target_freq); + return -EINVAL; + } + + clk = _get_opp_clk(dev); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + freq = clk_round_rate(clk, target_freq); + if ((long)freq <= 0) + freq = target_freq; + + old_freq = clk_get_rate(clk); + + /* Return early if nothing to do */ + if (old_freq == freq) { + dev_dbg(dev, "%s: old/new frequencies (%lu Hz) are same, nothing to do\n", + __func__, freq); + return 0; + } + + rcu_read_lock(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + dev_err(dev, "%s: device opp doesn't exist\n", __func__); + rcu_read_unlock(); + return PTR_ERR(dev_opp); + } + + old_opp = dev_pm_opp_find_freq_ceil(dev, &old_freq); + if (!IS_ERR(old_opp)) { + ou_volt = old_opp->u_volt; + ou_volt_min = old_opp->u_volt_min; + ou_volt_max = old_opp->u_volt_max; + } else { + dev_err(dev, "%s: failed to find current OPP for freq %lu (%ld)\n", + __func__, old_freq, PTR_ERR(old_opp)); + } + + opp = dev_pm_opp_find_freq_ceil(dev, &freq); + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + dev_err(dev, "%s: failed to find OPP for freq %lu (%d)\n", + __func__, freq, ret); + rcu_read_unlock(); + return ret; + } + + u_volt = opp->u_volt; + u_volt_min = opp->u_volt_min; + u_volt_max = opp->u_volt_max; + + reg = dev_opp->regulator; + + rcu_read_unlock(); + + /* Scaling up? Scale voltage before frequency */ + if (freq > old_freq) { + ret = _set_opp_voltage(dev, reg, u_volt, u_volt_min, + u_volt_max); + if (ret) + goto restore_voltage; + } + + /* Change frequency */ + + dev_dbg(dev, "%s: switching OPP: %lu Hz --> %lu Hz\n", + __func__, old_freq, freq); + + ret = clk_set_rate(clk, freq); + if (ret) { + dev_err(dev, "%s: failed to set clock rate: %d\n", __func__, + ret); + goto restore_voltage; + } + + /* Scaling down? Scale voltage after frequency */ + if (freq < old_freq) { + ret = _set_opp_voltage(dev, reg, u_volt, u_volt_min, + u_volt_max); + if (ret) + goto restore_freq; + } + + return 0; + +restore_freq: + if (clk_set_rate(clk, old_freq)) + dev_err(dev, "%s: failed to restore old-freq (%lu Hz)\n", + __func__, old_freq); +restore_voltage: + /* This shouldn't harm even if the voltages weren't updated earlier */ + if (!IS_ERR(old_opp)) + _set_opp_voltage(dev, reg, ou_volt, ou_volt_min, ou_volt_max); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_rate); + +/* List-dev Helpers */ +static void _kfree_list_dev_rcu(struct rcu_head *head) +{ + struct device_list_opp *list_dev; + + list_dev = container_of(head, struct device_list_opp, rcu_head); + kfree_rcu(list_dev, rcu_head); +} + +static void _remove_list_dev(struct device_list_opp *list_dev, + struct device_opp *dev_opp) +{ + list_del(&list_dev->node); + call_srcu(&dev_opp->srcu_head.srcu, &list_dev->rcu_head, + _kfree_list_dev_rcu); +} + +struct device_list_opp *_add_list_dev(const struct device *dev, + struct device_opp *dev_opp) +{ + struct device_list_opp *list_dev; + + list_dev = kzalloc(sizeof(*list_dev), GFP_KERNEL); + if (!list_dev) + return NULL; + + /* Initialize list-dev */ + list_dev->dev = dev; + list_add_rcu(&list_dev->node, &dev_opp->dev_list); + + return list_dev; +} + +/** + * _add_device_opp() - Find device OPP table or allocate a new one + * @dev: device for which we do this operation + * + * It tries to find an existing table first, if it couldn't find one, it + * allocates a new OPP table and returns that. + * + * Return: valid device_opp pointer if success, else NULL. + */ +static struct device_opp *_add_device_opp(struct device *dev) +{ + struct device_opp *dev_opp; + struct device_list_opp *list_dev; + struct device_node *np; + int ret; + + /* Check for existing list for 'dev' first */ + dev_opp = _find_device_opp(dev); + if (!IS_ERR(dev_opp)) + return dev_opp; + + /* + * Allocate a new device OPP table. In the infrequent case where a new + * device is needed to be added, we pay this penalty. + */ + dev_opp = kzalloc(sizeof(*dev_opp), GFP_KERNEL); + if (!dev_opp) + return NULL; + + INIT_LIST_HEAD(&dev_opp->dev_list); + + list_dev = _add_list_dev(dev, dev_opp); + if (!list_dev) { + kfree(dev_opp); + return NULL; + } + + /* + * Only required for backward compatibility with v1 bindings, but isn't + * harmful for other cases. And so we do it unconditionally. + */ + np = of_node_get(dev->of_node); + if (np) { + u32 val; + + if (!of_property_read_u32(np, "clock-latency", &val)) + dev_opp->clock_latency_ns_max = val; + of_property_read_u32(np, "voltage-tolerance", + &dev_opp->voltage_tolerance_v1); + of_node_put(np); + } + + /* Set regulator to a non-NULL error value */ + dev_opp->regulator = ERR_PTR(-ENXIO); + + /* Find clk for the device */ + dev_opp->clk = clk_get(dev, NULL); + if (IS_ERR(dev_opp->clk)) { + ret = PTR_ERR(dev_opp->clk); + if (ret != -EPROBE_DEFER) + dev_dbg(dev, "%s: Couldn't find clock: %d\n", __func__, + ret); + } + + srcu_init_notifier_head(&dev_opp->srcu_head); + INIT_LIST_HEAD(&dev_opp->opp_list); + + /* Secure the device list modification */ + list_add_rcu(&dev_opp->node, &dev_opp_list); + return dev_opp; +} + +/** + * _kfree_device_rcu() - Free device_opp RCU handler + * @head: RCU head + */ +static void _kfree_device_rcu(struct rcu_head *head) +{ + struct device_opp *device_opp = container_of(head, struct device_opp, rcu_head); + + kfree_rcu(device_opp, rcu_head); +} + +/** + * _remove_device_opp() - Removes a device OPP table + * @dev_opp: device OPP table to be removed. + * + * Removes/frees device OPP table it it doesn't contain any OPPs. + */ +static void _remove_device_opp(struct device_opp *dev_opp) +{ + struct device_list_opp *list_dev; + + if (!list_empty(&dev_opp->opp_list)) + return; + + if (dev_opp->supported_hw) + return; + + if (dev_opp->prop_name) + return; + + if (!IS_ERR(dev_opp->regulator)) + return; + + /* Release clk */ + if (!IS_ERR(dev_opp->clk)) + clk_put(dev_opp->clk); + + list_dev = list_first_entry(&dev_opp->dev_list, struct device_list_opp, + node); + + _remove_list_dev(list_dev, dev_opp); + + /* dev_list must be empty now */ + WARN_ON(!list_empty(&dev_opp->dev_list)); + + list_del_rcu(&dev_opp->node); + call_srcu(&dev_opp->srcu_head.srcu, &dev_opp->rcu_head, + _kfree_device_rcu); +} + +/** + * _kfree_opp_rcu() - Free OPP RCU handler + * @head: RCU head + */ +static void _kfree_opp_rcu(struct rcu_head *head) +{ + struct dev_pm_opp *opp = container_of(head, struct dev_pm_opp, rcu_head); + + kfree_rcu(opp, rcu_head); +} + +/** + * _opp_remove() - Remove an OPP from a table definition + * @dev_opp: points back to the device_opp struct this opp belongs to + * @opp: pointer to the OPP to remove + * @notify: OPP_EVENT_REMOVE notification should be sent or not + * + * This function removes an opp definition from the opp list. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * It is assumed that the caller holds required mutex for an RCU updater + * strategy. + */ +static void _opp_remove(struct device_opp *dev_opp, + struct dev_pm_opp *opp, bool notify) +{ + /* + * Notify the changes in the availability of the operable + * frequency/voltage list. + */ + if (notify) + srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_REMOVE, opp); + list_del_rcu(&opp->node); + call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu); + + _remove_device_opp(dev_opp); +} + +/** + * dev_pm_opp_remove() - Remove an OPP from OPP list + * @dev: device for which we do this operation + * @freq: OPP to remove with matching 'freq' + * + * This function removes an opp from the opp list. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_remove(struct device *dev, unsigned long freq) +{ + struct dev_pm_opp *opp; + struct device_opp *dev_opp; + bool found = false; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) + goto unlock; + + list_for_each_entry(opp, &dev_opp->opp_list, node) { + if (opp->rate == freq) { + found = true; + break; + } + } + + if (!found) { + dev_warn(dev, "%s: Couldn't find OPP with freq: %lu\n", + __func__, freq); + goto unlock; + } + + _opp_remove(dev_opp, opp, true); +unlock: + mutex_unlock(&dev_opp_list_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_remove); + +static struct dev_pm_opp *_allocate_opp(struct device *dev, + struct device_opp **dev_opp) +{ + struct dev_pm_opp *opp; + + /* allocate new OPP node */ + opp = kzalloc(sizeof(*opp), GFP_KERNEL); + if (!opp) + return NULL; + + INIT_LIST_HEAD(&opp->node); + + *dev_opp = _add_device_opp(dev); + if (!*dev_opp) { + kfree(opp); + return NULL; + } + + return opp; +} + +static int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, + struct device_opp *dev_opp) +{ + struct dev_pm_opp *opp; + struct list_head *head = &dev_opp->opp_list; + + /* + * Insert new OPP in order of increasing frequency and discard if + * already present. + * + * Need to use &dev_opp->opp_list in the condition part of the 'for' + * loop, don't replace it with head otherwise it will become an infinite + * loop. + */ + list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) { + if (new_opp->rate > opp->rate) { + head = &opp->node; + continue; + } + + if (new_opp->rate < opp->rate) + break; + + /* Duplicate OPPs */ + dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n", + __func__, opp->rate, opp->u_volt, opp->available, + new_opp->rate, new_opp->u_volt, new_opp->available); + + return opp->available && new_opp->u_volt == opp->u_volt ? + 0 : -EEXIST; + } + + new_opp->dev_opp = dev_opp; + list_add_rcu(&new_opp->node, head); + + return 0; +} + +/** + * _opp_add_v1() - Allocate a OPP based on v1 bindings. + * @dev: device for which we do this operation + * @freq: Frequency in Hz for this OPP + * @u_volt: Voltage in uVolts for this OPP + * @dynamic: Dynamically added OPPs. + * + * This function adds an opp definition to the opp list and returns status. + * The opp is made available by default and it can be controlled using + * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove. + * + * NOTE: "dynamic" parameter impacts OPPs added by the dev_pm_opp_of_add_table + * and freed by dev_pm_opp_of_remove_table. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + * + * Return: + * 0 On success OR + * Duplicate OPPs (both freq and volt are same) and opp->available + * -EEXIST Freq are same and volt are different OR + * Duplicate OPPs (both freq and volt are same) and !opp->available + * -ENOMEM Memory allocation failure + */ +static int _opp_add_v1(struct device *dev, unsigned long freq, long u_volt, + bool dynamic) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *new_opp; + unsigned long tol; + int ret; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + new_opp = _allocate_opp(dev, &dev_opp); + if (!new_opp) { + ret = -ENOMEM; + goto unlock; + } + + /* populate the opp table */ + new_opp->rate = freq; + tol = u_volt * dev_opp->voltage_tolerance_v1 / 100; + new_opp->u_volt = u_volt; + new_opp->u_volt_min = u_volt - tol; + new_opp->u_volt_max = u_volt + tol; + new_opp->available = true; + new_opp->dynamic = dynamic; + + ret = _opp_add(dev, new_opp, dev_opp); + if (ret) + goto free_opp; + + mutex_unlock(&dev_opp_list_lock); + + /* + * Notify the changes in the availability of the operable + * frequency/voltage list. + */ + srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp); + return 0; + +free_opp: + _opp_remove(dev_opp, new_opp, false); +unlock: + mutex_unlock(&dev_opp_list_lock); + return ret; +} + +/* TODO: Support multiple regulators */ +static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev, + struct device_opp *dev_opp) +{ + u32 microvolt[3] = {0}; + u32 val; + int count, ret; + struct property *prop = NULL; + char name[NAME_MAX]; + + /* Search for "opp-microvolt-<name>" */ + if (dev_opp->prop_name) { + snprintf(name, sizeof(name), "opp-microvolt-%s", + dev_opp->prop_name); + prop = of_find_property(opp->np, name, NULL); + } + + if (!prop) { + /* Search for "opp-microvolt" */ + sprintf(name, "opp-microvolt"); + prop = of_find_property(opp->np, name, NULL); + + /* Missing property isn't a problem, but an invalid entry is */ + if (!prop) + return 0; + } + + count = of_property_count_u32_elems(opp->np, name); + if (count < 0) { + dev_err(dev, "%s: Invalid %s property (%d)\n", + __func__, name, count); + return count; + } + + /* There can be one or three elements here */ + if (count != 1 && count != 3) { + dev_err(dev, "%s: Invalid number of elements in %s property (%d)\n", + __func__, name, count); + return -EINVAL; + } + + ret = of_property_read_u32_array(opp->np, name, microvolt, count); + if (ret) { + dev_err(dev, "%s: error parsing %s: %d\n", __func__, name, ret); + return -EINVAL; + } + + opp->u_volt = microvolt[0]; + + if (count == 1) { + opp->u_volt_min = opp->u_volt; + opp->u_volt_max = opp->u_volt; + } else { + opp->u_volt_min = microvolt[1]; + opp->u_volt_max = microvolt[2]; + } + + /* Search for "opp-microamp-<name>" */ + prop = NULL; + if (dev_opp->prop_name) { + snprintf(name, sizeof(name), "opp-microamp-%s", + dev_opp->prop_name); + prop = of_find_property(opp->np, name, NULL); + } + + if (!prop) { + /* Search for "opp-microamp" */ + sprintf(name, "opp-microamp"); + prop = of_find_property(opp->np, name, NULL); + } + + if (prop && !of_property_read_u32(opp->np, name, &val)) + opp->u_amp = val; + + return 0; +} + +/** + * dev_pm_opp_set_supported_hw() - Set supported platforms + * @dev: Device for which supported-hw has to be set. + * @versions: Array of hierarchy of versions to match. + * @count: Number of elements in the array. + * + * This is required only for the V2 bindings, and it enables a platform to + * specify the hierarchy of versions it supports. OPP layer will then enable + * OPPs, which are available for those versions, based on its 'opp-supported-hw' + * property. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +int dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, + unsigned int count) +{ + struct device_opp *dev_opp; + int ret = 0; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + dev_opp = _add_device_opp(dev); + if (!dev_opp) { + ret = -ENOMEM; + goto unlock; + } + + /* Make sure there are no concurrent readers while updating dev_opp */ + WARN_ON(!list_empty(&dev_opp->opp_list)); + + /* Do we already have a version hierarchy associated with dev_opp? */ + if (dev_opp->supported_hw) { + dev_err(dev, "%s: Already have supported hardware list\n", + __func__); + ret = -EBUSY; + goto err; + } + + dev_opp->supported_hw = kmemdup(versions, count * sizeof(*versions), + GFP_KERNEL); + if (!dev_opp->supported_hw) { + ret = -ENOMEM; + goto err; + } + + dev_opp->supported_hw_count = count; + mutex_unlock(&dev_opp_list_lock); + return 0; + +err: + _remove_device_opp(dev_opp); +unlock: + mutex_unlock(&dev_opp_list_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_supported_hw); + +/** + * dev_pm_opp_put_supported_hw() - Releases resources blocked for supported hw + * @dev: Device for which supported-hw has to be put. + * + * This is required only for the V2 bindings, and is called for a matching + * dev_pm_opp_set_supported_hw(). Until this is called, the device_opp structure + * will not be freed. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_put_supported_hw(struct device *dev) +{ + struct device_opp *dev_opp; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + /* Check for existing list for 'dev' first */ + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + dev_err(dev, "Failed to find dev_opp: %ld\n", PTR_ERR(dev_opp)); + goto unlock; + } + + /* Make sure there are no concurrent readers while updating dev_opp */ + WARN_ON(!list_empty(&dev_opp->opp_list)); + + if (!dev_opp->supported_hw) { + dev_err(dev, "%s: Doesn't have supported hardware list\n", + __func__); + goto unlock; + } + + kfree(dev_opp->supported_hw); + dev_opp->supported_hw = NULL; + dev_opp->supported_hw_count = 0; + + /* Try freeing device_opp if this was the last blocking resource */ + _remove_device_opp(dev_opp); + +unlock: + mutex_unlock(&dev_opp_list_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_put_supported_hw); + +/** + * dev_pm_opp_set_prop_name() - Set prop-extn name + * @dev: Device for which the prop-name has to be set. + * @name: name to postfix to properties. + * + * This is required only for the V2 bindings, and it enables a platform to + * specify the extn to be used for certain property names. The properties to + * which the extension will apply are opp-microvolt and opp-microamp. OPP core + * should postfix the property name with -<name> while looking for them. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +int dev_pm_opp_set_prop_name(struct device *dev, const char *name) +{ + struct device_opp *dev_opp; + int ret = 0; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + dev_opp = _add_device_opp(dev); + if (!dev_opp) { + ret = -ENOMEM; + goto unlock; + } + + /* Make sure there are no concurrent readers while updating dev_opp */ + WARN_ON(!list_empty(&dev_opp->opp_list)); + + /* Do we already have a prop-name associated with dev_opp? */ + if (dev_opp->prop_name) { + dev_err(dev, "%s: Already have prop-name %s\n", __func__, + dev_opp->prop_name); + ret = -EBUSY; + goto err; + } + + dev_opp->prop_name = kstrdup(name, GFP_KERNEL); + if (!dev_opp->prop_name) { + ret = -ENOMEM; + goto err; + } + + mutex_unlock(&dev_opp_list_lock); + return 0; + +err: + _remove_device_opp(dev_opp); +unlock: + mutex_unlock(&dev_opp_list_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_prop_name); + +/** + * dev_pm_opp_put_prop_name() - Releases resources blocked for prop-name + * @dev: Device for which the prop-name has to be put. + * + * This is required only for the V2 bindings, and is called for a matching + * dev_pm_opp_set_prop_name(). Until this is called, the device_opp structure + * will not be freed. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_put_prop_name(struct device *dev) +{ + struct device_opp *dev_opp; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + /* Check for existing list for 'dev' first */ + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + dev_err(dev, "Failed to find dev_opp: %ld\n", PTR_ERR(dev_opp)); + goto unlock; + } + + /* Make sure there are no concurrent readers while updating dev_opp */ + WARN_ON(!list_empty(&dev_opp->opp_list)); + + if (!dev_opp->prop_name) { + dev_err(dev, "%s: Doesn't have a prop-name\n", __func__); + goto unlock; + } + + kfree(dev_opp->prop_name); + dev_opp->prop_name = NULL; + + /* Try freeing device_opp if this was the last blocking resource */ + _remove_device_opp(dev_opp); + +unlock: + mutex_unlock(&dev_opp_list_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); + +/** + * dev_pm_opp_set_regulator() - Set regulator name for the device + * @dev: Device for which regulator name is being set. + * @name: Name of the regulator. + * + * In order to support OPP switching, OPP layer needs to know the name of the + * device's regulator, as the core would be required to switch voltages as well. + * + * This must be called before any OPPs are initialized for the device. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +int dev_pm_opp_set_regulator(struct device *dev, const char *name) +{ + struct device_opp *dev_opp; + struct regulator *reg; + int ret; + + mutex_lock(&dev_opp_list_lock); + + dev_opp = _add_device_opp(dev); + if (!dev_opp) { + ret = -ENOMEM; + goto unlock; + } + + /* This should be called before OPPs are initialized */ + if (WARN_ON(!list_empty(&dev_opp->opp_list))) { + ret = -EBUSY; + goto err; + } + + /* Already have a regulator set */ + if (WARN_ON(!IS_ERR(dev_opp->regulator))) { + ret = -EBUSY; + goto err; + } + /* Allocate the regulator */ + reg = regulator_get_optional(dev, name); + if (IS_ERR(reg)) { + ret = PTR_ERR(reg); + if (ret != -EPROBE_DEFER) + dev_err(dev, "%s: no regulator (%s) found: %d\n", + __func__, name, ret); + goto err; + } + + dev_opp->regulator = reg; + + mutex_unlock(&dev_opp_list_lock); + return 0; + +err: + _remove_device_opp(dev_opp); +unlock: + mutex_unlock(&dev_opp_list_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulator); + +/** + * dev_pm_opp_put_regulator() - Releases resources blocked for regulator + * @dev: Device for which regulator was set. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_put_regulator(struct device *dev) +{ + struct device_opp *dev_opp; + + mutex_lock(&dev_opp_list_lock); + + /* Check for existing list for 'dev' first */ + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + dev_err(dev, "Failed to find dev_opp: %ld\n", PTR_ERR(dev_opp)); + goto unlock; + } + + if (IS_ERR(dev_opp->regulator)) { + dev_err(dev, "%s: Doesn't have regulator set\n", __func__); + goto unlock; + } + + /* Make sure there are no concurrent readers while updating dev_opp */ + WARN_ON(!list_empty(&dev_opp->opp_list)); + + regulator_put(dev_opp->regulator); + dev_opp->regulator = ERR_PTR(-ENXIO); + + /* Try freeing device_opp if this was the last blocking resource */ + _remove_device_opp(dev_opp); + +unlock: + mutex_unlock(&dev_opp_list_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_put_regulator); + +static bool _opp_is_supported(struct device *dev, struct device_opp *dev_opp, + struct device_node *np) +{ + unsigned int count = dev_opp->supported_hw_count; + u32 version; + int ret; + + if (!dev_opp->supported_hw) + return true; + + while (count--) { + ret = of_property_read_u32_index(np, "opp-supported-hw", count, + &version); + if (ret) { + dev_warn(dev, "%s: failed to read opp-supported-hw property at index %d: %d\n", + __func__, count, ret); + return false; + } + + /* Both of these are bitwise masks of the versions */ + if (!(version & dev_opp->supported_hw[count])) + return false; + } + + return true; +} + +/** + * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings) + * @dev: device for which we do this operation + * @np: device node + * + * This function adds an opp definition to the opp list and returns status. The + * opp can be controlled using dev_pm_opp_enable/disable functions and may be + * removed by dev_pm_opp_remove. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + * + * Return: + * 0 On success OR + * Duplicate OPPs (both freq and volt are same) and opp->available + * -EEXIST Freq are same and volt are different OR + * Duplicate OPPs (both freq and volt are same) and !opp->available + * -ENOMEM Memory allocation failure + * -EINVAL Failed parsing the OPP node + */ +static int _opp_add_static_v2(struct device *dev, struct device_node *np) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *new_opp; + u64 rate; + u32 val; + int ret; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + new_opp = _allocate_opp(dev, &dev_opp); + if (!new_opp) { + ret = -ENOMEM; + goto unlock; + } + + ret = of_property_read_u64(np, "opp-hz", &rate); + if (ret < 0) { + dev_err(dev, "%s: opp-hz not found\n", __func__); + goto free_opp; + } + + /* Check if the OPP supports hardware's hierarchy of versions or not */ + if (!_opp_is_supported(dev, dev_opp, np)) { + dev_dbg(dev, "OPP not supported by hardware: %llu\n", rate); + goto free_opp; + } + + /* + * Rate is defined as an unsigned long in clk API, and so casting + * explicitly to its type. Must be fixed once rate is 64 bit + * guaranteed in clk API. + */ + new_opp->rate = (unsigned long)rate; + new_opp->turbo = of_property_read_bool(np, "turbo-mode"); + + new_opp->np = np; + new_opp->dynamic = false; + new_opp->available = true; + + if (!of_property_read_u32(np, "clock-latency-ns", &val)) + new_opp->clock_latency_ns = val; + + ret = opp_parse_supplies(new_opp, dev, dev_opp); + if (ret) + goto free_opp; + + ret = _opp_add(dev, new_opp, dev_opp); + if (ret) + goto free_opp; + + /* OPP to select on device suspend */ + if (of_property_read_bool(np, "opp-suspend")) { + if (dev_opp->suspend_opp) + dev_warn(dev, "%s: Multiple suspend OPPs found (%lu %lu)\n", + __func__, dev_opp->suspend_opp->rate, + new_opp->rate); + else + dev_opp->suspend_opp = new_opp; + } + + if (new_opp->clock_latency_ns > dev_opp->clock_latency_ns_max) + dev_opp->clock_latency_ns_max = new_opp->clock_latency_ns; + + mutex_unlock(&dev_opp_list_lock); + + pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n", + __func__, new_opp->turbo, new_opp->rate, new_opp->u_volt, + new_opp->u_volt_min, new_opp->u_volt_max, + new_opp->clock_latency_ns); + + /* + * Notify the changes in the availability of the operable + * frequency/voltage list. + */ + srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp); + return 0; + +free_opp: + _opp_remove(dev_opp, new_opp, false); +unlock: + mutex_unlock(&dev_opp_list_lock); + return ret; +} + +/** + * dev_pm_opp_add() - Add an OPP table from a table definitions + * @dev: device for which we do this operation + * @freq: Frequency in Hz for this OPP + * @u_volt: Voltage in uVolts for this OPP + * + * This function adds an opp definition to the opp list and returns status. + * The opp is made available by default and it can be controlled using + * dev_pm_opp_enable/disable functions. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + * + * Return: + * 0 On success OR + * Duplicate OPPs (both freq and volt are same) and opp->available + * -EEXIST Freq are same and volt are different OR + * Duplicate OPPs (both freq and volt are same) and !opp->available + * -ENOMEM Memory allocation failure + */ +int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt) +{ + return _opp_add_v1(dev, freq, u_volt, true); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_add); + +/** + * _opp_set_availability() - helper to set the availability of an opp + * @dev: device for which we do this operation + * @freq: OPP frequency to modify availability + * @availability_req: availability status requested for this opp + * + * Set the availability of an OPP with an RCU operation, opp_{enable,disable} + * share a common logic which is isolated here. + * + * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the + * copy operation, returns 0 if no modification was done OR modification was + * successful. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks to + * keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex locking or synchronize_rcu() blocking calls cannot be used. + */ +static int _opp_set_availability(struct device *dev, unsigned long freq, + bool availability_req) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *new_opp, *tmp_opp, *opp = ERR_PTR(-ENODEV); + int r = 0; + + /* keep the node allocated */ + new_opp = kmalloc(sizeof(*new_opp), GFP_KERNEL); + if (!new_opp) + return -ENOMEM; + + mutex_lock(&dev_opp_list_lock); + + /* Find the device_opp */ + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + r = PTR_ERR(dev_opp); + dev_warn(dev, "%s: Device OPP not found (%d)\n", __func__, r); + goto unlock; + } + + /* Do we have the frequency? */ + list_for_each_entry(tmp_opp, &dev_opp->opp_list, node) { + if (tmp_opp->rate == freq) { + opp = tmp_opp; + break; + } + } + if (IS_ERR(opp)) { + r = PTR_ERR(opp); + goto unlock; + } + + /* Is update really needed? */ + if (opp->available == availability_req) + goto unlock; + /* copy the old data over */ + *new_opp = *opp; + + /* plug in new node */ + new_opp->available = availability_req; + + list_replace_rcu(&opp->node, &new_opp->node); + mutex_unlock(&dev_opp_list_lock); + call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu); + + /* Notify the change of the OPP availability */ + if (availability_req) + srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ENABLE, + new_opp); + else + srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_DISABLE, + new_opp); + + return 0; + +unlock: + mutex_unlock(&dev_opp_list_lock); + kfree(new_opp); + return r; +} + +/** + * dev_pm_opp_enable() - Enable a specific OPP + * @dev: device for which we do this operation + * @freq: OPP frequency to enable + * + * Enables a provided opp. If the operation is valid, this returns 0, else the + * corresponding error value. It is meant to be used for users an OPP available + * after being temporarily made unavailable with dev_pm_opp_disable. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function indirectly uses RCU and mutex locks to keep the + * integrity of the internal data structures. Callers should ensure that + * this function is *NOT* called under RCU protection or in contexts where + * mutex locking or synchronize_rcu() blocking calls cannot be used. + * + * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the + * copy operation, returns 0 if no modification was done OR modification was + * successful. + */ +int dev_pm_opp_enable(struct device *dev, unsigned long freq) +{ + return _opp_set_availability(dev, freq, true); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_enable); + +/** + * dev_pm_opp_disable() - Disable a specific OPP + * @dev: device for which we do this operation + * @freq: OPP frequency to disable + * + * Disables a provided opp. If the operation is valid, this returns + * 0, else the corresponding error value. It is meant to be a temporary + * control by users to make this OPP not available until the circumstances are + * right to make it available again (with a call to dev_pm_opp_enable). + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function indirectly uses RCU and mutex locks to keep the + * integrity of the internal data structures. Callers should ensure that + * this function is *NOT* called under RCU protection or in contexts where + * mutex locking or synchronize_rcu() blocking calls cannot be used. + * + * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the + * copy operation, returns 0 if no modification was done OR modification was + * successful. + */ +int dev_pm_opp_disable(struct device *dev, unsigned long freq) +{ + return _opp_set_availability(dev, freq, false); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_disable); + +/** + * dev_pm_opp_get_notifier() - find notifier_head of the device with opp + * @dev: device pointer used to lookup device OPPs. + * + * Return: pointer to notifier head if found, otherwise -ENODEV or + * -EINVAL based on type of error casted as pointer. value must be checked + * with IS_ERR to determine valid pointer or error result. + * + * Locking: This function must be called under rcu_read_lock(). dev_opp is a RCU + * protected pointer. The reason for the same is that the opp pointer which is + * returned will remain valid for use with opp_get_{voltage, freq} only while + * under the locked area. The pointer returned must be used prior to unlocking + * with rcu_read_unlock() to maintain the integrity of the pointer. + */ +struct srcu_notifier_head *dev_pm_opp_get_notifier(struct device *dev) +{ + struct device_opp *dev_opp = _find_device_opp(dev); + + if (IS_ERR(dev_opp)) + return ERR_CAST(dev_opp); /* matching type */ + + return &dev_opp->srcu_head; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_notifier); + +#ifdef CONFIG_OF +/** + * dev_pm_opp_of_remove_table() - Free OPP table entries created from static DT + * entries + * @dev: device pointer used to lookup device OPPs. + * + * Free OPPs created using static entries present in DT. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function indirectly uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_of_remove_table(struct device *dev) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *opp, *tmp; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + /* Check for existing list for 'dev' */ + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + int error = PTR_ERR(dev_opp); + + if (error != -ENODEV) + WARN(1, "%s: dev_opp: %d\n", + IS_ERR_OR_NULL(dev) ? + "Invalid device" : dev_name(dev), + error); + goto unlock; + } + + /* Find if dev_opp manages a single device */ + if (list_is_singular(&dev_opp->dev_list)) { + /* Free static OPPs */ + list_for_each_entry_safe(opp, tmp, &dev_opp->opp_list, node) { + if (!opp->dynamic) + _opp_remove(dev_opp, opp, true); + } + } else { + _remove_list_dev(_find_list_dev(dev, dev_opp), dev_opp); + } + +unlock: + mutex_unlock(&dev_opp_list_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table); + +/* Returns opp descriptor node for a device, caller must do of_node_put() */ +struct device_node *_of_get_opp_desc_node(struct device *dev) +{ + /* + * TODO: Support for multiple OPP tables. + * + * There should be only ONE phandle present in "operating-points-v2" + * property. + */ + + return of_parse_phandle(dev->of_node, "operating-points-v2", 0); +} + +/* Initializes OPP tables based on new bindings */ +static int _of_add_opp_table_v2(struct device *dev, struct device_node *opp_np) +{ + struct device_node *np; + struct device_opp *dev_opp; + int ret = 0, count = 0; + + mutex_lock(&dev_opp_list_lock); + + dev_opp = _managed_opp(opp_np); + if (dev_opp) { + /* OPPs are already managed */ + if (!_add_list_dev(dev, dev_opp)) + ret = -ENOMEM; + mutex_unlock(&dev_opp_list_lock); + return ret; + } + mutex_unlock(&dev_opp_list_lock); + + /* We have opp-list node now, iterate over it and add OPPs */ + for_each_available_child_of_node(opp_np, np) { + count++; + + ret = _opp_add_static_v2(dev, np); + if (ret) { + dev_err(dev, "%s: Failed to add OPP, %d\n", __func__, + ret); + goto free_table; + } + } + + /* There should be one of more OPP defined */ + if (WARN_ON(!count)) + return -ENOENT; + + mutex_lock(&dev_opp_list_lock); + + dev_opp = _find_device_opp(dev); + if (WARN_ON(IS_ERR(dev_opp))) { + ret = PTR_ERR(dev_opp); + mutex_unlock(&dev_opp_list_lock); + goto free_table; + } + + dev_opp->np = opp_np; + dev_opp->shared_opp = of_property_read_bool(opp_np, "opp-shared"); + + mutex_unlock(&dev_opp_list_lock); + + return 0; + +free_table: + dev_pm_opp_of_remove_table(dev); + + return ret; +} + +/* Initializes OPP tables based on old-deprecated bindings */ +static int _of_add_opp_table_v1(struct device *dev) +{ + const struct property *prop; + const __be32 *val; + int nr; + + prop = of_find_property(dev->of_node, "operating-points", NULL); + if (!prop) + return -ENODEV; + if (!prop->value) + return -ENODATA; + + /* + * Each OPP is a set of tuples consisting of frequency and + * voltage like <freq-kHz vol-uV>. + */ + nr = prop->length / sizeof(u32); + if (nr % 2) { + dev_err(dev, "%s: Invalid OPP list\n", __func__); + return -EINVAL; + } + + val = prop->value; + while (nr) { + unsigned long freq = be32_to_cpup(val++) * 1000; + unsigned long volt = be32_to_cpup(val++); + + if (_opp_add_v1(dev, freq, volt, false)) + dev_warn(dev, "%s: Failed to add OPP %ld\n", + __func__, freq); + nr -= 2; + } + + return 0; +} + +/** + * dev_pm_opp_of_add_table() - Initialize opp table from device tree + * @dev: device pointer used to lookup device OPPs. + * + * Register the initial OPP table with the OPP library for given device. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function indirectly uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + * + * Return: + * 0 On success OR + * Duplicate OPPs (both freq and volt are same) and opp->available + * -EEXIST Freq are same and volt are different OR + * Duplicate OPPs (both freq and volt are same) and !opp->available + * -ENOMEM Memory allocation failure + * -ENODEV when 'operating-points' property is not found or is invalid data + * in device node. + * -ENODATA when empty 'operating-points' property is found + * -EINVAL when invalid entries are found in opp-v2 table + */ +int dev_pm_opp_of_add_table(struct device *dev) +{ + struct device_node *opp_np; + int ret; + + /* + * OPPs have two version of bindings now. The older one is deprecated, + * try for the new binding first. + */ + opp_np = _of_get_opp_desc_node(dev); + if (!opp_np) { + /* + * Try old-deprecated bindings for backward compatibility with + * older dtbs. + */ + return _of_add_opp_table_v1(dev); + } + + ret = _of_add_opp_table_v2(dev, opp_np); + of_node_put(opp_np); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table); +#endif diff --git a/drivers/base/power/opp/cpu.c b/drivers/base/power/opp/cpu.c new file mode 100644 index 000000000000..9f0c15570f64 --- /dev/null +++ b/drivers/base/power/opp/cpu.c @@ -0,0 +1,271 @@ +/* + * Generic OPP helper interface for CPU device + * + * Copyright (C) 2009-2014 Texas Instruments Incorporated. + * Nishanth Menon + * Romit Dasgupta + * Kevin Hilman + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/cpu.h> +#include <linux/cpufreq.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/of.h> +#include <linux/slab.h> + +#include "opp.h" + +#ifdef CONFIG_CPU_FREQ + +/** + * dev_pm_opp_init_cpufreq_table() - create a cpufreq table for a device + * @dev: device for which we do this operation + * @table: Cpufreq table returned back to caller + * + * Generate a cpufreq table for a provided device- this assumes that the + * opp list is already initialized and ready for usage. + * + * This function allocates required memory for the cpufreq table. It is + * expected that the caller does the required maintenance such as freeing + * the table as required. + * + * Returns -EINVAL for bad pointers, -ENODEV if the device is not found, -ENOMEM + * if no memory available for the operation (table is not populated), returns 0 + * if successful and table is populated. + * + * WARNING: It is important for the callers to ensure refreshing their copy of + * the table if any of the mentioned functions have been invoked in the interim. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Since we just use the regular accessor functions to access the internal data + * structures, we use RCU read lock inside this function. As a result, users of + * this function DONOT need to use explicit locks for invoking. + */ +int dev_pm_opp_init_cpufreq_table(struct device *dev, + struct cpufreq_frequency_table **table) +{ + struct dev_pm_opp *opp; + struct cpufreq_frequency_table *freq_table = NULL; + int i, max_opps, ret = 0; + unsigned long rate; + + rcu_read_lock(); + + max_opps = dev_pm_opp_get_opp_count(dev); + if (max_opps <= 0) { + ret = max_opps ? max_opps : -ENODATA; + goto out; + } + + freq_table = kcalloc((max_opps + 1), sizeof(*freq_table), GFP_ATOMIC); + if (!freq_table) { + ret = -ENOMEM; + goto out; + } + + for (i = 0, rate = 0; i < max_opps; i++, rate++) { + /* find next rate */ + opp = dev_pm_opp_find_freq_ceil(dev, &rate); + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + goto out; + } + freq_table[i].driver_data = i; + freq_table[i].frequency = rate / 1000; + + /* Is Boost/turbo opp ? */ + if (dev_pm_opp_is_turbo(opp)) + freq_table[i].flags = CPUFREQ_BOOST_FREQ; + } + + freq_table[i].driver_data = i; + freq_table[i].frequency = CPUFREQ_TABLE_END; + + *table = &freq_table[0]; + +out: + rcu_read_unlock(); + if (ret) + kfree(freq_table); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_init_cpufreq_table); + +/** + * dev_pm_opp_free_cpufreq_table() - free the cpufreq table + * @dev: device for which we do this operation + * @table: table to free + * + * Free up the table allocated by dev_pm_opp_init_cpufreq_table + */ +void dev_pm_opp_free_cpufreq_table(struct device *dev, + struct cpufreq_frequency_table **table) +{ + if (!table) + return; + + kfree(*table); + *table = NULL; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_free_cpufreq_table); +#endif /* CONFIG_CPU_FREQ */ + +/* Required only for V1 bindings, as v2 can manage it from DT itself */ +int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, cpumask_var_t cpumask) +{ + struct device_list_opp *list_dev; + struct device_opp *dev_opp; + struct device *dev; + int cpu, ret = 0; + + mutex_lock(&dev_opp_list_lock); + + dev_opp = _find_device_opp(cpu_dev); + if (IS_ERR(dev_opp)) { + ret = -EINVAL; + goto unlock; + } + + for_each_cpu(cpu, cpumask) { + if (cpu == cpu_dev->id) + continue; + + dev = get_cpu_device(cpu); + if (!dev) { + dev_err(cpu_dev, "%s: failed to get cpu%d device\n", + __func__, cpu); + continue; + } + + list_dev = _add_list_dev(dev, dev_opp); + if (!list_dev) { + dev_err(dev, "%s: failed to add list-dev for cpu%d device\n", + __func__, cpu); + continue; + } + } +unlock: + mutex_unlock(&dev_opp_list_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_sharing_cpus); + +#ifdef CONFIG_OF +void dev_pm_opp_of_cpumask_remove_table(cpumask_var_t cpumask) +{ + struct device *cpu_dev; + int cpu; + + WARN_ON(cpumask_empty(cpumask)); + + for_each_cpu(cpu, cpumask) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, + cpu); + continue; + } + + dev_pm_opp_of_remove_table(cpu_dev); + } +} +EXPORT_SYMBOL_GPL(dev_pm_opp_of_cpumask_remove_table); + +int dev_pm_opp_of_cpumask_add_table(cpumask_var_t cpumask) +{ + struct device *cpu_dev; + int cpu, ret = 0; + + WARN_ON(cpumask_empty(cpumask)); + + for_each_cpu(cpu, cpumask) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, + cpu); + continue; + } + + ret = dev_pm_opp_of_add_table(cpu_dev); + if (ret) { + pr_err("%s: couldn't find opp table for cpu:%d, %d\n", + __func__, cpu, ret); + + /* Free all other OPPs */ + dev_pm_opp_of_cpumask_remove_table(cpumask); + break; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_of_cpumask_add_table); + +/* + * Works only for OPP v2 bindings. + * + * Returns -ENOENT if operating-points-v2 bindings aren't supported. + */ +int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, cpumask_var_t cpumask) +{ + struct device_node *np, *tmp_np; + struct device *tcpu_dev; + int cpu, ret = 0; + + /* Get OPP descriptor node */ + np = _of_get_opp_desc_node(cpu_dev); + if (!np) { + dev_dbg(cpu_dev, "%s: Couldn't find cpu_dev node.\n", __func__); + return -ENOENT; + } + + cpumask_set_cpu(cpu_dev->id, cpumask); + + /* OPPs are shared ? */ + if (!of_property_read_bool(np, "opp-shared")) + goto put_cpu_node; + + for_each_possible_cpu(cpu) { + if (cpu == cpu_dev->id) + continue; + + tcpu_dev = get_cpu_device(cpu); + if (!tcpu_dev) { + dev_err(cpu_dev, "%s: failed to get cpu%d device\n", + __func__, cpu); + ret = -ENODEV; + goto put_cpu_node; + } + + /* Get OPP descriptor node */ + tmp_np = _of_get_opp_desc_node(tcpu_dev); + if (!tmp_np) { + dev_err(tcpu_dev, "%s: Couldn't find tcpu_dev node.\n", + __func__); + ret = -ENOENT; + goto put_cpu_node; + } + + /* CPUs are sharing opp node */ + if (np == tmp_np) + cpumask_set_cpu(cpu, cpumask); + + of_node_put(tmp_np); + } + +put_cpu_node: + of_node_put(np); + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_of_get_sharing_cpus); +#endif diff --git a/drivers/base/power/opp/opp.h b/drivers/base/power/opp/opp.h new file mode 100644 index 000000000000..d8e3999fbb2c --- /dev/null +++ b/drivers/base/power/opp/opp.h @@ -0,0 +1,172 @@ +/* + * Generic OPP Interface + * + * Copyright (C) 2009-2010 Texas Instruments Incorporated. + * Nishanth Menon + * Romit Dasgupta + * Kevin Hilman + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __DRIVER_OPP_H__ +#define __DRIVER_OPP_H__ + +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/pm_opp.h> +#include <linux/rculist.h> +#include <linux/rcupdate.h> +#include <linux/limits.h> + +struct clk; +struct regulator; + +/* Lock to allow exclusive modification to the device and opp lists */ +extern struct mutex dev_opp_list_lock; + +/* + * Internal data structure organization with the OPP layer library is as + * follows: + * dev_opp_list (root) + * |- device 1 (represents voltage domain 1) + * | |- opp 1 (availability, freq, voltage) + * | |- opp 2 .. + * ... ... + * | `- opp n .. + * |- device 2 (represents the next voltage domain) + * ... + * `- device m (represents mth voltage domain) + * device 1, 2.. are represented by dev_opp structure while each opp + * is represented by the opp structure. + */ + +/** + * struct dev_pm_opp - Generic OPP description structure + * @node: opp list node. The nodes are maintained throughout the lifetime + * of boot. It is expected only an optimal set of OPPs are + * added to the library by the SoC framework. + * RCU usage: opp list is traversed with RCU locks. node + * modification is possible realtime, hence the modifications + * are protected by the dev_opp_list_lock for integrity. + * IMPORTANT: the opp nodes should be maintained in increasing + * order. + * @available: true/false - marks if this OPP as available or not + * @dynamic: not-created from static DT entries. + * @turbo: true if turbo (boost) OPP + * @rate: Frequency in hertz + * @u_volt: Target voltage in microvolts corresponding to this OPP + * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP + * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP + * @u_amp: Maximum current drawn by the device in microamperes + * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's + * frequency from any other OPP's frequency. + * @dev_opp: points back to the device_opp struct this opp belongs to + * @rcu_head: RCU callback head used for deferred freeing + * @np: OPP's device node. + * + * This structure stores the OPP information for a given device. + */ +struct dev_pm_opp { + struct list_head node; + + bool available; + bool dynamic; + bool turbo; + unsigned long rate; + + unsigned long u_volt; + unsigned long u_volt_min; + unsigned long u_volt_max; + unsigned long u_amp; + unsigned long clock_latency_ns; + + struct device_opp *dev_opp; + struct rcu_head rcu_head; + + struct device_node *np; +}; + +/** + * struct device_list_opp - devices managed by 'struct device_opp' + * @node: list node + * @dev: device to which the struct object belongs + * @rcu_head: RCU callback head used for deferred freeing + * + * This is an internal data structure maintaining the list of devices that are + * managed by 'struct device_opp'. + */ +struct device_list_opp { + struct list_head node; + const struct device *dev; + struct rcu_head rcu_head; +}; + +/** + * struct device_opp - Device opp structure + * @node: list node - contains the devices with OPPs that + * have been registered. Nodes once added are not modified in this + * list. + * RCU usage: nodes are not modified in the list of device_opp, + * however addition is possible and is secured by dev_opp_list_lock + * @srcu_head: notifier head to notify the OPP availability changes. + * @rcu_head: RCU callback head used for deferred freeing + * @dev_list: list of devices that share these OPPs + * @opp_list: list of opps + * @np: struct device_node pointer for opp's DT node. + * @clock_latency_ns_max: Max clock latency in nanoseconds. + * @shared_opp: OPP is shared between multiple devices. + * @suspend_opp: Pointer to OPP to be used during device suspend. + * @supported_hw: Array of version number to support. + * @supported_hw_count: Number of elements in supported_hw array. + * @prop_name: A name to postfix to many DT properties, while parsing them. + * @clk: Device's clock handle + * @regulator: Supply regulator + * @dentry: debugfs dentry pointer of the real device directory (not links). + * @dentry_name: Name of the real dentry. + * + * @voltage_tolerance_v1: In percentage, for v1 bindings only. + * + * This is an internal data structure maintaining the link to opps attached to + * a device. This structure is not meant to be shared to users as it is + * meant for book keeping and private to OPP library. + * + * Because the opp structures can be used from both rcu and srcu readers, we + * need to wait for the grace period of both of them before freeing any + * resources. And so we have used kfree_rcu() from within call_srcu() handlers. + */ +struct device_opp { + struct list_head node; + + struct srcu_notifier_head srcu_head; + struct rcu_head rcu_head; + struct list_head dev_list; + struct list_head opp_list; + + struct device_node *np; + unsigned long clock_latency_ns_max; + + /* For backward compatibility with v1 bindings */ + unsigned int voltage_tolerance_v1; + + bool shared_opp; + struct dev_pm_opp *suspend_opp; + + unsigned int *supported_hw; + unsigned int supported_hw_count; + const char *prop_name; + struct clk *clk; + struct regulator *regulator; + +}; + +/* Routines internal to opp core */ +struct device_opp *_find_device_opp(struct device *dev); +struct device_list_opp *_add_list_dev(const struct device *dev, + struct device_opp *dev_opp); +struct device_node *_of_get_opp_desc_node(struct device *dev); + +#endif /* __DRIVER_OPP_H__ */ diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 40c53dc1937e..92eeae40fe72 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -1,6 +1,5 @@ # CPUfreq core obj-$(CONFIG_CPU_FREQ) += cpufreq.o freq_table.o -obj-$(CONFIG_PM_OPP) += cpufreq_opp.o # CPUfreq stats obj-$(CONFIG_CPU_FREQ_STAT) += cpufreq_stats.o diff --git a/drivers/cpufreq/arm_big_little.c b/drivers/cpufreq/arm_big_little.c index a46c223c2506..e1a6ba66a7f5 100644 --- a/drivers/cpufreq/arm_big_little.c +++ b/drivers/cpufreq/arm_big_little.c @@ -289,6 +289,8 @@ static void _put_cluster_clk_and_freq_table(struct device *cpu_dev) clk_put(clk[cluster]); dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]); + if (arm_bL_ops->free_opp_table) + arm_bL_ops->free_opp_table(cpu_dev); dev_dbg(cpu_dev, "%s: cluster: %d\n", __func__, cluster); } @@ -337,7 +339,7 @@ static int _get_cluster_clk_and_freq_table(struct device *cpu_dev) if (ret) { dev_err(cpu_dev, "%s: failed to init cpufreq table, cpu: %d, err: %d\n", __func__, cpu_dev->id, ret); - goto out; + goto free_opp_table; } name[12] = cluster + '0'; @@ -354,6 +356,9 @@ static int _get_cluster_clk_and_freq_table(struct device *cpu_dev) ret = PTR_ERR(clk[cluster]); dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]); +free_opp_table: + if (arm_bL_ops->free_opp_table) + arm_bL_ops->free_opp_table(cpu_dev); out: dev_err(cpu_dev, "%s: Failed to get data for cluster: %d\n", __func__, cluster); diff --git a/drivers/cpufreq/arm_big_little.h b/drivers/cpufreq/arm_big_little.h index 70f18fc12d4a..b88889d9387e 100644 --- a/drivers/cpufreq/arm_big_little.h +++ b/drivers/cpufreq/arm_big_little.h @@ -25,13 +25,16 @@ struct cpufreq_arm_bL_ops { char name[CPUFREQ_NAME_LEN]; - int (*get_transition_latency)(struct device *cpu_dev); /* * This must set opp table for cpu_dev in a similar way as done by - * of_init_opp_table(). + * dev_pm_opp_of_add_table(). */ int (*init_opp_table)(struct device *cpu_dev); + + /* Optional */ + int (*get_transition_latency)(struct device *cpu_dev); + void (*free_opp_table)(struct device *cpu_dev); }; int bL_cpufreq_register(struct cpufreq_arm_bL_ops *ops); diff --git a/drivers/cpufreq/arm_big_little_dt.c b/drivers/cpufreq/arm_big_little_dt.c index 4550f6976768..16ddeefe9443 100644 --- a/drivers/cpufreq/arm_big_little_dt.c +++ b/drivers/cpufreq/arm_big_little_dt.c @@ -54,7 +54,7 @@ static int dt_init_opp_table(struct device *cpu_dev) return -ENOENT; } - ret = of_init_opp_table(cpu_dev); + ret = dev_pm_opp_of_add_table(cpu_dev); of_node_put(np); return ret; @@ -82,6 +82,7 @@ static struct cpufreq_arm_bL_ops dt_bL_ops = { .name = "dt-bl", .get_transition_latency = dt_get_transition_latency, .init_opp_table = dt_init_opp_table, + .free_opp_table = dev_pm_opp_of_remove_table, }; static int generic_bL_probe(struct platform_device *pdev) @@ -105,7 +106,6 @@ static int generic_bL_remove(struct platform_device *pdev) static struct platform_driver generic_bL_platdrv = { .driver = { .name = "arm-bL-cpufreq-dt", - .owner = THIS_MODULE, }, .probe = generic_bL_probe, .remove = generic_bL_remove, diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index bdb6951d0978..90d64081ddb3 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -36,6 +36,12 @@ struct private_data { unsigned int voltage_tolerance; /* in percentage */ }; +static struct freq_attr *cpufreq_dt_attr[] = { + &cpufreq_freq_attr_scaling_available_freqs, + NULL, /* Extra space for boost-attr if required */ + NULL, +}; + static int set_target(struct cpufreq_policy *policy, unsigned int index) { struct dev_pm_opp *opp; @@ -58,6 +64,8 @@ static int set_target(struct cpufreq_policy *policy, unsigned int index) old_freq = clk_get_rate(cpu_clk) / 1000; if (!IS_ERR(cpu_reg)) { + unsigned long opp_freq; + rcu_read_lock(); opp = dev_pm_opp_find_freq_ceil(cpu_dev, &freq_Hz); if (IS_ERR(opp)) { @@ -67,13 +75,16 @@ static int set_target(struct cpufreq_policy *policy, unsigned int index) return PTR_ERR(opp); } volt = dev_pm_opp_get_voltage(opp); + opp_freq = dev_pm_opp_get_freq(opp); rcu_read_unlock(); tol = volt * priv->voltage_tolerance / 100; volt_old = regulator_get_voltage(cpu_reg); + dev_dbg(cpu_dev, "Found OPP: %ld kHz, %ld uV\n", + opp_freq / 1000, volt); } dev_dbg(cpu_dev, "%u MHz, %ld mV --> %u MHz, %ld mV\n", - old_freq / 1000, volt_old ? volt_old / 1000 : -1, + old_freq / 1000, (volt_old > 0) ? volt_old / 1000 : -1, new_freq / 1000, volt ? volt / 1000 : -1); /* scaling up? scale voltage before frequency */ @@ -89,7 +100,7 @@ static int set_target(struct cpufreq_policy *policy, unsigned int index) ret = clk_set_rate(cpu_clk, freq_exact); if (ret) { dev_err(cpu_dev, "failed to set clock rate: %d\n", ret); - if (!IS_ERR(cpu_reg)) + if (!IS_ERR(cpu_reg) && volt_old > 0) regulator_set_voltage_tol(cpu_reg, volt_old, tol); return ret; } @@ -179,21 +190,21 @@ try_again: static int cpufreq_init(struct cpufreq_policy *policy) { - struct cpufreq_dt_platform_data *pd; struct cpufreq_frequency_table *freq_table; - struct thermal_cooling_device *cdev; struct device_node *np; struct private_data *priv; struct device *cpu_dev; struct regulator *cpu_reg; struct clk *cpu_clk; + struct dev_pm_opp *suspend_opp; unsigned long min_uV = ~0, max_uV = 0; unsigned int transition_latency; + bool need_update = false; int ret; ret = allocate_resources(policy->cpu, &cpu_dev, &cpu_reg, &cpu_clk); if (ret) { - pr_err("%s: Failed to allocate resources\n: %d", __func__, ret); + pr_err("%s: Failed to allocate resources: %d\n", __func__, ret); return ret; } @@ -204,18 +215,71 @@ static int cpufreq_init(struct cpufreq_policy *policy) goto out_put_reg_clk; } - /* OPPs might be populated at runtime, don't check for error here */ - of_init_opp_table(cpu_dev); + /* Get OPP-sharing information from "operating-points-v2" bindings */ + ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, policy->cpus); + if (ret) { + /* + * operating-points-v2 not supported, fallback to old method of + * finding shared-OPPs for backward compatibility. + */ + if (ret == -ENOENT) + need_update = true; + else + goto out_node_put; + } + + /* + * Initialize OPP tables for all policy->cpus. They will be shared by + * all CPUs which have marked their CPUs shared with OPP bindings. + * + * For platforms not using operating-points-v2 bindings, we do this + * before updating policy->cpus. Otherwise, we will end up creating + * duplicate OPPs for policy->cpus. + * + * OPPs might be populated at runtime, don't check for error here + */ + dev_pm_opp_of_cpumask_add_table(policy->cpus); + + /* + * But we need OPP table to function so if it is not there let's + * give platform code chance to provide it for us. + */ + ret = dev_pm_opp_get_opp_count(cpu_dev); + if (ret <= 0) { + pr_debug("OPP table is not ready, deferring probe\n"); + ret = -EPROBE_DEFER; + goto out_free_opp; + } + + if (need_update) { + struct cpufreq_dt_platform_data *pd = cpufreq_get_driver_data(); + + if (!pd || !pd->independent_clocks) + cpumask_setall(policy->cpus); + + /* + * OPP tables are initialized only for policy->cpu, do it for + * others as well. + */ + ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); + if (ret) + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", + __func__, ret); + + of_property_read_u32(np, "clock-latency", &transition_latency); + } else { + transition_latency = dev_pm_opp_get_max_clock_latency(cpu_dev); + } priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) { ret = -ENOMEM; - goto out_put_node; + goto out_free_opp; } of_property_read_u32(np, "voltage-tolerance", &priv->voltage_tolerance); - if (of_property_read_u32(np, "clock-latency", &transition_latency)) + if (!transition_latency) transition_latency = CPUFREQ_ETERNAL; if (!IS_ERR(cpu_reg)) { @@ -265,48 +329,47 @@ static int cpufreq_init(struct cpufreq_policy *policy) goto out_free_priv; } - /* - * For now, just loading the cooling device; - * thermal DT code takes care of matching them. - */ - if (of_find_property(np, "#cooling-cells", NULL)) { - cdev = of_cpufreq_cooling_register(np, cpu_present_mask); - if (IS_ERR(cdev)) - dev_err(cpu_dev, - "running cpufreq without cooling device: %ld\n", - PTR_ERR(cdev)); - else - priv->cdev = cdev; - } - priv->cpu_dev = cpu_dev; priv->cpu_reg = cpu_reg; policy->driver_data = priv; policy->clk = cpu_clk; + + rcu_read_lock(); + suspend_opp = dev_pm_opp_get_suspend_opp(cpu_dev); + if (suspend_opp) + policy->suspend_freq = dev_pm_opp_get_freq(suspend_opp) / 1000; + rcu_read_unlock(); + ret = cpufreq_table_validate_and_show(policy, freq_table); if (ret) { dev_err(cpu_dev, "%s: invalid frequency table: %d\n", __func__, ret); - goto out_cooling_unregister; + goto out_free_cpufreq_table; } - policy->cpuinfo.transition_latency = transition_latency; + /* Support turbo/boost mode */ + if (policy_has_boost_freq(policy)) { + /* This gets disabled by core on driver unregister */ + ret = cpufreq_enable_boost_support(); + if (ret) + goto out_free_cpufreq_table; + cpufreq_dt_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; + } - pd = cpufreq_get_driver_data(); - if (!pd || !pd->independent_clocks) - cpumask_setall(policy->cpus); + policy->cpuinfo.transition_latency = transition_latency; of_node_put(np); return 0; -out_cooling_unregister: - cpufreq_cooling_unregister(priv->cdev); +out_free_cpufreq_table: dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); out_free_priv: kfree(priv); -out_put_node: +out_free_opp: + dev_pm_opp_of_cpumask_remove_table(policy->cpus); +out_node_put: of_node_put(np); out_put_reg_clk: clk_put(cpu_clk); @@ -322,6 +385,7 @@ static int cpufreq_exit(struct cpufreq_policy *policy) cpufreq_cooling_unregister(priv->cdev); dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); + dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); clk_put(policy->clk); if (!IS_ERR(priv->cpu_reg)) regulator_put(priv->cpu_reg); @@ -330,6 +394,33 @@ static int cpufreq_exit(struct cpufreq_policy *policy) return 0; } +static void cpufreq_ready(struct cpufreq_policy *policy) +{ + struct private_data *priv = policy->driver_data; + struct device_node *np = of_node_get(priv->cpu_dev->of_node); + + if (WARN_ON(!np)) + return; + + /* + * For now, just loading the cooling device; + * thermal DT code takes care of matching them. + */ + if (of_find_property(np, "#cooling-cells", NULL)) { + priv->cdev = of_cpufreq_cooling_register(np, + policy->related_cpus); + if (IS_ERR(priv->cdev)) { + dev_err(priv->cpu_dev, + "running cpufreq without cooling device: %ld\n", + PTR_ERR(priv->cdev)); + + priv->cdev = NULL; + } + } + + of_node_put(np); +} + static struct cpufreq_driver dt_cpufreq_driver = { .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, .verify = cpufreq_generic_frequency_table_verify, @@ -337,8 +428,10 @@ static struct cpufreq_driver dt_cpufreq_driver = { .get = cpufreq_generic_get, .init = cpufreq_init, .exit = cpufreq_exit, + .ready = cpufreq_ready, .name = "cpufreq-dt", - .attr = cpufreq_generic_attr, + .attr = cpufreq_dt_attr, + .suspend = cpufreq_generic_suspend, }; static int dt_cpufreq_probe(struct platform_device *pdev) @@ -381,13 +474,13 @@ static int dt_cpufreq_remove(struct platform_device *pdev) static struct platform_driver dt_cpufreq_platdrv = { .driver = { .name = "cpufreq-dt", - .owner = THIS_MODULE, }, .probe = dt_cpufreq_probe, .remove = dt_cpufreq_remove, }; module_platform_driver(dt_cpufreq_platdrv); +MODULE_ALIAS("platform:cpufreq-dt"); MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>"); MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>"); MODULE_DESCRIPTION("Generic cpufreq driver"); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 90e8deb6c15e..8277d62dd301 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1288,8 +1288,13 @@ static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) up_write(&policy->rwsem); kobject_uevent(&policy->kobj, KOBJ_ADD); + up_read(&cpufreq_rwsem); + /* Callback for handling stuff after policy is ready */ + if (cpufreq_driver->ready) + cpufreq_driver->ready(policy); + pr_debug("initialization complete\n"); return 0; @@ -2406,6 +2411,49 @@ int cpufreq_boost_supported(void) } EXPORT_SYMBOL_GPL(cpufreq_boost_supported); +static int create_boost_sysfs_file(void) +{ + int ret; + + if (!cpufreq_boost_supported()) + return 0; + + /* + * Check if driver provides function to enable boost - + * if not, use cpufreq_boost_set_sw as default + */ + if (!cpufreq_driver->set_boost) + cpufreq_driver->set_boost = cpufreq_boost_set_sw; + + ret = cpufreq_sysfs_create_file(&boost.attr); + if (ret) + pr_err("%s: cannot register global BOOST sysfs file\n", + __func__); + + return ret; +} + +static void remove_boost_sysfs_file(void) +{ + if (cpufreq_boost_supported()) + cpufreq_sysfs_remove_file(&boost.attr); +} + +int cpufreq_enable_boost_support(void) +{ + if (!cpufreq_driver) + return -EINVAL; + + if (cpufreq_boost_supported()) + return 0; + + cpufreq_driver->boost_supported = true; + + /* This will get removed on driver unregister */ + return create_boost_sysfs_file(); +} +EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support); + int cpufreq_boost_enabled(void) { return cpufreq_driver->boost_enabled; @@ -2455,21 +2503,12 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) cpufreq_driver = driver_data; write_unlock_irqrestore(&cpufreq_driver_lock, flags); - if (cpufreq_boost_supported()) { - /* - * Check if driver provides function to enable boost - - * if not, use cpufreq_boost_set_sw as default - */ - if (!cpufreq_driver->set_boost) - cpufreq_driver->set_boost = cpufreq_boost_set_sw; + if (driver_data->setpolicy) + driver_data->flags |= CPUFREQ_CONST_LOOPS; - ret = cpufreq_sysfs_create_file(&boost.attr); - if (ret) { - pr_err("%s: cannot register global BOOST sysfs file\n", - __func__); - goto err_null_driver; - } - } + ret = create_boost_sysfs_file(); + if (ret) + goto err_null_driver; ret = subsys_interface_register(&cpufreq_interface); if (ret) @@ -2501,8 +2540,7 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) err_if_unreg: subsys_interface_unregister(&cpufreq_interface); err_boost_unreg: - if (cpufreq_boost_supported()) - cpufreq_sysfs_remove_file(&boost.attr); + remove_boost_sysfs_file(); err_null_driver: write_lock_irqsave(&cpufreq_driver_lock, flags); cpufreq_driver = NULL; @@ -2529,9 +2567,7 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver) pr_debug("unregistering driver %s\n", driver->name); subsys_interface_unregister(&cpufreq_interface); - if (cpufreq_boost_supported()) - cpufreq_sysfs_remove_file(&boost.attr); - + remove_boost_sysfs_file(); unregister_hotcpu_notifier(&cpufreq_cpu_notifier); down_write(&cpufreq_rwsem); diff --git a/drivers/cpufreq/cpufreq_opp.c b/drivers/cpufreq/cpufreq_opp.c deleted file mode 100644 index 773bcde893c0..000000000000 --- a/drivers/cpufreq/cpufreq_opp.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Generic OPP helper interface for CPUFreq drivers - * - * Copyright (C) 2009-2014 Texas Instruments Incorporated. - * Nishanth Menon - * Romit Dasgupta - * Kevin Hilman - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -#include <linux/cpufreq.h> -#include <linux/device.h> -#include <linux/err.h> -#include <linux/errno.h> -#include <linux/export.h> -#include <linux/kernel.h> -#include <linux/pm_opp.h> -#include <linux/rcupdate.h> -#include <linux/slab.h> - -/** - * dev_pm_opp_init_cpufreq_table() - create a cpufreq table for a device - * @dev: device for which we do this operation - * @table: Cpufreq table returned back to caller - * - * Generate a cpufreq table for a provided device- this assumes that the - * opp list is already initialized and ready for usage. - * - * This function allocates required memory for the cpufreq table. It is - * expected that the caller does the required maintenance such as freeing - * the table as required. - * - * Returns -EINVAL for bad pointers, -ENODEV if the device is not found, -ENOMEM - * if no memory available for the operation (table is not populated), returns 0 - * if successful and table is populated. - * - * WARNING: It is important for the callers to ensure refreshing their copy of - * the table if any of the mentioned functions have been invoked in the interim. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Since we just use the regular accessor functions to access the internal data - * structures, we use RCU read lock inside this function. As a result, users of - * this function DONOT need to use explicit locks for invoking. - */ -int dev_pm_opp_init_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table) -{ - struct dev_pm_opp *opp; - struct cpufreq_frequency_table *freq_table = NULL; - int i, max_opps, ret = 0; - unsigned long rate; - - rcu_read_lock(); - - max_opps = dev_pm_opp_get_opp_count(dev); - if (max_opps <= 0) { - ret = max_opps ? max_opps : -ENODATA; - goto out; - } - - freq_table = kcalloc((max_opps + 1), sizeof(*freq_table), GFP_ATOMIC); - if (!freq_table) { - ret = -ENOMEM; - goto out; - } - - for (i = 0, rate = 0; i < max_opps; i++, rate++) { - /* find next rate */ - opp = dev_pm_opp_find_freq_ceil(dev, &rate); - if (IS_ERR(opp)) { - ret = PTR_ERR(opp); - goto out; - } - freq_table[i].driver_data = i; - freq_table[i].frequency = rate / 1000; - } - - freq_table[i].driver_data = i; - freq_table[i].frequency = CPUFREQ_TABLE_END; - - *table = &freq_table[0]; - -out: - rcu_read_unlock(); - if (ret) - kfree(freq_table); - - return ret; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_init_cpufreq_table); - -/** - * dev_pm_opp_free_cpufreq_table() - free the cpufreq table - * @dev: device for which we do this operation - * @table: table to free - * - * Free up the table allocated by dev_pm_opp_init_cpufreq_table - */ -void dev_pm_opp_free_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table) -{ - if (!table) - return; - - kfree(*table); - *table = NULL; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_free_cpufreq_table); diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 28a16dc6e02e..7e336d20c184 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -169,7 +169,6 @@ static int __exit davinci_cpufreq_remove(struct platform_device *pdev) static struct platform_driver davinci_cpufreq_driver = { .driver = { .name = "cpufreq-davinci", - .owner = THIS_MODULE, }, .remove = __exit_p(davinci_cpufreq_remove), }; diff --git a/drivers/cpufreq/dbx500-cpufreq.c b/drivers/cpufreq/dbx500-cpufreq.c index 4bebc1b5db48..5c3ec1dd4921 100644 --- a/drivers/cpufreq/dbx500-cpufreq.c +++ b/drivers/cpufreq/dbx500-cpufreq.c @@ -69,7 +69,6 @@ static int dbx500_cpufreq_probe(struct platform_device *pdev) static struct platform_driver dbx500_cpufreq_plat_driver = { .driver = { .name = "cpufreq-ux500", - .owner = THIS_MODULE, }, .probe = dbx500_cpufreq_probe, }; diff --git a/drivers/cpufreq/exynos-cpufreq.c b/drivers/cpufreq/exynos-cpufreq.c index 1e0ec57bf6e3..f99a0b0b7c06 100644 --- a/drivers/cpufreq/exynos-cpufreq.c +++ b/drivers/cpufreq/exynos-cpufreq.c @@ -211,7 +211,6 @@ err_vdd_arm: static struct platform_driver exynos_cpufreq_platdrv = { .driver = { .name = "exynos-cpufreq", - .owner = THIS_MODULE, }, .probe = exynos_cpufreq_probe, }; diff --git a/drivers/cpufreq/exynos5440-cpufreq.c b/drivers/cpufreq/exynos5440-cpufreq.c index f33f25b483ca..c0f3373706f4 100644 --- a/drivers/cpufreq/exynos5440-cpufreq.c +++ b/drivers/cpufreq/exynos5440-cpufreq.c @@ -360,7 +360,7 @@ static int exynos_cpufreq_probe(struct platform_device *pdev) goto err_put_node; } - ret = of_init_opp_table(dvfs_info->dev); + ret = dev_pm_opp_of_add_table(dvfs_info->dev); if (ret) { dev_err(dvfs_info->dev, "failed to init OPP table: %d\n", ret); goto err_put_node; @@ -371,7 +371,7 @@ static int exynos_cpufreq_probe(struct platform_device *pdev) if (ret) { dev_err(dvfs_info->dev, "failed to init cpufreq table: %d\n", ret); - goto err_put_node; + goto err_free_opp; } dvfs_info->freq_count = dev_pm_opp_get_opp_count(dvfs_info->dev); exynos_sort_descend_freq_table(); @@ -423,6 +423,8 @@ static int exynos_cpufreq_probe(struct platform_device *pdev) err_free_table: dev_pm_opp_free_cpufreq_table(dvfs_info->dev, &dvfs_info->freq_table); +err_free_opp: + dev_pm_opp_of_remove_table(dvfs_info->dev); err_put_node: of_node_put(np); dev_err(&pdev->dev, "%s: failed initialization\n", __func__); @@ -433,13 +435,13 @@ static int exynos_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&exynos_driver); dev_pm_opp_free_cpufreq_table(dvfs_info->dev, &dvfs_info->freq_table); + dev_pm_opp_of_remove_table(dvfs_info->dev); return 0; } static struct platform_driver exynos_cpufreq_platdrv = { .driver = { .name = "exynos5440-cpufreq", - .owner = THIS_MODULE, .of_match_table = exynos_cpufreq_match, }, .probe = exynos_cpufreq_probe, diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index df14766a8e06..d1a0cbfdba63 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -18,6 +18,21 @@ * FREQUENCY TABLE HELPERS * *********************************************************************/ +bool policy_has_boost_freq(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *pos, *table = policy->freq_table; + + if (!table) + return false; + + cpufreq_for_each_valid_entry(pos, table) + if (pos->flags & CPUFREQ_BOOST_FREQ) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(policy_has_boost_freq); + int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table) { diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index c2d30765bf3d..84fbc8e8fc56 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -31,6 +31,7 @@ static struct clk *step_clk; static struct clk *pll2_pfd2_396m_clk; static struct device *cpu_dev; +static bool free_opp; static struct cpufreq_frequency_table *freq_table; static unsigned int transition_latency; @@ -201,17 +202,20 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) */ num = dev_pm_opp_get_opp_count(cpu_dev); if (num < 0) { - ret = of_init_opp_table(cpu_dev); + ret = dev_pm_opp_of_add_table(cpu_dev); if (ret < 0) { dev_err(cpu_dev, "failed to init OPP table: %d\n", ret); goto put_reg; } + /* Because we have added the OPPs here, we must free them */ + free_opp = true; + num = dev_pm_opp_get_opp_count(cpu_dev); if (num < 0) { ret = num; dev_err(cpu_dev, "no OPP table is found: %d\n", ret); - goto put_reg; + goto out_free_opp; } } @@ -306,6 +310,9 @@ soc_opp_out: free_freq_table: dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); +out_free_opp: + if (free_opp) + dev_pm_opp_of_remove_table(cpu_dev); put_reg: if (!IS_ERR(arm_reg)) regulator_put(arm_reg); @@ -332,6 +339,8 @@ static int imx6q_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&imx6q_cpufreq_driver); dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); + if (free_opp) + dev_pm_opp_of_remove_table(cpu_dev); regulator_put(arm_reg); if (!IS_ERR(pu_reg)) regulator_put(pu_reg); @@ -348,7 +357,6 @@ static int imx6q_cpufreq_remove(struct platform_device *pdev) static struct platform_driver imx6q_cpufreq_platdrv = { .driver = { .name = "imx6q-cpufreq", - .owner = THIS_MODULE, }, .probe = imx6q_cpufreq_probe, .remove = imx6q_cpufreq_remove, diff --git a/drivers/cpufreq/integrator-cpufreq.c b/drivers/cpufreq/integrator-cpufreq.c index 6bd69adc3c5e..129e266f7621 100644 --- a/drivers/cpufreq/integrator-cpufreq.c +++ b/drivers/cpufreq/integrator-cpufreq.c @@ -226,7 +226,6 @@ static const struct of_device_id integrator_cpufreq_match[] = { static struct platform_driver integrator_cpufreq_driver = { .driver = { .name = "integrator-cpufreq", - .owner = THIS_MODULE, .of_match_table = integrator_cpufreq_match, }, .remove = __exit_p(integrator_cpufreq_remove), diff --git a/drivers/cpufreq/kirkwood-cpufreq.c b/drivers/cpufreq/kirkwood-cpufreq.c index 7906d4acfe40..be42f103db60 100644 --- a/drivers/cpufreq/kirkwood-cpufreq.c +++ b/drivers/cpufreq/kirkwood-cpufreq.c @@ -183,7 +183,6 @@ static struct platform_driver kirkwood_cpufreq_platform_driver = { .remove = kirkwood_cpufreq_remove, .driver = { .name = "kirkwood-cpufreq", - .owner = THIS_MODULE, }, }; diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c index 9fa177206032..fc897babab55 100644 --- a/drivers/cpufreq/loongson2_cpufreq.c +++ b/drivers/cpufreq/loongson2_cpufreq.c @@ -130,7 +130,6 @@ MODULE_DEVICE_TABLE(platform, platform_device_ids); static struct platform_driver platform_driver = { .driver = { .name = "loongson2_cpufreq", - .owner = THIS_MODULE, }, .id_table = platform_device_ids, }; diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c index 5f69c9aa703c..e3866e0d5bf8 100644 --- a/drivers/cpufreq/omap-cpufreq.c +++ b/drivers/cpufreq/omap-cpufreq.c @@ -195,7 +195,6 @@ static int omap_cpufreq_remove(struct platform_device *pdev) static struct platform_driver omap_cpufreq_platdrv = { .driver = { .name = "omap-cpufreq", - .owner = THIS_MODULE, }, .probe = omap_cpufreq_probe, .remove = omap_cpufreq_remove, diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index 567caa6313ff..b0dac7d6ba31 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -656,7 +656,6 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev) static struct platform_driver s5pv210_cpufreq_platdrv = { .driver = { .name = "s5pv210-cpufreq", - .owner = THIS_MODULE, }, .probe = s5pv210_cpufreq_probe, }; diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c index 38678396636d..4894924a3ca2 100644 --- a/drivers/cpufreq/spear-cpufreq.c +++ b/drivers/cpufreq/spear-cpufreq.c @@ -236,7 +236,6 @@ out_put_node: static struct platform_driver spear_cpufreq_platdrv = { .driver = { .name = "spear-cpufreq", - .owner = THIS_MODULE, }, .probe = spear_cpufreq_probe, }; diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c index 7f7c9c01b44e..433e93fd4900 100644 --- a/drivers/cpufreq/vexpress-spc-cpufreq.c +++ b/drivers/cpufreq/vexpress-spc-cpufreq.c @@ -60,7 +60,6 @@ static int ve_spc_cpufreq_remove(struct platform_device *pdev) static struct platform_driver ve_spc_cpufreq_platdrv = { .driver = { .name = "vexpress-spc-cpufreq", - .owner = THIS_MODULE, }, .probe = ve_spc_cpufreq_probe, .remove = ve_spc_cpufreq_remove, diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index 061407d59520..887196e9b50c 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -1009,9 +1009,9 @@ error_out: * notifier_to_errno() to decode this value */ static int nx842_OF_notifier(struct notifier_block *np, unsigned long action, - void *update) + void *data) { - struct of_prop_reconfig *upd = update; + struct of_reconfig_data *upd = data; struct nx842_devdata *local_devdata; struct device_node *node = NULL; diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile index aef6a95adef5..bef2579be1ec 100644 --- a/drivers/firmware/efi/Makefile +++ b/drivers/firmware/efi/Makefile @@ -1,6 +1,14 @@ # # Makefile for linux kernel # + +# +# ARM64 maps efi runtime services in userspace addresses +# which don't have KASAN shadow. So dereference of these addresses +# in efi_call_virt() will cause crash if this code instrumented. +# +KASAN_SANITIZE_runtime-wrappers.o := n + obj-$(CONFIG_EFI) += efi.o vars.o reboot.o obj-$(CONFIG_EFI_VARS) += efivars.o obj-$(CONFIG_EFI_VARS_PSTORE) += efi-pstore.o diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index b14bc2b9fb4d..c5533c76c202 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -19,6 +19,7 @@ KBUILD_CFLAGS := $(cflags-y) \ $(call cc-option,-fno-stack-protector) GCOV_PROFILE := n +KASAN_SANITIZE := n lib-y := efi-stub-helper.o lib-$(CONFIG_EFI_ARMSTUB) += arm-stub.o fdt.o diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index 304ab295ca1a..d1ba39c3ee4e 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -5,6 +5,10 @@ /* error code which can't be mistaken for valid address */ #define EFI_ERROR (~0UL) +#undef memcpy +#undef memset +#undef memmove + void efi_char16_printk(efi_system_table_t *, efi_char16_t *); efi_status_t efi_open_volume(efi_system_table_t *sys_table_arg, void *__image, diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 87743dd1df48..c91a4fb596e4 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -35,9 +35,6 @@ #include <drm/drm_fb_helper.h> #include <drm/drm_dp_mst_helper.h> -#define DIV_ROUND_CLOSEST_ULL(ll, d) \ -({ unsigned long long _tmp = (ll)+(d)/2; do_div(_tmp, d); _tmp; }) - /** * _wait_for - magic (register) wait macro * diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index 8bc193f81333..ce5a385150c8 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -30,6 +30,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <linux/kernel.h> #include <linux/moduleparam.h> #include "intel_drv.h" diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c index d16dbb33a531..e7c8bf9093ea 100644 --- a/drivers/hwmon/lm75.c +++ b/drivers/hwmon/lm75.c @@ -176,6 +176,10 @@ static struct attribute *lm75_attrs[] = { }; ATTRIBUTE_GROUPS(lm75); +static const struct thermal_zone_of_device_ops lm75_of_thermal_ops = { + .get_temp = lm75_read_temp, +}; + /*-----------------------------------------------------------------------*/ /* device probe and removal */ @@ -291,10 +295,9 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id) if (IS_ERR(data->hwmon_dev)) return PTR_ERR(data->hwmon_dev); - data->tz = thermal_zone_of_sensor_register(data->hwmon_dev, - 0, + data->tz = thermal_zone_of_sensor_register(data->hwmon_dev, 0, data->hwmon_dev, - lm75_read_temp, NULL); + &lm75_of_thermal_ops); if (IS_ERR(data->tz)) data->tz = NULL; diff --git a/drivers/hwmon/ntc_thermistor.c b/drivers/hwmon/ntc_thermistor.c index 31597c504879..b2c90bdc2dd1 100644 --- a/drivers/hwmon/ntc_thermistor.c +++ b/drivers/hwmon/ntc_thermistor.c @@ -495,6 +495,10 @@ static const struct attribute_group ntc_attr_group = { .attrs = ntc_attributes, }; +static const struct thermal_zone_of_device_ops ntc_of_thermal_ops = { + .get_temp = ntc_read_temp, +}; + static int ntc_thermistor_probe(struct platform_device *pdev) { const struct of_device_id *of_id = @@ -588,7 +592,7 @@ static int ntc_thermistor_probe(struct platform_device *pdev) pdev_id->name); data->tz = thermal_zone_of_sensor_register(data->dev, 0, data->dev, - ntc_read_temp, NULL); + &ntc_of_thermal_ops); if (IS_ERR(data->tz)) { dev_dbg(&pdev->dev, "Failed to register to thermal fw.\n"); data->tz = NULL; diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c index 51719956cc03..ba9f478f64ee 100644 --- a/drivers/hwmon/tmp102.c +++ b/drivers/hwmon/tmp102.c @@ -158,6 +158,10 @@ ATTRIBUTE_GROUPS(tmp102); #define TMP102_CONFIG (TMP102_CONF_TM | TMP102_CONF_EM | TMP102_CONF_CR1) #define TMP102_CONFIG_RD_ONLY (TMP102_CONF_R0 | TMP102_CONF_R1 | TMP102_CONF_AL) +static const struct thermal_zone_of_device_ops tmp102_of_thermal_ops = { + .get_temp = tmp102_read_temp, +}; + static int tmp102_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -215,7 +219,7 @@ static int tmp102_probe(struct i2c_client *client, } tmp102->hwmon_dev = hwmon_dev; tmp102->tz = thermal_zone_of_sensor_register(hwmon_dev, 0, hwmon_dev, - tmp102_read_temp, NULL); + &tmp102_of_thermal_ops); if (IS_ERR(tmp102->tz)) tmp102->tz = NULL; diff --git a/drivers/hwtracing/coresight/Kconfig b/drivers/hwtracing/coresight/Kconfig new file mode 100644 index 000000000000..fc1f1ae7a49d --- /dev/null +++ b/drivers/hwtracing/coresight/Kconfig @@ -0,0 +1,61 @@ +# +# Coresight configuration +# +menuconfig CORESIGHT + bool "CoreSight Tracing Support" + select ARM_AMBA + help + This framework provides a kernel interface for the CoreSight debug + and trace drivers to register themselves with. It's intended to build + a topological view of the CoreSight components based on a DT + specification and configure the right serie of components when a + trace source gets enabled. + +if CORESIGHT +config CORESIGHT_LINKS_AND_SINKS + bool "CoreSight Link and Sink drivers" + help + This enables support for CoreSight link and sink drivers that are + responsible for transporting and collecting the trace data + respectively. Link and sinks are dynamically aggregated with a trace + entity at run time to form a complete trace path. + +config CORESIGHT_LINK_AND_SINK_TMC + bool "Coresight generic TMC driver" + depends on CORESIGHT_LINKS_AND_SINKS + help + This enables support for the Trace Memory Controller driver. + Depending on its configuration the device can act as a link (embedded + trace router - ETR) or sink (embedded trace FIFO). The driver + complies with the generic implementation of the component without + special enhancement or added features. + +config CORESIGHT_SINK_TPIU + bool "Coresight generic TPIU driver" + depends on CORESIGHT_LINKS_AND_SINKS + help + This enables support for the Trace Port Interface Unit driver, + responsible for bridging the gap between the on-chip coresight + components and a trace for bridging the gap between the on-chip + coresight components and a trace port collection engine, typically + connected to an external host for use case capturing more traces than + the on-board coresight memory can handle. + +config CORESIGHT_SINK_ETBV10 + bool "Coresight ETBv1.0 driver" + depends on CORESIGHT_LINKS_AND_SINKS + help + This enables support for the Embedded Trace Buffer version 1.0 driver + that complies with the generic implementation of the component without + special enhancement or added features. + +config CORESIGHT_SOURCE_ETM3X + bool "CoreSight Embedded Trace Macrocell 3.x driver" + depends on !ARM64 + select CORESIGHT_LINKS_AND_SINKS + help + This driver provides support for processor ETM3.x and PTM1.x modules, + which allows tracing the instructions that a processor is executing + This is primarily useful for instruction level tracing. Depending + the ETM version data tracing may also be available. +endif diff --git a/drivers/hwtracing/coresight/Makefile b/drivers/hwtracing/coresight/Makefile new file mode 100644 index 000000000000..4b4bec890ef5 --- /dev/null +++ b/drivers/hwtracing/coresight/Makefile @@ -0,0 +1,11 @@ +# +# Makefile for CoreSight drivers. +# +obj-$(CONFIG_CORESIGHT) += coresight.o +obj-$(CONFIG_OF) += of_coresight.o +obj-$(CONFIG_CORESIGHT_LINK_AND_SINK_TMC) += coresight-tmc.o +obj-$(CONFIG_CORESIGHT_SINK_TPIU) += coresight-tpiu.o +obj-$(CONFIG_CORESIGHT_SINK_ETBV10) += coresight-etb10.o +obj-$(CONFIG_CORESIGHT_LINKS_AND_SINKS) += coresight-funnel.o \ + coresight-replicator.o +obj-$(CONFIG_CORESIGHT_SOURCE_ETM3X) += coresight-etm3x.o coresight-etm-cp14.o diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c new file mode 100644 index 000000000000..40049869aecd --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -0,0 +1,527 @@ +/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/io.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/miscdevice.h> +#include <linux/uaccess.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/clk.h> +#include <linux/seq_file.h> +#include <linux/coresight.h> +#include <linux/amba/bus.h> + +#include "coresight-priv.h" + +#define ETB_RAM_DEPTH_REG 0x004 +#define ETB_STATUS_REG 0x00c +#define ETB_RAM_READ_DATA_REG 0x010 +#define ETB_RAM_READ_POINTER 0x014 +#define ETB_RAM_WRITE_POINTER 0x018 +#define ETB_TRG 0x01c +#define ETB_CTL_REG 0x020 +#define ETB_RWD_REG 0x024 +#define ETB_FFSR 0x300 +#define ETB_FFCR 0x304 +#define ETB_ITMISCOP0 0xee0 +#define ETB_ITTRFLINACK 0xee4 +#define ETB_ITTRFLIN 0xee8 +#define ETB_ITATBDATA0 0xeeC +#define ETB_ITATBCTR2 0xef0 +#define ETB_ITATBCTR1 0xef4 +#define ETB_ITATBCTR0 0xef8 + +/* register description */ +/* STS - 0x00C */ +#define ETB_STATUS_RAM_FULL BIT(0) +/* CTL - 0x020 */ +#define ETB_CTL_CAPT_EN BIT(0) +/* FFCR - 0x304 */ +#define ETB_FFCR_EN_FTC BIT(0) +#define ETB_FFCR_FON_MAN BIT(6) +#define ETB_FFCR_STOP_FI BIT(12) +#define ETB_FFCR_STOP_TRIGGER BIT(13) + +#define ETB_FFCR_BIT 6 +#define ETB_FFSR_BIT 1 +#define ETB_FRAME_SIZE_WORDS 4 + +/** + * struct etb_drvdata - specifics associated to an ETB component + * @base: memory mapped base address for this component. + * @dev: the device entity associated to this component. + * @csdev: component vitals needed by the framework. + * @miscdev: specifics to handle "/dev/xyz.etb" entry. + * @clk: the clock this component is associated to. + * @spinlock: only one at a time pls. + * @in_use: synchronise user space access to etb buffer. + * @buf: area of memory where ETB buffer content gets sent. + * @buffer_depth: size of @buf. + * @enable: this ETB is being used. + * @trigger_cntr: amount of words to store after a trigger. + */ +struct etb_drvdata { + void __iomem *base; + struct device *dev; + struct coresight_device *csdev; + struct miscdevice miscdev; + struct clk *clk; + spinlock_t spinlock; + atomic_t in_use; + u8 *buf; + u32 buffer_depth; + bool enable; + u32 trigger_cntr; +}; + +static unsigned int etb_get_buffer_depth(struct etb_drvdata *drvdata) +{ + int ret; + u32 depth = 0; + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + /* RO registers don't need locking */ + depth = readl_relaxed(drvdata->base + ETB_RAM_DEPTH_REG); + + clk_disable_unprepare(drvdata->clk); + return depth; +} + +static void etb_enable_hw(struct etb_drvdata *drvdata) +{ + int i; + u32 depth; + + CS_UNLOCK(drvdata->base); + + depth = drvdata->buffer_depth; + /* reset write RAM pointer address */ + writel_relaxed(0x0, drvdata->base + ETB_RAM_WRITE_POINTER); + /* clear entire RAM buffer */ + for (i = 0; i < depth; i++) + writel_relaxed(0x0, drvdata->base + ETB_RWD_REG); + + /* reset write RAM pointer address */ + writel_relaxed(0x0, drvdata->base + ETB_RAM_WRITE_POINTER); + /* reset read RAM pointer address */ + writel_relaxed(0x0, drvdata->base + ETB_RAM_READ_POINTER); + + writel_relaxed(drvdata->trigger_cntr, drvdata->base + ETB_TRG); + writel_relaxed(ETB_FFCR_EN_FTC | ETB_FFCR_STOP_TRIGGER, + drvdata->base + ETB_FFCR); + /* ETB trace capture enable */ + writel_relaxed(ETB_CTL_CAPT_EN, drvdata->base + ETB_CTL_REG); + + CS_LOCK(drvdata->base); +} + +static int etb_enable(struct coresight_device *csdev) +{ + struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + int ret; + unsigned long flags; + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + spin_lock_irqsave(&drvdata->spinlock, flags); + etb_enable_hw(drvdata); + drvdata->enable = true; + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + dev_info(drvdata->dev, "ETB enabled\n"); + return 0; +} + +static void etb_disable_hw(struct etb_drvdata *drvdata) +{ + u32 ffcr; + + CS_UNLOCK(drvdata->base); + + ffcr = readl_relaxed(drvdata->base + ETB_FFCR); + /* stop formatter when a stop has completed */ + ffcr |= ETB_FFCR_STOP_FI; + writel_relaxed(ffcr, drvdata->base + ETB_FFCR); + /* manually generate a flush of the system */ + ffcr |= ETB_FFCR_FON_MAN; + writel_relaxed(ffcr, drvdata->base + ETB_FFCR); + + if (coresight_timeout(drvdata->base, ETB_FFCR, ETB_FFCR_BIT, 0)) { + dev_err(drvdata->dev, + "timeout observed when probing at offset %#x\n", + ETB_FFCR); + } + + /* disable trace capture */ + writel_relaxed(0x0, drvdata->base + ETB_CTL_REG); + + if (coresight_timeout(drvdata->base, ETB_FFSR, ETB_FFSR_BIT, 1)) { + dev_err(drvdata->dev, + "timeout observed when probing at offset %#x\n", + ETB_FFCR); + } + + CS_LOCK(drvdata->base); +} + +static void etb_dump_hw(struct etb_drvdata *drvdata) +{ + int i; + u8 *buf_ptr; + u32 read_data, depth; + u32 read_ptr, write_ptr; + u32 frame_off, frame_endoff; + + CS_UNLOCK(drvdata->base); + + read_ptr = readl_relaxed(drvdata->base + ETB_RAM_READ_POINTER); + write_ptr = readl_relaxed(drvdata->base + ETB_RAM_WRITE_POINTER); + + frame_off = write_ptr % ETB_FRAME_SIZE_WORDS; + frame_endoff = ETB_FRAME_SIZE_WORDS - frame_off; + if (frame_off) { + dev_err(drvdata->dev, + "write_ptr: %lu not aligned to formatter frame size\n", + (unsigned long)write_ptr); + dev_err(drvdata->dev, "frameoff: %lu, frame_endoff: %lu\n", + (unsigned long)frame_off, (unsigned long)frame_endoff); + write_ptr += frame_endoff; + } + + if ((readl_relaxed(drvdata->base + ETB_STATUS_REG) + & ETB_STATUS_RAM_FULL) == 0) + writel_relaxed(0x0, drvdata->base + ETB_RAM_READ_POINTER); + else + writel_relaxed(write_ptr, drvdata->base + ETB_RAM_READ_POINTER); + + depth = drvdata->buffer_depth; + buf_ptr = drvdata->buf; + for (i = 0; i < depth; i++) { + read_data = readl_relaxed(drvdata->base + + ETB_RAM_READ_DATA_REG); + *buf_ptr++ = read_data >> 0; + *buf_ptr++ = read_data >> 8; + *buf_ptr++ = read_data >> 16; + *buf_ptr++ = read_data >> 24; + } + + if (frame_off) { + buf_ptr -= (frame_endoff * 4); + for (i = 0; i < frame_endoff; i++) { + *buf_ptr++ = 0x0; + *buf_ptr++ = 0x0; + *buf_ptr++ = 0x0; + *buf_ptr++ = 0x0; + } + } + + writel_relaxed(read_ptr, drvdata->base + ETB_RAM_READ_POINTER); + + CS_LOCK(drvdata->base); +} + +static void etb_disable(struct coresight_device *csdev) +{ + struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + unsigned long flags; + + spin_lock_irqsave(&drvdata->spinlock, flags); + etb_disable_hw(drvdata); + etb_dump_hw(drvdata); + drvdata->enable = false; + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + clk_disable_unprepare(drvdata->clk); + + dev_info(drvdata->dev, "ETB disabled\n"); +} + +static const struct coresight_ops_sink etb_sink_ops = { + .enable = etb_enable, + .disable = etb_disable, +}; + +static const struct coresight_ops etb_cs_ops = { + .sink_ops = &etb_sink_ops, +}; + +static void etb_dump(struct etb_drvdata *drvdata) +{ + unsigned long flags; + + spin_lock_irqsave(&drvdata->spinlock, flags); + if (drvdata->enable) { + etb_disable_hw(drvdata); + etb_dump_hw(drvdata); + etb_enable_hw(drvdata); + } + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + dev_info(drvdata->dev, "ETB dumped\n"); +} + +static int etb_open(struct inode *inode, struct file *file) +{ + struct etb_drvdata *drvdata = container_of(file->private_data, + struct etb_drvdata, miscdev); + + if (atomic_cmpxchg(&drvdata->in_use, 0, 1)) + return -EBUSY; + + dev_dbg(drvdata->dev, "%s: successfully opened\n", __func__); + return 0; +} + +static ssize_t etb_read(struct file *file, char __user *data, + size_t len, loff_t *ppos) +{ + u32 depth; + struct etb_drvdata *drvdata = container_of(file->private_data, + struct etb_drvdata, miscdev); + + etb_dump(drvdata); + + depth = drvdata->buffer_depth; + if (*ppos + len > depth * 4) + len = depth * 4 - *ppos; + + if (copy_to_user(data, drvdata->buf + *ppos, len)) { + dev_dbg(drvdata->dev, "%s: copy_to_user failed\n", __func__); + return -EFAULT; + } + + *ppos += len; + + dev_dbg(drvdata->dev, "%s: %zu bytes copied, %d bytes left\n", + __func__, len, (int)(depth * 4 - *ppos)); + return len; +} + +static int etb_release(struct inode *inode, struct file *file) +{ + struct etb_drvdata *drvdata = container_of(file->private_data, + struct etb_drvdata, miscdev); + atomic_set(&drvdata->in_use, 0); + + dev_dbg(drvdata->dev, "%s: released\n", __func__); + return 0; +} + +static const struct file_operations etb_fops = { + .owner = THIS_MODULE, + .open = etb_open, + .read = etb_read, + .release = etb_release, + .llseek = no_llseek, +}; + +static ssize_t status_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + unsigned long flags; + u32 etb_rdr, etb_sr, etb_rrp, etb_rwp; + u32 etb_trg, etb_cr, etb_ffsr, etb_ffcr; + struct etb_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + goto out; + + spin_lock_irqsave(&drvdata->spinlock, flags); + CS_UNLOCK(drvdata->base); + + etb_rdr = readl_relaxed(drvdata->base + ETB_RAM_DEPTH_REG); + etb_sr = readl_relaxed(drvdata->base + ETB_STATUS_REG); + etb_rrp = readl_relaxed(drvdata->base + ETB_RAM_READ_POINTER); + etb_rwp = readl_relaxed(drvdata->base + ETB_RAM_WRITE_POINTER); + etb_trg = readl_relaxed(drvdata->base + ETB_TRG); + etb_cr = readl_relaxed(drvdata->base + ETB_CTL_REG); + etb_ffsr = readl_relaxed(drvdata->base + ETB_FFSR); + etb_ffcr = readl_relaxed(drvdata->base + ETB_FFCR); + + CS_LOCK(drvdata->base); + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + clk_disable_unprepare(drvdata->clk); + + return sprintf(buf, + "Depth:\t\t0x%x\n" + "Status:\t\t0x%x\n" + "RAM read ptr:\t0x%x\n" + "RAM wrt ptr:\t0x%x\n" + "Trigger cnt:\t0x%x\n" + "Control:\t0x%x\n" + "Flush status:\t0x%x\n" + "Flush ctrl:\t0x%x\n", + etb_rdr, etb_sr, etb_rrp, etb_rwp, + etb_trg, etb_cr, etb_ffsr, etb_ffcr); +out: + return -EINVAL; +} +static DEVICE_ATTR_RO(status); + +static ssize_t trigger_cntr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct etb_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val = drvdata->trigger_cntr; + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t trigger_cntr_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etb_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->trigger_cntr = val; + return size; +} +static DEVICE_ATTR_RW(trigger_cntr); + +static struct attribute *coresight_etb_attrs[] = { + &dev_attr_trigger_cntr.attr, + &dev_attr_status.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_etb); + +static int etb_probe(struct amba_device *adev, const struct amba_id *id) +{ + int ret; + void __iomem *base; + struct device *dev = &adev->dev; + struct coresight_platform_data *pdata = NULL; + struct etb_drvdata *drvdata; + struct resource *res = &adev->res; + struct coresight_desc *desc; + struct device_node *np = adev->dev.of_node; + + if (np) { + pdata = of_get_coresight_platform_data(dev, np); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + adev->dev.platform_data = pdata; + } + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + drvdata->dev = &adev->dev; + dev_set_drvdata(dev, drvdata); + + /* validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + drvdata->base = base; + + spin_lock_init(&drvdata->spinlock); + + drvdata->clk = adev->pclk; + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + drvdata->buffer_depth = etb_get_buffer_depth(drvdata); + clk_disable_unprepare(drvdata->clk); + + if (drvdata->buffer_depth < 0) + return -EINVAL; + + drvdata->buf = devm_kzalloc(dev, + drvdata->buffer_depth * 4, GFP_KERNEL); + if (!drvdata->buf) + return -ENOMEM; + + desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + + desc->type = CORESIGHT_DEV_TYPE_SINK; + desc->subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER; + desc->ops = &etb_cs_ops; + desc->pdata = pdata; + desc->dev = dev; + desc->groups = coresight_etb_groups; + drvdata->csdev = coresight_register(desc); + if (IS_ERR(drvdata->csdev)) + return PTR_ERR(drvdata->csdev); + + drvdata->miscdev.name = pdata->name; + drvdata->miscdev.minor = MISC_DYNAMIC_MINOR; + drvdata->miscdev.fops = &etb_fops; + ret = misc_register(&drvdata->miscdev); + if (ret) + goto err_misc_register; + + dev_info(dev, "ETB initialized\n"); + return 0; + +err_misc_register: + coresight_unregister(drvdata->csdev); + return ret; +} + +static int etb_remove(struct amba_device *adev) +{ + struct etb_drvdata *drvdata = amba_get_drvdata(adev); + + misc_deregister(&drvdata->miscdev); + coresight_unregister(drvdata->csdev); + return 0; +} + +static struct amba_id etb_ids[] = { + { + .id = 0x0003b907, + .mask = 0x0003ffff, + }, + { 0, 0}, +}; + +static struct amba_driver etb_driver = { + .drv = { + .name = "coresight-etb10", + .owner = THIS_MODULE, + }, + .probe = etb_probe, + .remove = etb_remove, + .id_table = etb_ids, +}; + +module_amba_driver(etb_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("CoreSight Embedded Trace Buffer driver"); diff --git a/drivers/hwtracing/coresight/coresight-etm-cp14.c b/drivers/hwtracing/coresight/coresight-etm-cp14.c new file mode 100644 index 000000000000..12a220682117 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-etm-cp14.c @@ -0,0 +1,591 @@ +/* Copyright (c) 2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/bug.h> +#include <asm/hardware/cp14.h> + +#include "coresight-etm.h" + +int etm_readl_cp14(u32 reg, unsigned int *val) +{ + switch (reg) { + case ETMCR: + *val = etm_read(ETMCR); + return 0; + case ETMCCR: + *val = etm_read(ETMCCR); + return 0; + case ETMTRIGGER: + *val = etm_read(ETMTRIGGER); + return 0; + case ETMSR: + *val = etm_read(ETMSR); + return 0; + case ETMSCR: + *val = etm_read(ETMSCR); + return 0; + case ETMTSSCR: + *val = etm_read(ETMTSSCR); + return 0; + case ETMTEEVR: + *val = etm_read(ETMTEEVR); + return 0; + case ETMTECR1: + *val = etm_read(ETMTECR1); + return 0; + case ETMFFLR: + *val = etm_read(ETMFFLR); + return 0; + case ETMACVRn(0): + *val = etm_read(ETMACVR0); + return 0; + case ETMACVRn(1): + *val = etm_read(ETMACVR1); + return 0; + case ETMACVRn(2): + *val = etm_read(ETMACVR2); + return 0; + case ETMACVRn(3): + *val = etm_read(ETMACVR3); + return 0; + case ETMACVRn(4): + *val = etm_read(ETMACVR4); + return 0; + case ETMACVRn(5): + *val = etm_read(ETMACVR5); + return 0; + case ETMACVRn(6): + *val = etm_read(ETMACVR6); + return 0; + case ETMACVRn(7): + *val = etm_read(ETMACVR7); + return 0; + case ETMACVRn(8): + *val = etm_read(ETMACVR8); + return 0; + case ETMACVRn(9): + *val = etm_read(ETMACVR9); + return 0; + case ETMACVRn(10): + *val = etm_read(ETMACVR10); + return 0; + case ETMACVRn(11): + *val = etm_read(ETMACVR11); + return 0; + case ETMACVRn(12): + *val = etm_read(ETMACVR12); + return 0; + case ETMACVRn(13): + *val = etm_read(ETMACVR13); + return 0; + case ETMACVRn(14): + *val = etm_read(ETMACVR14); + return 0; + case ETMACVRn(15): + *val = etm_read(ETMACVR15); + return 0; + case ETMACTRn(0): + *val = etm_read(ETMACTR0); + return 0; + case ETMACTRn(1): + *val = etm_read(ETMACTR1); + return 0; + case ETMACTRn(2): + *val = etm_read(ETMACTR2); + return 0; + case ETMACTRn(3): + *val = etm_read(ETMACTR3); + return 0; + case ETMACTRn(4): + *val = etm_read(ETMACTR4); + return 0; + case ETMACTRn(5): + *val = etm_read(ETMACTR5); + return 0; + case ETMACTRn(6): + *val = etm_read(ETMACTR6); + return 0; + case ETMACTRn(7): + *val = etm_read(ETMACTR7); + return 0; + case ETMACTRn(8): + *val = etm_read(ETMACTR8); + return 0; + case ETMACTRn(9): + *val = etm_read(ETMACTR9); + return 0; + case ETMACTRn(10): + *val = etm_read(ETMACTR10); + return 0; + case ETMACTRn(11): + *val = etm_read(ETMACTR11); + return 0; + case ETMACTRn(12): + *val = etm_read(ETMACTR12); + return 0; + case ETMACTRn(13): + *val = etm_read(ETMACTR13); + return 0; + case ETMACTRn(14): + *val = etm_read(ETMACTR14); + return 0; + case ETMACTRn(15): + *val = etm_read(ETMACTR15); + return 0; + case ETMCNTRLDVRn(0): + *val = etm_read(ETMCNTRLDVR0); + return 0; + case ETMCNTRLDVRn(1): + *val = etm_read(ETMCNTRLDVR1); + return 0; + case ETMCNTRLDVRn(2): + *val = etm_read(ETMCNTRLDVR2); + return 0; + case ETMCNTRLDVRn(3): + *val = etm_read(ETMCNTRLDVR3); + return 0; + case ETMCNTENRn(0): + *val = etm_read(ETMCNTENR0); + return 0; + case ETMCNTENRn(1): + *val = etm_read(ETMCNTENR1); + return 0; + case ETMCNTENRn(2): + *val = etm_read(ETMCNTENR2); + return 0; + case ETMCNTENRn(3): + *val = etm_read(ETMCNTENR3); + return 0; + case ETMCNTRLDEVRn(0): + *val = etm_read(ETMCNTRLDEVR0); + return 0; + case ETMCNTRLDEVRn(1): + *val = etm_read(ETMCNTRLDEVR1); + return 0; + case ETMCNTRLDEVRn(2): + *val = etm_read(ETMCNTRLDEVR2); + return 0; + case ETMCNTRLDEVRn(3): + *val = etm_read(ETMCNTRLDEVR3); + return 0; + case ETMCNTVRn(0): + *val = etm_read(ETMCNTVR0); + return 0; + case ETMCNTVRn(1): + *val = etm_read(ETMCNTVR1); + return 0; + case ETMCNTVRn(2): + *val = etm_read(ETMCNTVR2); + return 0; + case ETMCNTVRn(3): + *val = etm_read(ETMCNTVR3); + return 0; + case ETMSQ12EVR: + *val = etm_read(ETMSQ12EVR); + return 0; + case ETMSQ21EVR: + *val = etm_read(ETMSQ21EVR); + return 0; + case ETMSQ23EVR: + *val = etm_read(ETMSQ23EVR); + return 0; + case ETMSQ31EVR: + *val = etm_read(ETMSQ31EVR); + return 0; + case ETMSQ32EVR: + *val = etm_read(ETMSQ32EVR); + return 0; + case ETMSQ13EVR: + *val = etm_read(ETMSQ13EVR); + return 0; + case ETMSQR: + *val = etm_read(ETMSQR); + return 0; + case ETMEXTOUTEVRn(0): + *val = etm_read(ETMEXTOUTEVR0); + return 0; + case ETMEXTOUTEVRn(1): + *val = etm_read(ETMEXTOUTEVR1); + return 0; + case ETMEXTOUTEVRn(2): + *val = etm_read(ETMEXTOUTEVR2); + return 0; + case ETMEXTOUTEVRn(3): + *val = etm_read(ETMEXTOUTEVR3); + return 0; + case ETMCIDCVRn(0): + *val = etm_read(ETMCIDCVR0); + return 0; + case ETMCIDCVRn(1): + *val = etm_read(ETMCIDCVR1); + return 0; + case ETMCIDCVRn(2): + *val = etm_read(ETMCIDCVR2); + return 0; + case ETMCIDCMR: + *val = etm_read(ETMCIDCMR); + return 0; + case ETMIMPSPEC0: + *val = etm_read(ETMIMPSPEC0); + return 0; + case ETMIMPSPEC1: + *val = etm_read(ETMIMPSPEC1); + return 0; + case ETMIMPSPEC2: + *val = etm_read(ETMIMPSPEC2); + return 0; + case ETMIMPSPEC3: + *val = etm_read(ETMIMPSPEC3); + return 0; + case ETMIMPSPEC4: + *val = etm_read(ETMIMPSPEC4); + return 0; + case ETMIMPSPEC5: + *val = etm_read(ETMIMPSPEC5); + return 0; + case ETMIMPSPEC6: + *val = etm_read(ETMIMPSPEC6); + return 0; + case ETMIMPSPEC7: + *val = etm_read(ETMIMPSPEC7); + return 0; + case ETMSYNCFR: + *val = etm_read(ETMSYNCFR); + return 0; + case ETMIDR: + *val = etm_read(ETMIDR); + return 0; + case ETMCCER: + *val = etm_read(ETMCCER); + return 0; + case ETMEXTINSELR: + *val = etm_read(ETMEXTINSELR); + return 0; + case ETMTESSEICR: + *val = etm_read(ETMTESSEICR); + return 0; + case ETMEIBCR: + *val = etm_read(ETMEIBCR); + return 0; + case ETMTSEVR: + *val = etm_read(ETMTSEVR); + return 0; + case ETMAUXCR: + *val = etm_read(ETMAUXCR); + return 0; + case ETMTRACEIDR: + *val = etm_read(ETMTRACEIDR); + return 0; + case ETMVMIDCVR: + *val = etm_read(ETMVMIDCVR); + return 0; + case ETMOSLSR: + *val = etm_read(ETMOSLSR); + return 0; + case ETMOSSRR: + *val = etm_read(ETMOSSRR); + return 0; + case ETMPDCR: + *val = etm_read(ETMPDCR); + return 0; + case ETMPDSR: + *val = etm_read(ETMPDSR); + return 0; + default: + *val = 0; + return -EINVAL; + } +} + +int etm_writel_cp14(u32 reg, u32 val) +{ + switch (reg) { + case ETMCR: + etm_write(val, ETMCR); + break; + case ETMTRIGGER: + etm_write(val, ETMTRIGGER); + break; + case ETMSR: + etm_write(val, ETMSR); + break; + case ETMTSSCR: + etm_write(val, ETMTSSCR); + break; + case ETMTEEVR: + etm_write(val, ETMTEEVR); + break; + case ETMTECR1: + etm_write(val, ETMTECR1); + break; + case ETMFFLR: + etm_write(val, ETMFFLR); + break; + case ETMACVRn(0): + etm_write(val, ETMACVR0); + break; + case ETMACVRn(1): + etm_write(val, ETMACVR1); + break; + case ETMACVRn(2): + etm_write(val, ETMACVR2); + break; + case ETMACVRn(3): + etm_write(val, ETMACVR3); + break; + case ETMACVRn(4): + etm_write(val, ETMACVR4); + break; + case ETMACVRn(5): + etm_write(val, ETMACVR5); + break; + case ETMACVRn(6): + etm_write(val, ETMACVR6); + break; + case ETMACVRn(7): + etm_write(val, ETMACVR7); + break; + case ETMACVRn(8): + etm_write(val, ETMACVR8); + break; + case ETMACVRn(9): + etm_write(val, ETMACVR9); + break; + case ETMACVRn(10): + etm_write(val, ETMACVR10); + break; + case ETMACVRn(11): + etm_write(val, ETMACVR11); + break; + case ETMACVRn(12): + etm_write(val, ETMACVR12); + break; + case ETMACVRn(13): + etm_write(val, ETMACVR13); + break; + case ETMACVRn(14): + etm_write(val, ETMACVR14); + break; + case ETMACVRn(15): + etm_write(val, ETMACVR15); + break; + case ETMACTRn(0): + etm_write(val, ETMACTR0); + break; + case ETMACTRn(1): + etm_write(val, ETMACTR1); + break; + case ETMACTRn(2): + etm_write(val, ETMACTR2); + break; + case ETMACTRn(3): + etm_write(val, ETMACTR3); + break; + case ETMACTRn(4): + etm_write(val, ETMACTR4); + break; + case ETMACTRn(5): + etm_write(val, ETMACTR5); + break; + case ETMACTRn(6): + etm_write(val, ETMACTR6); + break; + case ETMACTRn(7): + etm_write(val, ETMACTR7); + break; + case ETMACTRn(8): + etm_write(val, ETMACTR8); + break; + case ETMACTRn(9): + etm_write(val, ETMACTR9); + break; + case ETMACTRn(10): + etm_write(val, ETMACTR10); + break; + case ETMACTRn(11): + etm_write(val, ETMACTR11); + break; + case ETMACTRn(12): + etm_write(val, ETMACTR12); + break; + case ETMACTRn(13): + etm_write(val, ETMACTR13); + break; + case ETMACTRn(14): + etm_write(val, ETMACTR14); + break; + case ETMACTRn(15): + etm_write(val, ETMACTR15); + break; + case ETMCNTRLDVRn(0): + etm_write(val, ETMCNTRLDVR0); + break; + case ETMCNTRLDVRn(1): + etm_write(val, ETMCNTRLDVR1); + break; + case ETMCNTRLDVRn(2): + etm_write(val, ETMCNTRLDVR2); + break; + case ETMCNTRLDVRn(3): + etm_write(val, ETMCNTRLDVR3); + break; + case ETMCNTENRn(0): + etm_write(val, ETMCNTENR0); + break; + case ETMCNTENRn(1): + etm_write(val, ETMCNTENR1); + break; + case ETMCNTENRn(2): + etm_write(val, ETMCNTENR2); + break; + case ETMCNTENRn(3): + etm_write(val, ETMCNTENR3); + break; + case ETMCNTRLDEVRn(0): + etm_write(val, ETMCNTRLDEVR0); + break; + case ETMCNTRLDEVRn(1): + etm_write(val, ETMCNTRLDEVR1); + break; + case ETMCNTRLDEVRn(2): + etm_write(val, ETMCNTRLDEVR2); + break; + case ETMCNTRLDEVRn(3): + etm_write(val, ETMCNTRLDEVR3); + break; + case ETMCNTVRn(0): + etm_write(val, ETMCNTVR0); + break; + case ETMCNTVRn(1): + etm_write(val, ETMCNTVR1); + break; + case ETMCNTVRn(2): + etm_write(val, ETMCNTVR2); + break; + case ETMCNTVRn(3): + etm_write(val, ETMCNTVR3); + break; + case ETMSQ12EVR: + etm_write(val, ETMSQ12EVR); + break; + case ETMSQ21EVR: + etm_write(val, ETMSQ21EVR); + break; + case ETMSQ23EVR: + etm_write(val, ETMSQ23EVR); + break; + case ETMSQ31EVR: + etm_write(val, ETMSQ31EVR); + break; + case ETMSQ32EVR: + etm_write(val, ETMSQ32EVR); + break; + case ETMSQ13EVR: + etm_write(val, ETMSQ13EVR); + break; + case ETMSQR: + etm_write(val, ETMSQR); + break; + case ETMEXTOUTEVRn(0): + etm_write(val, ETMEXTOUTEVR0); + break; + case ETMEXTOUTEVRn(1): + etm_write(val, ETMEXTOUTEVR1); + break; + case ETMEXTOUTEVRn(2): + etm_write(val, ETMEXTOUTEVR2); + break; + case ETMEXTOUTEVRn(3): + etm_write(val, ETMEXTOUTEVR3); + break; + case ETMCIDCVRn(0): + etm_write(val, ETMCIDCVR0); + break; + case ETMCIDCVRn(1): + etm_write(val, ETMCIDCVR1); + break; + case ETMCIDCVRn(2): + etm_write(val, ETMCIDCVR2); + break; + case ETMCIDCMR: + etm_write(val, ETMCIDCMR); + break; + case ETMIMPSPEC0: + etm_write(val, ETMIMPSPEC0); + break; + case ETMIMPSPEC1: + etm_write(val, ETMIMPSPEC1); + break; + case ETMIMPSPEC2: + etm_write(val, ETMIMPSPEC2); + break; + case ETMIMPSPEC3: + etm_write(val, ETMIMPSPEC3); + break; + case ETMIMPSPEC4: + etm_write(val, ETMIMPSPEC4); + break; + case ETMIMPSPEC5: + etm_write(val, ETMIMPSPEC5); + break; + case ETMIMPSPEC6: + etm_write(val, ETMIMPSPEC6); + break; + case ETMIMPSPEC7: + etm_write(val, ETMIMPSPEC7); + break; + case ETMSYNCFR: + etm_write(val, ETMSYNCFR); + break; + case ETMEXTINSELR: + etm_write(val, ETMEXTINSELR); + break; + case ETMTESSEICR: + etm_write(val, ETMTESSEICR); + break; + case ETMEIBCR: + etm_write(val, ETMEIBCR); + break; + case ETMTSEVR: + etm_write(val, ETMTSEVR); + break; + case ETMAUXCR: + etm_write(val, ETMAUXCR); + break; + case ETMTRACEIDR: + etm_write(val, ETMTRACEIDR); + break; + case ETMVMIDCVR: + etm_write(val, ETMVMIDCVR); + break; + case ETMOSLAR: + etm_write(val, ETMOSLAR); + break; + case ETMOSSRR: + etm_write(val, ETMOSSRR); + break; + case ETMPDCR: + etm_write(val, ETMPDCR); + break; + case ETMPDSR: + etm_write(val, ETMPDSR); + break; + default: + return -EINVAL; + } + + return 0; +} diff --git a/drivers/hwtracing/coresight/coresight-etm.h b/drivers/hwtracing/coresight/coresight-etm.h new file mode 100644 index 000000000000..501c5fac8a45 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-etm.h @@ -0,0 +1,251 @@ +/* Copyright (c) 2014-2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CORESIGHT_CORESIGHT_ETM_H +#define _CORESIGHT_CORESIGHT_ETM_H + +#include <linux/spinlock.h> +#include "coresight-priv.h" + +/* + * Device registers: + * 0x000 - 0x2FC: Trace registers + * 0x300 - 0x314: Management registers + * 0x318 - 0xEFC: Trace registers + * + * Coresight registers + * 0xF00 - 0xF9C: Management registers + * 0xFA0 - 0xFA4: Management registers in PFTv1.0 + * Trace registers in PFTv1.1 + * 0xFA8 - 0xFFC: Management registers + */ + +/* Trace registers (0x000-0x2FC) */ +#define ETMCR 0x000 +#define ETMCCR 0x004 +#define ETMTRIGGER 0x008 +#define ETMSR 0x010 +#define ETMSCR 0x014 +#define ETMTSSCR 0x018 +#define ETMTECR2 0x01c +#define ETMTEEVR 0x020 +#define ETMTECR1 0x024 +#define ETMFFLR 0x02c +#define ETMACVRn(n) (0x040 + (n * 4)) +#define ETMACTRn(n) (0x080 + (n * 4)) +#define ETMCNTRLDVRn(n) (0x140 + (n * 4)) +#define ETMCNTENRn(n) (0x150 + (n * 4)) +#define ETMCNTRLDEVRn(n) (0x160 + (n * 4)) +#define ETMCNTVRn(n) (0x170 + (n * 4)) +#define ETMSQ12EVR 0x180 +#define ETMSQ21EVR 0x184 +#define ETMSQ23EVR 0x188 +#define ETMSQ31EVR 0x18c +#define ETMSQ32EVR 0x190 +#define ETMSQ13EVR 0x194 +#define ETMSQR 0x19c +#define ETMEXTOUTEVRn(n) (0x1a0 + (n * 4)) +#define ETMCIDCVRn(n) (0x1b0 + (n * 4)) +#define ETMCIDCMR 0x1bc +#define ETMIMPSPEC0 0x1c0 +#define ETMIMPSPEC1 0x1c4 +#define ETMIMPSPEC2 0x1c8 +#define ETMIMPSPEC3 0x1cc +#define ETMIMPSPEC4 0x1d0 +#define ETMIMPSPEC5 0x1d4 +#define ETMIMPSPEC6 0x1d8 +#define ETMIMPSPEC7 0x1dc +#define ETMSYNCFR 0x1e0 +#define ETMIDR 0x1e4 +#define ETMCCER 0x1e8 +#define ETMEXTINSELR 0x1ec +#define ETMTESSEICR 0x1f0 +#define ETMEIBCR 0x1f4 +#define ETMTSEVR 0x1f8 +#define ETMAUXCR 0x1fc +#define ETMTRACEIDR 0x200 +#define ETMVMIDCVR 0x240 +/* Management registers (0x300-0x314) */ +#define ETMOSLAR 0x300 +#define ETMOSLSR 0x304 +#define ETMOSSRR 0x308 +#define ETMPDCR 0x310 +#define ETMPDSR 0x314 +#define ETM_MAX_ADDR_CMP 16 +#define ETM_MAX_CNTR 4 +#define ETM_MAX_CTXID_CMP 3 + +/* Register definition */ +/* ETMCR - 0x00 */ +#define ETMCR_PWD_DWN BIT(0) +#define ETMCR_STALL_MODE BIT(7) +#define ETMCR_ETM_PRG BIT(10) +#define ETMCR_ETM_EN BIT(11) +#define ETMCR_CYC_ACC BIT(12) +#define ETMCR_CTXID_SIZE (BIT(14)|BIT(15)) +#define ETMCR_TIMESTAMP_EN BIT(28) +/* ETMCCR - 0x04 */ +#define ETMCCR_FIFOFULL BIT(23) +/* ETMPDCR - 0x310 */ +#define ETMPDCR_PWD_UP BIT(3) +/* ETMTECR1 - 0x024 */ +#define ETMTECR1_ADDR_COMP_1 BIT(0) +#define ETMTECR1_INC_EXC BIT(24) +#define ETMTECR1_START_STOP BIT(25) +/* ETMCCER - 0x1E8 */ +#define ETMCCER_TIMESTAMP BIT(22) + +#define ETM_MODE_EXCLUDE BIT(0) +#define ETM_MODE_CYCACC BIT(1) +#define ETM_MODE_STALL BIT(2) +#define ETM_MODE_TIMESTAMP BIT(3) +#define ETM_MODE_CTXID BIT(4) +#define ETM_MODE_ALL 0x1f + +#define ETM_SQR_MASK 0x3 +#define ETM_TRACEID_MASK 0x3f +#define ETM_EVENT_MASK 0x1ffff +#define ETM_SYNC_MASK 0xfff +#define ETM_ALL_MASK 0xffffffff + +#define ETMSR_PROG_BIT 1 +#define ETM_SEQ_STATE_MAX_VAL (0x2) +#define PORT_SIZE_MASK (GENMASK(21, 21) | GENMASK(6, 4)) + +#define ETM_HARD_WIRE_RES_A /* Hard wired, always true */ \ + ((0x0f << 0) | \ + /* Resource index A */ \ + (0x06 << 4)) + +#define ETM_ADD_COMP_0 /* Single addr comparator 1 */ \ + ((0x00 << 7) | \ + /* Resource index B */ \ + (0x00 << 11)) + +#define ETM_EVENT_NOT_A BIT(14) /* NOT(A) */ + +#define ETM_DEFAULT_EVENT_VAL (ETM_HARD_WIRE_RES_A | \ + ETM_ADD_COMP_0 | \ + ETM_EVENT_NOT_A) +/** + * struct etm_drvdata - specifics associated to an ETM component + * @base: memory mapped base address for this component. + * @dev: the device entity associated to this component. + * @csdev: component vitals needed by the framework. + * @clk: the clock this component is associated to. + * @spinlock: only one at a time pls. + * @cpu: the cpu this component is affined to. + * @port_size: port size as reported by ETMCR bit 4-6 and 21. + * @arch: ETM/PTM version number. + * @use_cpu14: true if management registers need to be accessed via CP14. + * @enable: is this ETM/PTM currently tracing. + * @sticky_enable: true if ETM base configuration has been done. + * @boot_enable:true if we should start tracing at boot time. + * @os_unlock: true if access to management registers is allowed. + * @nr_addr_cmp:Number of pairs of address comparators as found in ETMCCR. + * @nr_cntr: Number of counters as found in ETMCCR bit 13-15. + * @nr_ext_inp: Number of external input as found in ETMCCR bit 17-19. + * @nr_ext_out: Number of external output as found in ETMCCR bit 20-22. + * @nr_ctxid_cmp: Number of contextID comparators as found in ETMCCR bit 24-25. + * @etmccr: value of register ETMCCR. + * @etmccer: value of register ETMCCER. + * @traceid: value of the current ID for this component. + * @mode: controls various modes supported by this ETM/PTM. + * @ctrl: used in conjunction with @mode. + * @trigger_event: setting for register ETMTRIGGER. + * @startstop_ctrl: setting for register ETMTSSCR. + * @enable_event: setting for register ETMTEEVR. + * @enable_ctrl1: setting for register ETMTECR1. + * @fifofull_level: setting for register ETMFFLR. + * @addr_idx: index for the address comparator selection. + * @addr_val: value for address comparator register. + * @addr_acctype: access type for address comparator register. + * @addr_type: current status of the comparator register. + * @cntr_idx: index for the counter register selection. + * @cntr_rld_val: reload value of a counter register. + * @cntr_event: control for counter enable register. + * @cntr_rld_event: value for counter reload event register. + * @cntr_val: counter value register. + * @seq_12_event: event causing the transition from 1 to 2. + * @seq_21_event: event causing the transition from 2 to 1. + * @seq_23_event: event causing the transition from 2 to 3. + * @seq_31_event: event causing the transition from 3 to 1. + * @seq_32_event: event causing the transition from 3 to 2. + * @seq_13_event: event causing the transition from 1 to 3. + * @seq_curr_state: current value of the sequencer register. + * @ctxid_idx: index for the context ID registers. + * @ctxid_val: value for the context ID to trigger on. + * @ctxid_mask: mask applicable to all the context IDs. + * @sync_freq: Synchronisation frequency. + * @timestamp_event: Defines an event that requests the insertion + of a timestamp into the trace stream. + */ +struct etm_drvdata { + void __iomem *base; + struct device *dev; + struct coresight_device *csdev; + struct clk *clk; + spinlock_t spinlock; + int cpu; + int port_size; + u8 arch; + bool use_cp14; + bool enable; + bool sticky_enable; + bool boot_enable; + bool os_unlock; + u8 nr_addr_cmp; + u8 nr_cntr; + u8 nr_ext_inp; + u8 nr_ext_out; + u8 nr_ctxid_cmp; + u32 etmccr; + u32 etmccer; + u32 traceid; + u32 mode; + u32 ctrl; + u32 trigger_event; + u32 startstop_ctrl; + u32 enable_event; + u32 enable_ctrl1; + u32 fifofull_level; + u8 addr_idx; + u32 addr_val[ETM_MAX_ADDR_CMP]; + u32 addr_acctype[ETM_MAX_ADDR_CMP]; + u32 addr_type[ETM_MAX_ADDR_CMP]; + u8 cntr_idx; + u32 cntr_rld_val[ETM_MAX_CNTR]; + u32 cntr_event[ETM_MAX_CNTR]; + u32 cntr_rld_event[ETM_MAX_CNTR]; + u32 cntr_val[ETM_MAX_CNTR]; + u32 seq_12_event; + u32 seq_21_event; + u32 seq_23_event; + u32 seq_31_event; + u32 seq_32_event; + u32 seq_13_event; + u32 seq_curr_state; + u8 ctxid_idx; + u32 ctxid_val[ETM_MAX_CTXID_CMP]; + u32 ctxid_mask; + u32 sync_freq; + u32 timestamp_event; +}; + +enum etm_addr_type { + ETM_ADDR_TYPE_NONE, + ETM_ADDR_TYPE_SINGLE, + ETM_ADDR_TYPE_RANGE, + ETM_ADDR_TYPE_START, + ETM_ADDR_TYPE_STOP, +}; +#endif diff --git a/drivers/hwtracing/coresight/coresight-etm3x.c b/drivers/hwtracing/coresight/coresight-etm3x.c new file mode 100644 index 000000000000..c965f5724abd --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-etm3x.c @@ -0,0 +1,1932 @@ +/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/io.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/delay.h> +#include <linux/smp.h> +#include <linux/sysfs.h> +#include <linux/stat.h> +#include <linux/clk.h> +#include <linux/cpu.h> +#include <linux/of.h> +#include <linux/coresight.h> +#include <linux/amba/bus.h> +#include <linux/seq_file.h> +#include <linux/uaccess.h> +#include <asm/sections.h> + +#include "coresight-etm.h" + +static int boot_enable; +module_param_named(boot_enable, boot_enable, int, S_IRUGO); + +/* The number of ETM/PTM currently registered */ +static int etm_count; +static struct etm_drvdata *etmdrvdata[NR_CPUS]; + +static inline void etm_writel(struct etm_drvdata *drvdata, + u32 val, u32 off) +{ + if (drvdata->use_cp14) { + if (etm_writel_cp14(off, val)) { + dev_err(drvdata->dev, + "invalid CP14 access to ETM reg: %#x", off); + } + } else { + writel_relaxed(val, drvdata->base + off); + } +} + +static inline unsigned int etm_readl(struct etm_drvdata *drvdata, u32 off) +{ + u32 val; + + if (drvdata->use_cp14) { + if (etm_readl_cp14(off, &val)) { + dev_err(drvdata->dev, + "invalid CP14 access to ETM reg: %#x", off); + } + } else { + val = readl_relaxed(drvdata->base + off); + } + + return val; +} + +/* + * Memory mapped writes to clear os lock are not supported on some processors + * and OS lock must be unlocked before any memory mapped access on such + * processors, otherwise memory mapped reads/writes will be invalid. + */ +static void etm_os_unlock(void *info) +{ + struct etm_drvdata *drvdata = (struct etm_drvdata *)info; + /* Writing any value to ETMOSLAR unlocks the trace registers */ + etm_writel(drvdata, 0x0, ETMOSLAR); + isb(); +} + +static void etm_set_pwrdwn(struct etm_drvdata *drvdata) +{ + u32 etmcr; + + /* Ensure pending cp14 accesses complete before setting pwrdwn */ + mb(); + isb(); + etmcr = etm_readl(drvdata, ETMCR); + etmcr |= ETMCR_PWD_DWN; + etm_writel(drvdata, etmcr, ETMCR); +} + +static void etm_clr_pwrdwn(struct etm_drvdata *drvdata) +{ + u32 etmcr; + + etmcr = etm_readl(drvdata, ETMCR); + etmcr &= ~ETMCR_PWD_DWN; + etm_writel(drvdata, etmcr, ETMCR); + /* Ensure pwrup completes before subsequent cp14 accesses */ + mb(); + isb(); +} + +static void etm_set_pwrup(struct etm_drvdata *drvdata) +{ + u32 etmpdcr; + + etmpdcr = readl_relaxed(drvdata->base + ETMPDCR); + etmpdcr |= ETMPDCR_PWD_UP; + writel_relaxed(etmpdcr, drvdata->base + ETMPDCR); + /* Ensure pwrup completes before subsequent cp14 accesses */ + mb(); + isb(); +} + +static void etm_clr_pwrup(struct etm_drvdata *drvdata) +{ + u32 etmpdcr; + + /* Ensure pending cp14 accesses complete before clearing pwrup */ + mb(); + isb(); + etmpdcr = readl_relaxed(drvdata->base + ETMPDCR); + etmpdcr &= ~ETMPDCR_PWD_UP; + writel_relaxed(etmpdcr, drvdata->base + ETMPDCR); +} + +/** + * coresight_timeout_etm - loop until a bit has changed to a specific state. + * @drvdata: etm's private data structure. + * @offset: address of a register, starting from @addr. + * @position: the position of the bit of interest. + * @value: the value the bit should have. + * + * Basically the same as @coresight_timeout except for the register access + * method where we have to account for CP14 configurations. + + * Return: 0 as soon as the bit has taken the desired state or -EAGAIN if + * TIMEOUT_US has elapsed, which ever happens first. + */ + +static int coresight_timeout_etm(struct etm_drvdata *drvdata, u32 offset, + int position, int value) +{ + int i; + u32 val; + + for (i = TIMEOUT_US; i > 0; i--) { + val = etm_readl(drvdata, offset); + /* Waiting on the bit to go from 0 to 1 */ + if (value) { + if (val & BIT(position)) + return 0; + /* Waiting on the bit to go from 1 to 0 */ + } else { + if (!(val & BIT(position))) + return 0; + } + + /* + * Delay is arbitrary - the specification doesn't say how long + * we are expected to wait. Extra check required to make sure + * we don't wait needlessly on the last iteration. + */ + if (i - 1) + udelay(1); + } + + return -EAGAIN; +} + + +static void etm_set_prog(struct etm_drvdata *drvdata) +{ + u32 etmcr; + + etmcr = etm_readl(drvdata, ETMCR); + etmcr |= ETMCR_ETM_PRG; + etm_writel(drvdata, etmcr, ETMCR); + /* + * Recommended by spec for cp14 accesses to ensure etmcr write is + * complete before polling etmsr + */ + isb(); + if (coresight_timeout_etm(drvdata, ETMSR, ETMSR_PROG_BIT, 1)) { + dev_err(drvdata->dev, + "timeout observed when probing at offset %#x\n", ETMSR); + } +} + +static void etm_clr_prog(struct etm_drvdata *drvdata) +{ + u32 etmcr; + + etmcr = etm_readl(drvdata, ETMCR); + etmcr &= ~ETMCR_ETM_PRG; + etm_writel(drvdata, etmcr, ETMCR); + /* + * Recommended by spec for cp14 accesses to ensure etmcr write is + * complete before polling etmsr + */ + isb(); + if (coresight_timeout_etm(drvdata, ETMSR, ETMSR_PROG_BIT, 0)) { + dev_err(drvdata->dev, + "timeout observed when probing at offset %#x\n", ETMSR); + } +} + +static void etm_set_default(struct etm_drvdata *drvdata) +{ + int i; + + drvdata->trigger_event = ETM_DEFAULT_EVENT_VAL; + drvdata->enable_event = ETM_HARD_WIRE_RES_A; + + drvdata->seq_12_event = ETM_DEFAULT_EVENT_VAL; + drvdata->seq_21_event = ETM_DEFAULT_EVENT_VAL; + drvdata->seq_23_event = ETM_DEFAULT_EVENT_VAL; + drvdata->seq_31_event = ETM_DEFAULT_EVENT_VAL; + drvdata->seq_32_event = ETM_DEFAULT_EVENT_VAL; + drvdata->seq_13_event = ETM_DEFAULT_EVENT_VAL; + drvdata->timestamp_event = ETM_DEFAULT_EVENT_VAL; + + for (i = 0; i < drvdata->nr_cntr; i++) { + drvdata->cntr_rld_val[i] = 0x0; + drvdata->cntr_event[i] = ETM_DEFAULT_EVENT_VAL; + drvdata->cntr_rld_event[i] = ETM_DEFAULT_EVENT_VAL; + drvdata->cntr_val[i] = 0x0; + } + + drvdata->seq_curr_state = 0x0; + drvdata->ctxid_idx = 0x0; + for (i = 0; i < drvdata->nr_ctxid_cmp; i++) + drvdata->ctxid_val[i] = 0x0; + drvdata->ctxid_mask = 0x0; +} + +static void etm_enable_hw(void *info) +{ + int i; + u32 etmcr; + struct etm_drvdata *drvdata = info; + + CS_UNLOCK(drvdata->base); + + /* Turn engine on */ + etm_clr_pwrdwn(drvdata); + /* Apply power to trace registers */ + etm_set_pwrup(drvdata); + /* Make sure all registers are accessible */ + etm_os_unlock(drvdata); + + etm_set_prog(drvdata); + + etmcr = etm_readl(drvdata, ETMCR); + etmcr &= (ETMCR_PWD_DWN | ETMCR_ETM_PRG); + etmcr |= drvdata->port_size; + etm_writel(drvdata, drvdata->ctrl | etmcr, ETMCR); + etm_writel(drvdata, drvdata->trigger_event, ETMTRIGGER); + etm_writel(drvdata, drvdata->startstop_ctrl, ETMTSSCR); + etm_writel(drvdata, drvdata->enable_event, ETMTEEVR); + etm_writel(drvdata, drvdata->enable_ctrl1, ETMTECR1); + etm_writel(drvdata, drvdata->fifofull_level, ETMFFLR); + for (i = 0; i < drvdata->nr_addr_cmp; i++) { + etm_writel(drvdata, drvdata->addr_val[i], ETMACVRn(i)); + etm_writel(drvdata, drvdata->addr_acctype[i], ETMACTRn(i)); + } + for (i = 0; i < drvdata->nr_cntr; i++) { + etm_writel(drvdata, drvdata->cntr_rld_val[i], ETMCNTRLDVRn(i)); + etm_writel(drvdata, drvdata->cntr_event[i], ETMCNTENRn(i)); + etm_writel(drvdata, drvdata->cntr_rld_event[i], + ETMCNTRLDEVRn(i)); + etm_writel(drvdata, drvdata->cntr_val[i], ETMCNTVRn(i)); + } + etm_writel(drvdata, drvdata->seq_12_event, ETMSQ12EVR); + etm_writel(drvdata, drvdata->seq_21_event, ETMSQ21EVR); + etm_writel(drvdata, drvdata->seq_23_event, ETMSQ23EVR); + etm_writel(drvdata, drvdata->seq_31_event, ETMSQ31EVR); + etm_writel(drvdata, drvdata->seq_32_event, ETMSQ32EVR); + etm_writel(drvdata, drvdata->seq_13_event, ETMSQ13EVR); + etm_writel(drvdata, drvdata->seq_curr_state, ETMSQR); + for (i = 0; i < drvdata->nr_ext_out; i++) + etm_writel(drvdata, ETM_DEFAULT_EVENT_VAL, ETMEXTOUTEVRn(i)); + for (i = 0; i < drvdata->nr_ctxid_cmp; i++) + etm_writel(drvdata, drvdata->ctxid_val[i], ETMCIDCVRn(i)); + etm_writel(drvdata, drvdata->ctxid_mask, ETMCIDCMR); + etm_writel(drvdata, drvdata->sync_freq, ETMSYNCFR); + /* No external input selected */ + etm_writel(drvdata, 0x0, ETMEXTINSELR); + etm_writel(drvdata, drvdata->timestamp_event, ETMTSEVR); + /* No auxiliary control selected */ + etm_writel(drvdata, 0x0, ETMAUXCR); + etm_writel(drvdata, drvdata->traceid, ETMTRACEIDR); + /* No VMID comparator value selected */ + etm_writel(drvdata, 0x0, ETMVMIDCVR); + + /* Ensures trace output is enabled from this ETM */ + etm_writel(drvdata, drvdata->ctrl | ETMCR_ETM_EN | etmcr, ETMCR); + + etm_clr_prog(drvdata); + CS_LOCK(drvdata->base); + + dev_dbg(drvdata->dev, "cpu: %d enable smp call done\n", drvdata->cpu); +} + +static int etm_trace_id_simple(struct etm_drvdata *drvdata) +{ + if (!drvdata->enable) + return drvdata->traceid; + + return (etm_readl(drvdata, ETMTRACEIDR) & ETM_TRACEID_MASK); +} + +static int etm_trace_id(struct coresight_device *csdev) +{ + struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + unsigned long flags; + int trace_id = -1; + + if (!drvdata->enable) + return drvdata->traceid; + + if (clk_prepare_enable(drvdata->clk)) + goto out; + + spin_lock_irqsave(&drvdata->spinlock, flags); + + CS_UNLOCK(drvdata->base); + trace_id = (etm_readl(drvdata, ETMTRACEIDR) & ETM_TRACEID_MASK); + CS_LOCK(drvdata->base); + + spin_unlock_irqrestore(&drvdata->spinlock, flags); + clk_disable_unprepare(drvdata->clk); +out: + return trace_id; +} + +static int etm_enable(struct coresight_device *csdev) +{ + struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + int ret; + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + goto err_clk; + + spin_lock(&drvdata->spinlock); + + /* + * Configure the ETM only if the CPU is online. If it isn't online + * hw configuration will take place when 'CPU_STARTING' is received + * in @etm_cpu_callback. + */ + if (cpu_online(drvdata->cpu)) { + ret = smp_call_function_single(drvdata->cpu, + etm_enable_hw, drvdata, 1); + if (ret) + goto err; + } + + drvdata->enable = true; + drvdata->sticky_enable = true; + + spin_unlock(&drvdata->spinlock); + + dev_info(drvdata->dev, "ETM tracing enabled\n"); + return 0; +err: + spin_unlock(&drvdata->spinlock); + clk_disable_unprepare(drvdata->clk); +err_clk: + return ret; +} + +static void etm_disable_hw(void *info) +{ + int i; + struct etm_drvdata *drvdata = info; + + CS_UNLOCK(drvdata->base); + etm_set_prog(drvdata); + + /* Program trace enable to low by using always false event */ + etm_writel(drvdata, ETM_HARD_WIRE_RES_A | ETM_EVENT_NOT_A, ETMTEEVR); + + /* Read back sequencer and counters for post trace analysis */ + drvdata->seq_curr_state = (etm_readl(drvdata, ETMSQR) & ETM_SQR_MASK); + + for (i = 0; i < drvdata->nr_cntr; i++) + drvdata->cntr_val[i] = etm_readl(drvdata, ETMCNTVRn(i)); + + etm_set_pwrdwn(drvdata); + CS_LOCK(drvdata->base); + + dev_dbg(drvdata->dev, "cpu: %d disable smp call done\n", drvdata->cpu); +} + +static void etm_disable(struct coresight_device *csdev) +{ + struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + /* + * Taking hotplug lock here protects from clocks getting disabled + * with tracing being left on (crash scenario) if user disable occurs + * after cpu online mask indicates the cpu is offline but before the + * DYING hotplug callback is serviced by the ETM driver. + */ + get_online_cpus(); + spin_lock(&drvdata->spinlock); + + /* + * Executing etm_disable_hw on the cpu whose ETM is being disabled + * ensures that register writes occur when cpu is powered. + */ + smp_call_function_single(drvdata->cpu, etm_disable_hw, drvdata, 1); + drvdata->enable = false; + + spin_unlock(&drvdata->spinlock); + put_online_cpus(); + + clk_disable_unprepare(drvdata->clk); + + dev_info(drvdata->dev, "ETM tracing disabled\n"); +} + +static const struct coresight_ops_source etm_source_ops = { + .trace_id = etm_trace_id, + .enable = etm_enable, + .disable = etm_disable, +}; + +static const struct coresight_ops etm_cs_ops = { + .source_ops = &etm_source_ops, +}; + +static ssize_t nr_addr_cmp_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->nr_addr_cmp; + return sprintf(buf, "%#lx\n", val); +} +static DEVICE_ATTR_RO(nr_addr_cmp); + +static ssize_t nr_cntr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->nr_cntr; + return sprintf(buf, "%#lx\n", val); +} +static DEVICE_ATTR_RO(nr_cntr); + +static ssize_t nr_ctxid_cmp_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->nr_ctxid_cmp; + return sprintf(buf, "%#lx\n", val); +} +static DEVICE_ATTR_RO(nr_ctxid_cmp); + +static ssize_t etmsr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + unsigned long flags, val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + spin_lock_irqsave(&drvdata->spinlock, flags); + CS_UNLOCK(drvdata->base); + + val = etm_readl(drvdata, ETMSR); + + CS_LOCK(drvdata->base); + spin_unlock_irqrestore(&drvdata->spinlock, flags); + clk_disable_unprepare(drvdata->clk); + + return sprintf(buf, "%#lx\n", val); +} +static DEVICE_ATTR_RO(etmsr); + +static ssize_t reset_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int i, ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + if (val) { + spin_lock(&drvdata->spinlock); + drvdata->mode = ETM_MODE_EXCLUDE; + drvdata->ctrl = 0x0; + drvdata->trigger_event = ETM_DEFAULT_EVENT_VAL; + drvdata->startstop_ctrl = 0x0; + drvdata->addr_idx = 0x0; + for (i = 0; i < drvdata->nr_addr_cmp; i++) { + drvdata->addr_val[i] = 0x0; + drvdata->addr_acctype[i] = 0x0; + drvdata->addr_type[i] = ETM_ADDR_TYPE_NONE; + } + drvdata->cntr_idx = 0x0; + + etm_set_default(drvdata); + spin_unlock(&drvdata->spinlock); + } + + return size; +} +static DEVICE_ATTR_WO(reset); + +static ssize_t mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->mode; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + drvdata->mode = val & ETM_MODE_ALL; + + if (drvdata->mode & ETM_MODE_EXCLUDE) + drvdata->enable_ctrl1 |= ETMTECR1_INC_EXC; + else + drvdata->enable_ctrl1 &= ~ETMTECR1_INC_EXC; + + if (drvdata->mode & ETM_MODE_CYCACC) + drvdata->ctrl |= ETMCR_CYC_ACC; + else + drvdata->ctrl &= ~ETMCR_CYC_ACC; + + if (drvdata->mode & ETM_MODE_STALL) { + if (!(drvdata->etmccr & ETMCCR_FIFOFULL)) { + dev_warn(drvdata->dev, "stall mode not supported\n"); + ret = -EINVAL; + goto err_unlock; + } + drvdata->ctrl |= ETMCR_STALL_MODE; + } else + drvdata->ctrl &= ~ETMCR_STALL_MODE; + + if (drvdata->mode & ETM_MODE_TIMESTAMP) { + if (!(drvdata->etmccer & ETMCCER_TIMESTAMP)) { + dev_warn(drvdata->dev, "timestamp not supported\n"); + ret = -EINVAL; + goto err_unlock; + } + drvdata->ctrl |= ETMCR_TIMESTAMP_EN; + } else + drvdata->ctrl &= ~ETMCR_TIMESTAMP_EN; + + if (drvdata->mode & ETM_MODE_CTXID) + drvdata->ctrl |= ETMCR_CTXID_SIZE; + else + drvdata->ctrl &= ~ETMCR_CTXID_SIZE; + spin_unlock(&drvdata->spinlock); + + return size; + +err_unlock: + spin_unlock(&drvdata->spinlock); + return ret; +} +static DEVICE_ATTR_RW(mode); + +static ssize_t trigger_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->trigger_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t trigger_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->trigger_event = val & ETM_EVENT_MASK; + + return size; +} +static DEVICE_ATTR_RW(trigger_event); + +static ssize_t enable_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->enable_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t enable_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->enable_event = val & ETM_EVENT_MASK; + + return size; +} +static DEVICE_ATTR_RW(enable_event); + +static ssize_t fifofull_level_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->fifofull_level; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t fifofull_level_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->fifofull_level = val; + + return size; +} +static DEVICE_ATTR_RW(fifofull_level); + +static ssize_t addr_idx_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->addr_idx; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t addr_idx_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + if (val >= drvdata->nr_addr_cmp) + return -EINVAL; + + /* + * Use spinlock to ensure index doesn't change while it gets + * dereferenced multiple times within a spinlock block elsewhere. + */ + spin_lock(&drvdata->spinlock); + drvdata->addr_idx = val; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(addr_idx); + +static ssize_t addr_single_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u8 idx; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + idx = drvdata->addr_idx; + if (!(drvdata->addr_type[idx] == ETM_ADDR_TYPE_NONE || + drvdata->addr_type[idx] == ETM_ADDR_TYPE_SINGLE)) { + spin_unlock(&drvdata->spinlock); + return -EINVAL; + } + + val = drvdata->addr_val[idx]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t addr_single_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + u8 idx; + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + idx = drvdata->addr_idx; + if (!(drvdata->addr_type[idx] == ETM_ADDR_TYPE_NONE || + drvdata->addr_type[idx] == ETM_ADDR_TYPE_SINGLE)) { + spin_unlock(&drvdata->spinlock); + return -EINVAL; + } + + drvdata->addr_val[idx] = val; + drvdata->addr_type[idx] = ETM_ADDR_TYPE_SINGLE; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(addr_single); + +static ssize_t addr_range_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u8 idx; + unsigned long val1, val2; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + idx = drvdata->addr_idx; + if (idx % 2 != 0) { + spin_unlock(&drvdata->spinlock); + return -EPERM; + } + if (!((drvdata->addr_type[idx] == ETM_ADDR_TYPE_NONE && + drvdata->addr_type[idx + 1] == ETM_ADDR_TYPE_NONE) || + (drvdata->addr_type[idx] == ETM_ADDR_TYPE_RANGE && + drvdata->addr_type[idx + 1] == ETM_ADDR_TYPE_RANGE))) { + spin_unlock(&drvdata->spinlock); + return -EPERM; + } + + val1 = drvdata->addr_val[idx]; + val2 = drvdata->addr_val[idx + 1]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx %#lx\n", val1, val2); +} + +static ssize_t addr_range_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + u8 idx; + unsigned long val1, val2; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + if (sscanf(buf, "%lx %lx", &val1, &val2) != 2) + return -EINVAL; + /* Lower address comparator cannot have a higher address value */ + if (val1 > val2) + return -EINVAL; + + spin_lock(&drvdata->spinlock); + idx = drvdata->addr_idx; + if (idx % 2 != 0) { + spin_unlock(&drvdata->spinlock); + return -EPERM; + } + if (!((drvdata->addr_type[idx] == ETM_ADDR_TYPE_NONE && + drvdata->addr_type[idx + 1] == ETM_ADDR_TYPE_NONE) || + (drvdata->addr_type[idx] == ETM_ADDR_TYPE_RANGE && + drvdata->addr_type[idx + 1] == ETM_ADDR_TYPE_RANGE))) { + spin_unlock(&drvdata->spinlock); + return -EPERM; + } + + drvdata->addr_val[idx] = val1; + drvdata->addr_type[idx] = ETM_ADDR_TYPE_RANGE; + drvdata->addr_val[idx + 1] = val2; + drvdata->addr_type[idx + 1] = ETM_ADDR_TYPE_RANGE; + drvdata->enable_ctrl1 |= (1 << (idx/2)); + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(addr_range); + +static ssize_t addr_start_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u8 idx; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + idx = drvdata->addr_idx; + if (!(drvdata->addr_type[idx] == ETM_ADDR_TYPE_NONE || + drvdata->addr_type[idx] == ETM_ADDR_TYPE_START)) { + spin_unlock(&drvdata->spinlock); + return -EPERM; + } + + val = drvdata->addr_val[idx]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t addr_start_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + u8 idx; + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + idx = drvdata->addr_idx; + if (!(drvdata->addr_type[idx] == ETM_ADDR_TYPE_NONE || + drvdata->addr_type[idx] == ETM_ADDR_TYPE_START)) { + spin_unlock(&drvdata->spinlock); + return -EPERM; + } + + drvdata->addr_val[idx] = val; + drvdata->addr_type[idx] = ETM_ADDR_TYPE_START; + drvdata->startstop_ctrl |= (1 << idx); + drvdata->enable_ctrl1 |= BIT(25); + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(addr_start); + +static ssize_t addr_stop_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u8 idx; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + idx = drvdata->addr_idx; + if (!(drvdata->addr_type[idx] == ETM_ADDR_TYPE_NONE || + drvdata->addr_type[idx] == ETM_ADDR_TYPE_STOP)) { + spin_unlock(&drvdata->spinlock); + return -EPERM; + } + + val = drvdata->addr_val[idx]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t addr_stop_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + u8 idx; + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + idx = drvdata->addr_idx; + if (!(drvdata->addr_type[idx] == ETM_ADDR_TYPE_NONE || + drvdata->addr_type[idx] == ETM_ADDR_TYPE_STOP)) { + spin_unlock(&drvdata->spinlock); + return -EPERM; + } + + drvdata->addr_val[idx] = val; + drvdata->addr_type[idx] = ETM_ADDR_TYPE_STOP; + drvdata->startstop_ctrl |= (1 << (idx + 16)); + drvdata->enable_ctrl1 |= ETMTECR1_START_STOP; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(addr_stop); + +static ssize_t addr_acctype_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + val = drvdata->addr_acctype[drvdata->addr_idx]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t addr_acctype_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + drvdata->addr_acctype[drvdata->addr_idx] = val; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(addr_acctype); + +static ssize_t cntr_idx_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->cntr_idx; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t cntr_idx_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + if (val >= drvdata->nr_cntr) + return -EINVAL; + /* + * Use spinlock to ensure index doesn't change while it gets + * dereferenced multiple times within a spinlock block elsewhere. + */ + spin_lock(&drvdata->spinlock); + drvdata->cntr_idx = val; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(cntr_idx); + +static ssize_t cntr_rld_val_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + val = drvdata->cntr_rld_val[drvdata->cntr_idx]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t cntr_rld_val_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + drvdata->cntr_rld_val[drvdata->cntr_idx] = val; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(cntr_rld_val); + +static ssize_t cntr_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + val = drvdata->cntr_event[drvdata->cntr_idx]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t cntr_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + drvdata->cntr_event[drvdata->cntr_idx] = val & ETM_EVENT_MASK; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(cntr_event); + +static ssize_t cntr_rld_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + val = drvdata->cntr_rld_event[drvdata->cntr_idx]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t cntr_rld_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + drvdata->cntr_rld_event[drvdata->cntr_idx] = val & ETM_EVENT_MASK; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(cntr_rld_event); + +static ssize_t cntr_val_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int i, ret = 0; + u32 val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + if (!drvdata->enable) { + spin_lock(&drvdata->spinlock); + for (i = 0; i < drvdata->nr_cntr; i++) + ret += sprintf(buf, "counter %d: %x\n", + i, drvdata->cntr_val[i]); + spin_unlock(&drvdata->spinlock); + return ret; + } + + for (i = 0; i < drvdata->nr_cntr; i++) { + val = etm_readl(drvdata, ETMCNTVRn(i)); + ret += sprintf(buf, "counter %d: %x\n", i, val); + } + + return ret; +} + +static ssize_t cntr_val_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + drvdata->cntr_val[drvdata->cntr_idx] = val; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(cntr_val); + +static ssize_t seq_12_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->seq_12_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t seq_12_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->seq_12_event = val & ETM_EVENT_MASK; + return size; +} +static DEVICE_ATTR_RW(seq_12_event); + +static ssize_t seq_21_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->seq_21_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t seq_21_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->seq_21_event = val & ETM_EVENT_MASK; + return size; +} +static DEVICE_ATTR_RW(seq_21_event); + +static ssize_t seq_23_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->seq_23_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t seq_23_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->seq_23_event = val & ETM_EVENT_MASK; + return size; +} +static DEVICE_ATTR_RW(seq_23_event); + +static ssize_t seq_31_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->seq_31_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t seq_31_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->seq_31_event = val & ETM_EVENT_MASK; + return size; +} +static DEVICE_ATTR_RW(seq_31_event); + +static ssize_t seq_32_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->seq_32_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t seq_32_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->seq_32_event = val & ETM_EVENT_MASK; + return size; +} +static DEVICE_ATTR_RW(seq_32_event); + +static ssize_t seq_13_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->seq_13_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t seq_13_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->seq_13_event = val & ETM_EVENT_MASK; + return size; +} +static DEVICE_ATTR_RW(seq_13_event); + +static ssize_t seq_curr_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + unsigned long val, flags; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + if (!drvdata->enable) { + val = drvdata->seq_curr_state; + goto out; + } + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + spin_lock_irqsave(&drvdata->spinlock, flags); + + CS_UNLOCK(drvdata->base); + val = (etm_readl(drvdata, ETMSQR) & ETM_SQR_MASK); + CS_LOCK(drvdata->base); + + spin_unlock_irqrestore(&drvdata->spinlock, flags); + clk_disable_unprepare(drvdata->clk); +out: + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t seq_curr_state_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + if (val > ETM_SEQ_STATE_MAX_VAL) + return -EINVAL; + + drvdata->seq_curr_state = val; + + return size; +} +static DEVICE_ATTR_RW(seq_curr_state); + +static ssize_t ctxid_idx_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->ctxid_idx; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t ctxid_idx_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + if (val >= drvdata->nr_ctxid_cmp) + return -EINVAL; + + /* + * Use spinlock to ensure index doesn't change while it gets + * dereferenced multiple times within a spinlock block elsewhere. + */ + spin_lock(&drvdata->spinlock); + drvdata->ctxid_idx = val; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(ctxid_idx); + +static ssize_t ctxid_val_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + spin_lock(&drvdata->spinlock); + val = drvdata->ctxid_val[drvdata->ctxid_idx]; + spin_unlock(&drvdata->spinlock); + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t ctxid_val_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + spin_lock(&drvdata->spinlock); + drvdata->ctxid_val[drvdata->ctxid_idx] = val; + spin_unlock(&drvdata->spinlock); + + return size; +} +static DEVICE_ATTR_RW(ctxid_val); + +static ssize_t ctxid_mask_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->ctxid_mask; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t ctxid_mask_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->ctxid_mask = val; + return size; +} +static DEVICE_ATTR_RW(ctxid_mask); + +static ssize_t sync_freq_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->sync_freq; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t sync_freq_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->sync_freq = val & ETM_SYNC_MASK; + return size; +} +static DEVICE_ATTR_RW(sync_freq); + +static ssize_t timestamp_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + val = drvdata->timestamp_event; + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t timestamp_event_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->timestamp_event = val & ETM_EVENT_MASK; + return size; +} +static DEVICE_ATTR_RW(timestamp_event); + +static ssize_t status_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + unsigned long flags; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + spin_lock_irqsave(&drvdata->spinlock, flags); + + CS_UNLOCK(drvdata->base); + ret = sprintf(buf, + "ETMCCR: 0x%08x\n" + "ETMCCER: 0x%08x\n" + "ETMSCR: 0x%08x\n" + "ETMIDR: 0x%08x\n" + "ETMCR: 0x%08x\n" + "ETMTRACEIDR: 0x%08x\n" + "Enable event: 0x%08x\n" + "Enable start/stop: 0x%08x\n" + "Enable control: CR1 0x%08x CR2 0x%08x\n" + "CPU affinity: %d\n", + drvdata->etmccr, drvdata->etmccer, + etm_readl(drvdata, ETMSCR), etm_readl(drvdata, ETMIDR), + etm_readl(drvdata, ETMCR), etm_trace_id_simple(drvdata), + etm_readl(drvdata, ETMTEEVR), + etm_readl(drvdata, ETMTSSCR), + etm_readl(drvdata, ETMTECR1), + etm_readl(drvdata, ETMTECR2), + drvdata->cpu); + CS_LOCK(drvdata->base); + + spin_unlock_irqrestore(&drvdata->spinlock, flags); + clk_disable_unprepare(drvdata->clk); + + return ret; +} +static DEVICE_ATTR_RO(status); + +static ssize_t traceid_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + unsigned long val, flags; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + if (!drvdata->enable) { + val = drvdata->traceid; + goto out; + } + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + spin_lock_irqsave(&drvdata->spinlock, flags); + CS_UNLOCK(drvdata->base); + + val = (etm_readl(drvdata, ETMTRACEIDR) & ETM_TRACEID_MASK); + + CS_LOCK(drvdata->base); + spin_unlock_irqrestore(&drvdata->spinlock, flags); + clk_disable_unprepare(drvdata->clk); +out: + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t traceid_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->traceid = val & ETM_TRACEID_MASK; + return size; +} +static DEVICE_ATTR_RW(traceid); + +static struct attribute *coresight_etm_attrs[] = { + &dev_attr_nr_addr_cmp.attr, + &dev_attr_nr_cntr.attr, + &dev_attr_nr_ctxid_cmp.attr, + &dev_attr_etmsr.attr, + &dev_attr_reset.attr, + &dev_attr_mode.attr, + &dev_attr_trigger_event.attr, + &dev_attr_enable_event.attr, + &dev_attr_fifofull_level.attr, + &dev_attr_addr_idx.attr, + &dev_attr_addr_single.attr, + &dev_attr_addr_range.attr, + &dev_attr_addr_start.attr, + &dev_attr_addr_stop.attr, + &dev_attr_addr_acctype.attr, + &dev_attr_cntr_idx.attr, + &dev_attr_cntr_rld_val.attr, + &dev_attr_cntr_event.attr, + &dev_attr_cntr_rld_event.attr, + &dev_attr_cntr_val.attr, + &dev_attr_seq_12_event.attr, + &dev_attr_seq_21_event.attr, + &dev_attr_seq_23_event.attr, + &dev_attr_seq_31_event.attr, + &dev_attr_seq_32_event.attr, + &dev_attr_seq_13_event.attr, + &dev_attr_seq_curr_state.attr, + &dev_attr_ctxid_idx.attr, + &dev_attr_ctxid_val.attr, + &dev_attr_ctxid_mask.attr, + &dev_attr_sync_freq.attr, + &dev_attr_timestamp_event.attr, + &dev_attr_status.attr, + &dev_attr_traceid.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_etm); + +static int etm_cpu_callback(struct notifier_block *nfb, unsigned long action, + void *hcpu) +{ + unsigned int cpu = (unsigned long)hcpu; + + if (!etmdrvdata[cpu]) + goto out; + + switch (action & (~CPU_TASKS_FROZEN)) { + case CPU_STARTING: + spin_lock(&etmdrvdata[cpu]->spinlock); + if (!etmdrvdata[cpu]->os_unlock) { + etm_os_unlock(etmdrvdata[cpu]); + etmdrvdata[cpu]->os_unlock = true; + } + + if (etmdrvdata[cpu]->enable) + etm_enable_hw(etmdrvdata[cpu]); + spin_unlock(&etmdrvdata[cpu]->spinlock); + break; + + case CPU_ONLINE: + if (etmdrvdata[cpu]->boot_enable && + !etmdrvdata[cpu]->sticky_enable) + coresight_enable(etmdrvdata[cpu]->csdev); + break; + + case CPU_DYING: + spin_lock(&etmdrvdata[cpu]->spinlock); + if (etmdrvdata[cpu]->enable) + etm_disable_hw(etmdrvdata[cpu]); + spin_unlock(&etmdrvdata[cpu]->spinlock); + break; + } +out: + return NOTIFY_OK; +} + +static struct notifier_block etm_cpu_notifier = { + .notifier_call = etm_cpu_callback, +}; + +static bool etm_arch_supported(u8 arch) +{ + switch (arch) { + case ETM_ARCH_V3_3: + break; + case ETM_ARCH_V3_5: + break; + case PFT_ARCH_V1_0: + break; + case PFT_ARCH_V1_1: + break; + default: + return false; + } + return true; +} + +static void etm_init_arch_data(void *info) +{ + u32 etmidr; + u32 etmccr; + struct etm_drvdata *drvdata = info; + + CS_UNLOCK(drvdata->base); + + /* First dummy read */ + (void)etm_readl(drvdata, ETMPDSR); + /* Provide power to ETM: ETMPDCR[3] == 1 */ + etm_set_pwrup(drvdata); + /* + * Clear power down bit since when this bit is set writes to + * certain registers might be ignored. + */ + etm_clr_pwrdwn(drvdata); + /* + * Set prog bit. It will be set from reset but this is included to + * ensure it is set + */ + etm_set_prog(drvdata); + + /* Find all capabilities */ + etmidr = etm_readl(drvdata, ETMIDR); + drvdata->arch = BMVAL(etmidr, 4, 11); + drvdata->port_size = etm_readl(drvdata, ETMCR) & PORT_SIZE_MASK; + + drvdata->etmccer = etm_readl(drvdata, ETMCCER); + etmccr = etm_readl(drvdata, ETMCCR); + drvdata->etmccr = etmccr; + drvdata->nr_addr_cmp = BMVAL(etmccr, 0, 3) * 2; + drvdata->nr_cntr = BMVAL(etmccr, 13, 15); + drvdata->nr_ext_inp = BMVAL(etmccr, 17, 19); + drvdata->nr_ext_out = BMVAL(etmccr, 20, 22); + drvdata->nr_ctxid_cmp = BMVAL(etmccr, 24, 25); + + etm_set_pwrdwn(drvdata); + etm_clr_pwrup(drvdata); + CS_LOCK(drvdata->base); +} + +static void etm_init_default_data(struct etm_drvdata *drvdata) +{ + /* + * A trace ID of value 0 is invalid, so let's start at some + * random value that fits in 7 bits and will be just as good. + */ + static int etm3x_traceid = 0x10; + + u32 flags = (1 << 0 | /* instruction execute*/ + 3 << 3 | /* ARM instruction */ + 0 << 5 | /* No data value comparison */ + 0 << 7 | /* No exact mach */ + 0 << 8 | /* Ignore context ID */ + 0 << 10); /* Security ignored */ + + /* + * Initial configuration only - guarantees sources handled by + * this driver have a unique ID at startup time but not between + * all other types of sources. For that we lean on the core + * framework. + */ + drvdata->traceid = etm3x_traceid++; + drvdata->ctrl = (ETMCR_CYC_ACC | ETMCR_TIMESTAMP_EN); + drvdata->enable_ctrl1 = ETMTECR1_ADDR_COMP_1; + if (drvdata->nr_addr_cmp >= 2) { + drvdata->addr_val[0] = (u32) _stext; + drvdata->addr_val[1] = (u32) _etext; + drvdata->addr_acctype[0] = flags; + drvdata->addr_acctype[1] = flags; + drvdata->addr_type[0] = ETM_ADDR_TYPE_RANGE; + drvdata->addr_type[1] = ETM_ADDR_TYPE_RANGE; + } + + etm_set_default(drvdata); +} + +static int etm_probe(struct amba_device *adev, const struct amba_id *id) +{ + int ret; + void __iomem *base; + struct device *dev = &adev->dev; + struct coresight_platform_data *pdata = NULL; + struct etm_drvdata *drvdata; + struct resource *res = &adev->res; + struct coresight_desc *desc; + struct device_node *np = adev->dev.of_node; + + desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + if (np) { + pdata = of_get_coresight_platform_data(dev, np); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + + adev->dev.platform_data = pdata; + drvdata->use_cp14 = of_property_read_bool(np, "arm,cp14"); + } + + drvdata->dev = &adev->dev; + dev_set_drvdata(dev, drvdata); + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + drvdata->base = base; + + spin_lock_init(&drvdata->spinlock); + + drvdata->clk = adev->pclk; + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + drvdata->cpu = pdata ? pdata->cpu : 0; + + get_online_cpus(); + etmdrvdata[drvdata->cpu] = drvdata; + + if (!smp_call_function_single(drvdata->cpu, etm_os_unlock, drvdata, 1)) + drvdata->os_unlock = true; + + if (smp_call_function_single(drvdata->cpu, + etm_init_arch_data, drvdata, 1)) + dev_err(dev, "ETM arch init failed\n"); + + if (!etm_count++) + register_hotcpu_notifier(&etm_cpu_notifier); + + put_online_cpus(); + + if (etm_arch_supported(drvdata->arch) == false) { + ret = -EINVAL; + goto err_arch_supported; + } + etm_init_default_data(drvdata); + + clk_disable_unprepare(drvdata->clk); + + desc->type = CORESIGHT_DEV_TYPE_SOURCE; + desc->subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; + desc->ops = &etm_cs_ops; + desc->pdata = pdata; + desc->dev = dev; + desc->groups = coresight_etm_groups; + drvdata->csdev = coresight_register(desc); + if (IS_ERR(drvdata->csdev)) { + ret = PTR_ERR(drvdata->csdev); + goto err_arch_supported; + } + + dev_info(dev, "ETM initialized\n"); + + if (boot_enable) { + coresight_enable(drvdata->csdev); + drvdata->boot_enable = true; + } + + return 0; + +err_arch_supported: + clk_disable_unprepare(drvdata->clk); + if (--etm_count == 0) + unregister_hotcpu_notifier(&etm_cpu_notifier); + return ret; +} + +static int etm_remove(struct amba_device *adev) +{ + struct etm_drvdata *drvdata = amba_get_drvdata(adev); + + coresight_unregister(drvdata->csdev); + if (--etm_count == 0) + unregister_hotcpu_notifier(&etm_cpu_notifier); + + return 0; +} + +static struct amba_id etm_ids[] = { + { /* ETM 3.3 */ + .id = 0x0003b921, + .mask = 0x0003ffff, + }, + { /* ETM 3.5 */ + .id = 0x0003b956, + .mask = 0x0003ffff, + }, + { /* PTM 1.0 */ + .id = 0x0003b950, + .mask = 0x0003ffff, + }, + { /* PTM 1.1 */ + .id = 0x0003b95f, + .mask = 0x0003ffff, + }, + { 0, 0}, +}; + +static struct amba_driver etm_driver = { + .drv = { + .name = "coresight-etm3x", + .owner = THIS_MODULE, + }, + .probe = etm_probe, + .remove = etm_remove, + .id_table = etm_ids, +}; + +int __init etm_init(void) +{ + return amba_driver_register(&etm_driver); +} +module_init(etm_init); + +void __exit etm_exit(void) +{ + amba_driver_unregister(&etm_driver); +} +module_exit(etm_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("CoreSight Program Flow Trace driver"); diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c new file mode 100644 index 000000000000..3db36f70b666 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-funnel.c @@ -0,0 +1,258 @@ +/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/clk.h> +#include <linux/coresight.h> +#include <linux/amba/bus.h> + +#include "coresight-priv.h" + +#define FUNNEL_FUNCTL 0x000 +#define FUNNEL_PRICTL 0x004 + +#define FUNNEL_HOLDTIME_MASK 0xf00 +#define FUNNEL_HOLDTIME_SHFT 0x8 +#define FUNNEL_HOLDTIME (0x7 << FUNNEL_HOLDTIME_SHFT) + +/** + * struct funnel_drvdata - specifics associated to a funnel component + * @base: memory mapped base address for this component. + * @dev: the device entity associated to this component. + * @csdev: component vitals needed by the framework. + * @clk: the clock this component is associated to. + * @priority: port selection order. + */ +struct funnel_drvdata { + void __iomem *base; + struct device *dev; + struct coresight_device *csdev; + struct clk *clk; + unsigned long priority; +}; + +static void funnel_enable_hw(struct funnel_drvdata *drvdata, int port) +{ + u32 functl; + + CS_UNLOCK(drvdata->base); + + functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); + functl &= ~FUNNEL_HOLDTIME_MASK; + functl |= FUNNEL_HOLDTIME; + functl |= (1 << port); + writel_relaxed(functl, drvdata->base + FUNNEL_FUNCTL); + writel_relaxed(drvdata->priority, drvdata->base + FUNNEL_PRICTL); + + CS_LOCK(drvdata->base); +} + +static int funnel_enable(struct coresight_device *csdev, int inport, + int outport) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + int ret; + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + funnel_enable_hw(drvdata, inport); + + dev_info(drvdata->dev, "FUNNEL inport %d enabled\n", inport); + return 0; +} + +static void funnel_disable_hw(struct funnel_drvdata *drvdata, int inport) +{ + u32 functl; + + CS_UNLOCK(drvdata->base); + + functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); + functl &= ~(1 << inport); + writel_relaxed(functl, drvdata->base + FUNNEL_FUNCTL); + + CS_LOCK(drvdata->base); +} + +static void funnel_disable(struct coresight_device *csdev, int inport, + int outport) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + funnel_disable_hw(drvdata, inport); + + clk_disable_unprepare(drvdata->clk); + + dev_info(drvdata->dev, "FUNNEL inport %d disabled\n", inport); +} + +static const struct coresight_ops_link funnel_link_ops = { + .enable = funnel_enable, + .disable = funnel_disable, +}; + +static const struct coresight_ops funnel_cs_ops = { + .link_ops = &funnel_link_ops, +}; + +static ssize_t priority_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val = drvdata->priority; + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t priority_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct funnel_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->priority = val; + return size; +} +static DEVICE_ATTR_RW(priority); + +static u32 get_funnel_ctrl_hw(struct funnel_drvdata *drvdata) +{ + u32 functl; + + CS_UNLOCK(drvdata->base); + functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); + CS_LOCK(drvdata->base); + + return functl; +} + +static ssize_t funnel_ctrl_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + u32 val; + struct funnel_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + val = get_funnel_ctrl_hw(drvdata); + clk_disable_unprepare(drvdata->clk); + + return sprintf(buf, "%#x\n", val); +} +static DEVICE_ATTR_RO(funnel_ctrl); + +static struct attribute *coresight_funnel_attrs[] = { + &dev_attr_funnel_ctrl.attr, + &dev_attr_priority.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_funnel); + +static int funnel_probe(struct amba_device *adev, const struct amba_id *id) +{ + void __iomem *base; + struct device *dev = &adev->dev; + struct coresight_platform_data *pdata = NULL; + struct funnel_drvdata *drvdata; + struct resource *res = &adev->res; + struct coresight_desc *desc; + struct device_node *np = adev->dev.of_node; + + if (np) { + pdata = of_get_coresight_platform_data(dev, np); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + adev->dev.platform_data = pdata; + } + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + drvdata->dev = &adev->dev; + dev_set_drvdata(dev, drvdata); + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + drvdata->base = base; + + drvdata->clk = adev->pclk; + + desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + + desc->type = CORESIGHT_DEV_TYPE_LINK; + desc->subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_MERG; + desc->ops = &funnel_cs_ops; + desc->pdata = pdata; + desc->dev = dev; + desc->groups = coresight_funnel_groups; + drvdata->csdev = coresight_register(desc); + if (IS_ERR(drvdata->csdev)) + return PTR_ERR(drvdata->csdev); + + dev_info(dev, "FUNNEL initialized\n"); + return 0; +} + +static int funnel_remove(struct amba_device *adev) +{ + struct funnel_drvdata *drvdata = amba_get_drvdata(adev); + + coresight_unregister(drvdata->csdev); + return 0; +} + +static struct amba_id funnel_ids[] = { + { + .id = 0x0003b908, + .mask = 0x0003ffff, + }, + { 0, 0}, +}; + +static struct amba_driver funnel_driver = { + .drv = { + .name = "coresight-funnel", + .owner = THIS_MODULE, + }, + .probe = funnel_probe, + .remove = funnel_remove, + .id_table = funnel_ids, +}; + +module_amba_driver(funnel_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("CoreSight Funnel driver"); diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h new file mode 100644 index 000000000000..62fcd98cc7cf --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CORESIGHT_PRIV_H +#define _CORESIGHT_PRIV_H + +#include <linux/bitops.h> +#include <linux/io.h> +#include <linux/coresight.h> + +/* + * Coresight management registers (0xf00-0xfcc) + * 0xfa0 - 0xfa4: Management registers in PFTv1.0 + * Trace registers in PFTv1.1 + */ +#define CORESIGHT_ITCTRL 0xf00 +#define CORESIGHT_CLAIMSET 0xfa0 +#define CORESIGHT_CLAIMCLR 0xfa4 +#define CORESIGHT_LAR 0xfb0 +#define CORESIGHT_LSR 0xfb4 +#define CORESIGHT_AUTHSTATUS 0xfb8 +#define CORESIGHT_DEVID 0xfc8 +#define CORESIGHT_DEVTYPE 0xfcc + +#define TIMEOUT_US 100 +#define BMVAL(val, lsb, msb) ((val & GENMASK(msb, lsb)) >> lsb) + +static inline void CS_LOCK(void __iomem *addr) +{ + do { + /* Wait for things to settle */ + mb(); + writel_relaxed(0x0, addr + CORESIGHT_LAR); + } while (0); +} + +static inline void CS_UNLOCK(void __iomem *addr) +{ + do { + writel_relaxed(CORESIGHT_UNLOCK, addr + CORESIGHT_LAR); + /* Make sure everyone has seen this */ + mb(); + } while (0); +} + +#ifdef CONFIG_CORESIGHT_SOURCE_ETM3X +extern int etm_readl_cp14(u32 off, unsigned int *val); +extern int etm_writel_cp14(u32 off, u32 val); +#else +static inline int etm_readl_cp14(u32 off, unsigned int *val) { return 0; } +static inline int etm_writel_cp14(u32 off, u32 val) { return 0; } +#endif + +#endif diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c new file mode 100644 index 000000000000..cdf05537d574 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -0,0 +1,137 @@ +/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/io.h> +#include <linux/err.h> +#include <linux/slab.h> +#include <linux/clk.h> +#include <linux/of.h> +#include <linux/coresight.h> + +#include "coresight-priv.h" + +/** + * struct replicator_drvdata - specifics associated to a replicator component + * @dev: the device entity associated with this component + * @csdev: component vitals needed by the framework + */ +struct replicator_drvdata { + struct device *dev; + struct coresight_device *csdev; +}; + +static int replicator_enable(struct coresight_device *csdev, int inport, + int outport) +{ + struct replicator_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + dev_info(drvdata->dev, "REPLICATOR enabled\n"); + return 0; +} + +static void replicator_disable(struct coresight_device *csdev, int inport, + int outport) +{ + struct replicator_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + dev_info(drvdata->dev, "REPLICATOR disabled\n"); +} + +static const struct coresight_ops_link replicator_link_ops = { + .enable = replicator_enable, + .disable = replicator_disable, +}; + +static const struct coresight_ops replicator_cs_ops = { + .link_ops = &replicator_link_ops, +}; + +static int replicator_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct coresight_platform_data *pdata = NULL; + struct replicator_drvdata *drvdata; + struct coresight_desc *desc; + struct device_node *np = pdev->dev.of_node; + + if (np) { + pdata = of_get_coresight_platform_data(dev, np); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + pdev->dev.platform_data = pdata; + } + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + drvdata->dev = &pdev->dev; + platform_set_drvdata(pdev, drvdata); + + desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + + desc->type = CORESIGHT_DEV_TYPE_LINK; + desc->subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_SPLIT; + desc->ops = &replicator_cs_ops; + desc->pdata = pdev->dev.platform_data; + desc->dev = &pdev->dev; + drvdata->csdev = coresight_register(desc); + if (IS_ERR(drvdata->csdev)) + return PTR_ERR(drvdata->csdev); + + dev_info(dev, "REPLICATOR initialized\n"); + return 0; +} + +static int replicator_remove(struct platform_device *pdev) +{ + struct replicator_drvdata *drvdata = platform_get_drvdata(pdev); + + coresight_unregister(drvdata->csdev); + return 0; +} + +static struct of_device_id replicator_match[] = { + {.compatible = "arm,coresight-replicator"}, + {} +}; + +static struct platform_driver replicator_driver = { + .probe = replicator_probe, + .remove = replicator_remove, + .driver = { + .name = "coresight-replicator", + .of_match_table = replicator_match, + }, +}; + +static int __init replicator_init(void) +{ + return platform_driver_register(&replicator_driver); +} +module_init(replicator_init); + +static void __exit replicator_exit(void) +{ + platform_driver_unregister(&replicator_driver); +} +module_exit(replicator_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("CoreSight Replicator driver"); diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c new file mode 100644 index 000000000000..7147f3dd363c --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-tmc.c @@ -0,0 +1,822 @@ +/* Copyright (c) 2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/io.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/miscdevice.h> +#include <linux/uaccess.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/clk.h> +#include <linux/of.h> +#include <linux/coresight.h> +#include <linux/amba/bus.h> + +#include "coresight-priv.h" + +#define TMC_RSZ 0x004 +#define TMC_STS 0x00c +#define TMC_RRD 0x010 +#define TMC_RRP 0x014 +#define TMC_RWP 0x018 +#define TMC_TRG 0x01c +#define TMC_CTL 0x020 +#define TMC_RWD 0x024 +#define TMC_MODE 0x028 +#define TMC_LBUFLEVEL 0x02c +#define TMC_CBUFLEVEL 0x030 +#define TMC_BUFWM 0x034 +#define TMC_RRPHI 0x038 +#define TMC_RWPHI 0x03c +#define TMC_AXICTL 0x110 +#define TMC_DBALO 0x118 +#define TMC_DBAHI 0x11c +#define TMC_FFSR 0x300 +#define TMC_FFCR 0x304 +#define TMC_PSCR 0x308 +#define TMC_ITMISCOP0 0xee0 +#define TMC_ITTRFLIN 0xee8 +#define TMC_ITATBDATA0 0xeec +#define TMC_ITATBCTR2 0xef0 +#define TMC_ITATBCTR1 0xef4 +#define TMC_ITATBCTR0 0xef8 + +/* register description */ +/* TMC_CTL - 0x020 */ +#define TMC_CTL_CAPT_EN BIT(0) +/* TMC_STS - 0x00C */ +#define TMC_STS_TRIGGERED BIT(1) +/* TMC_AXICTL - 0x110 */ +#define TMC_AXICTL_PROT_CTL_B0 BIT(0) +#define TMC_AXICTL_PROT_CTL_B1 BIT(1) +#define TMC_AXICTL_SCT_GAT_MODE BIT(7) +#define TMC_AXICTL_WR_BURST_LEN 0xF00 +/* TMC_FFCR - 0x304 */ +#define TMC_FFCR_EN_FMT BIT(0) +#define TMC_FFCR_EN_TI BIT(1) +#define TMC_FFCR_FON_FLIN BIT(4) +#define TMC_FFCR_FON_TRIG_EVT BIT(5) +#define TMC_FFCR_FLUSHMAN BIT(6) +#define TMC_FFCR_TRIGON_TRIGIN BIT(8) +#define TMC_FFCR_STOP_ON_FLUSH BIT(12) + +#define TMC_STS_TRIGGERED_BIT 2 +#define TMC_FFCR_FLUSHMAN_BIT 6 + +enum tmc_config_type { + TMC_CONFIG_TYPE_ETB, + TMC_CONFIG_TYPE_ETR, + TMC_CONFIG_TYPE_ETF, +}; + +enum tmc_mode { + TMC_MODE_CIRCULAR_BUFFER, + TMC_MODE_SOFTWARE_FIFO, + TMC_MODE_HARDWARE_FIFO, +}; + +enum tmc_mem_intf_width { + TMC_MEM_INTF_WIDTH_32BITS = 0x2, + TMC_MEM_INTF_WIDTH_64BITS = 0x3, + TMC_MEM_INTF_WIDTH_128BITS = 0x4, + TMC_MEM_INTF_WIDTH_256BITS = 0x5, +}; + +/** + * struct tmc_drvdata - specifics associated to an TMC component + * @base: memory mapped base address for this component. + * @dev: the device entity associated to this component. + * @csdev: component vitals needed by the framework. + * @miscdev: specifics to handle "/dev/xyz.tmc" entry. + * @clk: the clock this component is associated to. + * @spinlock: only one at a time pls. + * @read_count: manages preparation of buffer for reading. + * @buf: area of memory where trace data get sent. + * @paddr: DMA start location in RAM. + * @vaddr: virtual representation of @paddr. + * @size: @buf size. + * @enable: this TMC is being used. + * @config_type: TMC variant, must be of type @tmc_config_type. + * @trigger_cntr: amount of words to store after a trigger. + */ +struct tmc_drvdata { + void __iomem *base; + struct device *dev; + struct coresight_device *csdev; + struct miscdevice miscdev; + struct clk *clk; + spinlock_t spinlock; + int read_count; + bool reading; + char *buf; + dma_addr_t paddr; + void __iomem *vaddr; + u32 size; + bool enable; + enum tmc_config_type config_type; + u32 trigger_cntr; +}; + +static void tmc_wait_for_ready(struct tmc_drvdata *drvdata) +{ + /* Ensure formatter, unformatter and hardware fifo are empty */ + if (coresight_timeout(drvdata->base, + TMC_STS, TMC_STS_TRIGGERED_BIT, 1)) { + dev_err(drvdata->dev, + "timeout observed when probing at offset %#x\n", + TMC_STS); + } +} + +static void tmc_flush_and_stop(struct tmc_drvdata *drvdata) +{ + u32 ffcr; + + ffcr = readl_relaxed(drvdata->base + TMC_FFCR); + ffcr |= TMC_FFCR_STOP_ON_FLUSH; + writel_relaxed(ffcr, drvdata->base + TMC_FFCR); + ffcr |= TMC_FFCR_FLUSHMAN; + writel_relaxed(ffcr, drvdata->base + TMC_FFCR); + /* Ensure flush completes */ + if (coresight_timeout(drvdata->base, + TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0)) { + dev_err(drvdata->dev, + "timeout observed when probing at offset %#x\n", + TMC_FFCR); + } + + tmc_wait_for_ready(drvdata); +} + +static void tmc_enable_hw(struct tmc_drvdata *drvdata) +{ + writel_relaxed(TMC_CTL_CAPT_EN, drvdata->base + TMC_CTL); +} + +static void tmc_disable_hw(struct tmc_drvdata *drvdata) +{ + writel_relaxed(0x0, drvdata->base + TMC_CTL); +} + +static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +{ + /* Zero out the memory to help with debug */ + memset(drvdata->buf, 0, drvdata->size); + + CS_UNLOCK(drvdata->base); + + writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE); + writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | + TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT | + TMC_FFCR_TRIGON_TRIGIN, + drvdata->base + TMC_FFCR); + + writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG); + tmc_enable_hw(drvdata); + + CS_LOCK(drvdata->base); +} + +static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata) +{ + u32 axictl; + + /* Zero out the memory to help with debug */ + memset(drvdata->vaddr, 0, drvdata->size); + + CS_UNLOCK(drvdata->base); + + writel_relaxed(drvdata->size / 4, drvdata->base + TMC_RSZ); + writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE); + + axictl = readl_relaxed(drvdata->base + TMC_AXICTL); + axictl |= TMC_AXICTL_WR_BURST_LEN; + writel_relaxed(axictl, drvdata->base + TMC_AXICTL); + axictl &= ~TMC_AXICTL_SCT_GAT_MODE; + writel_relaxed(axictl, drvdata->base + TMC_AXICTL); + axictl = (axictl & + ~(TMC_AXICTL_PROT_CTL_B0 | TMC_AXICTL_PROT_CTL_B1)) | + TMC_AXICTL_PROT_CTL_B1; + writel_relaxed(axictl, drvdata->base + TMC_AXICTL); + + writel_relaxed(drvdata->paddr, drvdata->base + TMC_DBALO); + writel_relaxed(0x0, drvdata->base + TMC_DBAHI); + writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | + TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT | + TMC_FFCR_TRIGON_TRIGIN, + drvdata->base + TMC_FFCR); + writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG); + tmc_enable_hw(drvdata); + + CS_LOCK(drvdata->base); +} + +static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +{ + CS_UNLOCK(drvdata->base); + + writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE); + writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI, + drvdata->base + TMC_FFCR); + writel_relaxed(0x0, drvdata->base + TMC_BUFWM); + tmc_enable_hw(drvdata); + + CS_LOCK(drvdata->base); +} + +static int tmc_enable(struct tmc_drvdata *drvdata, enum tmc_mode mode) +{ + int ret; + unsigned long flags; + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + spin_lock_irqsave(&drvdata->spinlock, flags); + if (drvdata->reading) { + spin_unlock_irqrestore(&drvdata->spinlock, flags); + clk_disable_unprepare(drvdata->clk); + return -EBUSY; + } + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETB) { + tmc_etb_enable_hw(drvdata); + } else if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + tmc_etr_enable_hw(drvdata); + } else { + if (mode == TMC_MODE_CIRCULAR_BUFFER) + tmc_etb_enable_hw(drvdata); + else + tmc_etf_enable_hw(drvdata); + } + drvdata->enable = true; + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + dev_info(drvdata->dev, "TMC enabled\n"); + return 0; +} + +static int tmc_enable_sink(struct coresight_device *csdev) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + return tmc_enable(drvdata, TMC_MODE_CIRCULAR_BUFFER); +} + +static int tmc_enable_link(struct coresight_device *csdev, int inport, + int outport) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + return tmc_enable(drvdata, TMC_MODE_HARDWARE_FIFO); +} + +static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata) +{ + enum tmc_mem_intf_width memwidth; + u8 memwords; + char *bufp; + u32 read_data; + int i; + + memwidth = BMVAL(readl_relaxed(drvdata->base + CORESIGHT_DEVID), 8, 10); + if (memwidth == TMC_MEM_INTF_WIDTH_32BITS) + memwords = 1; + else if (memwidth == TMC_MEM_INTF_WIDTH_64BITS) + memwords = 2; + else if (memwidth == TMC_MEM_INTF_WIDTH_128BITS) + memwords = 4; + else + memwords = 8; + + bufp = drvdata->buf; + while (1) { + for (i = 0; i < memwords; i++) { + read_data = readl_relaxed(drvdata->base + TMC_RRD); + if (read_data == 0xFFFFFFFF) + return; + memcpy(bufp, &read_data, 4); + bufp += 4; + } + } +} + +static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) +{ + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + tmc_etb_dump_hw(drvdata); + tmc_disable_hw(drvdata); + + CS_LOCK(drvdata->base); +} + +static void tmc_etr_dump_hw(struct tmc_drvdata *drvdata) +{ + u32 rwp, val; + + rwp = readl_relaxed(drvdata->base + TMC_RWP); + val = readl_relaxed(drvdata->base + TMC_STS); + + /* How much memory do we still have */ + if (val & BIT(0)) + drvdata->buf = drvdata->vaddr + rwp - drvdata->paddr; + else + drvdata->buf = drvdata->vaddr; +} + +static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) +{ + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + tmc_etr_dump_hw(drvdata); + tmc_disable_hw(drvdata); + + CS_LOCK(drvdata->base); +} + +static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) +{ + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + tmc_disable_hw(drvdata); + + CS_LOCK(drvdata->base); +} + +static void tmc_disable(struct tmc_drvdata *drvdata, enum tmc_mode mode) +{ + unsigned long flags; + + spin_lock_irqsave(&drvdata->spinlock, flags); + if (drvdata->reading) + goto out; + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETB) { + tmc_etb_disable_hw(drvdata); + } else if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + tmc_etr_disable_hw(drvdata); + } else { + if (mode == TMC_MODE_CIRCULAR_BUFFER) + tmc_etb_disable_hw(drvdata); + else + tmc_etf_disable_hw(drvdata); + } +out: + drvdata->enable = false; + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + clk_disable_unprepare(drvdata->clk); + + dev_info(drvdata->dev, "TMC disabled\n"); +} + +static void tmc_disable_sink(struct coresight_device *csdev) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + tmc_disable(drvdata, TMC_MODE_CIRCULAR_BUFFER); +} + +static void tmc_disable_link(struct coresight_device *csdev, int inport, + int outport) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + tmc_disable(drvdata, TMC_MODE_HARDWARE_FIFO); +} + +static const struct coresight_ops_sink tmc_sink_ops = { + .enable = tmc_enable_sink, + .disable = tmc_disable_sink, +}; + +static const struct coresight_ops_link tmc_link_ops = { + .enable = tmc_enable_link, + .disable = tmc_disable_link, +}; + +static const struct coresight_ops tmc_etb_cs_ops = { + .sink_ops = &tmc_sink_ops, +}; + +static const struct coresight_ops tmc_etr_cs_ops = { + .sink_ops = &tmc_sink_ops, +}; + +static const struct coresight_ops tmc_etf_cs_ops = { + .sink_ops = &tmc_sink_ops, + .link_ops = &tmc_link_ops, +}; + +static int tmc_read_prepare(struct tmc_drvdata *drvdata) +{ + int ret; + unsigned long flags; + enum tmc_mode mode; + + spin_lock_irqsave(&drvdata->spinlock, flags); + if (!drvdata->enable) + goto out; + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETB) { + tmc_etb_disable_hw(drvdata); + } else if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + tmc_etr_disable_hw(drvdata); + } else { + mode = readl_relaxed(drvdata->base + TMC_MODE); + if (mode == TMC_MODE_CIRCULAR_BUFFER) { + tmc_etb_disable_hw(drvdata); + } else { + ret = -ENODEV; + goto err; + } + } +out: + drvdata->reading = true; + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + dev_info(drvdata->dev, "TMC read start\n"); + return 0; +err: + spin_unlock_irqrestore(&drvdata->spinlock, flags); + return ret; +} + +static void tmc_read_unprepare(struct tmc_drvdata *drvdata) +{ + unsigned long flags; + enum tmc_mode mode; + + spin_lock_irqsave(&drvdata->spinlock, flags); + if (!drvdata->enable) + goto out; + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETB) { + tmc_etb_enable_hw(drvdata); + } else if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + tmc_etr_enable_hw(drvdata); + } else { + mode = readl_relaxed(drvdata->base + TMC_MODE); + if (mode == TMC_MODE_CIRCULAR_BUFFER) + tmc_etb_enable_hw(drvdata); + } +out: + drvdata->reading = false; + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + dev_info(drvdata->dev, "TMC read end\n"); +} + +static int tmc_open(struct inode *inode, struct file *file) +{ + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, miscdev); + int ret = 0; + + if (drvdata->read_count++) + goto out; + + ret = tmc_read_prepare(drvdata); + if (ret) + return ret; +out: + nonseekable_open(inode, file); + + dev_dbg(drvdata->dev, "%s: successfully opened\n", __func__); + return 0; +} + +static ssize_t tmc_read(struct file *file, char __user *data, size_t len, + loff_t *ppos) +{ + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, miscdev); + char *bufp = drvdata->buf + *ppos; + + if (*ppos + len > drvdata->size) + len = drvdata->size - *ppos; + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + if (bufp == (char *)(drvdata->vaddr + drvdata->size)) + bufp = drvdata->vaddr; + else if (bufp > (char *)(drvdata->vaddr + drvdata->size)) + bufp -= drvdata->size; + if ((bufp + len) > (char *)(drvdata->vaddr + drvdata->size)) + len = (char *)(drvdata->vaddr + drvdata->size) - bufp; + } + + if (copy_to_user(data, bufp, len)) { + dev_dbg(drvdata->dev, "%s: copy_to_user failed\n", __func__); + return -EFAULT; + } + + *ppos += len; + + dev_dbg(drvdata->dev, "%s: %zu bytes copied, %d bytes left\n", + __func__, len, (int)(drvdata->size - *ppos)); + return len; +} + +static int tmc_release(struct inode *inode, struct file *file) +{ + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, miscdev); + + if (--drvdata->read_count) { + if (drvdata->read_count < 0) { + dev_err(drvdata->dev, "mismatched close\n"); + drvdata->read_count = 0; + } + goto out; + } + + tmc_read_unprepare(drvdata); +out: + dev_dbg(drvdata->dev, "%s: released\n", __func__); + return 0; +} + +static const struct file_operations tmc_fops = { + .owner = THIS_MODULE, + .open = tmc_open, + .read = tmc_read, + .release = tmc_release, + .llseek = no_llseek, +}; + +static ssize_t status_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + unsigned long flags; + u32 tmc_rsz, tmc_sts, tmc_rrp, tmc_rwp, tmc_trg; + u32 tmc_ctl, tmc_ffsr, tmc_ffcr, tmc_mode, tmc_pscr; + u32 devid; + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + goto out; + + spin_lock_irqsave(&drvdata->spinlock, flags); + CS_UNLOCK(drvdata->base); + + tmc_rsz = readl_relaxed(drvdata->base + TMC_RSZ); + tmc_sts = readl_relaxed(drvdata->base + TMC_STS); + tmc_rrp = readl_relaxed(drvdata->base + TMC_RRP); + tmc_rwp = readl_relaxed(drvdata->base + TMC_RWP); + tmc_trg = readl_relaxed(drvdata->base + TMC_TRG); + tmc_ctl = readl_relaxed(drvdata->base + TMC_CTL); + tmc_ffsr = readl_relaxed(drvdata->base + TMC_FFSR); + tmc_ffcr = readl_relaxed(drvdata->base + TMC_FFCR); + tmc_mode = readl_relaxed(drvdata->base + TMC_MODE); + tmc_pscr = readl_relaxed(drvdata->base + TMC_PSCR); + devid = readl_relaxed(drvdata->base + CORESIGHT_DEVID); + + CS_LOCK(drvdata->base); + spin_unlock_irqrestore(&drvdata->spinlock, flags); + + clk_disable_unprepare(drvdata->clk); + + return sprintf(buf, + "Depth:\t\t0x%x\n" + "Status:\t\t0x%x\n" + "RAM read ptr:\t0x%x\n" + "RAM wrt ptr:\t0x%x\n" + "Trigger cnt:\t0x%x\n" + "Control:\t0x%x\n" + "Flush status:\t0x%x\n" + "Flush ctrl:\t0x%x\n" + "Mode:\t\t0x%x\n" + "PSRC:\t\t0x%x\n" + "DEVID:\t\t0x%x\n", + tmc_rsz, tmc_sts, tmc_rrp, tmc_rwp, tmc_trg, + tmc_ctl, tmc_ffsr, tmc_ffcr, tmc_mode, tmc_pscr, devid); +out: + return -EINVAL; +} +static DEVICE_ATTR_RO(status); + +static ssize_t trigger_cntr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val = drvdata->trigger_cntr; + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t trigger_cntr_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->trigger_cntr = val; + return size; +} +static DEVICE_ATTR_RW(trigger_cntr); + +static struct attribute *coresight_etb_attrs[] = { + &dev_attr_trigger_cntr.attr, + &dev_attr_status.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_etb); + +static struct attribute *coresight_etr_attrs[] = { + &dev_attr_trigger_cntr.attr, + &dev_attr_status.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_etr); + +static struct attribute *coresight_etf_attrs[] = { + &dev_attr_trigger_cntr.attr, + &dev_attr_status.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_etf); + +static int tmc_probe(struct amba_device *adev, const struct amba_id *id) +{ + int ret = 0; + u32 devid; + void __iomem *base; + struct device *dev = &adev->dev; + struct coresight_platform_data *pdata = NULL; + struct tmc_drvdata *drvdata; + struct resource *res = &adev->res; + struct coresight_desc *desc; + struct device_node *np = adev->dev.of_node; + + if (np) { + pdata = of_get_coresight_platform_data(dev, np); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + adev->dev.platform_data = pdata; + } + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + drvdata->dev = &adev->dev; + dev_set_drvdata(dev, drvdata); + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + drvdata->base = base; + + spin_lock_init(&drvdata->spinlock); + + drvdata->clk = adev->pclk; + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + devid = readl_relaxed(drvdata->base + CORESIGHT_DEVID); + drvdata->config_type = BMVAL(devid, 6, 7); + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + if (np) + ret = of_property_read_u32(np, + "arm,buffer-size", + &drvdata->size); + if (ret) + drvdata->size = SZ_1M; + } else { + drvdata->size = readl_relaxed(drvdata->base + TMC_RSZ) * 4; + } + + clk_disable_unprepare(drvdata->clk); + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + drvdata->vaddr = dma_alloc_coherent(dev, drvdata->size, + &drvdata->paddr, GFP_KERNEL); + if (!drvdata->vaddr) + return -ENOMEM; + + memset(drvdata->vaddr, 0, drvdata->size); + drvdata->buf = drvdata->vaddr; + } else { + drvdata->buf = devm_kzalloc(dev, drvdata->size, GFP_KERNEL); + if (!drvdata->buf) + return -ENOMEM; + } + + desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); + if (!desc) { + ret = -ENOMEM; + goto err_devm_kzalloc; + } + + desc->pdata = pdata; + desc->dev = dev; + desc->subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER; + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETB) { + desc->type = CORESIGHT_DEV_TYPE_SINK; + desc->ops = &tmc_etb_cs_ops; + desc->groups = coresight_etb_groups; + } else if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + desc->type = CORESIGHT_DEV_TYPE_SINK; + desc->ops = &tmc_etr_cs_ops; + desc->groups = coresight_etr_groups; + } else { + desc->type = CORESIGHT_DEV_TYPE_LINKSINK; + desc->subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_FIFO; + desc->ops = &tmc_etf_cs_ops; + desc->groups = coresight_etf_groups; + } + + drvdata->csdev = coresight_register(desc); + if (IS_ERR(drvdata->csdev)) { + ret = PTR_ERR(drvdata->csdev); + goto err_devm_kzalloc; + } + + drvdata->miscdev.name = pdata->name; + drvdata->miscdev.minor = MISC_DYNAMIC_MINOR; + drvdata->miscdev.fops = &tmc_fops; + ret = misc_register(&drvdata->miscdev); + if (ret) + goto err_misc_register; + + dev_info(dev, "TMC initialized\n"); + return 0; + +err_misc_register: + coresight_unregister(drvdata->csdev); +err_devm_kzalloc: + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) + dma_free_coherent(dev, drvdata->size, + &drvdata->paddr, GFP_KERNEL); + return ret; +} + +static int tmc_remove(struct amba_device *adev) +{ + struct tmc_drvdata *drvdata = amba_get_drvdata(adev); + + misc_deregister(&drvdata->miscdev); + coresight_unregister(drvdata->csdev); + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) + dma_free_coherent(drvdata->dev, drvdata->size, + &drvdata->paddr, GFP_KERNEL); + + return 0; +} + +static struct amba_id tmc_ids[] = { + { + .id = 0x0003b961, + .mask = 0x0003ffff, + }, + { 0, 0}, +}; + +static struct amba_driver tmc_driver = { + .drv = { + .name = "coresight-tmc", + .owner = THIS_MODULE, + }, + .probe = tmc_probe, + .remove = tmc_remove, + .id_table = tmc_ids, +}; + +module_amba_driver(tmc_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("CoreSight Trace Memory Controller driver"); diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c new file mode 100644 index 000000000000..3b33af2416bb --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -0,0 +1,207 @@ +/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/device.h> +#include <linux/io.h> +#include <linux/err.h> +#include <linux/slab.h> +#include <linux/clk.h> +#include <linux/coresight.h> +#include <linux/amba/bus.h> + +#include "coresight-priv.h" + +#define TPIU_SUPP_PORTSZ 0x000 +#define TPIU_CURR_PORTSZ 0x004 +#define TPIU_SUPP_TRIGMODES 0x100 +#define TPIU_TRIG_CNTRVAL 0x104 +#define TPIU_TRIG_MULT 0x108 +#define TPIU_SUPP_TESTPATM 0x200 +#define TPIU_CURR_TESTPATM 0x204 +#define TPIU_TEST_PATREPCNTR 0x208 +#define TPIU_FFSR 0x300 +#define TPIU_FFCR 0x304 +#define TPIU_FSYNC_CNTR 0x308 +#define TPIU_EXTCTL_INPORT 0x400 +#define TPIU_EXTCTL_OUTPORT 0x404 +#define TPIU_ITTRFLINACK 0xee4 +#define TPIU_ITTRFLIN 0xee8 +#define TPIU_ITATBDATA0 0xeec +#define TPIU_ITATBCTR2 0xef0 +#define TPIU_ITATBCTR1 0xef4 +#define TPIU_ITATBCTR0 0xef8 + +/** register definition **/ +/* FFCR - 0x304 */ +#define FFCR_FON_MAN BIT(6) + +/** + * @base: memory mapped base address for this component. + * @dev: the device entity associated to this component. + * @csdev: component vitals needed by the framework. + * @clk: the clock this component is associated to. + */ +struct tpiu_drvdata { + void __iomem *base; + struct device *dev; + struct coresight_device *csdev; + struct clk *clk; +}; + +static void tpiu_enable_hw(struct tpiu_drvdata *drvdata) +{ + CS_UNLOCK(drvdata->base); + + /* TODO: fill this up */ + + CS_LOCK(drvdata->base); +} + +static int tpiu_enable(struct coresight_device *csdev) +{ + struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + int ret; + + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + tpiu_enable_hw(drvdata); + + dev_info(drvdata->dev, "TPIU enabled\n"); + return 0; +} + +static void tpiu_disable_hw(struct tpiu_drvdata *drvdata) +{ + CS_UNLOCK(drvdata->base); + + /* Clear formatter controle reg. */ + writel_relaxed(0x0, drvdata->base + TPIU_FFCR); + /* Generate manual flush */ + writel_relaxed(FFCR_FON_MAN, drvdata->base + TPIU_FFCR); + + CS_LOCK(drvdata->base); +} + +static void tpiu_disable(struct coresight_device *csdev) +{ + struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + + tpiu_disable_hw(drvdata); + + clk_disable_unprepare(drvdata->clk); + + dev_info(drvdata->dev, "TPIU disabled\n"); +} + +static const struct coresight_ops_sink tpiu_sink_ops = { + .enable = tpiu_enable, + .disable = tpiu_disable, +}; + +static const struct coresight_ops tpiu_cs_ops = { + .sink_ops = &tpiu_sink_ops, +}; + +static int tpiu_probe(struct amba_device *adev, const struct amba_id *id) +{ + int ret; + void __iomem *base; + struct device *dev = &adev->dev; + struct coresight_platform_data *pdata = NULL; + struct tpiu_drvdata *drvdata; + struct resource *res = &adev->res; + struct coresight_desc *desc; + struct device_node *np = adev->dev.of_node; + + if (np) { + pdata = of_get_coresight_platform_data(dev, np); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + adev->dev.platform_data = pdata; + } + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + drvdata->dev = &adev->dev; + dev_set_drvdata(dev, drvdata); + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + drvdata->base = base; + + drvdata->clk = adev->pclk; + ret = clk_prepare_enable(drvdata->clk); + if (ret) + return ret; + + /* Disable tpiu to support older devices */ + tpiu_disable_hw(drvdata); + + clk_disable_unprepare(drvdata->clk); + + desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + + desc->type = CORESIGHT_DEV_TYPE_SINK; + desc->subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PORT; + desc->ops = &tpiu_cs_ops; + desc->pdata = pdata; + desc->dev = dev; + drvdata->csdev = coresight_register(desc); + if (IS_ERR(drvdata->csdev)) + return PTR_ERR(drvdata->csdev); + + dev_info(dev, "TPIU initialized\n"); + return 0; +} + +static int tpiu_remove(struct amba_device *adev) +{ + struct tpiu_drvdata *drvdata = amba_get_drvdata(adev); + + coresight_unregister(drvdata->csdev); + return 0; +} + +static struct amba_id tpiu_ids[] = { + { + .id = 0x0003b912, + .mask = 0x0003ffff, + }, + { 0, 0}, +}; + +static struct amba_driver tpiu_driver = { + .drv = { + .name = "coresight-tpiu", + .owner = THIS_MODULE, + }, + .probe = tpiu_probe, + .remove = tpiu_remove, + .id_table = tpiu_ids, +}; + +module_amba_driver(tpiu_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("CoreSight Trace Port Interface Unit driver"); diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c new file mode 100644 index 000000000000..894531d315b8 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight.c @@ -0,0 +1,720 @@ +/* Copyright (c) 2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/io.h> +#include <linux/err.h> +#include <linux/export.h> +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/clk.h> +#include <linux/coresight.h> +#include <linux/of_platform.h> +#include <linux/delay.h> + +#include "coresight-priv.h" + +static DEFINE_MUTEX(coresight_mutex); + +static int coresight_id_match(struct device *dev, void *data) +{ + int trace_id, i_trace_id; + struct coresight_device *csdev, *i_csdev; + + csdev = data; + i_csdev = to_coresight_device(dev); + + /* + * No need to care about oneself and components that are not + * sources or not enabled + */ + if (i_csdev == csdev || !i_csdev->enable || + i_csdev->type != CORESIGHT_DEV_TYPE_SOURCE) + return 0; + + /* Get the source ID for both compoment */ + trace_id = source_ops(csdev)->trace_id(csdev); + i_trace_id = source_ops(i_csdev)->trace_id(i_csdev); + + /* All you need is one */ + if (trace_id == i_trace_id) + return 1; + + return 0; +} + +static int coresight_source_is_unique(struct coresight_device *csdev) +{ + int trace_id = source_ops(csdev)->trace_id(csdev); + + /* this shouldn't happen */ + if (trace_id < 0) + return 0; + + return !bus_for_each_dev(&coresight_bustype, NULL, + csdev, coresight_id_match); +} + +static int coresight_find_link_inport(struct coresight_device *csdev) +{ + int i; + struct coresight_device *parent; + struct coresight_connection *conn; + + parent = container_of(csdev->path_link.next, + struct coresight_device, path_link); + + for (i = 0; i < parent->nr_outport; i++) { + conn = &parent->conns[i]; + if (conn->child_dev == csdev) + return conn->child_port; + } + + dev_err(&csdev->dev, "couldn't find inport, parent: %s, child: %s\n", + dev_name(&parent->dev), dev_name(&csdev->dev)); + + return 0; +} + +static int coresight_find_link_outport(struct coresight_device *csdev) +{ + int i; + struct coresight_device *child; + struct coresight_connection *conn; + + child = container_of(csdev->path_link.prev, + struct coresight_device, path_link); + + for (i = 0; i < csdev->nr_outport; i++) { + conn = &csdev->conns[i]; + if (conn->child_dev == child) + return conn->outport; + } + + dev_err(&csdev->dev, "couldn't find outport, parent: %s, child: %s\n", + dev_name(&csdev->dev), dev_name(&child->dev)); + + return 0; +} + +static int coresight_enable_sink(struct coresight_device *csdev) +{ + int ret; + + if (!csdev->enable) { + if (sink_ops(csdev)->enable) { + ret = sink_ops(csdev)->enable(csdev); + if (ret) + return ret; + } + csdev->enable = true; + } + + atomic_inc(csdev->refcnt); + + return 0; +} + +static void coresight_disable_sink(struct coresight_device *csdev) +{ + if (atomic_dec_return(csdev->refcnt) == 0) { + if (sink_ops(csdev)->disable) { + sink_ops(csdev)->disable(csdev); + csdev->enable = false; + } + } +} + +static int coresight_enable_link(struct coresight_device *csdev) +{ + int ret; + int link_subtype; + int refport, inport, outport; + + inport = coresight_find_link_inport(csdev); + outport = coresight_find_link_outport(csdev); + link_subtype = csdev->subtype.link_subtype; + + if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_MERG) + refport = inport; + else if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_SPLIT) + refport = outport; + else + refport = 0; + + if (atomic_inc_return(&csdev->refcnt[refport]) == 1) { + if (link_ops(csdev)->enable) { + ret = link_ops(csdev)->enable(csdev, inport, outport); + if (ret) + return ret; + } + } + + csdev->enable = true; + + return 0; +} + +static void coresight_disable_link(struct coresight_device *csdev) +{ + int i, nr_conns; + int link_subtype; + int refport, inport, outport; + + inport = coresight_find_link_inport(csdev); + outport = coresight_find_link_outport(csdev); + link_subtype = csdev->subtype.link_subtype; + + if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_MERG) { + refport = inport; + nr_conns = csdev->nr_inport; + } else if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_SPLIT) { + refport = outport; + nr_conns = csdev->nr_outport; + } else { + refport = 0; + nr_conns = 1; + } + + if (atomic_dec_return(&csdev->refcnt[refport]) == 0) { + if (link_ops(csdev)->disable) + link_ops(csdev)->disable(csdev, inport, outport); + } + + for (i = 0; i < nr_conns; i++) + if (atomic_read(&csdev->refcnt[i]) != 0) + return; + + csdev->enable = false; +} + +static int coresight_enable_source(struct coresight_device *csdev) +{ + int ret; + + if (!coresight_source_is_unique(csdev)) { + dev_warn(&csdev->dev, "traceID %d not unique\n", + source_ops(csdev)->trace_id(csdev)); + return -EINVAL; + } + + if (!csdev->enable) { + if (source_ops(csdev)->enable) { + ret = source_ops(csdev)->enable(csdev); + if (ret) + return ret; + } + csdev->enable = true; + } + + atomic_inc(csdev->refcnt); + + return 0; +} + +static void coresight_disable_source(struct coresight_device *csdev) +{ + if (atomic_dec_return(csdev->refcnt) == 0) { + if (source_ops(csdev)->disable) { + source_ops(csdev)->disable(csdev); + csdev->enable = false; + } + } +} + +static int coresight_enable_path(struct list_head *path) +{ + int ret = 0; + struct coresight_device *cd; + + list_for_each_entry(cd, path, path_link) { + if (cd == list_first_entry(path, struct coresight_device, + path_link)) { + ret = coresight_enable_sink(cd); + } else if (list_is_last(&cd->path_link, path)) { + /* + * Don't enable the source just yet - this needs to + * happen at the very end when all links and sink + * along the path have been configured properly. + */ + ; + } else { + ret = coresight_enable_link(cd); + } + if (ret) + goto err; + } + + return 0; +err: + list_for_each_entry_continue_reverse(cd, path, path_link) { + if (cd == list_first_entry(path, struct coresight_device, + path_link)) { + coresight_disable_sink(cd); + } else if (list_is_last(&cd->path_link, path)) { + ; + } else { + coresight_disable_link(cd); + } + } + + return ret; +} + +static int coresight_disable_path(struct list_head *path) +{ + struct coresight_device *cd; + + list_for_each_entry_reverse(cd, path, path_link) { + if (cd == list_first_entry(path, struct coresight_device, + path_link)) { + coresight_disable_sink(cd); + } else if (list_is_last(&cd->path_link, path)) { + /* + * The source has already been stopped, no need + * to do it again here. + */ + ; + } else { + coresight_disable_link(cd); + } + } + + return 0; +} + +static int coresight_build_paths(struct coresight_device *csdev, + struct list_head *path, + bool enable) +{ + int i, ret = -EINVAL; + struct coresight_connection *conn; + + list_add(&csdev->path_link, path); + + if ((csdev->type == CORESIGHT_DEV_TYPE_SINK || + csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) && + csdev->activated) { + if (enable) + ret = coresight_enable_path(path); + else + ret = coresight_disable_path(path); + } else { + for (i = 0; i < csdev->nr_outport; i++) { + conn = &csdev->conns[i]; + if (coresight_build_paths(conn->child_dev, + path, enable) == 0) + ret = 0; + } + } + + if (list_first_entry(path, struct coresight_device, path_link) != csdev) + dev_err(&csdev->dev, "wrong device in %s\n", __func__); + + list_del(&csdev->path_link); + + return ret; +} + +int coresight_enable(struct coresight_device *csdev) +{ + int ret = 0; + LIST_HEAD(path); + + mutex_lock(&coresight_mutex); + if (csdev->type != CORESIGHT_DEV_TYPE_SOURCE) { + ret = -EINVAL; + dev_err(&csdev->dev, "wrong device type in %s\n", __func__); + goto out; + } + if (csdev->enable) + goto out; + + if (coresight_build_paths(csdev, &path, true)) { + dev_err(&csdev->dev, "building path(s) failed\n"); + goto out; + } + + if (coresight_enable_source(csdev)) + dev_err(&csdev->dev, "source enable failed\n"); +out: + mutex_unlock(&coresight_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(coresight_enable); + +void coresight_disable(struct coresight_device *csdev) +{ + LIST_HEAD(path); + + mutex_lock(&coresight_mutex); + if (csdev->type != CORESIGHT_DEV_TYPE_SOURCE) { + dev_err(&csdev->dev, "wrong device type in %s\n", __func__); + goto out; + } + if (!csdev->enable) + goto out; + + coresight_disable_source(csdev); + if (coresight_build_paths(csdev, &path, false)) + dev_err(&csdev->dev, "releasing path(s) failed\n"); + +out: + mutex_unlock(&coresight_mutex); +} +EXPORT_SYMBOL_GPL(coresight_disable); + +static ssize_t enable_sink_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct coresight_device *csdev = to_coresight_device(dev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", (unsigned)csdev->activated); +} + +static ssize_t enable_sink_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct coresight_device *csdev = to_coresight_device(dev); + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + + if (val) + csdev->activated = true; + else + csdev->activated = false; + + return size; + +} +static DEVICE_ATTR_RW(enable_sink); + +static ssize_t enable_source_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct coresight_device *csdev = to_coresight_device(dev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", (unsigned)csdev->enable); +} + +static ssize_t enable_source_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret = 0; + unsigned long val; + struct coresight_device *csdev = to_coresight_device(dev); + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + + if (val) { + ret = coresight_enable(csdev); + if (ret) + return ret; + } else { + coresight_disable(csdev); + } + + return size; +} +static DEVICE_ATTR_RW(enable_source); + +static struct attribute *coresight_sink_attrs[] = { + &dev_attr_enable_sink.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_sink); + +static struct attribute *coresight_source_attrs[] = { + &dev_attr_enable_source.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_source); + +static struct device_type coresight_dev_type[] = { + { + .name = "none", + }, + { + .name = "sink", + .groups = coresight_sink_groups, + }, + { + .name = "link", + }, + { + .name = "linksink", + .groups = coresight_sink_groups, + }, + { + .name = "source", + .groups = coresight_source_groups, + }, +}; + +static void coresight_device_release(struct device *dev) +{ + struct coresight_device *csdev = to_coresight_device(dev); + + kfree(csdev); +} + +static int coresight_orphan_match(struct device *dev, void *data) +{ + int i; + bool still_orphan = false; + struct coresight_device *csdev, *i_csdev; + struct coresight_connection *conn; + + csdev = data; + i_csdev = to_coresight_device(dev); + + /* No need to check oneself */ + if (csdev == i_csdev) + return 0; + + /* Move on to another component if no connection is orphan */ + if (!i_csdev->orphan) + return 0; + /* + * Circle throuch all the connection of that component. If we find + * an orphan connection whose name matches @csdev, link it. + */ + for (i = 0; i < i_csdev->nr_outport; i++) { + conn = &i_csdev->conns[i]; + + /* We have found at least one orphan connection */ + if (conn->child_dev == NULL) { + /* Does it match this newly added device? */ + if (!strcmp(dev_name(&csdev->dev), conn->child_name)) { + conn->child_dev = csdev; + } else { + /* This component still has an orphan */ + still_orphan = true; + } + } + } + + i_csdev->orphan = still_orphan; + + /* + * Returning '0' ensures that all known component on the + * bus will be checked. + */ + return 0; +} + +static void coresight_fixup_orphan_conns(struct coresight_device *csdev) +{ + /* + * No need to check for a return value as orphan connection(s) + * are hooked-up with each newly added component. + */ + bus_for_each_dev(&coresight_bustype, NULL, + csdev, coresight_orphan_match); +} + + +static int coresight_name_match(struct device *dev, void *data) +{ + char *to_match; + struct coresight_device *i_csdev; + + to_match = data; + i_csdev = to_coresight_device(dev); + + if (!strcmp(to_match, dev_name(&i_csdev->dev))) + return 1; + + return 0; +} + +static void coresight_fixup_device_conns(struct coresight_device *csdev) +{ + int i; + struct device *dev = NULL; + struct coresight_connection *conn; + + for (i = 0; i < csdev->nr_outport; i++) { + conn = &csdev->conns[i]; + dev = bus_find_device(&coresight_bustype, NULL, + (void *)conn->child_name, + coresight_name_match); + + if (dev) { + conn->child_dev = to_coresight_device(dev); + } else { + csdev->orphan = true; + conn->child_dev = NULL; + } + } +} + +/** + * coresight_timeout - loop until a bit has changed to a specific state. + * @addr: base address of the area of interest. + * @offset: address of a register, starting from @addr. + * @position: the position of the bit of interest. + * @value: the value the bit should have. + * + * Return: 0 as soon as the bit has taken the desired state or -EAGAIN if + * TIMEOUT_US has elapsed, which ever happens first. + */ + +int coresight_timeout(void __iomem *addr, u32 offset, int position, int value) +{ + int i; + u32 val; + + for (i = TIMEOUT_US; i > 0; i--) { + val = __raw_readl(addr + offset); + /* waiting on the bit to go from 0 to 1 */ + if (value) { + if (val & BIT(position)) + return 0; + /* waiting on the bit to go from 1 to 0 */ + } else { + if (!(val & BIT(position))) + return 0; + } + + /* + * Delay is arbitrary - the specification doesn't say how long + * we are expected to wait. Extra check required to make sure + * we don't wait needlessly on the last iteration. + */ + if (i - 1) + udelay(1); + } + + return -EAGAIN; +} + +struct bus_type coresight_bustype = { + .name = "coresight", +}; + +static int __init coresight_init(void) +{ + return bus_register(&coresight_bustype); +} +postcore_initcall(coresight_init); + +struct coresight_device *coresight_register(struct coresight_desc *desc) +{ + int i; + int ret; + int link_subtype; + int nr_refcnts = 1; + atomic_t *refcnts = NULL; + struct coresight_device *csdev; + struct coresight_connection *conns; + + csdev = kzalloc(sizeof(*csdev), GFP_KERNEL); + if (!csdev) { + ret = -ENOMEM; + goto err_kzalloc_csdev; + } + + if (desc->type == CORESIGHT_DEV_TYPE_LINK || + desc->type == CORESIGHT_DEV_TYPE_LINKSINK) { + link_subtype = desc->subtype.link_subtype; + + if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_MERG) + nr_refcnts = desc->pdata->nr_inport; + else if (link_subtype == CORESIGHT_DEV_SUBTYPE_LINK_SPLIT) + nr_refcnts = desc->pdata->nr_outport; + } + + refcnts = kcalloc(nr_refcnts, sizeof(*refcnts), GFP_KERNEL); + if (!refcnts) { + ret = -ENOMEM; + goto err_kzalloc_refcnts; + } + + csdev->refcnt = refcnts; + + csdev->nr_inport = desc->pdata->nr_inport; + csdev->nr_outport = desc->pdata->nr_outport; + conns = kcalloc(csdev->nr_outport, sizeof(*conns), GFP_KERNEL); + if (!conns) { + ret = -ENOMEM; + goto err_kzalloc_conns; + } + + for (i = 0; i < csdev->nr_outport; i++) { + conns[i].outport = desc->pdata->outports[i]; + conns[i].child_name = desc->pdata->child_names[i]; + conns[i].child_port = desc->pdata->child_ports[i]; + } + + csdev->conns = conns; + + csdev->type = desc->type; + csdev->subtype = desc->subtype; + csdev->ops = desc->ops; + csdev->orphan = false; + + csdev->dev.type = &coresight_dev_type[desc->type]; + csdev->dev.groups = desc->groups; + csdev->dev.parent = desc->dev; + csdev->dev.release = coresight_device_release; + csdev->dev.bus = &coresight_bustype; + dev_set_name(&csdev->dev, "%s", desc->pdata->name); + + ret = device_register(&csdev->dev); + if (ret) + goto err_device_register; + + mutex_lock(&coresight_mutex); + + coresight_fixup_device_conns(csdev); + coresight_fixup_orphan_conns(csdev); + + mutex_unlock(&coresight_mutex); + + return csdev; + +err_device_register: + kfree(conns); +err_kzalloc_conns: + kfree(refcnts); +err_kzalloc_refcnts: + kfree(csdev); +err_kzalloc_csdev: + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(coresight_register); + +void coresight_unregister(struct coresight_device *csdev) +{ + mutex_lock(&coresight_mutex); + + kfree(csdev->conns); + device_unregister(&csdev->dev); + + mutex_unlock(&coresight_mutex); +} +EXPORT_SYMBOL_GPL(coresight_unregister); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/hwtracing/coresight/of_coresight.c b/drivers/hwtracing/coresight/of_coresight.c new file mode 100644 index 000000000000..f3cc8e97a0f8 --- /dev/null +++ b/drivers/hwtracing/coresight/of_coresight.c @@ -0,0 +1,200 @@ +/* Copyright (c) 2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/module.h> +#include <linux/types.h> +#include <linux/err.h> +#include <linux/slab.h> +#include <linux/clk.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_graph.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/amba/bus.h> +#include <linux/coresight.h> +#include <linux/cpumask.h> +#include <asm/smp_plat.h> + + +static int of_dev_node_match(struct device *dev, void *data) +{ + return dev->of_node == data; +} + +static struct device * +of_coresight_get_endpoint_device(struct device_node *endpoint) +{ + struct device *dev = NULL; + + /* + * If we have a non-configuable replicator, it will be found on the + * platform bus. + */ + dev = bus_find_device(&platform_bus_type, NULL, + endpoint, of_dev_node_match); + if (dev) + return dev; + + /* + * We have a configurable component - circle through the AMBA bus + * looking for the device that matches the endpoint node. + */ + return bus_find_device(&amba_bustype, NULL, + endpoint, of_dev_node_match); +} + +static struct device_node *of_get_coresight_endpoint( + const struct device_node *parent, struct device_node *prev) +{ + struct device_node *node = of_graph_get_next_endpoint(parent, prev); + + of_node_put(prev); + return node; +} + +static void of_coresight_get_ports(struct device_node *node, + int *nr_inport, int *nr_outport) +{ + struct device_node *ep = NULL; + int in = 0, out = 0; + + do { + ep = of_get_coresight_endpoint(node, ep); + if (!ep) + break; + + if (of_property_read_bool(ep, "slave-mode")) + in++; + else + out++; + + } while (ep); + + *nr_inport = in; + *nr_outport = out; +} + +static int of_coresight_alloc_memory(struct device *dev, + struct coresight_platform_data *pdata) +{ + /* List of output port on this component */ + pdata->outports = devm_kzalloc(dev, pdata->nr_outport * + sizeof(*pdata->outports), + GFP_KERNEL); + if (!pdata->outports) + return -ENOMEM; + + /* Children connected to this component via @outports */ + pdata->child_names = devm_kzalloc(dev, pdata->nr_outport * + sizeof(*pdata->child_names), + GFP_KERNEL); + if (!pdata->child_names) + return -ENOMEM; + + /* Port number on the child this component is connected to */ + pdata->child_ports = devm_kzalloc(dev, pdata->nr_outport * + sizeof(*pdata->child_ports), + GFP_KERNEL); + if (!pdata->child_ports) + return -ENOMEM; + + return 0; +} + +struct coresight_platform_data *of_get_coresight_platform_data( + struct device *dev, struct device_node *node) +{ + int i = 0, ret = 0, cpu; + struct coresight_platform_data *pdata; + struct of_endpoint endpoint, rendpoint; + struct device *rdev; + struct device_node *dn; + struct device_node *ep = NULL; + struct device_node *rparent = NULL; + struct device_node *rport = NULL; + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return ERR_PTR(-ENOMEM); + + /* Use device name as sysfs handle */ + pdata->name = dev_name(dev); + + /* Get the number of input and output port for this component */ + of_coresight_get_ports(node, &pdata->nr_inport, &pdata->nr_outport); + + if (pdata->nr_outport) { + ret = of_coresight_alloc_memory(dev, pdata); + if (ret) + return ERR_PTR(ret); + + /* Iterate through each port to discover topology */ + do { + /* Get a handle on a port */ + ep = of_get_coresight_endpoint(node, ep); + if (!ep) + break; + + /* + * No need to deal with input ports, processing for as + * processing for output ports will deal with them. + */ + if (of_find_property(ep, "slave-mode", NULL)) + continue; + + /* Get a handle on the local endpoint */ + ret = of_graph_parse_endpoint(ep, &endpoint); + + if (ret) + continue; + + /* The local out port number */ + pdata->outports[i] = endpoint.id; + + /* + * Get a handle on the remote port and parent + * attached to it. + */ + rparent = of_graph_get_remote_port_parent(ep); + rport = of_graph_get_remote_port(ep); + + if (!rparent || !rport) + continue; + + if (of_graph_parse_endpoint(rport, &rendpoint)) + continue; + + rdev = of_coresight_get_endpoint_device(rparent); + if (!rdev) + continue; + + pdata->child_names[i] = dev_name(rdev); + pdata->child_ports[i] = rendpoint.id; + + i++; + } while (ep); + } + + /* Affinity defaults to CPU0 */ + pdata->cpu = 0; + dn = of_parse_phandle(node, "cpu", 0); + for (cpu = 0; dn && cpu < nr_cpu_ids; cpu++) { + if (dn == of_get_cpu_node(cpu, NULL)) { + pdata->cpu = cpu; + break; + } + } + + return pdata; +} +EXPORT_SYMBOL_GPL(of_get_coresight_platform_data); diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index dd5112265cc9..6d13f962f156 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -15,7 +15,7 @@ if IOMMU_SUPPORT config OF_IOMMU def_bool y - depends on OF + depends on OF && IOMMU_API config FSL_PAMU bool "Freescale IOMMU support" diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 74a1767c89b5..2c3f5ad01098 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -56,19 +56,13 @@ static int do_setup_msi_irqs(struct pci_dev *dev, int nvec) unsigned int irq; struct msi_desc *msidesc; - WARN_ON(!list_is_singular(&dev->msi_list)); msidesc = list_entry(dev->msi_list.next, struct msi_desc, list); - WARN_ON(msidesc->irq); - WARN_ON(msidesc->msi_attrib.multiple); - WARN_ON(msidesc->nvec_used); irq = irq_alloc_hwirqs(nvec, dev_to_node(&dev->dev)); if (irq == 0) return -ENOSPC; nvec_pow2 = __roundup_pow_of_two(nvec); - msidesc->nvec_used = nvec; - msidesc->msi_attrib.multiple = ilog2(nvec_pow2); for (sub_handle = 0; sub_handle < nvec; sub_handle++) { if (!sub_handle) { index = msi_alloc_remapped_irq(dev, irq, nvec_pow2); @@ -96,8 +90,6 @@ error: * IRQs from tearing down again in default_teardown_msi_irqs() */ msidesc->irq = 0; - msidesc->nvec_used = 0; - msidesc->msi_attrib.multiple = 0; return ret; } diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index e550ccb7634e..43429ab62228 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -18,9 +18,14 @@ */ #include <linux/export.h> +#include <linux/iommu.h> #include <linux/limits.h> #include <linux/of.h> #include <linux/of_iommu.h> +#include <linux/slab.h> + +static const struct of_device_id __iommu_of_table_sentinel + __used __section(__iommu_of_table_end); /** * of_get_dma_window - Parse *dma-window property and returns 0 if found. @@ -89,3 +94,93 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index, return 0; } EXPORT_SYMBOL_GPL(of_get_dma_window); + +struct of_iommu_node { + struct list_head list; + struct device_node *np; + struct iommu_ops *ops; +}; +static LIST_HEAD(of_iommu_list); +static DEFINE_SPINLOCK(of_iommu_lock); + +void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops) +{ + struct of_iommu_node *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); + + if (WARN_ON(!iommu)) + return; + + INIT_LIST_HEAD(&iommu->list); + iommu->np = np; + iommu->ops = ops; + spin_lock(&of_iommu_lock); + list_add_tail(&iommu->list, &of_iommu_list); + spin_unlock(&of_iommu_lock); +} + +struct iommu_ops *of_iommu_get_ops(struct device_node *np) +{ + struct of_iommu_node *node; + struct iommu_ops *ops = NULL; + + spin_lock(&of_iommu_lock); + list_for_each_entry(node, &of_iommu_list, list) + if (node->np == np) { + ops = node->ops; + break; + } + spin_unlock(&of_iommu_lock); + return ops; +} + +struct iommu_ops *of_iommu_configure(struct device *dev, + struct device_node *master_np) +{ + struct of_phandle_args iommu_spec; + struct device_node *np; + struct iommu_ops *ops = NULL; + int idx = 0; + + if (dev_is_pci(dev)) { + dev_err(dev, "IOMMU is currently not supported for PCI\n"); + return NULL; + } + + /* + * We don't currently walk up the tree looking for a parent IOMMU. + * See the `Notes:' section of + * Documentation/devicetree/bindings/iommu/iommu.txt + */ + while (!of_parse_phandle_with_args(master_np, "iommus", + "#iommu-cells", idx, + &iommu_spec)) { + np = iommu_spec.np; + ops = of_iommu_get_ops(np); + + if (!ops || !ops->of_xlate || ops->of_xlate(dev, &iommu_spec)) + goto err_put_node; + + of_node_put(np); + idx++; + } + + return ops; + +err_put_node: + of_node_put(np); + return NULL; +} + +void __init of_iommu_init(void) +{ + struct device_node *np; + const struct of_device_id *match, *matches = &__iommu_of_table; + + for_each_matching_node_and_match(np, matches, &match) { + const of_iommu_init_fn init_fn = match->data; + + if (init_fn(np)) + pr_err("Failed to initialise IOMMU %s\n", + of_node_full_name(np)); + } +} diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index b21f12f1766d..e72e23960632 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -5,8 +5,15 @@ config IRQCHIP config ARM_GIC bool select IRQ_DOMAIN + select IRQ_DOMAIN_HIERARCHY select MULTI_IRQ_HANDLER +config ARM_GIC_V2M + bool + depends on ARM_GIC + depends on PCI && PCI_MSI + select PCI_MSI_IRQ_DOMAIN + config GIC_NON_BANKED bool @@ -14,6 +21,11 @@ config ARM_GIC_V3 bool select IRQ_DOMAIN select MULTI_IRQ_HANDLER + select IRQ_DOMAIN_HIERARCHY + +config ARM_GIC_V3_ITS + bool + select PCI_MSI_IRQ_DOMAIN config ARM_NVIC bool diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 173bb5fa2cc9..a64d861b4031 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -19,7 +19,9 @@ obj-$(CONFIG_ARCH_SUNXI) += irq-sun4i.o obj-$(CONFIG_ARCH_SUNXI) += irq-sunxi-nmi.o obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o +obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-common.o +obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v3-its-pci-msi.o obj-$(CONFIG_ARM_NVIC) += irq-nvic.o obj-$(CONFIG_ARM_VIC) += irq-vic.o obj-$(CONFIG_ATMEL_AIC_IRQ) += irq-atmel-aic-common.o irq-atmel-aic.o @@ -38,3 +40,4 @@ obj-$(CONFIG_IRQ_CROSSBAR) += irq-crossbar.o obj-$(CONFIG_BRCMSTB_L2_IRQ) += irq-brcmstb-l2.o \ irq-bcm7120-l2.o obj-$(CONFIG_KEYSTONE_IRQ) += irq-keystone.o +obj-$(CONFIG_ARCH_MEDIATEK) += irq-mtk-sysirq.o diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index 41ac85af043e..615075db5fcf 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -131,7 +131,7 @@ static void armada_370_xp_free_msi(int hwirq) mutex_unlock(&msi_used_lock); } -static int armada_370_xp_setup_msi_irq(struct msi_chip *chip, +static int armada_370_xp_setup_msi_irq(struct msi_controller *chip, struct pci_dev *pdev, struct msi_desc *desc) { @@ -158,11 +158,11 @@ static int armada_370_xp_setup_msi_irq(struct msi_chip *chip, msg.address_hi = 0; msg.data = 0xf00 | (hwirq + 16); - write_msi_msg(virq, &msg); + pci_write_msi_msg(virq, &msg); return 0; } -static void armada_370_xp_teardown_msi_irq(struct msi_chip *chip, +static void armada_370_xp_teardown_msi_irq(struct msi_controller *chip, unsigned int irq) { struct irq_data *d = irq_get_irq_data(irq); @@ -174,10 +174,10 @@ static void armada_370_xp_teardown_msi_irq(struct msi_chip *chip, static struct irq_chip armada_370_xp_msi_irq_chip = { .name = "armada_370_xp_msi_irq", - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, }; static int armada_370_xp_msi_map(struct irq_domain *domain, unsigned int virq, @@ -197,7 +197,7 @@ static const struct irq_domain_ops armada_370_xp_msi_irq_ops = { static int armada_370_xp_msi_init(struct device_node *node, phys_addr_t main_int_phys_base) { - struct msi_chip *msi_chip; + struct msi_controller *msi_chip; u32 reg; int ret; diff --git a/drivers/irqchip/irq-gic-common.c b/drivers/irqchip/irq-gic-common.c index 61541ff24397..ad96ebb0c7ab 100644 --- a/drivers/irqchip/irq-gic-common.c +++ b/drivers/irqchip/irq-gic-common.c @@ -21,7 +21,7 @@ #include "irq-gic-common.h" -void gic_configure_irq(unsigned int irq, unsigned int type, +int gic_configure_irq(unsigned int irq, unsigned int type, void __iomem *base, void (*sync_access)(void)) { u32 enablemask = 1 << (irq % 32); @@ -29,16 +29,17 @@ void gic_configure_irq(unsigned int irq, unsigned int type, u32 confmask = 0x2 << ((irq % 16) * 2); u32 confoff = (irq / 16) * 4; bool enabled = false; - u32 val; + u32 val, oldval; + int ret = 0; /* * Read current configuration register, and insert the config * for "irq", depending on "type". */ - val = readl_relaxed(base + GIC_DIST_CONFIG + confoff); - if (type == IRQ_TYPE_LEVEL_HIGH) + val = oldval = readl_relaxed(base + GIC_DIST_CONFIG + confoff); + if (type & IRQ_TYPE_LEVEL_MASK) val &= ~confmask; - else if (type == IRQ_TYPE_EDGE_RISING) + else if (type & IRQ_TYPE_EDGE_BOTH) val |= confmask; /* @@ -54,15 +55,20 @@ void gic_configure_irq(unsigned int irq, unsigned int type, /* * Write back the new configuration, and possibly re-enable - * the interrupt. + * the interrupt. If we tried to write a new configuration and failed, + * return an error. */ writel_relaxed(val, base + GIC_DIST_CONFIG + confoff); + if (readl_relaxed(base + GIC_DIST_CONFIG + confoff) != val && val != oldval) + ret = -EINVAL; if (enabled) writel_relaxed(enablemask, base + GIC_DIST_ENABLE_SET + enableoff); if (sync_access) sync_access(); + + return ret; } void __init gic_dist_config(void __iomem *base, int gic_irqs, diff --git a/drivers/irqchip/irq-gic-common.h b/drivers/irqchip/irq-gic-common.h index b41f02481c3a..35a9884778bd 100644 --- a/drivers/irqchip/irq-gic-common.h +++ b/drivers/irqchip/irq-gic-common.h @@ -20,7 +20,7 @@ #include <linux/of.h> #include <linux/irqdomain.h> -void gic_configure_irq(unsigned int irq, unsigned int type, +int gic_configure_irq(unsigned int irq, unsigned int type, void __iomem *base, void (*sync_access)(void)); void gic_dist_config(void __iomem *base, int gic_irqs, void (*sync_access)(void)); diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c new file mode 100644 index 000000000000..ec9c37674a0b --- /dev/null +++ b/drivers/irqchip/irq-gic-v2m.c @@ -0,0 +1,326 @@ +/* + * ARM GIC v2m MSI(-X) support + * Support for Message Signaled Interrupts for systems that + * implement ARM Generic Interrupt Controller: GICv2m. + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + * Authors: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> + * Harish Kasiviswanathan <harish.kasiviswanathan@amd.com> + * Brandon Anderson <brandon.anderson@amd.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#define pr_fmt(fmt) "GICv2m: " fmt + +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +/* +* MSI_TYPER: +* [31:26] Reserved +* [25:16] lowest SPI assigned to MSI +* [15:10] Reserved +* [9:0] Numer of SPIs assigned to MSI +*/ +#define V2M_MSI_TYPER 0x008 +#define V2M_MSI_TYPER_BASE_SHIFT 16 +#define V2M_MSI_TYPER_BASE_MASK 0x3FF +#define V2M_MSI_TYPER_NUM_MASK 0x3FF +#define V2M_MSI_SETSPI_NS 0x040 +#define V2M_MIN_SPI 32 +#define V2M_MAX_SPI 1019 + +#define V2M_MSI_TYPER_BASE_SPI(x) \ + (((x) >> V2M_MSI_TYPER_BASE_SHIFT) & V2M_MSI_TYPER_BASE_MASK) + +#define V2M_MSI_TYPER_NUM_SPI(x) ((x) & V2M_MSI_TYPER_NUM_MASK) + +struct v2m_data { + spinlock_t msi_cnt_lock; + struct resource res; /* GICv2m resource */ + void __iomem *base; /* GICv2m virt address */ + u32 spi_start; /* The SPI number that MSIs start */ + u32 nr_spis; /* The number of SPIs for MSIs */ + unsigned long *bm; /* MSI vector bitmap */ + struct irq_domain *domain; +}; + +static void gicv2m_mask_msi_irq(struct irq_data *d) +{ + pci_msi_mask_irq(d); + irq_chip_mask_parent(d); +} + +static void gicv2m_unmask_msi_irq(struct irq_data *d) +{ + pci_msi_unmask_irq(d); + irq_chip_unmask_parent(d); +} + +static struct irq_chip gicv2m_msi_irq_chip = { + .name = "MSI", + .irq_mask = gicv2m_mask_msi_irq, + .irq_unmask = gicv2m_unmask_msi_irq, + .irq_eoi = irq_chip_eoi_parent, + .irq_write_msi_msg = pci_msi_domain_write_msg, +}; + +static struct msi_domain_info gicv2m_msi_domain_info = { + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSIX), + .chip = &gicv2m_msi_irq_chip, +}; + +static int gicv2m_set_affinity(struct irq_data *irq_data, + const struct cpumask *mask, bool force) +{ + int ret; + + ret = irq_chip_set_affinity_parent(irq_data, mask, force); + if (ret == IRQ_SET_MASK_OK) + ret = IRQ_SET_MASK_OK_DONE; + + return ret; +} + +static void gicv2m_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct v2m_data *v2m = irq_data_get_irq_chip_data(data); + phys_addr_t addr = v2m->res.start + V2M_MSI_SETSPI_NS; + + msg->address_hi = (u32) (addr >> 32); + msg->address_lo = (u32) (addr); + msg->data = data->hwirq; +} + +static struct irq_chip gicv2m_irq_chip = { + .name = "GICv2m", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = gicv2m_set_affinity, + .irq_compose_msi_msg = gicv2m_compose_msi_msg, +}; + +static int gicv2m_irq_gic_domain_alloc(struct irq_domain *domain, + unsigned int virq, + irq_hw_number_t hwirq) +{ + struct of_phandle_args args; + struct irq_data *d; + int err; + + args.np = domain->parent->of_node; + args.args_count = 3; + args.args[0] = 0; + args.args[1] = hwirq - 32; + args.args[2] = IRQ_TYPE_EDGE_RISING; + + err = irq_domain_alloc_irqs_parent(domain, virq, 1, &args); + if (err) + return err; + + /* Configure the interrupt line to be edge */ + d = irq_domain_get_irq_data(domain->parent, virq); + d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING); + return 0; +} + +static void gicv2m_unalloc_msi(struct v2m_data *v2m, unsigned int hwirq) +{ + int pos; + + pos = hwirq - v2m->spi_start; + if (pos < 0 || pos >= v2m->nr_spis) { + pr_err("Failed to teardown msi. Invalid hwirq %d\n", hwirq); + return; + } + + spin_lock(&v2m->msi_cnt_lock); + __clear_bit(pos, v2m->bm); + spin_unlock(&v2m->msi_cnt_lock); +} + +static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct v2m_data *v2m = domain->host_data; + int hwirq, offset, err = 0; + + spin_lock(&v2m->msi_cnt_lock); + offset = find_first_zero_bit(v2m->bm, v2m->nr_spis); + if (offset < v2m->nr_spis) + __set_bit(offset, v2m->bm); + else + err = -ENOSPC; + spin_unlock(&v2m->msi_cnt_lock); + + if (err) + return err; + + hwirq = v2m->spi_start + offset; + + err = gicv2m_irq_gic_domain_alloc(domain, virq, hwirq); + if (err) { + gicv2m_unalloc_msi(v2m, hwirq); + return err; + } + + irq_domain_set_hwirq_and_chip(domain, virq, hwirq, + &gicv2m_irq_chip, v2m); + + return 0; +} + +static void gicv2m_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct v2m_data *v2m = irq_data_get_irq_chip_data(d); + + BUG_ON(nr_irqs != 1); + gicv2m_unalloc_msi(v2m, d->hwirq); + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops gicv2m_domain_ops = { + .alloc = gicv2m_irq_domain_alloc, + .free = gicv2m_irq_domain_free, +}; + +static bool is_msi_spi_valid(u32 base, u32 num) +{ + if (base < V2M_MIN_SPI) { + pr_err("Invalid MSI base SPI (base:%u)\n", base); + return false; + } + + if ((num == 0) || (base + num > V2M_MAX_SPI)) { + pr_err("Number of SPIs (%u) exceed maximum (%u)\n", + num, V2M_MAX_SPI - V2M_MIN_SPI + 1); + return false; + } + + return true; +} + +static int __init gicv2m_init_one(struct device_node *node, + struct irq_domain *parent) +{ + int ret; + struct v2m_data *v2m; + struct irq_domain *inner_domain; + + v2m = kzalloc(sizeof(struct v2m_data), GFP_KERNEL); + if (!v2m) { + pr_err("Failed to allocate struct v2m_data.\n"); + return -ENOMEM; + } + + ret = of_address_to_resource(node, 0, &v2m->res); + if (ret) { + pr_err("Failed to allocate v2m resource.\n"); + goto err_free_v2m; + } + + v2m->base = ioremap(v2m->res.start, resource_size(&v2m->res)); + if (!v2m->base) { + pr_err("Failed to map GICv2m resource\n"); + ret = -ENOMEM; + goto err_free_v2m; + } + + if (!of_property_read_u32(node, "arm,msi-base-spi", &v2m->spi_start) && + !of_property_read_u32(node, "arm,msi-num-spis", &v2m->nr_spis)) { + pr_info("Overriding V2M MSI_TYPER (base:%u, num:%u)\n", + v2m->spi_start, v2m->nr_spis); + } else { + u32 typer = readl_relaxed(v2m->base + V2M_MSI_TYPER); + + v2m->spi_start = V2M_MSI_TYPER_BASE_SPI(typer); + v2m->nr_spis = V2M_MSI_TYPER_NUM_SPI(typer); + } + + if (!is_msi_spi_valid(v2m->spi_start, v2m->nr_spis)) { + ret = -EINVAL; + goto err_iounmap; + } + + v2m->bm = kzalloc(sizeof(long) * BITS_TO_LONGS(v2m->nr_spis), + GFP_KERNEL); + if (!v2m->bm) { + ret = -ENOMEM; + goto err_iounmap; + } + + inner_domain = irq_domain_add_tree(node, &gicv2m_domain_ops, v2m); + if (!inner_domain) { + pr_err("Failed to create GICv2m domain\n"); + ret = -ENOMEM; + goto err_free_bm; + } + + inner_domain->bus_token = DOMAIN_BUS_NEXUS; + inner_domain->parent = parent; + v2m->domain = pci_msi_create_irq_domain(node, &gicv2m_msi_domain_info, + inner_domain); + if (!v2m->domain) { + pr_err("Failed to create MSI domain\n"); + ret = -ENOMEM; + goto err_free_domains; + } + + spin_lock_init(&v2m->msi_cnt_lock); + + pr_info("Node %s: range[%#lx:%#lx], SPI[%d:%d]\n", node->name, + (unsigned long)v2m->res.start, (unsigned long)v2m->res.end, + v2m->spi_start, (v2m->spi_start + v2m->nr_spis)); + + return 0; + +err_free_domains: + if (v2m->domain) + irq_domain_remove(v2m->domain); + if (inner_domain) + irq_domain_remove(inner_domain); +err_free_bm: + kfree(v2m->bm); +err_iounmap: + iounmap(v2m->base); +err_free_v2m: + kfree(v2m); + return ret; +} + +static struct of_device_id gicv2m_device_id[] = { + { .compatible = "arm,gic-v2m-frame", }, + {}, +}; + +int __init gicv2m_of_init(struct device_node *node, struct irq_domain *parent) +{ + int ret = 0; + struct device_node *child; + + for (child = of_find_matching_node(node, gicv2m_device_id); child; + child = of_find_matching_node(child, gicv2m_device_id)) { + if (!of_find_property(child, "msi-controller", NULL)) + continue; + + ret = gicv2m_init_one(child, parent); + if (ret) { + of_node_put(node); + break; + } + } + + return ret; +} diff --git a/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/drivers/irqchip/irq-gic-v3-its-pci-msi.c new file mode 100644 index 000000000000..cf351c637464 --- /dev/null +++ b/drivers/irqchip/irq-gic-v3-its-pci-msi.c @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2013-2015 ARM Limited, All Rights Reserved. + * Author: Marc Zyngier <marc.zyngier@arm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/msi.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> + +static void its_mask_msi_irq(struct irq_data *d) +{ + pci_msi_mask_irq(d); + irq_chip_mask_parent(d); +} + +static void its_unmask_msi_irq(struct irq_data *d) +{ + pci_msi_unmask_irq(d); + irq_chip_unmask_parent(d); +} + +static struct irq_chip its_msi_irq_chip = { + .name = "ITS-MSI", + .irq_unmask = its_unmask_msi_irq, + .irq_mask = its_mask_msi_irq, + .irq_eoi = irq_chip_eoi_parent, + .irq_write_msi_msg = pci_msi_domain_write_msg, +}; + +struct its_pci_alias { + struct pci_dev *pdev; + u32 dev_id; + u32 count; +}; + +static int its_pci_msi_vec_count(struct pci_dev *pdev) +{ + int msi, msix; + + msi = max(pci_msi_vec_count(pdev), 0); + msix = max(pci_msix_vec_count(pdev), 0); + + return max(msi, msix); +} + +static int its_get_pci_alias(struct pci_dev *pdev, u16 alias, void *data) +{ + struct its_pci_alias *dev_alias = data; + + dev_alias->dev_id = alias; + if (pdev != dev_alias->pdev) + dev_alias->count += its_pci_msi_vec_count(dev_alias->pdev); + + return 0; +} + +static int its_pci_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + struct pci_dev *pdev; + struct its_pci_alias dev_alias; + struct msi_domain_info *msi_info; + + if (!dev_is_pci(dev)) + return -EINVAL; + + msi_info = msi_get_domain_info(domain->parent); + + pdev = to_pci_dev(dev); + dev_alias.pdev = pdev; + dev_alias.count = nvec; + + pci_for_each_dma_alias(pdev, its_get_pci_alias, &dev_alias); + + /* ITS specific DeviceID, as the core ITS ignores dev. */ + info->scratchpad[0].ul = dev_alias.dev_id; + + return msi_info->ops->msi_prepare(domain->parent, + dev, dev_alias.count, info); +} + +static struct msi_domain_ops its_pci_msi_ops = { + .msi_prepare = its_pci_msi_prepare, +}; + +static struct msi_domain_info its_pci_msi_domain_info = { + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), + .ops = &its_pci_msi_ops, + .chip = &its_msi_irq_chip, +}; + +static struct of_device_id its_device_id[] = { + { .compatible = "arm,gic-v3-its", }, + {}, +}; + +static int __init its_pci_msi_init(void) +{ + struct device_node *np; + struct irq_domain *parent; + + for (np = of_find_matching_node(NULL, its_device_id); np; + np = of_find_matching_node(np, its_device_id)) { + if (!of_property_read_bool(np, "msi-controller")) + continue; + + parent = irq_find_matching_host(np, DOMAIN_BUS_NEXUS); + if (!parent || !msi_get_domain_info(parent)) { + pr_err("%s: unable to locate ITS domain\n", + np->full_name); + continue; + } + + if (!pci_msi_create_irq_domain(np, &its_pci_msi_domain_info, + parent)) { + pr_err("%s: unable to create PCI domain\n", + np->full_name); + continue; + } + + pr_info("PCI/MSI: %s domain created\n", np->full_name); + } + + return 0; +} +early_initcall(its_pci_msi_init); diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c new file mode 100644 index 000000000000..78db25293384 --- /dev/null +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -0,0 +1,1555 @@ +/* + * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. + * Author: Marc Zyngier <marc.zyngier@arm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/bitmap.h> +#include <linux/cpu.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/log2.h> +#include <linux/mm.h> +#include <linux/msi.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/percpu.h> +#include <linux/slab.h> + +#include <linux/irqchip/arm-gic-v3.h> + +#include <asm/cacheflush.h> +#include <asm/cputype.h> +#include <asm/exception.h> + +#include "irqchip.h" + +#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1 << 0) + +#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) + +/* + * Collection structure - just an ID, and a redistributor address to + * ping. We use one per CPU as a bag of interrupts assigned to this + * CPU. + */ +struct its_collection { + u64 target_address; + u16 col_id; +}; + +/* + * The ITS structure - contains most of the infrastructure, with the + * top-level MSI domain, the command queue, the collections, and the + * list of devices writing to it. + */ +struct its_node { + raw_spinlock_t lock; + struct list_head entry; + void __iomem *base; + unsigned long phys_base; + struct its_cmd_block *cmd_base; + struct its_cmd_block *cmd_write; + void *tables[GITS_BASER_NR_REGS]; + struct its_collection *collections; + struct list_head its_device_list; + u64 flags; + u32 ite_size; +}; + +#define ITS_ITT_ALIGN SZ_256 + +struct event_lpi_map { + unsigned long *lpi_map; + u16 *col_map; + irq_hw_number_t lpi_base; + int nr_lpis; +}; + +/* + * The ITS view of a device - belongs to an ITS, a collection, owns an + * interrupt translation table, and a list of interrupts. + */ +struct its_device { + struct list_head entry; + struct its_node *its; + struct event_lpi_map event_map; + void *itt; + u32 nr_ites; + u32 device_id; +}; + +static LIST_HEAD(its_nodes); +static DEFINE_SPINLOCK(its_lock); +static struct device_node *gic_root_node; +static struct rdists *gic_rdists; + +#define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) +#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) + +static struct its_collection *dev_event_to_col(struct its_device *its_dev, + u32 event) +{ + struct its_node *its = its_dev->its; + + return its->collections + its_dev->event_map.col_map[event]; +} + +/* + * ITS command descriptors - parameters to be encoded in a command + * block. + */ +struct its_cmd_desc { + union { + struct { + struct its_device *dev; + u32 event_id; + } its_inv_cmd; + + struct { + struct its_device *dev; + u32 event_id; + } its_int_cmd; + + struct { + struct its_device *dev; + int valid; + } its_mapd_cmd; + + struct { + struct its_collection *col; + int valid; + } its_mapc_cmd; + + struct { + struct its_device *dev; + u32 phys_id; + u32 event_id; + } its_mapvi_cmd; + + struct { + struct its_device *dev; + struct its_collection *col; + u32 event_id; + } its_movi_cmd; + + struct { + struct its_device *dev; + u32 event_id; + } its_discard_cmd; + + struct { + struct its_collection *col; + } its_invall_cmd; + }; +}; + +/* + * The ITS command block, which is what the ITS actually parses. + */ +struct its_cmd_block { + u64 raw_cmd[4]; +}; + +#define ITS_CMD_QUEUE_SZ SZ_64K +#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) + +typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, + struct its_cmd_desc *); + +static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) +{ + cmd->raw_cmd[0] &= ~0xffUL; + cmd->raw_cmd[0] |= cmd_nr; +} + +static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) +{ + cmd->raw_cmd[0] &= BIT_ULL(32) - 1; + cmd->raw_cmd[0] |= ((u64)devid) << 32; +} + +static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) +{ + cmd->raw_cmd[1] &= ~0xffffffffUL; + cmd->raw_cmd[1] |= id; +} + +static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) +{ + cmd->raw_cmd[1] &= 0xffffffffUL; + cmd->raw_cmd[1] |= ((u64)phys_id) << 32; +} + +static void its_encode_size(struct its_cmd_block *cmd, u8 size) +{ + cmd->raw_cmd[1] &= ~0x1fUL; + cmd->raw_cmd[1] |= size & 0x1f; +} + +static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) +{ + cmd->raw_cmd[2] &= ~0xffffffffffffUL; + cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL; +} + +static void its_encode_valid(struct its_cmd_block *cmd, int valid) +{ + cmd->raw_cmd[2] &= ~(1UL << 63); + cmd->raw_cmd[2] |= ((u64)!!valid) << 63; +} + +static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) +{ + cmd->raw_cmd[2] &= ~(0xffffffffUL << 16); + cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16)); +} + +static void its_encode_collection(struct its_cmd_block *cmd, u16 col) +{ + cmd->raw_cmd[2] &= ~0xffffUL; + cmd->raw_cmd[2] |= col; +} + +static inline void its_fixup_cmd(struct its_cmd_block *cmd) +{ + /* Let's fixup BE commands */ + cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); + cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); + cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); + cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); +} + +static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + unsigned long itt_addr; + u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); + + itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); + itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); + + its_encode_cmd(cmd, GITS_CMD_MAPD); + its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); + its_encode_size(cmd, size - 1); + its_encode_itt(cmd, itt_addr); + its_encode_valid(cmd, desc->its_mapd_cmd.valid); + + its_fixup_cmd(cmd); + + return NULL; +} + +static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_MAPC); + its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); + its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); + its_encode_valid(cmd, desc->its_mapc_cmd.valid); + + its_fixup_cmd(cmd); + + return desc->its_mapc_cmd.col; +} + +static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_collection *col; + + col = dev_event_to_col(desc->its_mapvi_cmd.dev, + desc->its_mapvi_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_MAPVI); + its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id); + its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id); + its_encode_collection(cmd, col->col_id); + + its_fixup_cmd(cmd); + + return col; +} + +static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_collection *col; + + col = dev_event_to_col(desc->its_movi_cmd.dev, + desc->its_movi_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_MOVI); + its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_movi_cmd.event_id); + its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); + + its_fixup_cmd(cmd); + + return col; +} + +static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_collection *col; + + col = dev_event_to_col(desc->its_discard_cmd.dev, + desc->its_discard_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_DISCARD); + its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_discard_cmd.event_id); + + its_fixup_cmd(cmd); + + return col; +} + +static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_collection *col; + + col = dev_event_to_col(desc->its_inv_cmd.dev, + desc->its_inv_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_INV); + its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_inv_cmd.event_id); + + its_fixup_cmd(cmd); + + return col; +} + +static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_INVALL); + its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); + + its_fixup_cmd(cmd); + + return NULL; +} + +static u64 its_cmd_ptr_to_offset(struct its_node *its, + struct its_cmd_block *ptr) +{ + return (ptr - its->cmd_base) * sizeof(*ptr); +} + +static int its_queue_full(struct its_node *its) +{ + int widx; + int ridx; + + widx = its->cmd_write - its->cmd_base; + ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); + + /* This is incredibly unlikely to happen, unless the ITS locks up. */ + if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) + return 1; + + return 0; +} + +static struct its_cmd_block *its_allocate_entry(struct its_node *its) +{ + struct its_cmd_block *cmd; + u32 count = 1000000; /* 1s! */ + + while (its_queue_full(its)) { + count--; + if (!count) { + pr_err_ratelimited("ITS queue not draining\n"); + return NULL; + } + cpu_relax(); + udelay(1); + } + + cmd = its->cmd_write++; + + /* Handle queue wrapping */ + if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) + its->cmd_write = its->cmd_base; + + return cmd; +} + +static struct its_cmd_block *its_post_commands(struct its_node *its) +{ + u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); + + writel_relaxed(wr, its->base + GITS_CWRITER); + + return its->cmd_write; +} + +static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) +{ + /* + * Make sure the commands written to memory are observable by + * the ITS. + */ + if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) + __flush_dcache_area(cmd, sizeof(*cmd)); + else + dsb(ishst); +} + +static void its_wait_for_range_completion(struct its_node *its, + struct its_cmd_block *from, + struct its_cmd_block *to) +{ + u64 rd_idx, from_idx, to_idx; + u32 count = 1000000; /* 1s! */ + + from_idx = its_cmd_ptr_to_offset(its, from); + to_idx = its_cmd_ptr_to_offset(its, to); + + while (1) { + rd_idx = readl_relaxed(its->base + GITS_CREADR); + if (rd_idx >= to_idx || rd_idx < from_idx) + break; + + count--; + if (!count) { + pr_err_ratelimited("ITS queue timeout\n"); + return; + } + cpu_relax(); + udelay(1); + } +} + +static void its_send_single_command(struct its_node *its, + its_cmd_builder_t builder, + struct its_cmd_desc *desc) +{ + struct its_cmd_block *cmd, *sync_cmd, *next_cmd; + struct its_collection *sync_col; + unsigned long flags; + + raw_spin_lock_irqsave(&its->lock, flags); + + cmd = its_allocate_entry(its); + if (!cmd) { /* We're soooooo screewed... */ + pr_err_ratelimited("ITS can't allocate, dropping command\n"); + raw_spin_unlock_irqrestore(&its->lock, flags); + return; + } + sync_col = builder(cmd, desc); + its_flush_cmd(its, cmd); + + if (sync_col) { + sync_cmd = its_allocate_entry(its); + if (!sync_cmd) { + pr_err_ratelimited("ITS can't SYNC, skipping\n"); + goto post; + } + its_encode_cmd(sync_cmd, GITS_CMD_SYNC); + its_encode_target(sync_cmd, sync_col->target_address); + its_fixup_cmd(sync_cmd); + its_flush_cmd(its, sync_cmd); + } + +post: + next_cmd = its_post_commands(its); + raw_spin_unlock_irqrestore(&its->lock, flags); + + its_wait_for_range_completion(its, cmd, next_cmd); +} + +static void its_send_inv(struct its_device *dev, u32 event_id) +{ + struct its_cmd_desc desc; + + desc.its_inv_cmd.dev = dev; + desc.its_inv_cmd.event_id = event_id; + + its_send_single_command(dev->its, its_build_inv_cmd, &desc); +} + +static void its_send_mapd(struct its_device *dev, int valid) +{ + struct its_cmd_desc desc; + + desc.its_mapd_cmd.dev = dev; + desc.its_mapd_cmd.valid = !!valid; + + its_send_single_command(dev->its, its_build_mapd_cmd, &desc); +} + +static void its_send_mapc(struct its_node *its, struct its_collection *col, + int valid) +{ + struct its_cmd_desc desc; + + desc.its_mapc_cmd.col = col; + desc.its_mapc_cmd.valid = !!valid; + + its_send_single_command(its, its_build_mapc_cmd, &desc); +} + +static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_mapvi_cmd.dev = dev; + desc.its_mapvi_cmd.phys_id = irq_id; + desc.its_mapvi_cmd.event_id = id; + + its_send_single_command(dev->its, its_build_mapvi_cmd, &desc); +} + +static void its_send_movi(struct its_device *dev, + struct its_collection *col, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_movi_cmd.dev = dev; + desc.its_movi_cmd.col = col; + desc.its_movi_cmd.event_id = id; + + its_send_single_command(dev->its, its_build_movi_cmd, &desc); +} + +static void its_send_discard(struct its_device *dev, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_discard_cmd.dev = dev; + desc.its_discard_cmd.event_id = id; + + its_send_single_command(dev->its, its_build_discard_cmd, &desc); +} + +static void its_send_invall(struct its_node *its, struct its_collection *col) +{ + struct its_cmd_desc desc; + + desc.its_invall_cmd.col = col; + + its_send_single_command(its, its_build_invall_cmd, &desc); +} + +/* + * irqchip functions - assumes MSI, mostly. + */ + +static inline u32 its_get_event_id(struct irq_data *d) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + return d->hwirq - its_dev->event_map.lpi_base; +} + +static void lpi_set_config(struct irq_data *d, bool enable) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + irq_hw_number_t hwirq = d->hwirq; + u32 id = its_get_event_id(d); + u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192; + + if (enable) + *cfg |= LPI_PROP_ENABLED; + else + *cfg &= ~LPI_PROP_ENABLED; + + /* + * Make the above write visible to the redistributors. + * And yes, we're flushing exactly: One. Single. Byte. + * Humpf... + */ + if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) + __flush_dcache_area(cfg, sizeof(*cfg)); + else + dsb(ishst); + its_send_inv(its_dev, id); +} + +static void its_mask_irq(struct irq_data *d) +{ + lpi_set_config(d, false); +} + +static void its_unmask_irq(struct irq_data *d) +{ + lpi_set_config(d, true); +} + +static void its_eoi_irq(struct irq_data *d) +{ + gic_write_eoir(d->hwirq); +} + +static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, + bool force) +{ + unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask); + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + struct its_collection *target_col; + u32 id = its_get_event_id(d); + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + target_col = &its_dev->its->collections[cpu]; + its_send_movi(its_dev, target_col, id); + its_dev->event_map.col_map[id] = cpu; + + return IRQ_SET_MASK_OK_DONE; +} + +static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + struct its_node *its; + u64 addr; + + its = its_dev->its; + addr = its->phys_base + GITS_TRANSLATER; + + msg->address_lo = addr & ((1UL << 32) - 1); + msg->address_hi = addr >> 32; + msg->data = its_get_event_id(d); +} + +static struct irq_chip its_irq_chip = { + .name = "ITS", + .irq_mask = its_mask_irq, + .irq_unmask = its_unmask_irq, + .irq_eoi = its_eoi_irq, + .irq_set_affinity = its_set_affinity, + .irq_compose_msi_msg = its_irq_compose_msi_msg, +}; + +/* + * How we allocate LPIs: + * + * The GIC has id_bits bits for interrupt identifiers. From there, we + * must subtract 8192 which are reserved for SGIs/PPIs/SPIs. Then, as + * we allocate LPIs by chunks of 32, we can shift the whole thing by 5 + * bits to the right. + * + * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations. + */ +#define IRQS_PER_CHUNK_SHIFT 5 +#define IRQS_PER_CHUNK (1 << IRQS_PER_CHUNK_SHIFT) + +static unsigned long *lpi_bitmap; +static u32 lpi_chunks; +static DEFINE_SPINLOCK(lpi_lock); + +static int its_lpi_to_chunk(int lpi) +{ + return (lpi - 8192) >> IRQS_PER_CHUNK_SHIFT; +} + +static int its_chunk_to_lpi(int chunk) +{ + return (chunk << IRQS_PER_CHUNK_SHIFT) + 8192; +} + +static int its_lpi_init(u32 id_bits) +{ + lpi_chunks = its_lpi_to_chunk(1UL << id_bits); + + lpi_bitmap = kzalloc(BITS_TO_LONGS(lpi_chunks) * sizeof(long), + GFP_KERNEL); + if (!lpi_bitmap) { + lpi_chunks = 0; + return -ENOMEM; + } + + pr_info("ITS: Allocated %d chunks for LPIs\n", (int)lpi_chunks); + return 0; +} + +static unsigned long *its_lpi_alloc_chunks(int nr_irqs, int *base, int *nr_ids) +{ + unsigned long *bitmap = NULL; + int chunk_id; + int nr_chunks; + int i; + + nr_chunks = DIV_ROUND_UP(nr_irqs, IRQS_PER_CHUNK); + + spin_lock(&lpi_lock); + + do { + chunk_id = bitmap_find_next_zero_area(lpi_bitmap, lpi_chunks, + 0, nr_chunks, 0); + if (chunk_id < lpi_chunks) + break; + + nr_chunks--; + } while (nr_chunks > 0); + + if (!nr_chunks) + goto out; + + bitmap = kzalloc(BITS_TO_LONGS(nr_chunks * IRQS_PER_CHUNK) * sizeof (long), + GFP_ATOMIC); + if (!bitmap) + goto out; + + for (i = 0; i < nr_chunks; i++) + set_bit(chunk_id + i, lpi_bitmap); + + *base = its_chunk_to_lpi(chunk_id); + *nr_ids = nr_chunks * IRQS_PER_CHUNK; + +out: + spin_unlock(&lpi_lock); + + if (!bitmap) + *base = *nr_ids = 0; + + return bitmap; +} + +static void its_lpi_free(struct event_lpi_map *map) +{ + int base = map->lpi_base; + int nr_ids = map->nr_lpis; + int lpi; + + spin_lock(&lpi_lock); + + for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) { + int chunk = its_lpi_to_chunk(lpi); + BUG_ON(chunk > lpi_chunks); + if (test_bit(chunk, lpi_bitmap)) { + clear_bit(chunk, lpi_bitmap); + } else { + pr_err("Bad LPI chunk %d\n", chunk); + } + } + + spin_unlock(&lpi_lock); + + kfree(map->lpi_map); + kfree(map->col_map); +} + +/* + * We allocate 64kB for PROPBASE. That gives us at most 64K LPIs to + * deal with (one configuration byte per interrupt). PENDBASE has to + * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). + */ +#define LPI_PROPBASE_SZ SZ_64K +#define LPI_PENDBASE_SZ (LPI_PROPBASE_SZ / 8 + SZ_1K) + +/* + * This is how many bits of ID we need, including the useless ones. + */ +#define LPI_NRBITS ilog2(LPI_PROPBASE_SZ + SZ_8K) + +#define LPI_PROP_DEFAULT_PRIO 0xa0 + +static int __init its_alloc_lpi_tables(void) +{ + phys_addr_t paddr; + + gic_rdists->prop_page = alloc_pages(GFP_NOWAIT, + get_order(LPI_PROPBASE_SZ)); + if (!gic_rdists->prop_page) { + pr_err("Failed to allocate PROPBASE\n"); + return -ENOMEM; + } + + paddr = page_to_phys(gic_rdists->prop_page); + pr_info("GIC: using LPI property table @%pa\n", &paddr); + + /* Priority 0xa0, Group-1, disabled */ + memset(page_address(gic_rdists->prop_page), + LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, + LPI_PROPBASE_SZ); + + /* Make sure the GIC will observe the written configuration */ + __flush_dcache_area(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ); + + return 0; +} + +static const char *its_base_type_string[] = { + [GITS_BASER_TYPE_DEVICE] = "Devices", + [GITS_BASER_TYPE_VCPU] = "Virtual CPUs", + [GITS_BASER_TYPE_CPU] = "Physical CPUs", + [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections", + [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)", + [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)", + [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)", +}; + +static void its_free_tables(struct its_node *its) +{ + int i; + + for (i = 0; i < GITS_BASER_NR_REGS; i++) { + if (its->tables[i]) { + free_page((unsigned long)its->tables[i]); + its->tables[i] = NULL; + } + } +} + +static int its_alloc_tables(const char *node_name, struct its_node *its) +{ + int err; + int i; + int psz = SZ_64K; + u64 shr = GITS_BASER_InnerShareable; + u64 cache = GITS_BASER_WaWb; + + for (i = 0; i < GITS_BASER_NR_REGS; i++) { + u64 val = readq_relaxed(its->base + GITS_BASER + i * 8); + u64 type = GITS_BASER_TYPE(val); + u64 entry_size = GITS_BASER_ENTRY_SIZE(val); + int order = get_order(psz); + int alloc_size; + u64 tmp; + void *base; + + if (type == GITS_BASER_TYPE_NONE) + continue; + + /* + * Allocate as many entries as required to fit the + * range of device IDs that the ITS can grok... The ID + * space being incredibly sparse, this results in a + * massive waste of memory. + * + * For other tables, only allocate a single page. + */ + if (type == GITS_BASER_TYPE_DEVICE) { + u64 typer = readq_relaxed(its->base + GITS_TYPER); + u32 ids = GITS_TYPER_DEVBITS(typer); + + /* + * 'order' was initialized earlier to the default page + * granule of the the ITS. We can't have an allocation + * smaller than that. If the requested allocation + * is smaller, round up to the default page granule. + */ + order = max(get_order((1UL << ids) * entry_size), + order); + if (order >= MAX_ORDER) { + order = MAX_ORDER - 1; + pr_warn("%s: Device Table too large, reduce its page order to %u\n", + node_name, order); + } + } + + alloc_size = (1 << order) * PAGE_SIZE; + base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!base) { + err = -ENOMEM; + goto out_free; + } + + its->tables[i] = base; + +retry_baser: + val = (virt_to_phys(base) | + (type << GITS_BASER_TYPE_SHIFT) | + ((entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | + cache | + shr | + GITS_BASER_VALID); + + switch (psz) { + case SZ_4K: + val |= GITS_BASER_PAGE_SIZE_4K; + break; + case SZ_16K: + val |= GITS_BASER_PAGE_SIZE_16K; + break; + case SZ_64K: + val |= GITS_BASER_PAGE_SIZE_64K; + break; + } + + val |= (alloc_size / psz) - 1; + + writeq_relaxed(val, its->base + GITS_BASER + i * 8); + tmp = readq_relaxed(its->base + GITS_BASER + i * 8); + + if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { + /* + * Shareability didn't stick. Just use + * whatever the read reported, which is likely + * to be the only thing this redistributor + * supports. If that's zero, make it + * non-cacheable as well. + */ + shr = tmp & GITS_BASER_SHAREABILITY_MASK; + if (!shr) { + cache = GITS_BASER_nC; + __flush_dcache_area(base, alloc_size); + } + goto retry_baser; + } + + if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) { + /* + * Page size didn't stick. Let's try a smaller + * size and retry. If we reach 4K, then + * something is horribly wrong... + */ + switch (psz) { + case SZ_16K: + psz = SZ_4K; + goto retry_baser; + case SZ_64K: + psz = SZ_16K; + goto retry_baser; + } + } + + if (val != tmp) { + pr_err("ITS: %s: GITS_BASER%d doesn't stick: %lx %lx\n", + node_name, i, + (unsigned long) val, (unsigned long) tmp); + err = -ENXIO; + goto out_free; + } + + pr_info("ITS: allocated %d %s @%lx (psz %dK, shr %d)\n", + (int)(alloc_size / entry_size), + its_base_type_string[type], + (unsigned long)virt_to_phys(base), + psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); + } + + return 0; + +out_free: + its_free_tables(its); + + return err; +} + +static int its_alloc_collections(struct its_node *its) +{ + its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections), + GFP_KERNEL); + if (!its->collections) + return -ENOMEM; + + return 0; +} + +static void its_cpu_init_lpis(void) +{ + void __iomem *rbase = gic_data_rdist_rd_base(); + struct page *pend_page; + u64 val, tmp; + + /* If we didn't allocate the pending table yet, do it now */ + pend_page = gic_data_rdist()->pend_page; + if (!pend_page) { + phys_addr_t paddr; + /* + * The pending pages have to be at least 64kB aligned, + * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below. + */ + pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO, + get_order(max(LPI_PENDBASE_SZ, SZ_64K))); + if (!pend_page) { + pr_err("Failed to allocate PENDBASE for CPU%d\n", + smp_processor_id()); + return; + } + + /* Make sure the GIC will observe the zero-ed page */ + __flush_dcache_area(page_address(pend_page), LPI_PENDBASE_SZ); + + paddr = page_to_phys(pend_page); + pr_info("CPU%d: using LPI pending table @%pa\n", + smp_processor_id(), &paddr); + gic_data_rdist()->pend_page = pend_page; + } + + /* Disable LPIs */ + val = readl_relaxed(rbase + GICR_CTLR); + val &= ~GICR_CTLR_ENABLE_LPIS; + writel_relaxed(val, rbase + GICR_CTLR); + + /* + * Make sure any change to the table is observable by the GIC. + */ + dsb(sy); + + /* set PROPBASE */ + val = (page_to_phys(gic_rdists->prop_page) | + GICR_PROPBASER_InnerShareable | + GICR_PROPBASER_WaWb | + ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); + + writeq_relaxed(val, rbase + GICR_PROPBASER); + tmp = readq_relaxed(rbase + GICR_PROPBASER); + + if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) { + if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) { + /* + * The HW reports non-shareable, we must + * remove the cacheability attributes as + * well. + */ + val &= ~(GICR_PROPBASER_SHAREABILITY_MASK | + GICR_PROPBASER_CACHEABILITY_MASK); + val |= GICR_PROPBASER_nC; + writeq_relaxed(val, rbase + GICR_PROPBASER); + } + pr_info_once("GIC: using cache flushing for LPI property table\n"); + gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; + } + + /* set PENDBASE */ + val = (page_to_phys(pend_page) | + GICR_PENDBASER_InnerShareable | + GICR_PENDBASER_WaWb); + + writeq_relaxed(val, rbase + GICR_PENDBASER); + tmp = readq_relaxed(rbase + GICR_PENDBASER); + + if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) { + /* + * The HW reports non-shareable, we must remove the + * cacheability attributes as well. + */ + val &= ~(GICR_PENDBASER_SHAREABILITY_MASK | + GICR_PENDBASER_CACHEABILITY_MASK); + val |= GICR_PENDBASER_nC; + writeq_relaxed(val, rbase + GICR_PENDBASER); + } + + /* Enable LPIs */ + val = readl_relaxed(rbase + GICR_CTLR); + val |= GICR_CTLR_ENABLE_LPIS; + writel_relaxed(val, rbase + GICR_CTLR); + + /* Make sure the GIC has seen the above */ + dsb(sy); +} + +static void its_cpu_init_collection(void) +{ + struct its_node *its; + int cpu; + + spin_lock(&its_lock); + cpu = smp_processor_id(); + + list_for_each_entry(its, &its_nodes, entry) { + u64 target; + + /* + * We now have to bind each collection to its target + * redistributor. + */ + if (readq_relaxed(its->base + GITS_TYPER) & GITS_TYPER_PTA) { + /* + * This ITS wants the physical address of the + * redistributor. + */ + target = gic_data_rdist()->phys_base; + } else { + /* + * This ITS wants a linear CPU number. + */ + target = readq_relaxed(gic_data_rdist_rd_base() + GICR_TYPER); + target = GICR_TYPER_CPU_NUMBER(target) << 16; + } + + /* Perform collection mapping */ + its->collections[cpu].target_address = target; + its->collections[cpu].col_id = cpu; + + its_send_mapc(its, &its->collections[cpu], 1); + its_send_invall(its, &its->collections[cpu]); + } + + spin_unlock(&its_lock); +} + +static struct its_device *its_find_device(struct its_node *its, u32 dev_id) +{ + struct its_device *its_dev = NULL, *tmp; + unsigned long flags; + + raw_spin_lock_irqsave(&its->lock, flags); + + list_for_each_entry(tmp, &its->its_device_list, entry) { + if (tmp->device_id == dev_id) { + its_dev = tmp; + break; + } + } + + raw_spin_unlock_irqrestore(&its->lock, flags); + + return its_dev; +} + +static struct its_device *its_create_device(struct its_node *its, u32 dev_id, + int nvecs) +{ + struct its_device *dev; + unsigned long *lpi_map; + unsigned long flags; + u16 *col_map = NULL; + void *itt; + int lpi_base; + int nr_lpis; + int nr_ites; + int sz; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + /* + * At least one bit of EventID is being used, hence a minimum + * of two entries. No, the architecture doesn't let you + * express an ITT with a single entry. + */ + nr_ites = max(2UL, roundup_pow_of_two(nvecs)); + sz = nr_ites * its->ite_size; + sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; + itt = kzalloc(sz, GFP_KERNEL); + lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis); + if (lpi_map) + col_map = kzalloc(sizeof(*col_map) * nr_lpis, GFP_KERNEL); + + if (!dev || !itt || !lpi_map || !col_map) { + kfree(dev); + kfree(itt); + kfree(lpi_map); + kfree(col_map); + return NULL; + } + + __flush_dcache_area(itt, sz); + + dev->its = its; + dev->itt = itt; + dev->nr_ites = nr_ites; + dev->event_map.lpi_map = lpi_map; + dev->event_map.col_map = col_map; + dev->event_map.lpi_base = lpi_base; + dev->event_map.nr_lpis = nr_lpis; + dev->device_id = dev_id; + INIT_LIST_HEAD(&dev->entry); + + raw_spin_lock_irqsave(&its->lock, flags); + list_add(&dev->entry, &its->its_device_list); + raw_spin_unlock_irqrestore(&its->lock, flags); + + /* Map device to its ITT */ + its_send_mapd(dev, 1); + + return dev; +} + +static void its_free_device(struct its_device *its_dev) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&its_dev->its->lock, flags); + list_del(&its_dev->entry); + raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); + kfree(its_dev->itt); + kfree(its_dev); +} + +static int its_alloc_device_irq(struct its_device *dev, irq_hw_number_t *hwirq) +{ + int idx; + + idx = find_first_zero_bit(dev->event_map.lpi_map, + dev->event_map.nr_lpis); + if (idx == dev->event_map.nr_lpis) + return -ENOSPC; + + *hwirq = dev->event_map.lpi_base + idx; + set_bit(idx, dev->event_map.lpi_map); + + return 0; +} + +static int its_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + struct its_node *its; + struct its_device *its_dev; + struct msi_domain_info *msi_info; + u32 dev_id; + + /* + * We ignore "dev" entierely, and rely on the dev_id that has + * been passed via the scratchpad. This limits this domain's + * usefulness to upper layers that definitely know that they + * are built on top of the ITS. + */ + dev_id = info->scratchpad[0].ul; + + msi_info = msi_get_domain_info(domain); + its = msi_info->data; + + its_dev = its_find_device(its, dev_id); + if (its_dev) { + /* + * We already have seen this ID, probably through + * another alias (PCI bridge of some sort). No need to + * create the device. + */ + pr_debug("Reusing ITT for devID %x\n", dev_id); + goto out; + } + + its_dev = its_create_device(its, dev_id, nvec); + if (!its_dev) + return -ENOMEM; + + pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec)); +out: + info->scratchpad[0].ptr = its_dev; + return 0; +} + +static struct msi_domain_ops its_msi_domain_ops = { + .msi_prepare = its_msi_prepare, +}; + +static int its_irq_gic_domain_alloc(struct irq_domain *domain, + unsigned int virq, + irq_hw_number_t hwirq) +{ + struct of_phandle_args args; + + args.np = domain->parent->of_node; + args.args_count = 3; + args.args[0] = GIC_IRQ_TYPE_LPI; + args.args[1] = hwirq; + args.args[2] = IRQ_TYPE_EDGE_RISING; + + return irq_domain_alloc_irqs_parent(domain, virq, 1, &args); +} + +static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + msi_alloc_info_t *info = args; + struct its_device *its_dev = info->scratchpad[0].ptr; + irq_hw_number_t hwirq; + int err; + int i; + + for (i = 0; i < nr_irqs; i++) { + err = its_alloc_device_irq(its_dev, &hwirq); + if (err) + return err; + + err = its_irq_gic_domain_alloc(domain, virq + i, hwirq); + if (err) + return err; + + irq_domain_set_hwirq_and_chip(domain, virq + i, + hwirq, &its_irq_chip, its_dev); + pr_debug("ID:%d pID:%d vID:%d\n", + (int)(hwirq - its_dev->event_map.lpi_base), + (int) hwirq, virq + i); + } + + return 0; +} + +static void its_irq_domain_activate(struct irq_domain *domain, + struct irq_data *d) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + + /* Bind the LPI to the first possible CPU */ + its_dev->event_map.col_map[event] = cpumask_first(cpu_online_mask); + + /* Map the GIC IRQ and event to the device */ + its_send_mapvi(its_dev, d->hwirq, event); +} + +static void its_irq_domain_deactivate(struct irq_domain *domain, + struct irq_data *d) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + + /* Stop the delivery of interrupts */ + its_send_discard(its_dev, event); +} + +static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *data = irq_domain_get_irq_data(domain, + virq + i); + u32 event = its_get_event_id(data); + + /* Mark interrupt index as unused */ + clear_bit(event, its_dev->event_map.lpi_map); + + /* Nuke the entry in the domain */ + irq_domain_reset_irq_data(data); + } + + /* If all interrupts have been freed, start mopping the floor */ + if (bitmap_empty(its_dev->event_map.lpi_map, + its_dev->event_map.nr_lpis)) { + its_lpi_free(&its_dev->event_map); + + /* Unmap device/itt */ + its_send_mapd(its_dev, 0); + its_free_device(its_dev); + } + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops its_domain_ops = { + .alloc = its_irq_domain_alloc, + .free = its_irq_domain_free, + .activate = its_irq_domain_activate, + .deactivate = its_irq_domain_deactivate, +}; + +static int its_force_quiescent(void __iomem *base) +{ + u32 count = 1000000; /* 1s */ + u32 val; + + val = readl_relaxed(base + GITS_CTLR); + if (val & GITS_CTLR_QUIESCENT) + return 0; + + /* Disable the generation of all interrupts to this ITS */ + val &= ~GITS_CTLR_ENABLE; + writel_relaxed(val, base + GITS_CTLR); + + /* Poll GITS_CTLR and wait until ITS becomes quiescent */ + while (1) { + val = readl_relaxed(base + GITS_CTLR); + if (val & GITS_CTLR_QUIESCENT) + return 0; + + count--; + if (!count) + return -EBUSY; + + cpu_relax(); + udelay(1); + } +} + +static int its_probe(struct device_node *node, struct irq_domain *parent) +{ + struct resource res; + struct its_node *its; + void __iomem *its_base; + struct irq_domain *inner_domain; + u32 val; + u64 baser, tmp; + int err; + + err = of_address_to_resource(node, 0, &res); + if (err) { + pr_warn("%s: no regs?\n", node->full_name); + return -ENXIO; + } + + its_base = ioremap(res.start, resource_size(&res)); + if (!its_base) { + pr_warn("%s: unable to map registers\n", node->full_name); + return -ENOMEM; + } + + val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK; + if (val != 0x30 && val != 0x40) { + pr_warn("%s: no ITS detected, giving up\n", node->full_name); + err = -ENODEV; + goto out_unmap; + } + + err = its_force_quiescent(its_base); + if (err) { + pr_warn("%s: failed to quiesce, giving up\n", + node->full_name); + goto out_unmap; + } + + pr_info("ITS: %s\n", node->full_name); + + its = kzalloc(sizeof(*its), GFP_KERNEL); + if (!its) { + err = -ENOMEM; + goto out_unmap; + } + + raw_spin_lock_init(&its->lock); + INIT_LIST_HEAD(&its->entry); + INIT_LIST_HEAD(&its->its_device_list); + its->base = its_base; + its->phys_base = res.start; + its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1; + + its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL); + if (!its->cmd_base) { + err = -ENOMEM; + goto out_free_its; + } + its->cmd_write = its->cmd_base; + + err = its_alloc_tables(node->full_name, its); + if (err) + goto out_free_cmd; + + err = its_alloc_collections(its); + if (err) + goto out_free_tables; + + baser = (virt_to_phys(its->cmd_base) | + GITS_CBASER_WaWb | + GITS_CBASER_InnerShareable | + (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | + GITS_CBASER_VALID); + + writeq_relaxed(baser, its->base + GITS_CBASER); + tmp = readq_relaxed(its->base + GITS_CBASER); + + if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) { + if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) { + /* + * The HW reports non-shareable, we must + * remove the cacheability attributes as + * well. + */ + baser &= ~(GITS_CBASER_SHAREABILITY_MASK | + GITS_CBASER_CACHEABILITY_MASK); + baser |= GITS_CBASER_nC; + writeq_relaxed(baser, its->base + GITS_CBASER); + } + pr_info("ITS: using cache flushing for cmd queue\n"); + its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; + } + + writeq_relaxed(0, its->base + GITS_CWRITER); + writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR); + + if (of_property_read_bool(node, "msi-controller")) { + struct msi_domain_info *info; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + err = -ENOMEM; + goto out_free_tables; + } + + inner_domain = irq_domain_add_tree(node, &its_domain_ops, its); + if (!inner_domain) { + err = -ENOMEM; + kfree(info); + goto out_free_tables; + } + + inner_domain->parent = parent; + inner_domain->bus_token = DOMAIN_BUS_NEXUS; + info->ops = &its_msi_domain_ops; + info->data = its; + inner_domain->host_data = info; + } + + spin_lock(&its_lock); + list_add(&its->entry, &its_nodes); + spin_unlock(&its_lock); + + return 0; + +out_free_tables: + its_free_tables(its); +out_free_cmd: + kfree(its->cmd_base); +out_free_its: + kfree(its); +out_unmap: + iounmap(its_base); + pr_err("ITS: failed probing %s (%d)\n", node->full_name, err); + return err; +} + +static bool gic_rdists_supports_plpis(void) +{ + return !!(readl_relaxed(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS); +} + +int its_cpu_init(void) +{ + if (!list_empty(&its_nodes)) { + if (!gic_rdists_supports_plpis()) { + pr_info("CPU%d: LPIs not supported\n", smp_processor_id()); + return -ENXIO; + } + its_cpu_init_lpis(); + its_cpu_init_collection(); + } + + return 0; +} + +static struct of_device_id its_device_id[] = { + { .compatible = "arm,gic-v3-its", }, + {}, +}; + +int its_init(struct device_node *node, struct rdists *rdists, + struct irq_domain *parent_domain) +{ + struct device_node *np; + + for (np = of_find_matching_node(node, its_device_id); np; + np = of_find_matching_node(np, its_device_id)) { + its_probe(np, parent_domain); + } + + if (list_empty(&its_nodes)) { + pr_warn("ITS: No ITS available, not enabling LPIs\n"); + return -ENXIO; + } + + gic_rdists = rdists; + gic_root_node = node; + + its_alloc_lpi_tables(); + its_lpi_init(rdists->id_bits); + + return 0; +} diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 30b93dd1e612..95e8cf408688 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -34,20 +34,25 @@ #include "irq-gic-common.h" #include "irqchip.h" +struct redist_region { + void __iomem *redist_base; + phys_addr_t phys_base; +}; + struct gic_chip_data { void __iomem *dist_base; - void __iomem **redist_base; - void __iomem * __percpu *rdist; + struct redist_region *redist_regions; + struct rdists rdists; struct irq_domain *domain; u64 redist_stride; - u32 redist_regions; + u32 nr_redist_regions; unsigned int irq_nr; }; static struct gic_chip_data gic_data __read_mostly; -#define gic_data_rdist() (this_cpu_ptr(gic_data.rdist)) -#define gic_data_rdist_rd_base() (*gic_data_rdist()) +#define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) +#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) /* Our default, arbitrary priority value. Linux only uses one anyway. */ @@ -71,9 +76,6 @@ static inline void __iomem *gic_dist_base(struct irq_data *d) if (d->hwirq <= 1023) /* SPI -> dist_base */ return gic_data.dist_base; - if (d->hwirq >= 8192) - BUG(); /* LPI Detected!!! */ - return NULL; } @@ -236,7 +238,9 @@ static int gic_set_type(struct irq_data *d, unsigned int type) if (irq < 16) return -EINVAL; - if (type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) + /* SPIs have restrictions on the supported types */ + if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH && + type != IRQ_TYPE_EDGE_RISING) return -EINVAL; if (gic_irq_in_rdist(d)) { @@ -247,9 +251,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type) rwp_wait = gic_dist_wait_for_rwp; } - gic_configure_irq(irq, type, base, rwp_wait); - - return 0; + return gic_configure_irq(irq, type, base, rwp_wait); } static u64 gic_mpidr_to_affinity(u64 mpidr) @@ -271,11 +273,11 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs do { irqnr = gic_read_iar(); - if (likely(irqnr > 15 && irqnr < 1020)) { + if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) { int err; err = handle_domain_irq(gic_data.domain, irqnr, regs); if (err) { - WARN_ONCE(true, "Unexpected SPI received!\n"); + WARN_ONCE(true, "Unexpected interrupt received!\n"); gic_write_eoir(irqnr); } continue; @@ -340,8 +342,8 @@ static int gic_populate_rdist(void) MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | MPIDR_AFFINITY_LEVEL(mpidr, 0)); - for (i = 0; i < gic_data.redist_regions; i++) { - void __iomem *ptr = gic_data.redist_base[i]; + for (i = 0; i < gic_data.nr_redist_regions; i++) { + void __iomem *ptr = gic_data.redist_regions[i].redist_base; u32 reg; reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; @@ -354,10 +356,13 @@ static int gic_populate_rdist(void) do { typer = readq_relaxed(ptr + GICR_TYPER); if ((typer >> 32) == aff) { + u64 offset = ptr - gic_data.redist_regions[i].redist_base; gic_data_rdist_rd_base() = ptr; - pr_info("CPU%d: found redistributor %llx @%p\n", + gic_data_rdist()->phys_base = gic_data.redist_regions[i].phys_base + offset; + pr_info("CPU%d: found redistributor %llx region %d:%pa\n", smp_processor_id(), - (unsigned long long)mpidr, ptr); + (unsigned long long)mpidr, + i, &gic_data_rdist()->phys_base); return 0; } @@ -392,6 +397,11 @@ static void gic_cpu_sys_reg_init(void) gic_write_grpen1(1); } +static int gic_dist_supports_lpis(void) +{ + return !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS); +} + static void gic_cpu_init(void) { void __iomem *rbase; @@ -406,6 +416,10 @@ static void gic_cpu_init(void) gic_cpu_config(rbase, gic_redist_wait_for_rwp); + /* Give LPIs a spin */ + if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis()) + its_cpu_init(); + /* initialise system registers */ gic_cpu_sys_reg_init(); } @@ -459,7 +473,7 @@ static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, tlist |= 1 << (mpidr & 0xf); cpu = cpumask_next(cpu, mask); - if (cpu == nr_cpu_ids) + if (cpu >= nr_cpu_ids) goto out; mpidr = cpu_logical_map(cpu); @@ -599,26 +613,43 @@ static struct irq_chip gic_chip = { .irq_set_affinity = gic_set_affinity, }; +#define GIC_ID_NR (1U << gic_data.rdists.id_bits) + static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) { /* SGIs are private to the core kernel */ if (hw < 16) return -EPERM; + /* Nothing here */ + if (hw >= gic_data.irq_nr && hw < 8192) + return -EPERM; + /* Off limits */ + if (hw >= GIC_ID_NR) + return -EPERM; + /* PPIs */ if (hw < 32) { irq_set_percpu_devid(irq); - irq_set_chip_and_handler(irq, &gic_chip, - handle_percpu_devid_irq); + irq_domain_set_info(d, irq, hw, &gic_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN); } /* SPIs */ if (hw >= 32 && hw < gic_data.irq_nr) { - irq_set_chip_and_handler(irq, &gic_chip, - handle_fasteoi_irq); + irq_domain_set_info(d, irq, hw, &gic_chip, d->host_data, + handle_fasteoi_irq, NULL, NULL); set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); } - irq_set_chip_data(irq, d->host_data); + /* LPIs */ + if (hw >= 8192 && hw < GIC_ID_NR) { + if (!gic_dist_supports_lpis()) + return -EPERM; + irq_domain_set_info(d, irq, hw, &gic_chip, d->host_data, + handle_fasteoi_irq, NULL, NULL); + set_irq_flags(irq, IRQF_VALID); + } + return 0; } @@ -639,6 +670,9 @@ static int gic_irq_domain_xlate(struct irq_domain *d, case 1: /* PPI */ *out_hwirq = intspec[1] + 16; break; + case GIC_IRQ_TYPE_LPI: /* LPI */ + *out_hwirq = intspec[1]; + break; default: return -EINVAL; } @@ -647,17 +681,50 @@ static int gic_irq_domain_xlate(struct irq_domain *d, return 0; } +static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int i, ret; + irq_hw_number_t hwirq; + unsigned int type = IRQ_TYPE_NONE; + struct of_phandle_args *irq_data = arg; + + ret = gic_irq_domain_xlate(domain, irq_data->np, irq_data->args, + irq_data->args_count, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) + gic_irq_domain_map(domain, virq + i, hwirq + i); + + return 0; +} + +static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + static const struct irq_domain_ops gic_irq_domain_ops = { - .map = gic_irq_domain_map, .xlate = gic_irq_domain_xlate, + .alloc = gic_irq_domain_alloc, + .free = gic_irq_domain_free, }; static int __init gic_of_init(struct device_node *node, struct device_node *parent) { void __iomem *dist_base; - void __iomem **redist_base; + struct redist_region *rdist_regs; u64 redist_stride; - u32 redist_regions; + u32 nr_redist_regions; + u32 typer; u32 reg; int gic_irqs; int err; @@ -678,54 +745,63 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare goto out_unmap_dist; } - if (of_property_read_u32(node, "#redistributor-regions", &redist_regions)) - redist_regions = 1; + if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) + nr_redist_regions = 1; - redist_base = kzalloc(sizeof(*redist_base) * redist_regions, GFP_KERNEL); - if (!redist_base) { + rdist_regs = kzalloc(sizeof(*rdist_regs) * nr_redist_regions, GFP_KERNEL); + if (!rdist_regs) { err = -ENOMEM; goto out_unmap_dist; } - for (i = 0; i < redist_regions; i++) { - redist_base[i] = of_iomap(node, 1 + i); - if (!redist_base[i]) { + for (i = 0; i < nr_redist_regions; i++) { + struct resource res; + int ret; + + ret = of_address_to_resource(node, 1 + i, &res); + rdist_regs[i].redist_base = of_iomap(node, 1 + i); + if (ret || !rdist_regs[i].redist_base) { pr_err("%s: couldn't map region %d\n", node->full_name, i); err = -ENODEV; goto out_unmap_rdist; } + rdist_regs[i].phys_base = res.start; } if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) redist_stride = 0; gic_data.dist_base = dist_base; - gic_data.redist_base = redist_base; - gic_data.redist_regions = redist_regions; + gic_data.redist_regions = rdist_regs; + gic_data.nr_redist_regions = nr_redist_regions; gic_data.redist_stride = redist_stride; /* * Find out how many interrupts are supported. * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI) */ - gic_irqs = readl_relaxed(gic_data.dist_base + GICD_TYPER) & 0x1f; - gic_irqs = (gic_irqs + 1) * 32; + typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); + gic_data.rdists.id_bits = GICD_TYPER_ID_BITS(typer); + gic_irqs = GICD_TYPER_IRQS(typer); if (gic_irqs > 1020) gic_irqs = 1020; gic_data.irq_nr = gic_irqs; gic_data.domain = irq_domain_add_tree(node, &gic_irq_domain_ops, &gic_data); - gic_data.rdist = alloc_percpu(typeof(*gic_data.rdist)); + gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); - if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdist)) { + if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { err = -ENOMEM; goto out_free; } set_handle_irq(gic_handle_irq); + if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis()) + its_init(node, &gic_data.rdists, gic_data.domain); + gic_smp_init(); gic_dist_init(); gic_cpu_init(); @@ -736,12 +812,12 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare out_free: if (gic_data.domain) irq_domain_remove(gic_data.domain); - free_percpu(gic_data.rdist); + free_percpu(gic_data.rdists.rdist); out_unmap_rdist: - for (i = 0; i < redist_regions; i++) - if (redist_base[i]) - iounmap(redist_base[i]); - kfree(redist_base); + for (i = 0; i < nr_redist_regions; i++) + if (rdist_regs[i].redist_base) + iounmap(rdist_regs[i].redist_base); + kfree(rdist_regs); out_unmap_dist: iounmap(dist_base); return err; diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 1107051df379..d67507911f9f 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -154,23 +154,25 @@ static inline unsigned int gic_irq(struct irq_data *d) static void gic_mask_irq(struct irq_data *d) { u32 mask = 1 << (gic_irq(d) % 32); + unsigned long flags; - raw_spin_lock(&irq_controller_lock); + raw_spin_lock_irqsave(&irq_controller_lock, flags); writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_CLEAR + (gic_irq(d) / 32) * 4); if (gic_arch_extn.irq_mask) gic_arch_extn.irq_mask(d); - raw_spin_unlock(&irq_controller_lock); + raw_spin_unlock_irqrestore(&irq_controller_lock, flags); } static void gic_unmask_irq(struct irq_data *d) { u32 mask = 1 << (gic_irq(d) % 32); + unsigned long flags; - raw_spin_lock(&irq_controller_lock); + raw_spin_lock_irqsave(&irq_controller_lock, flags); if (gic_arch_extn.irq_unmask) gic_arch_extn.irq_unmask(d); writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_SET + (gic_irq(d) / 32) * 4); - raw_spin_unlock(&irq_controller_lock); + raw_spin_unlock_irqrestore(&irq_controller_lock, flags); } static void gic_eoi_irq(struct irq_data *d) @@ -188,24 +190,28 @@ static int gic_set_type(struct irq_data *d, unsigned int type) { void __iomem *base = gic_dist_base(d); unsigned int gicirq = gic_irq(d); + unsigned long flags; + int ret; /* Interrupt configuration for SGIs can't be changed */ if (gicirq < 16) return -EINVAL; - if (type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) + /* SPIs have restrictions on the supported types */ + if (gicirq >= 32 && type != IRQ_TYPE_LEVEL_HIGH && + type != IRQ_TYPE_EDGE_RISING) return -EINVAL; - raw_spin_lock(&irq_controller_lock); + raw_spin_lock_irqsave(&irq_controller_lock, flags); if (gic_arch_extn.irq_set_type) gic_arch_extn.irq_set_type(d, type); - gic_configure_irq(gicirq, type, base, NULL); + ret = gic_configure_irq(gicirq, type, base, NULL); - raw_spin_unlock(&irq_controller_lock); + raw_spin_unlock_irqrestore(&irq_controller_lock, flags); - return 0; + return ret; } static int gic_retrigger(struct irq_data *d) @@ -224,6 +230,7 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + (gic_irq(d) & ~3); unsigned int cpu, shift = (gic_irq(d) % 4) * 8; u32 val, mask, bit; + unsigned long flags; if (!force) cpu = cpumask_any_and(mask_val, cpu_online_mask); @@ -233,12 +240,12 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids) return -EINVAL; - raw_spin_lock(&irq_controller_lock); + raw_spin_lock_irqsave(&irq_controller_lock, flags); mask = 0xff << shift; bit = gic_cpu_map[cpu] << shift; val = readl_relaxed(reg) & ~mask; writel_relaxed(val | bit, reg); - raw_spin_unlock(&irq_controller_lock); + raw_spin_unlock_irqrestore(&irq_controller_lock, flags); return IRQ_SET_MASK_OK; } @@ -796,17 +803,16 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, { if (hw < 32) { irq_set_percpu_devid(irq); - irq_set_chip_and_handler(irq, &gic_chip, - handle_percpu_devid_irq); + irq_domain_set_info(d, irq, hw, &gic_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN); } else { - irq_set_chip_and_handler(irq, &gic_chip, - handle_fasteoi_irq); + irq_domain_set_info(d, irq, hw, &gic_chip, d->host_data, + handle_fasteoi_irq, NULL, NULL); set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); gic_routable_irq_domain_ops->map(d, irq, hw); } - irq_set_chip_data(irq, d->host_data); return 0; } @@ -866,6 +872,31 @@ static struct notifier_block gic_cpu_notifier = { }; #endif +static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int i, ret; + irq_hw_number_t hwirq; + unsigned int type = IRQ_TYPE_NONE; + struct of_phandle_args *irq_data = arg; + + ret = gic_irq_domain_xlate(domain, irq_data->np, irq_data->args, + irq_data->args_count, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) + gic_irq_domain_map(domain, virq + i, hwirq + i); + + return 0; +} + +static const struct irq_domain_ops gic_irq_domain_hierarchy_ops = { + .xlate = gic_irq_domain_xlate, + .alloc = gic_irq_domain_alloc, + .free = irq_domain_free_irqs_top, +}; + static const struct irq_domain_ops gic_irq_domain_ops = { .map = gic_irq_domain_map, .unmap = gic_irq_domain_unmap, @@ -956,18 +987,6 @@ void __init gic_init_bases(unsigned int gic_nr, int irq_start, gic_cpu_map[i] = 0xff; /* - * For primary GICs, skip over SGIs. - * For secondary GICs, skip over PPIs, too. - */ - if (gic_nr == 0 && (irq_start & 31) > 0) { - hwirq_base = 16; - if (irq_start != -1) - irq_start = (irq_start & ~31) + 16; - } else { - hwirq_base = 32; - } - - /* * Find out how many interrupts are supported. * The GIC only supports up to 1020 interrupt sources. */ @@ -977,10 +996,31 @@ void __init gic_init_bases(unsigned int gic_nr, int irq_start, gic_irqs = 1020; gic->gic_irqs = gic_irqs; - gic_irqs -= hwirq_base; /* calculate # of irqs to allocate */ + if (node) { /* DT case */ + const struct irq_domain_ops *ops = &gic_irq_domain_hierarchy_ops; + + if (!of_property_read_u32(node, "arm,routable-irqs", + &nr_routable_irqs)) { + ops = &gic_irq_domain_ops; + gic_irqs = nr_routable_irqs; + } + + gic->domain = irq_domain_add_linear(node, gic_irqs, ops, gic); + } else { /* Non-DT case */ + /* + * For primary GICs, skip over SGIs. + * For secondary GICs, skip over PPIs, too. + */ + if (gic_nr == 0 && (irq_start & 31) > 0) { + hwirq_base = 16; + if (irq_start != -1) + irq_start = (irq_start & ~31) + 16; + } else { + hwirq_base = 32; + } + + gic_irqs -= hwirq_base; /* calculate # of irqs to allocate */ - if (of_property_read_u32(node, "arm,routable-irqs", - &nr_routable_irqs)) { irq_base = irq_alloc_descs(irq_start, 16, gic_irqs, numa_node_id()); if (IS_ERR_VALUE(irq_base)) { @@ -991,10 +1031,6 @@ void __init gic_init_bases(unsigned int gic_nr, int irq_start, gic->domain = irq_domain_add_legacy(node, gic_irqs, irq_base, hwirq_base, &gic_irq_domain_ops, gic); - } else { - gic->domain = irq_domain_add_linear(node, nr_routable_irqs, - &gic_irq_domain_ops, - gic); } if (WARN_ON(!gic->domain)) @@ -1045,6 +1081,10 @@ gic_of_init(struct device_node *node, struct device_node *parent) irq = irq_of_parse_and_map(node, 0); gic_cascade_irq(gic_cnt, irq); } + + if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) + gicv2m_of_init(node, gic_data[gic_cnt].domain); + gic_cnt++; return 0; } diff --git a/drivers/irqchip/irq-hip04.c b/drivers/irqchip/irq-hip04.c index 9c8f833522e6..5507a0c9a61d 100644 --- a/drivers/irqchip/irq-hip04.c +++ b/drivers/irqchip/irq-hip04.c @@ -120,21 +120,24 @@ static int hip04_irq_set_type(struct irq_data *d, unsigned int type) { void __iomem *base = hip04_dist_base(d); unsigned int irq = hip04_irq(d); + int ret; /* Interrupt configuration for SGIs can't be changed */ if (irq < 16) return -EINVAL; - if (type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) + /* SPIs have restrictions on the supported types */ + if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH && + type != IRQ_TYPE_EDGE_RISING) return -EINVAL; raw_spin_lock(&irq_controller_lock); - gic_configure_irq(irq, type, base, NULL); + ret = gic_configure_irq(irq, type, base, NULL); raw_spin_unlock(&irq_controller_lock); - return 0; + return ret; } #ifdef CONFIG_SMP diff --git a/drivers/irqchip/irq-mtk-sysirq.c b/drivers/irqchip/irq-mtk-sysirq.c new file mode 100644 index 000000000000..7e342df6a62f --- /dev/null +++ b/drivers/irqchip/irq-mtk-sysirq.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2014 MediaTek Inc. + * Author: Joe.C <yingjoe.chen@mediatek.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/of_address.h> +#include <linux/io.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +#include "irqchip.h" + +#define MT6577_SYS_INTPOL_NUM (224) + +struct mtk_sysirq_chip_data { + spinlock_t lock; + void __iomem *intpol_base; +}; + +static int mtk_sysirq_set_type(struct irq_data *data, unsigned int type) +{ + irq_hw_number_t hwirq = data->hwirq; + struct mtk_sysirq_chip_data *chip_data = data->chip_data; + u32 offset, reg_index, value; + unsigned long flags; + int ret; + + offset = hwirq & 0x1f; + reg_index = hwirq >> 5; + + spin_lock_irqsave(&chip_data->lock, flags); + value = readl_relaxed(chip_data->intpol_base + reg_index * 4); + if (type == IRQ_TYPE_LEVEL_LOW || type == IRQ_TYPE_EDGE_FALLING) { + if (type == IRQ_TYPE_LEVEL_LOW) + type = IRQ_TYPE_LEVEL_HIGH; + else + type = IRQ_TYPE_EDGE_RISING; + value |= (1 << offset); + } else { + value &= ~(1 << offset); + } + writel(value, chip_data->intpol_base + reg_index * 4); + + data = data->parent_data; + ret = data->chip->irq_set_type(data, type); + spin_unlock_irqrestore(&chip_data->lock, flags); + return ret; +} + +static struct irq_chip mtk_sysirq_chip = { + .name = "MT_SYSIRQ", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_type = mtk_sysirq_set_type, + .irq_retrigger = irq_chip_retrigger_hierarchy, + .irq_set_affinity = irq_chip_set_affinity_parent, +}; + +static int mtk_sysirq_domain_xlate(struct irq_domain *d, + struct device_node *controller, + const u32 *intspec, unsigned int intsize, + unsigned long *out_hwirq, + unsigned int *out_type) +{ + if (intsize != 3) + return -EINVAL; + + /* sysirq doesn't support PPI */ + if (intspec[0]) + return -EINVAL; + + *out_hwirq = intspec[1]; + *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; + return 0; +} + +static int mtk_sysirq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int i; + irq_hw_number_t hwirq; + struct of_phandle_args *irq_data = arg; + struct of_phandle_args gic_data = *irq_data; + + if (irq_data->args_count != 3) + return -EINVAL; + + /* sysirq doesn't support PPI */ + if (irq_data->args[0]) + return -EINVAL; + + hwirq = irq_data->args[1]; + for (i = 0; i < nr_irqs; i++) + irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + &mtk_sysirq_chip, + domain->host_data); + + gic_data.np = domain->parent->of_node; + return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &gic_data); +} + +static struct irq_domain_ops sysirq_domain_ops = { + .xlate = mtk_sysirq_domain_xlate, + .alloc = mtk_sysirq_domain_alloc, + .free = irq_domain_free_irqs_common, +}; + +static int __init mtk_sysirq_of_init(struct device_node *node, + struct device_node *parent) +{ + struct irq_domain *domain, *domain_parent; + struct mtk_sysirq_chip_data *chip_data; + int ret = 0; + + domain_parent = irq_find_host(parent); + if (!domain_parent) { + pr_err("mtk_sysirq: interrupt-parent not found\n"); + return -EINVAL; + } + + chip_data = kzalloc(sizeof(*chip_data), GFP_KERNEL); + if (!chip_data) + return -ENOMEM; + + chip_data->intpol_base = of_io_request_and_map(node, 0, "intpol"); + if (!chip_data->intpol_base) { + pr_err("mtk_sysirq: unable to map sysirq register\n"); + ret = -ENOMEM; + goto out_free; + } + + domain = irq_domain_add_hierarchy(domain_parent, 0, + MT6577_SYS_INTPOL_NUM, node, + &sysirq_domain_ops, chip_data); + if (!domain) { + ret = -ENOMEM; + goto out_unmap; + } + spin_lock_init(&chip_data->lock); + + return 0; + +out_unmap: + iounmap(chip_data->intpol_base); +out_free: + kfree(chip_data); + return ret; +} +IRQCHIP_DECLARE(mtk_sysirq, "mediatek,mt6577-sysirq", mtk_sysirq_of_init); diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 9c4836f5fea7..3f56ab28cee0 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -18,9 +18,11 @@ #include <linux/slab.h> #include <linux/crypto.h> #include <linux/workqueue.h> +#include <linux/kthread.h> #include <linux/backing-dev.h> #include <linux/atomic.h> #include <linux/scatterlist.h> +#include <linux/rbtree.h> #include <asm/page.h> #include <asm/unaligned.h> #include <crypto/hash.h> @@ -58,7 +60,8 @@ struct dm_crypt_io { atomic_t io_pending; int error; sector_t sector; - struct dm_crypt_io *base_io; + + struct rb_node rb_node; } CRYPTO_MINALIGN_ATTR; struct dm_crypt_request { @@ -108,7 +111,8 @@ struct iv_tcw_private { * Crypt: maps a linear range of a block device * and encrypts / decrypts at the same time. */ -enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; +enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID, + DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD }; /* * The fields in here must be read only after initialization. @@ -121,14 +125,18 @@ struct crypt_config { * pool for per bio private data, crypto requests and * encryption requeusts/buffer pages */ - mempool_t *io_pool; mempool_t *req_pool; mempool_t *page_pool; struct bio_set *bs; + struct mutex bio_alloc_lock; struct workqueue_struct *io_queue; struct workqueue_struct *crypt_queue; + struct task_struct *write_thread; + wait_queue_head_t write_thread_wait; + struct rb_root write_tree; + char *cipher; char *cipher_string; @@ -172,9 +180,6 @@ struct crypt_config { }; #define MIN_IOS 16 -#define MIN_POOL_PAGES 32 - -static struct kmem_cache *_crypt_io_pool; static void clone_init(struct dm_crypt_io *, struct bio *); static void kcryptd_queue_crypt(struct dm_crypt_io *io); @@ -223,7 +228,7 @@ static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc) * * tcw: Compatible implementation of the block chaining mode used * by the TrueCrypt device encryption system (prior to version 4.1). - * For more info see: http://www.truecrypt.org + * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat * It operates on full 512 byte sectors and uses CBC * with an IV derived from initial key and the sector number. * In addition, whitening value is applied on every sector, whitening @@ -946,57 +951,70 @@ static int crypt_convert(struct crypt_config *cc, return 0; } +static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone); + /* * Generate a new unfragmented bio with the given size * This should never violate the device limitations - * May return a smaller bio when running out of pages, indicated by - * *out_of_pages set to 1. + * + * This function may be called concurrently. If we allocate from the mempool + * concurrently, there is a possibility of deadlock. For example, if we have + * mempool of 256 pages, two processes, each wanting 256, pages allocate from + * the mempool concurrently, it may deadlock in a situation where both processes + * have allocated 128 pages and the mempool is exhausted. + * + * In order to avoid this scenario we allocate the pages under a mutex. + * + * In order to not degrade performance with excessive locking, we try + * non-blocking allocations without a mutex first but on failure we fallback + * to blocking allocations with a mutex. */ -static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, - unsigned *out_of_pages) +static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size) { struct crypt_config *cc = io->cc; struct bio *clone; unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; - gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; - unsigned i, len; + gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM; + unsigned i, len, remaining_size; struct page *page; + struct bio_vec *bvec; + +retry: + if (unlikely(gfp_mask & __GFP_WAIT)) + mutex_lock(&cc->bio_alloc_lock); clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); if (!clone) - return NULL; + goto return_clone; clone_init(io, clone); - *out_of_pages = 0; + + remaining_size = size; for (i = 0; i < nr_iovecs; i++) { page = mempool_alloc(cc->page_pool, gfp_mask); if (!page) { - *out_of_pages = 1; - break; + crypt_free_buffer_pages(cc, clone); + bio_put(clone); + gfp_mask |= __GFP_WAIT; + goto retry; } - /* - * If additional pages cannot be allocated without waiting, - * return a partially-allocated bio. The caller will then try - * to allocate more bios while submitting this partial bio. - */ - gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; + len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size; - len = (size > PAGE_SIZE) ? PAGE_SIZE : size; + bvec = &clone->bi_io_vec[clone->bi_vcnt++]; + bvec->bv_page = page; + bvec->bv_len = len; + bvec->bv_offset = 0; - if (!bio_add_page(clone, page, len, 0)) { - mempool_free(page, cc->page_pool); - break; - } + clone->bi_iter.bi_size += len; - size -= len; + remaining_size -= len; } - if (!clone->bi_iter.bi_size) { - bio_put(clone); - return NULL; - } +return_clone: + if (unlikely(gfp_mask & __GFP_WAIT)) + mutex_unlock(&cc->bio_alloc_lock); return clone; } @@ -1020,7 +1038,6 @@ static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc, io->base_bio = bio; io->sector = sector; io->error = 0; - io->base_io = NULL; io->ctx.req = NULL; atomic_set(&io->io_pending, 0); } @@ -1033,13 +1050,11 @@ static void crypt_inc_pending(struct dm_crypt_io *io) /* * One of the bios was finished. Check for completion of * the whole request and correctly clean up the buffer. - * If base_io is set, wait for the last fragment to complete. */ static void crypt_dec_pending(struct dm_crypt_io *io) { struct crypt_config *cc = io->cc; struct bio *base_bio = io->base_bio; - struct dm_crypt_io *base_io = io->base_io; int error = io->error; if (!atomic_dec_and_test(&io->io_pending)) @@ -1047,16 +1062,8 @@ static void crypt_dec_pending(struct dm_crypt_io *io) if (io->ctx.req) crypt_free_req(cc, io->ctx.req, base_bio); - if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size)) - mempool_free(io, cc->io_pool); - - if (likely(!base_io)) - bio_endio(base_bio, error); - else { - if (error && !base_io->error) - base_io->error = error; - crypt_dec_pending(base_io); - } + + bio_endio(base_bio, error); } /* @@ -1117,15 +1124,15 @@ static void clone_init(struct dm_crypt_io *io, struct bio *clone) static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) { struct crypt_config *cc = io->cc; - struct bio *base_bio = io->base_bio; struct bio *clone; /* - * The block layer might modify the bvec array, so always - * copy the required bvecs because we need the original - * one in order to decrypt the whole bio data *afterwards*. + * We need the original biovec array in order to decrypt + * the whole bio data *afterwards* -- thanks to immutable + * biovecs we don't need to worry about the block layer + * modifying the biovec array; so leverage bio_clone_fast(). */ - clone = bio_clone_bioset(base_bio, gfp, cc->bs); + clone = bio_clone_fast(io->base_bio, gfp, cc->bs); if (!clone) return 1; @@ -1138,37 +1145,97 @@ static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) return 0; } +static void kcryptd_io_read_work(struct work_struct *work) +{ + struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); + + crypt_inc_pending(io); + if (kcryptd_io_read(io, GFP_NOIO)) + io->error = -ENOMEM; + crypt_dec_pending(io); +} + +static void kcryptd_queue_read(struct dm_crypt_io *io) +{ + struct crypt_config *cc = io->cc; + + INIT_WORK(&io->work, kcryptd_io_read_work); + queue_work(cc->io_queue, &io->work); +} + static void kcryptd_io_write(struct dm_crypt_io *io) { struct bio *clone = io->ctx.bio_out; + generic_make_request(clone); } -static void kcryptd_io(struct work_struct *work) +#define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node) + +static int dmcrypt_write(void *data) { - struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); + struct crypt_config *cc = data; + struct dm_crypt_io *io; - if (bio_data_dir(io->base_bio) == READ) { - crypt_inc_pending(io); - if (kcryptd_io_read(io, GFP_NOIO)) - io->error = -ENOMEM; - crypt_dec_pending(io); - } else - kcryptd_io_write(io); -} + while (1) { + struct rb_root write_tree; + struct blk_plug plug; -static void kcryptd_queue_io(struct dm_crypt_io *io) -{ - struct crypt_config *cc = io->cc; + DECLARE_WAITQUEUE(wait, current); - INIT_WORK(&io->work, kcryptd_io); - queue_work(cc->io_queue, &io->work); + spin_lock_irq(&cc->write_thread_wait.lock); +continue_locked: + + if (!RB_EMPTY_ROOT(&cc->write_tree)) + goto pop_from_list; + + __set_current_state(TASK_INTERRUPTIBLE); + __add_wait_queue(&cc->write_thread_wait, &wait); + + spin_unlock_irq(&cc->write_thread_wait.lock); + + if (unlikely(kthread_should_stop())) { + set_task_state(current, TASK_RUNNING); + remove_wait_queue(&cc->write_thread_wait, &wait); + break; + } + + schedule(); + + set_task_state(current, TASK_RUNNING); + spin_lock_irq(&cc->write_thread_wait.lock); + __remove_wait_queue(&cc->write_thread_wait, &wait); + goto continue_locked; + +pop_from_list: + write_tree = cc->write_tree; + cc->write_tree = RB_ROOT; + spin_unlock_irq(&cc->write_thread_wait.lock); + + BUG_ON(rb_parent(write_tree.rb_node)); + + /* + * Note: we cannot walk the tree here with rb_next because + * the structures may be freed when kcryptd_io_write is called. + */ + blk_start_plug(&plug); + do { + io = crypt_io_from_node(rb_first(&write_tree)); + rb_erase(&io->rb_node, &write_tree); + kcryptd_io_write(io); + } while (!RB_EMPTY_ROOT(&write_tree)); + blk_finish_plug(&plug); + } + return 0; } static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) { struct bio *clone = io->ctx.bio_out; struct crypt_config *cc = io->cc; + unsigned long flags; + sector_t sector; + struct rb_node **rbp, *parent; if (unlikely(io->error < 0)) { crypt_free_buffer_pages(cc, clone); @@ -1182,20 +1249,34 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) clone->bi_iter.bi_sector = cc->start + io->sector; - if (async) - kcryptd_queue_io(io); - else + if (likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) { generic_make_request(clone); + return; + } + + spin_lock_irqsave(&cc->write_thread_wait.lock, flags); + rbp = &cc->write_tree.rb_node; + parent = NULL; + sector = io->sector; + while (*rbp) { + parent = *rbp; + if (sector < crypt_io_from_node(parent)->sector) + rbp = &(*rbp)->rb_left; + else + rbp = &(*rbp)->rb_right; + } + rb_link_node(&io->rb_node, parent, rbp); + rb_insert_color(&io->rb_node, &cc->write_tree); + + wake_up_locked(&cc->write_thread_wait); + spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags); } static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) { struct crypt_config *cc = io->cc; struct bio *clone; - struct dm_crypt_io *new_io; int crypt_finished; - unsigned out_of_pages = 0; - unsigned remaining = io->base_bio->bi_iter.bi_size; sector_t sector = io->sector; int r; @@ -1205,80 +1286,30 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) crypt_inc_pending(io); crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); - /* - * The allocated buffers can be smaller than the whole bio, - * so repeat the whole process until all the data can be handled. - */ - while (remaining) { - clone = crypt_alloc_buffer(io, remaining, &out_of_pages); - if (unlikely(!clone)) { - io->error = -ENOMEM; - break; - } - - io->ctx.bio_out = clone; - io->ctx.iter_out = clone->bi_iter; - - remaining -= clone->bi_iter.bi_size; - sector += bio_sectors(clone); - - crypt_inc_pending(io); - - r = crypt_convert(cc, &io->ctx); - if (r < 0) - io->error = -EIO; - - crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending); - - /* Encryption was already finished, submit io now */ - if (crypt_finished) { - kcryptd_crypt_write_io_submit(io, 0); + clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size); + if (unlikely(!clone)) { + io->error = -EIO; + goto dec; + } - /* - * If there was an error, do not try next fragments. - * For async, error is processed in async handler. - */ - if (unlikely(r < 0)) - break; + io->ctx.bio_out = clone; + io->ctx.iter_out = clone->bi_iter; - io->sector = sector; - } + sector += bio_sectors(clone); - /* - * Out of memory -> run queues - * But don't wait if split was due to the io size restriction - */ - if (unlikely(out_of_pages)) - congestion_wait(BLK_RW_ASYNC, HZ/100); - - /* - * With async crypto it is unsafe to share the crypto context - * between fragments, so switch to a new dm_crypt_io structure. - */ - if (unlikely(!crypt_finished && remaining)) { - new_io = mempool_alloc(cc->io_pool, GFP_NOIO); - crypt_io_init(new_io, io->cc, io->base_bio, sector); - crypt_inc_pending(new_io); - crypt_convert_init(cc, &new_io->ctx, NULL, - io->base_bio, sector); - new_io->ctx.iter_in = io->ctx.iter_in; - - /* - * Fragments after the first use the base_io - * pending count. - */ - if (!io->base_io) - new_io->base_io = io; - else { - new_io->base_io = io->base_io; - crypt_inc_pending(io->base_io); - crypt_dec_pending(io); - } + crypt_inc_pending(io); + r = crypt_convert(cc, &io->ctx); + if (r) + io->error = -EIO; + crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending); - io = new_io; - } + /* Encryption was already finished, submit io now */ + if (crypt_finished) { + kcryptd_crypt_write_io_submit(io, 0); + io->sector = sector; } +dec: crypt_dec_pending(io); } @@ -1484,6 +1515,9 @@ static void crypt_dtr(struct dm_target *ti) if (!cc) return; + if (cc->write_thread) + kthread_stop(cc->write_thread); + if (cc->io_queue) destroy_workqueue(cc->io_queue); if (cc->crypt_queue) @@ -1498,8 +1532,6 @@ static void crypt_dtr(struct dm_target *ti) mempool_destroy(cc->page_pool); if (cc->req_pool) mempool_destroy(cc->req_pool); - if (cc->io_pool) - mempool_destroy(cc->io_pool); if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) cc->iv_gen_ops->dtr(cc); @@ -1691,7 +1723,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) char dummy; static struct dm_arg _args[] = { - {0, 1, "Invalid number of feature args"}, + {0, 3, "Invalid number of feature args"}, }; if (argc < 5) { @@ -1713,13 +1745,6 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (ret < 0) goto bad; - ret = -ENOMEM; - cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); - if (!cc->io_pool) { - ti->error = "Cannot allocate crypt io mempool"; - goto bad; - } - cc->dmreq_start = sizeof(struct ablkcipher_request); cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc)); cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request)); @@ -1737,6 +1762,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) iv_size_padding = crypto_ablkcipher_alignmask(any_tfm(cc)); } + ret = -ENOMEM; cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + sizeof(struct dm_crypt_request) + iv_size_padding + cc->iv_size); if (!cc->req_pool) { @@ -1749,7 +1775,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) sizeof(struct dm_crypt_request) + iv_size_padding + cc->iv_size, ARCH_KMALLOC_MINALIGN); - cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); + cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0); if (!cc->page_pool) { ti->error = "Cannot allocate page mempool"; goto bad; @@ -1761,6 +1787,8 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) goto bad; } + mutex_init(&cc->bio_alloc_lock); + ret = -EINVAL; if (sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) { ti->error = "Invalid iv_offset sector"; @@ -1791,15 +1819,27 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (ret) goto bad; - opt_string = dm_shift_arg(&as); + ret = -EINVAL; + while (opt_params--) { + opt_string = dm_shift_arg(&as); + if (!opt_string) { + ti->error = "Not enough feature arguments"; + goto bad; + } - if (opt_params == 1 && opt_string && - !strcasecmp(opt_string, "allow_discards")) - ti->num_discard_bios = 1; - else if (opt_params) { - ret = -EINVAL; - ti->error = "Invalid feature arguments"; - goto bad; + if (!strcasecmp(opt_string, "allow_discards")) + ti->num_discard_bios = 1; + + else if (!strcasecmp(opt_string, "same_cpu_crypt")) + set_bit(DM_CRYPT_SAME_CPU, &cc->flags); + + else if (!strcasecmp(opt_string, "submit_from_crypt_cpus")) + set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); + + else { + ti->error = "Invalid feature arguments"; + goto bad; + } } } @@ -1810,13 +1850,28 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) goto bad; } - cc->crypt_queue = alloc_workqueue("kcryptd", - WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); + if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) + cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); + else + cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, + num_online_cpus()); if (!cc->crypt_queue) { ti->error = "Couldn't create kcryptd queue"; goto bad; } + init_waitqueue_head(&cc->write_thread_wait); + cc->write_tree = RB_ROOT; + + cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write"); + if (IS_ERR(cc->write_thread)) { + ret = PTR_ERR(cc->write_thread); + cc->write_thread = NULL; + ti->error = "Couldn't spawn write thread"; + goto bad; + } + wake_up_process(cc->write_thread); + ti->num_flush_bios = 1; ti->discard_zeroes_data_unsupported = true; @@ -1858,7 +1913,7 @@ static int crypt_map(struct dm_target *ti, struct bio *bio) if (bio_data_dir(io->base_bio) == READ) { if (kcryptd_io_read(io, GFP_NOWAIT)) - kcryptd_queue_io(io); + kcryptd_queue_read(io); } else kcryptd_queue_crypt(io); @@ -1870,6 +1925,7 @@ static void crypt_status(struct dm_target *ti, status_type_t type, { struct crypt_config *cc = ti->private; unsigned i, sz = 0; + int num_feature_args = 0; switch (type) { case STATUSTYPE_INFO: @@ -1888,8 +1944,18 @@ static void crypt_status(struct dm_target *ti, status_type_t type, DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, cc->dev->name, (unsigned long long)cc->start); - if (ti->num_discard_bios) - DMEMIT(" 1 allow_discards"); + num_feature_args += !!ti->num_discard_bios; + num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags); + num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); + if (num_feature_args) { + DMEMIT(" %d", num_feature_args); + if (ti->num_discard_bios) + DMEMIT(" allow_discards"); + if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) + DMEMIT(" same_cpu_crypt"); + if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) + DMEMIT(" submit_from_crypt_cpus"); + } break; } @@ -1986,7 +2052,7 @@ static int crypt_iterate_devices(struct dm_target *ti, static struct target_type crypt_target = { .name = "crypt", - .version = {1, 13, 0}, + .version = {1, 14, 0}, .module = THIS_MODULE, .ctr = crypt_ctr, .dtr = crypt_dtr, @@ -2004,15 +2070,9 @@ static int __init dm_crypt_init(void) { int r; - _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); - if (!_crypt_io_pool) - return -ENOMEM; - r = dm_register_target(&crypt_target); - if (r < 0) { + if (r < 0) DMERR("register failed %d", r); - kmem_cache_destroy(_crypt_io_pool); - } return r; } @@ -2020,7 +2080,6 @@ static int __init dm_crypt_init(void) static void __exit dm_crypt_exit(void) { dm_unregister_target(&crypt_target); - kmem_cache_destroy(_crypt_io_pool); } module_init(dm_crypt_init); diff --git a/drivers/mfd/vexpress-sysreg.c b/drivers/mfd/vexpress-sysreg.c index 9e21e4fc9599..8f43ab8fd2d6 100644 --- a/drivers/mfd/vexpress-sysreg.c +++ b/drivers/mfd/vexpress-sysreg.c @@ -223,7 +223,7 @@ static int vexpress_sysreg_probe(struct platform_device *pdev) vexpress_config_set_master(vexpress_sysreg_get_master()); /* Confirm board type against DT property, if available */ - if (of_property_read_u32(of_allnodes, "arm,hbi", &dt_hbi) == 0) { + if (of_property_read_u32(of_root, "arm,hbi", &dt_hbi) == 0) { u32 id = vexpress_get_procid(VEXPRESS_SITE_MASTER); u32 hbi = (id >> SYS_PROCIDx_HBI_SHIFT) & SYS_HBI_MASK; diff --git a/drivers/of/Kconfig b/drivers/of/Kconfig index 1a13f5b722c5..7bcaeec876c0 100644 --- a/drivers/of/Kconfig +++ b/drivers/of/Kconfig @@ -7,10 +7,9 @@ config OF menu "Device Tree and Open Firmware support" depends on OF -config OF_SELFTEST - bool "Device Tree Runtime self tests" +config OF_UNITTEST + bool "Device Tree runtime unit tests" depends on OF_IRQ && OF_EARLY_FLATTREE - select OF_DYNAMIC select OF_RESOLVE help This option builds in test cases for the device tree infrastructure @@ -23,6 +22,7 @@ config OF_FLATTREE bool select DTC select LIBFDT + select CRC32 config OF_EARLY_FLATTREE bool @@ -83,4 +83,9 @@ config OF_RESERVED_MEM config OF_RESOLVE bool +config OF_OVERLAY + bool "Device Tree overlays" + select OF_DYNAMIC + select OF_RESOLVE + endmenu # OF diff --git a/drivers/of/Makefile b/drivers/of/Makefile index ca9209ce50cd..7563f36c71db 100644 --- a/drivers/of/Makefile +++ b/drivers/of/Makefile @@ -6,14 +6,15 @@ obj-$(CONFIG_OF_PROMTREE) += pdt.o obj-$(CONFIG_OF_ADDRESS) += address.o obj-$(CONFIG_OF_IRQ) += irq.o obj-$(CONFIG_OF_NET) += of_net.o -obj-$(CONFIG_OF_SELFTEST) += of_selftest.o -of_selftest-objs := selftest.o testcase-data/testcases.dtb.o +obj-$(CONFIG_OF_UNITTEST) += of_unittest.o +of_unittest-objs := unittest.o unittest-data/testcases.dtb.o obj-$(CONFIG_OF_MDIO) += of_mdio.o obj-$(CONFIG_OF_PCI) += of_pci.o obj-$(CONFIG_OF_PCI_IRQ) += of_pci_irq.o obj-$(CONFIG_OF_MTD) += of_mtd.o obj-$(CONFIG_OF_RESERVED_MEM) += of_reserved_mem.o obj-$(CONFIG_OF_RESOLVE) += resolver.o +obj-$(CONFIG_OF_OVERLAY) += overlay.o CFLAGS_fdt.o = -I$(src)/../../scripts/dtc/libfdt CFLAGS_fdt_address.o = -I$(src)/../../scripts/dtc/libfdt diff --git a/drivers/of/address.c b/drivers/of/address.c index 1f5a9b38250b..eca588862ab9 100644 --- a/drivers/of/address.c +++ b/drivers/of/address.c @@ -889,7 +889,7 @@ EXPORT_SYMBOL(of_iomap); * return PTR_ERR(base); */ void __iomem *of_io_request_and_map(struct device_node *np, int index, - char *name) + const char *name) { struct resource res; void __iomem *mem; diff --git a/drivers/of/base.c b/drivers/of/base.c index 4e0e8ee2d17f..08e243924f9f 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -32,11 +32,12 @@ LIST_HEAD(aliases_lookup); -struct device_node *of_allnodes; -EXPORT_SYMBOL(of_allnodes); +struct device_node *of_root; +EXPORT_SYMBOL(of_root); struct device_node *of_chosen; struct device_node *of_aliases; struct device_node *of_stdout; +static const char *of_stdout_options; struct kset *of_kset; @@ -48,7 +49,7 @@ struct kset *of_kset; */ DEFINE_MUTEX(of_mutex); -/* use when traversing tree through the allnext, child, sibling, +/* use when traversing tree through the child, sibling, * or parent members of struct device_node. */ DEFINE_RAW_SPINLOCK(devtree_lock); @@ -210,7 +211,7 @@ static int __init of_init(void) mutex_unlock(&of_mutex); /* Symlink in /proc as required by userspace ABI */ - if (of_allnodes) + if (of_root) proc_symlink("device-tree", NULL, "/sys/firmware/devicetree/base"); return 0; @@ -251,6 +252,23 @@ struct property *of_find_property(const struct device_node *np, } EXPORT_SYMBOL(of_find_property); +struct device_node *__of_find_all_nodes(struct device_node *prev) +{ + struct device_node *np; + if (!prev) { + np = of_root; + } else if (prev->child) { + np = prev->child; + } else { + /* Walk back up looking for a sibling, or the end of the structure */ + np = prev; + while (np->parent && !np->sibling) + np = np->parent; + np = np->sibling; /* Might be null at the end of the tree */ + } + return np; +} + /** * of_find_all_nodes - Get next node in global list * @prev: Previous node or NULL to start iteration @@ -265,10 +283,8 @@ struct device_node *of_find_all_nodes(struct device_node *prev) unsigned long flags; raw_spin_lock_irqsave(&devtree_lock, flags); - np = prev ? prev->allnext : of_allnodes; - for (; np != NULL; np = np->allnext) - if (of_node_get(np)) - break; + np = __of_find_all_nodes(prev); + of_node_get(np); of_node_put(prev); raw_spin_unlock_irqrestore(&devtree_lock, flags); return np; @@ -513,27 +529,27 @@ EXPORT_SYMBOL(of_machine_is_compatible); * * @device: Node to check for availability, with locks already held * - * Returns 1 if the status property is absent or set to "okay" or "ok", - * 0 otherwise + * Returns true if the status property is absent or set to "okay" or "ok", + * false otherwise */ -static int __of_device_is_available(const struct device_node *device) +static bool __of_device_is_available(const struct device_node *device) { const char *status; int statlen; if (!device) - return 0; + return false; status = __of_get_property(device, "status", &statlen); if (status == NULL) - return 1; + return true; if (statlen > 0) { if (!strcmp(status, "okay") || !strcmp(status, "ok")) - return 1; + return true; } - return 0; + return false; } /** @@ -541,13 +557,13 @@ static int __of_device_is_available(const struct device_node *device) * * @device: Node to check for availability * - * Returns 1 if the status property is absent or set to "okay" or "ok", - * 0 otherwise + * Returns true if the status property is absent or set to "okay" or "ok", + * false otherwise */ -int of_device_is_available(const struct device_node *device) +bool of_device_is_available(const struct device_node *device) { unsigned long flags; - int res; + bool res; raw_spin_lock_irqsave(&devtree_lock, flags); res = __of_device_is_available(device); @@ -704,8 +720,9 @@ static struct device_node *__of_find_node_by_path(struct device_node *parent, const char *path) { struct device_node *child; - int len = strchrnul(path, '/') - path; + int len; + len = strcspn(path, "/:"); if (!len) return NULL; @@ -721,11 +738,14 @@ static struct device_node *__of_find_node_by_path(struct device_node *parent, } /** - * of_find_node_by_path - Find a node matching a full OF path + * of_find_node_opts_by_path - Find a node matching a full OF path * @path: Either the full path to match, or if the path does not * start with '/', the name of a property of the /aliases * node (an alias). In the case of an alias, the node * matching the alias' value will be returned. + * @opts: Address of a pointer into which to store the start of + * an options string appended to the end of the path with + * a ':' separator. * * Valid paths: * /foo/bar Full path @@ -735,19 +755,27 @@ static struct device_node *__of_find_node_by_path(struct device_node *parent, * Returns a node pointer with refcount incremented, use * of_node_put() on it when done. */ -struct device_node *of_find_node_by_path(const char *path) +struct device_node *of_find_node_opts_by_path(const char *path, const char **opts) { struct device_node *np = NULL; struct property *pp; unsigned long flags; + const char *separator = strchr(path, ':'); + + if (opts) + *opts = separator ? separator + 1 : NULL; if (strcmp(path, "/") == 0) - return of_node_get(of_allnodes); + return of_node_get(of_root); /* The path could begin with an alias */ if (*path != '/') { - char *p = strchrnul(path, '/'); - int len = p - path; + int len; + const char *p = separator; + + if (!p) + p = strchrnul(path, '/'); + len = p - path; /* of_aliases must not be NULL */ if (!of_aliases) @@ -767,16 +795,18 @@ struct device_node *of_find_node_by_path(const char *path) /* Step down the tree matching path components */ raw_spin_lock_irqsave(&devtree_lock, flags); if (!np) - np = of_node_get(of_allnodes); + np = of_node_get(of_root); while (np && *path == '/') { path++; /* Increment past '/' delimiter */ np = __of_find_node_by_path(np, path); path = strchrnul(path, '/'); + if (separator && separator < path) + break; } raw_spin_unlock_irqrestore(&devtree_lock, flags); return np; } -EXPORT_SYMBOL(of_find_node_by_path); +EXPORT_SYMBOL(of_find_node_opts_by_path); /** * of_find_node_by_name - Find a node by its "name" property @@ -796,8 +826,7 @@ struct device_node *of_find_node_by_name(struct device_node *from, unsigned long flags; raw_spin_lock_irqsave(&devtree_lock, flags); - np = from ? from->allnext : of_allnodes; - for (; np; np = np->allnext) + for_each_of_allnodes_from(from, np) if (np->name && (of_node_cmp(np->name, name) == 0) && of_node_get(np)) break; @@ -826,8 +855,7 @@ struct device_node *of_find_node_by_type(struct device_node *from, unsigned long flags; raw_spin_lock_irqsave(&devtree_lock, flags); - np = from ? from->allnext : of_allnodes; - for (; np; np = np->allnext) + for_each_of_allnodes_from(from, np) if (np->type && (of_node_cmp(np->type, type) == 0) && of_node_get(np)) break; @@ -858,12 +886,10 @@ struct device_node *of_find_compatible_node(struct device_node *from, unsigned long flags; raw_spin_lock_irqsave(&devtree_lock, flags); - np = from ? from->allnext : of_allnodes; - for (; np; np = np->allnext) { + for_each_of_allnodes_from(from, np) if (__of_device_is_compatible(np, compatible, type, NULL) && of_node_get(np)) break; - } of_node_put(from); raw_spin_unlock_irqrestore(&devtree_lock, flags); return np; @@ -890,8 +916,7 @@ struct device_node *of_find_node_with_property(struct device_node *from, unsigned long flags; raw_spin_lock_irqsave(&devtree_lock, flags); - np = from ? from->allnext : of_allnodes; - for (; np; np = np->allnext) { + for_each_of_allnodes_from(from, np) { for (pp = np->properties; pp; pp = pp->next) { if (of_prop_cmp(pp->name, prop_name) == 0) { of_node_get(np); @@ -973,8 +998,7 @@ struct device_node *of_find_matching_node_and_match(struct device_node *from, *match = NULL; raw_spin_lock_irqsave(&devtree_lock, flags); - np = from ? from->allnext : of_allnodes; - for (; np; np = np->allnext) { + for_each_of_allnodes_from(from, np) { m = __of_match_node(matches, np); if (m && of_node_get(np)) { if (match) @@ -1031,7 +1055,7 @@ struct device_node *of_find_node_by_phandle(phandle handle) return NULL; raw_spin_lock_irqsave(&devtree_lock, flags); - for (np = of_allnodes; np; np = np->allnext) + for_each_of_allnodes(np) if (np->phandle == handle) break; of_node_get(np); @@ -1287,6 +1311,7 @@ int of_property_read_u64_array(const struct device_node *np, } return 0; } +EXPORT_SYMBOL_GPL(of_property_read_u64_array); /** * of_property_read_string - Find and read a string from a property @@ -1356,7 +1381,7 @@ int of_property_match_string(struct device_node *np, const char *propname, EXPORT_SYMBOL_GPL(of_property_match_string); /** - * of_property_read_string_util() - Utility helper for parsing string properties + * of_property_read_string_helper() - Utility helper for parsing string properties * @np: device node from which the property value is to be read. * @propname: name of the property to be searched. * @out_strs: output array of string pointers. @@ -1876,7 +1901,7 @@ void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align)) if (IS_ENABLED(CONFIG_PPC) && !name) name = of_get_property(of_aliases, "stdout", NULL); if (name) - of_stdout = of_find_node_by_path(name); + of_stdout = of_find_node_opts_by_path(name, &of_stdout_options); } if (!of_aliases) @@ -2002,7 +2027,8 @@ bool of_console_check(struct device_node *dn, char *name, int index) { if (!dn || dn != of_stdout || console_set_on_cmdline) return false; - return !add_preferred_console(name, index, NULL); + return !add_preferred_console(name, index, + kstrdup(of_stdout_options, GFP_KERNEL)); } EXPORT_SYMBOL_GPL(of_console_check); diff --git a/drivers/of/device.c b/drivers/of/device.c index 2ef5ca45a136..6601cc62a990 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -2,6 +2,9 @@ #include <linux/kernel.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/of_address.h> +#include <linux/of_iommu.h> +#include <linux/dma-mapping.h> #include <linux/init.h> #include <linux/module.h> #include <linux/mod_devicetable.h> @@ -66,6 +69,87 @@ int of_device_add(struct platform_device *ofdev) return device_add(&ofdev->dev); } +/** + * of_dma_configure - Setup DMA configuration + * @dev: Device to apply DMA configuration + * @np: Pointer to OF node having DMA configuration + * + * Try to get devices's DMA configuration from DT and update it + * accordingly. + * + * If platform code needs to use its own special DMA configuration, it + * can use a platform bus notifier and handle BUS_NOTIFY_ADD_DEVICE events + * to fix up DMA configuration. + */ +void of_dma_configure(struct device *dev, struct device_node *np) +{ + u64 dma_addr, paddr, size; + int ret; + bool coherent; + unsigned long offset; + struct iommu_ops *iommu; + + /* + * Set default coherent_dma_mask to 32 bit. Drivers are expected to + * setup the correct supported mask. + */ + if (!dev->coherent_dma_mask) + dev->coherent_dma_mask = DMA_BIT_MASK(32); + + /* + * Set it to coherent_dma_mask by default if the architecture + * code has not set it. + */ + if (!dev->dma_mask) + dev->dma_mask = &dev->coherent_dma_mask; + + ret = of_dma_get_range(np, &dma_addr, &paddr, &size); + if (ret < 0) { + dma_addr = offset = 0; + size = dev->coherent_dma_mask + 1; + } else { + offset = PFN_DOWN(paddr - dma_addr); + + /* + * Add a work around to treat the size as mask + 1 in case + * it is defined in DT as a mask. + */ + if (size & 1) { + dev_warn(dev, "Invalid size 0x%llx for dma-range\n", + size); + size = size + 1; + } + + if (!size) { + dev_err(dev, "Adjusted size 0x%llx invalid\n", size); + return; + } + dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", offset); + } + + dev->dma_pfn_offset = offset; + + /* + * Limit coherent and dma mask based on size and default mask + * set by the driver. + */ + dev->coherent_dma_mask = min(dev->coherent_dma_mask, + DMA_BIT_MASK(ilog2(dma_addr + size))); + *dev->dma_mask = min((*dev->dma_mask), + DMA_BIT_MASK(ilog2(dma_addr + size))); + + coherent = of_dma_is_coherent(np); + dev_dbg(dev, "device is%sdma coherent\n", + coherent ? " " : " not "); + + iommu = of_iommu_configure(dev, np); + dev_dbg(dev, "device is%sbehind an iommu\n", + iommu ? " " : " not "); + + arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent); +} +EXPORT_SYMBOL_GPL(of_dma_configure); + int of_device_register(struct platform_device *pdev) { device_initialize(&pdev->dev); diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index ab7374960393..57e25f67131b 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -77,18 +77,132 @@ int of_reconfig_notifier_unregister(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(of_reconfig_notifier_unregister); -int of_reconfig_notify(unsigned long action, void *p) +#ifdef DEBUG +const char *action_names[] = { + [OF_RECONFIG_ATTACH_NODE] = "ATTACH_NODE", + [OF_RECONFIG_DETACH_NODE] = "DETACH_NODE", + [OF_RECONFIG_ADD_PROPERTY] = "ADD_PROPERTY", + [OF_RECONFIG_REMOVE_PROPERTY] = "REMOVE_PROPERTY", + [OF_RECONFIG_UPDATE_PROPERTY] = "UPDATE_PROPERTY", +}; +#endif + +int of_reconfig_notify(unsigned long action, struct of_reconfig_data *p) { int rc; +#ifdef DEBUG + struct of_reconfig_data *pr = p; + + switch (action) { + case OF_RECONFIG_ATTACH_NODE: + case OF_RECONFIG_DETACH_NODE: + pr_debug("of/notify %-15s %s\n", action_names[action], + pr->dn->full_name); + break; + case OF_RECONFIG_ADD_PROPERTY: + case OF_RECONFIG_REMOVE_PROPERTY: + case OF_RECONFIG_UPDATE_PROPERTY: + pr_debug("of/notify %-15s %s:%s\n", action_names[action], + pr->dn->full_name, pr->prop->name); + break; + } +#endif rc = blocking_notifier_call_chain(&of_reconfig_chain, action, p); return notifier_to_errno(rc); } +/* + * of_reconfig_get_state_change() - Returns new state of device + * @action - action of the of notifier + * @arg - argument of the of notifier + * + * Returns the new state of a device based on the notifier used. + * Returns 0 on device going from enabled to disabled, 1 on device + * going from disabled to enabled and -1 on no change. + */ +int of_reconfig_get_state_change(unsigned long action, struct of_reconfig_data *pr) +{ + struct property *prop, *old_prop = NULL; + int is_status, status_state, old_status_state, prev_state, new_state; + + /* figure out if a device should be created or destroyed */ + switch (action) { + case OF_RECONFIG_ATTACH_NODE: + case OF_RECONFIG_DETACH_NODE: + prop = of_find_property(pr->dn, "status", NULL); + break; + case OF_RECONFIG_ADD_PROPERTY: + case OF_RECONFIG_REMOVE_PROPERTY: + prop = pr->prop; + break; + case OF_RECONFIG_UPDATE_PROPERTY: + prop = pr->prop; + old_prop = pr->old_prop; + break; + default: + return OF_RECONFIG_NO_CHANGE; + } + + is_status = 0; + status_state = -1; + old_status_state = -1; + prev_state = -1; + new_state = -1; + + if (prop && !strcmp(prop->name, "status")) { + is_status = 1; + status_state = !strcmp(prop->value, "okay") || + !strcmp(prop->value, "ok"); + if (old_prop) + old_status_state = !strcmp(old_prop->value, "okay") || + !strcmp(old_prop->value, "ok"); + } + + switch (action) { + case OF_RECONFIG_ATTACH_NODE: + prev_state = 0; + /* -1 & 0 status either missing or okay */ + new_state = status_state != 0; + break; + case OF_RECONFIG_DETACH_NODE: + /* -1 & 0 status either missing or okay */ + prev_state = status_state != 0; + new_state = 0; + break; + case OF_RECONFIG_ADD_PROPERTY: + if (is_status) { + /* no status property -> enabled (legacy) */ + prev_state = 1; + new_state = status_state; + } + break; + case OF_RECONFIG_REMOVE_PROPERTY: + if (is_status) { + prev_state = status_state; + /* no status property -> enabled (legacy) */ + new_state = 1; + } + break; + case OF_RECONFIG_UPDATE_PROPERTY: + if (is_status) { + prev_state = old_status_state != 0; + new_state = status_state != 0; + } + break; + } + + if (prev_state == new_state) + return OF_RECONFIG_NO_CHANGE; + + return new_state ? OF_RECONFIG_CHANGE_ADD : OF_RECONFIG_CHANGE_REMOVE; +} +EXPORT_SYMBOL_GPL(of_reconfig_get_state_change); + int of_property_notify(int action, struct device_node *np, struct property *prop, struct property *oldprop) { - struct of_prop_reconfig pr; + struct of_reconfig_data pr; /* only call notifiers if the node is attached */ if (!of_node_is_attached(np)) @@ -117,8 +231,6 @@ void __of_attach_node(struct device_node *np) np->child = NULL; np->sibling = np->parent->child; - np->allnext = np->parent->allnext; - np->parent->allnext = np; np->parent->child = np; of_node_clear_flag(np, OF_DETACHED); } @@ -128,8 +240,12 @@ void __of_attach_node(struct device_node *np) */ int of_attach_node(struct device_node *np) { + struct of_reconfig_data rd; unsigned long flags; + memset(&rd, 0, sizeof(rd)); + rd.dn = np; + mutex_lock(&of_mutex); raw_spin_lock_irqsave(&devtree_lock, flags); __of_attach_node(np); @@ -138,7 +254,7 @@ int of_attach_node(struct device_node *np) __of_attach_node_sysfs(np); mutex_unlock(&of_mutex); - of_reconfig_notify(OF_RECONFIG_ATTACH_NODE, np); + of_reconfig_notify(OF_RECONFIG_ATTACH_NODE, &rd); return 0; } @@ -154,17 +270,6 @@ void __of_detach_node(struct device_node *np) if (WARN_ON(!parent)) return; - if (of_allnodes == np) - of_allnodes = np->allnext; - else { - struct device_node *prev; - for (prev = of_allnodes; - prev->allnext != np; - prev = prev->allnext) - ; - prev->allnext = np->allnext; - } - if (parent->child == np) parent->child = np->sibling; else { @@ -187,9 +292,13 @@ void __of_detach_node(struct device_node *np) */ int of_detach_node(struct device_node *np) { + struct of_reconfig_data rd; unsigned long flags; int rc = 0; + memset(&rd, 0, sizeof(rd)); + rd.dn = np; + mutex_lock(&of_mutex); raw_spin_lock_irqsave(&devtree_lock, flags); __of_detach_node(np); @@ -198,7 +307,7 @@ int of_detach_node(struct device_node *np) __of_detach_node_sysfs(np); mutex_unlock(&of_mutex); - of_reconfig_notify(OF_RECONFIG_DETACH_NODE, np); + of_reconfig_notify(OF_RECONFIG_DETACH_NODE, &rd); return rc; } @@ -285,36 +394,54 @@ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags) } /** - * __of_node_alloc() - Create an empty device node dynamically. - * @full_name: Full name of the new device node - * @allocflags: Allocation flags (typically pass GFP_KERNEL) + * __of_node_dup() - Duplicate or create an empty device node dynamically. + * @fmt: Format string (plus vargs) for new full name of the device node * - * Create an empty device tree node, suitable for further modification. - * The node data are dynamically allocated and all the node flags - * have the OF_DYNAMIC & OF_DETACHED bits set. - * Returns the newly allocated node or NULL on out of memory error. + * Create an device tree node, either by duplicating an empty node or by allocating + * an empty one suitable for further modification. The node data are + * dynamically allocated and all the node flags have the OF_DYNAMIC & + * OF_DETACHED bits set. Returns the newly allocated node or NULL on out of + * memory error. */ -struct device_node *__of_node_alloc(const char *full_name, gfp_t allocflags) +struct device_node *__of_node_dup(const struct device_node *np, const char *fmt, ...) { + va_list vargs; struct device_node *node; - node = kzalloc(sizeof(*node), allocflags); + node = kzalloc(sizeof(*node), GFP_KERNEL); if (!node) return NULL; + va_start(vargs, fmt); + node->full_name = kvasprintf(GFP_KERNEL, fmt, vargs); + va_end(vargs); + if (!node->full_name) { + kfree(node); + return NULL; + } - node->full_name = kstrdup(full_name, allocflags); of_node_set_flag(node, OF_DYNAMIC); of_node_set_flag(node, OF_DETACHED); - if (!node->full_name) - goto err_free; - of_node_init(node); + /* Iterate over and duplicate all properties */ + if (np) { + struct property *pp, *new_pp; + for_each_property_of_node(np, pp) { + new_pp = __of_prop_dup(pp, GFP_KERNEL); + if (!new_pp) + goto err_prop; + if (__of_add_property(node, new_pp)) { + kfree(new_pp->name); + kfree(new_pp->value); + kfree(new_pp); + goto err_prop; + } + } + } return node; - err_free: - kfree(node->full_name); - kfree(node); + err_prop: + of_node_put(node); /* Frees the node and properties */ return NULL; } @@ -330,27 +457,15 @@ static void __of_changeset_entry_dump(struct of_changeset_entry *ce) { switch (ce->action) { case OF_RECONFIG_ADD_PROPERTY: - pr_debug("%p: %s %s/%s\n", - ce, "ADD_PROPERTY ", ce->np->full_name, - ce->prop->name); - break; case OF_RECONFIG_REMOVE_PROPERTY: - pr_debug("%p: %s %s/%s\n", - ce, "REMOVE_PROPERTY", ce->np->full_name, - ce->prop->name); - break; case OF_RECONFIG_UPDATE_PROPERTY: - pr_debug("%p: %s %s/%s\n", - ce, "UPDATE_PROPERTY", ce->np->full_name, - ce->prop->name); + pr_debug("of/cset<%p> %-15s %s/%s\n", ce, action_names[ce->action], + ce->np->full_name, ce->prop->name); break; case OF_RECONFIG_ATTACH_NODE: - pr_debug("%p: %s %s\n", - ce, "ATTACH_NODE ", ce->np->full_name); - break; case OF_RECONFIG_DETACH_NODE: - pr_debug("%p: %s %s\n", - ce, "DETACH_NODE ", ce->np->full_name); + pr_debug("of/cset<%p> %-15s %s\n", ce, action_names[ce->action], + ce->np->full_name); break; } } @@ -388,6 +503,7 @@ static void __of_changeset_entry_invert(struct of_changeset_entry *ce, static void __of_changeset_entry_notify(struct of_changeset_entry *ce, bool revert) { + struct of_reconfig_data rd; struct of_changeset_entry ce_inverted; int ret; @@ -399,7 +515,9 @@ static void __of_changeset_entry_notify(struct of_changeset_entry *ce, bool reve switch (ce->action) { case OF_RECONFIG_ATTACH_NODE: case OF_RECONFIG_DETACH_NODE: - ret = of_reconfig_notify(ce->action, ce->np); + memset(&rd, 0, sizeof(rd)); + rd.dn = ce->np; + ret = of_reconfig_notify(ce->action, &rd); break; case OF_RECONFIG_ADD_PROPERTY: case OF_RECONFIG_REMOVE_PROPERTY: diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index 63a6e1e6abdc..e6665ee1846f 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -9,6 +9,7 @@ * version 2 as published by the Free Software Foundation. */ +#include <linux/crc32.h> #include <linux/kernel.h> #include <linux/initrd.h> #include <linux/memblock.h> @@ -22,6 +23,7 @@ #include <linux/libfdt.h> #include <linux/debugfs.h> #include <linux/serial_core.h> +#include <linux/sysfs.h> #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ #include <asm/page.h> @@ -145,15 +147,15 @@ static void *unflatten_dt_alloc(void **mem, unsigned long size, * @mem: Memory chunk to use for allocating device nodes and properties * @p: pointer to node in flat tree * @dad: Parent struct device_node - * @allnextpp: pointer to ->allnext from last allocated device_node * @fpsize: Size of the node path up at the current depth. */ static void * unflatten_dt_node(void *blob, void *mem, int *poffset, struct device_node *dad, - struct device_node ***allnextpp, - unsigned long fpsize) + struct device_node **nodepp, + unsigned long fpsize, + bool dryrun) { const __be32 *p; struct device_node *np; @@ -200,7 +202,7 @@ static void * unflatten_dt_node(void *blob, np = unflatten_dt_alloc(&mem, sizeof(struct device_node) + allocl, __alignof__(struct device_node)); - if (allnextpp) { + if (!dryrun) { char *fn; of_node_init(np); np->full_name = fn = ((char *)np) + sizeof(*np); @@ -222,16 +224,10 @@ static void * unflatten_dt_node(void *blob, memcpy(fn, pathp, l); prev_pp = &np->properties; - **allnextpp = np; - *allnextpp = &np->allnext; if (dad != NULL) { np->parent = dad; - /* we temporarily use the next field as `last_child'*/ - if (dad->next == NULL) - dad->child = np; - else - dad->next->sibling = np; - dad->next = np; + np->sibling = dad->child; + dad->child = np; } } /* process properties */ @@ -254,7 +250,7 @@ static void * unflatten_dt_node(void *blob, has_name = 1; pp = unflatten_dt_alloc(&mem, sizeof(struct property), __alignof__(struct property)); - if (allnextpp) { + if (!dryrun) { /* We accept flattened tree phandles either in * ePAPR-style "phandle" properties, or the * legacy "linux,phandle" properties. If both @@ -296,7 +292,7 @@ static void * unflatten_dt_node(void *blob, sz = (pa - ps) + 1; pp = unflatten_dt_alloc(&mem, sizeof(struct property) + sz, __alignof__(struct property)); - if (allnextpp) { + if (!dryrun) { pp->name = "name"; pp->length = sz; pp->value = pp + 1; @@ -308,7 +304,7 @@ static void * unflatten_dt_node(void *blob, (char *)pp->value); } } - if (allnextpp) { + if (!dryrun) { *prev_pp = NULL; np->name = of_get_property(np, "name", NULL); np->type = of_get_property(np, "device_type", NULL); @@ -324,12 +320,30 @@ static void * unflatten_dt_node(void *blob, if (depth < 0) depth = 0; while (*poffset > 0 && depth > old_depth) - mem = unflatten_dt_node(blob, mem, poffset, np, allnextpp, - fpsize); + mem = unflatten_dt_node(blob, mem, poffset, np, NULL, + fpsize, dryrun); if (*poffset < 0 && *poffset != -FDT_ERR_NOTFOUND) pr_err("unflatten: error %d processing FDT\n", *poffset); + /* + * Reverse the child list. Some drivers assumes node order matches .dts + * node order + */ + if (!dryrun && np->child) { + struct device_node *child = np->child; + np->child = NULL; + while (child) { + struct device_node *next = child->sibling; + child->sibling = np->child; + np->child = child; + child = next; + } + } + + if (nodepp) + *nodepp = np; + return mem; } @@ -352,7 +366,6 @@ static void __unflatten_device_tree(void *blob, unsigned long size; int start; void *mem; - struct device_node **allnextp = mynodes; pr_debug(" -> unflatten_device_tree()\n"); @@ -373,7 +386,7 @@ static void __unflatten_device_tree(void *blob, /* First pass, scan for size */ start = 0; - size = (unsigned long)unflatten_dt_node(blob, NULL, &start, NULL, NULL, 0); + size = (unsigned long)unflatten_dt_node(blob, NULL, &start, NULL, NULL, 0, true); size = ALIGN(size, 4); pr_debug(" size is %lx, allocating...\n", size); @@ -391,11 +404,10 @@ static void __unflatten_device_tree(void *blob, /* Second pass, do actual unflattening */ start = 0; - unflatten_dt_node(blob, mem, &start, NULL, &allnextp, 0); + unflatten_dt_node(blob, mem, &start, NULL, mynodes, 0, false); if (be32_to_cpup(mem + size) != 0xdeadbeef) pr_warning("End of tree marker overwritten: %08x\n", be32_to_cpup(mem + size)); - *allnextp = NULL; pr_debug(" <- unflatten_device_tree()\n"); } @@ -428,6 +440,8 @@ void *initial_boot_params; #ifdef CONFIG_OF_EARLY_FLATTREE +static u32 of_fdt_crc32; + /** * res_mem_reserve_reg() - reserve all memory described in 'reg' property */ @@ -754,7 +768,7 @@ static inline void early_init_dt_check_for_initrd(unsigned long node) #ifdef CONFIG_SERIAL_EARLYCON extern struct of_device_id __earlycon_of_table[]; -int __init early_init_dt_scan_chosen_serial(void) +static int __init early_init_dt_scan_chosen_serial(void) { int offset; const char *p; @@ -936,6 +950,11 @@ void __init __weak early_init_dt_add_memory_arch(u64 base, u64 size) const u64 phys_offset = __pa(PAGE_OFFSET); if (!PAGE_ALIGNED(base)) { + if (size < PAGE_SIZE - (base & ~PAGE_MASK)) { + pr_warn("Ignoring memory block 0x%llx - 0x%llx\n", + base, base + size); + return; + } size -= PAGE_SIZE - (base & ~PAGE_MASK); base = PAGE_ALIGN(base); } @@ -998,15 +1017,14 @@ bool __init early_init_dt_verify(void *params) if (!params) return false; - /* Setup flat device-tree pointer */ - initial_boot_params = params; - /* check device tree validity */ - if (fdt_check_header(params)) { - initial_boot_params = NULL; + if (fdt_check_header(params)) return false; - } + /* Setup flat device-tree pointer */ + initial_boot_params = params; + of_fdt_crc32 = crc32_be(~0, initial_boot_params, + fdt_totalsize(initial_boot_params)); return true; } @@ -1045,7 +1063,7 @@ bool __init early_init_dt_scan(void *params) */ void __init unflatten_device_tree(void) { - __unflatten_device_tree(initial_boot_params, &of_allnodes, + __unflatten_device_tree(initial_boot_params, &of_root, early_init_dt_alloc_memory_arch); /* Get pointer to "/chosen" and "/aliases" nodes for use everywhere */ @@ -1084,27 +1102,32 @@ void __init unflatten_and_copy_device_tree(void) unflatten_device_tree(); } -#if defined(CONFIG_DEBUG_FS) && defined(DEBUG) -static struct debugfs_blob_wrapper flat_dt_blob; - -static int __init of_flat_dt_debugfs_export_fdt(void) +#ifdef CONFIG_SYSFS +static ssize_t of_fdt_raw_read(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, + char *buf, loff_t off, size_t count) { - struct dentry *d = debugfs_create_dir("device-tree", NULL); - - if (!d) - return -ENOENT; + memcpy(buf, initial_boot_params + off, count); + return count; +} - flat_dt_blob.data = initial_boot_params; - flat_dt_blob.size = fdt_totalsize(initial_boot_params); +static int __init of_fdt_raw_init(void) +{ + static struct bin_attribute of_fdt_raw_attr = + __BIN_ATTR(fdt, S_IRUSR, of_fdt_raw_read, NULL, 0); - d = debugfs_create_blob("flat-device-tree", S_IFREG | S_IRUSR, - d, &flat_dt_blob); - if (!d) - return -ENOENT; + if (!initial_boot_params) + return 0; - return 0; + if (of_fdt_crc32 != crc32_be(~0, initial_boot_params, + fdt_totalsize(initial_boot_params))) { + pr_warn("fdt: not creating '/sys/firmware/fdt': CRC check failed\n"); + return 0; + } + of_fdt_raw_attr.size = fdt_totalsize(initial_boot_params); + return sysfs_create_bin_file(firmware_kobj, &of_fdt_raw_attr); } -module_init(of_flat_dt_debugfs_export_fdt); +late_initcall(of_fdt_raw_init); #endif #endif /* CONFIG_OF_EARLY_FLATTREE */ diff --git a/drivers/of/irq.c b/drivers/of/irq.c index 8e557ebb8d84..10145d37931f 100644 --- a/drivers/of/irq.c +++ b/drivers/of/irq.c @@ -18,6 +18,7 @@ * driver. */ +#include <linux/device.h> #include <linux/errno.h> #include <linux/list.h> #include <linux/module.h> @@ -409,6 +410,7 @@ int of_irq_get(struct device_node *dev, int index) return irq_create_of_mapping(&oirq); } +EXPORT_SYMBOL_GPL(of_irq_get); /** * of_irq_get_byname - Decode a node's IRQ and return it as a Linux IRQ number @@ -577,3 +579,23 @@ err: kfree(desc); } } + +/** + * of_msi_configure - Set the msi_domain field of a device + * @dev: device structure to associate with an MSI irq domain + * @np: device node for that device + */ +void of_msi_configure(struct device *dev, struct device_node *np) +{ + struct device_node *msi_np; + struct irq_domain *d; + + msi_np = of_parse_phandle(np, "msi-parent", 0); + if (!msi_np) + return; + + d = irq_find_matching_host(msi_np, DOMAIN_BUS_PLATFORM_MSI); + if (!d) + d = irq_find_host(msi_np); + dev_set_msi_domain(dev, d); +} diff --git a/drivers/of/of_pci.c b/drivers/of/of_pci.c index ecc5fa5640d2..b249fdf5ecf8 100644 --- a/drivers/of/of_pci.c +++ b/drivers/of/of_pci.c @@ -2,6 +2,7 @@ #include <linux/export.h> #include <linux/of.h> #include <linux/of_address.h> +#include <linux/of_device.h> #include <linux/of_pci.h> #include <linux/slab.h> @@ -116,6 +117,26 @@ int of_get_pci_domain_nr(struct device_node *node) } EXPORT_SYMBOL_GPL(of_get_pci_domain_nr); +/** + * of_pci_dma_configure - Setup DMA configuration + * @dev: ptr to pci_dev struct of the PCI device + * + * Function to update PCI devices's DMA configuration using the same + * info from the OF node of host bridge's parent (if any). + */ +void of_pci_dma_configure(struct pci_dev *pci_dev) +{ + struct device *dev = &pci_dev->dev; + struct device *bridge = pci_get_host_bridge_device(pci_dev); + + if (!bridge->parent) + return; + + of_dma_configure(dev, bridge->parent->of_node); + pci_put_host_bridge_device(bridge); +} +EXPORT_SYMBOL_GPL(of_pci_dma_configure); + #if defined(CONFIG_OF_ADDRESS) /** * of_pci_get_host_bridge_resources - Parse PCI host bridge resources from DT @@ -229,7 +250,6 @@ parse_failed: list_for_each_entry(window, resources, list) kfree(window->res); pci_free_resource_list(resources); - kfree(bus_range); return err; } EXPORT_SYMBOL_GPL(of_pci_get_host_bridge_resources); @@ -240,7 +260,7 @@ EXPORT_SYMBOL_GPL(of_pci_get_host_bridge_resources); static LIST_HEAD(of_pci_msi_chip_list); static DEFINE_MUTEX(of_pci_msi_chip_mutex); -int of_pci_msi_chip_add(struct msi_chip *chip) +int of_pci_msi_chip_add(struct msi_controller *chip) { if (!of_property_read_bool(chip->of_node, "msi-controller")) return -EINVAL; @@ -253,7 +273,7 @@ int of_pci_msi_chip_add(struct msi_chip *chip) } EXPORT_SYMBOL_GPL(of_pci_msi_chip_add); -void of_pci_msi_chip_remove(struct msi_chip *chip) +void of_pci_msi_chip_remove(struct msi_controller *chip) { mutex_lock(&of_pci_msi_chip_mutex); list_del(&chip->list); @@ -261,9 +281,9 @@ void of_pci_msi_chip_remove(struct msi_chip *chip) } EXPORT_SYMBOL_GPL(of_pci_msi_chip_remove); -struct msi_chip *of_pci_find_msi_chip_by_node(struct device_node *of_node) +struct msi_controller *of_pci_find_msi_chip_by_node(struct device_node *of_node) { - struct msi_chip *c; + struct msi_controller *c; mutex_lock(&of_pci_msi_chip_mutex); list_for_each_entry(c, &of_pci_msi_chip_list, list) { diff --git a/drivers/of/of_private.h b/drivers/of/of_private.h index 21bf733eb4ad..46ddbee22ce3 100644 --- a/drivers/of/of_private.h +++ b/drivers/of/of_private.h @@ -61,7 +61,7 @@ static inline int of_property_notify(int action, struct device_node *np, * own the devtree lock or work on detached trees only. */ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags); -struct device_node *__of_node_alloc(const char *full_name, gfp_t allocflags); +__printf(2, 3) struct device_node *__of_node_dup(const struct device_node *np, const char *fmt, ...); extern const void *__of_get_property(const struct device_node *np, const char *name, int *lenp); diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 2ede604ff17a..26fb7b0b856c 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -267,6 +267,7 @@ int of_reserved_mem_device_init(struct device *dev) return ret; } +EXPORT_SYMBOL_GPL(of_reserved_mem_device_init); /** * of_reserved_mem_device_release() - release reserved memory device structures @@ -291,3 +292,4 @@ void of_reserved_mem_device_release(struct device *dev) rmem->ops->device_release(rmem, dev); } +EXPORT_SYMBOL_GPL(of_reserved_mem_device_release); diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c new file mode 100644 index 000000000000..dee9270ba547 --- /dev/null +++ b/drivers/of/overlay.c @@ -0,0 +1,552 @@ +/* + * Functions for working with device tree overlays + * + * Copyright (C) 2012 Pantelis Antoniou <panto@antoniou-consulting.com> + * Copyright (C) 2012 Texas Instruments Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * version 2 as published by the Free Software Foundation. + */ +#undef DEBUG +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/string.h> +#include <linux/ctype.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/slab.h> +#include <linux/err.h> +#include <linux/idr.h> + +#include "of_private.h" + +/** + * struct of_overlay_info - Holds a single overlay info + * @target: target of the overlay operation + * @overlay: pointer to the overlay contents node + * + * Holds a single overlay state, including all the overlay logs & + * records. + */ +struct of_overlay_info { + struct device_node *target; + struct device_node *overlay; +}; + +/** + * struct of_overlay - Holds a complete overlay transaction + * @node: List on which we are located + * @count: Count of ovinfo structures + * @ovinfo_tab: Overlay info table (count sized) + * @cset: Changeset to be used + * + * Holds a complete overlay transaction + */ +struct of_overlay { + int id; + struct list_head node; + int count; + struct of_overlay_info *ovinfo_tab; + struct of_changeset cset; +}; + +static int of_overlay_apply_one(struct of_overlay *ov, + struct device_node *target, const struct device_node *overlay); + +static int of_overlay_apply_single_property(struct of_overlay *ov, + struct device_node *target, struct property *prop) +{ + struct property *propn, *tprop; + + /* NOTE: Multiple changes of single properties not supported */ + tprop = of_find_property(target, prop->name, NULL); + + /* special properties are not meant to be updated (silent NOP) */ + if (of_prop_cmp(prop->name, "name") == 0 || + of_prop_cmp(prop->name, "phandle") == 0 || + of_prop_cmp(prop->name, "linux,phandle") == 0) + return 0; + + propn = __of_prop_dup(prop, GFP_KERNEL); + if (propn == NULL) + return -ENOMEM; + + /* not found? add */ + if (tprop == NULL) + return of_changeset_add_property(&ov->cset, target, propn); + + /* found? update */ + return of_changeset_update_property(&ov->cset, target, propn); +} + +static int of_overlay_apply_single_device_node(struct of_overlay *ov, + struct device_node *target, struct device_node *child) +{ + const char *cname; + struct device_node *tchild; + int ret = 0; + + cname = kbasename(child->full_name); + if (cname == NULL) + return -ENOMEM; + + /* NOTE: Multiple mods of created nodes not supported */ + tchild = of_get_child_by_name(target, cname); + if (tchild != NULL) { + /* apply overlay recursively */ + ret = of_overlay_apply_one(ov, tchild, child); + of_node_put(tchild); + } else { + /* create empty tree as a target */ + tchild = __of_node_dup(child, "%s/%s", target->full_name, cname); + if (!tchild) + return -ENOMEM; + + /* point to parent */ + tchild->parent = target; + + ret = of_changeset_attach_node(&ov->cset, tchild); + if (ret) + return ret; + + ret = of_overlay_apply_one(ov, tchild, child); + if (ret) + return ret; + } + + return ret; +} + +/* + * Apply a single overlay node recursively. + * + * Note that the in case of an error the target node is left + * in a inconsistent state. Error recovery should be performed + * by using the changeset. + */ +static int of_overlay_apply_one(struct of_overlay *ov, + struct device_node *target, const struct device_node *overlay) +{ + struct device_node *child; + struct property *prop; + int ret; + + for_each_property_of_node(overlay, prop) { + ret = of_overlay_apply_single_property(ov, target, prop); + if (ret) { + pr_err("%s: Failed to apply prop @%s/%s\n", + __func__, target->full_name, prop->name); + return ret; + } + } + + for_each_child_of_node(overlay, child) { + ret = of_overlay_apply_single_device_node(ov, target, child); + if (ret != 0) { + pr_err("%s: Failed to apply single node @%s/%s\n", + __func__, target->full_name, + child->name); + return ret; + } + } + + return 0; +} + +/** + * of_overlay_apply() - Apply @count overlays pointed at by @ovinfo_tab + * @ov: Overlay to apply + * + * Applies the overlays given, while handling all error conditions + * appropriately. Either the operation succeeds, or if it fails the + * live tree is reverted to the state before the attempt. + * Returns 0, or an error if the overlay attempt failed. + */ +static int of_overlay_apply(struct of_overlay *ov) +{ + int i, err; + + /* first we apply the overlays atomically */ + for (i = 0; i < ov->count; i++) { + struct of_overlay_info *ovinfo = &ov->ovinfo_tab[i]; + + err = of_overlay_apply_one(ov, ovinfo->target, ovinfo->overlay); + if (err != 0) { + pr_err("%s: overlay failed '%s'\n", + __func__, ovinfo->target->full_name); + return err; + } + } + + return 0; +} + +/* + * Find the target node using a number of different strategies + * in order of preference + * + * "target" property containing the phandle of the target + * "target-path" property containing the path of the target + */ +static struct device_node *find_target_node(struct device_node *info_node) +{ + const char *path; + u32 val; + int ret; + + /* first try to go by using the target as a phandle */ + ret = of_property_read_u32(info_node, "target", &val); + if (ret == 0) + return of_find_node_by_phandle(val); + + /* now try to locate by path */ + ret = of_property_read_string(info_node, "target-path", &path); + if (ret == 0) + return of_find_node_by_path(path); + + pr_err("%s: Failed to find target for node %p (%s)\n", __func__, + info_node, info_node->name); + + return NULL; +} + +/** + * of_fill_overlay_info() - Fill an overlay info structure + * @ov Overlay to fill + * @info_node: Device node containing the overlay + * @ovinfo: Pointer to the overlay info structure to fill + * + * Fills an overlay info structure with the overlay information + * from a device node. This device node must have a target property + * which contains a phandle of the overlay target node, and an + * __overlay__ child node which has the overlay contents. + * Both ovinfo->target & ovinfo->overlay have their references taken. + * + * Returns 0 on success, or a negative error value. + */ +static int of_fill_overlay_info(struct of_overlay *ov, + struct device_node *info_node, struct of_overlay_info *ovinfo) +{ + ovinfo->overlay = of_get_child_by_name(info_node, "__overlay__"); + if (ovinfo->overlay == NULL) + goto err_fail; + + ovinfo->target = find_target_node(info_node); + if (ovinfo->target == NULL) + goto err_fail; + + return 0; + +err_fail: + of_node_put(ovinfo->target); + of_node_put(ovinfo->overlay); + + memset(ovinfo, 0, sizeof(*ovinfo)); + return -EINVAL; +} + +/** + * of_build_overlay_info() - Build an overlay info array + * @ov Overlay to build + * @tree: Device node containing all the overlays + * + * Helper function that given a tree containing overlay information, + * allocates and builds an overlay info array containing it, ready + * for use using of_overlay_apply. + * + * Returns 0 on success with the @cntp @ovinfop pointers valid, + * while on error a negative error value is returned. + */ +static int of_build_overlay_info(struct of_overlay *ov, + struct device_node *tree) +{ + struct device_node *node; + struct of_overlay_info *ovinfo; + int cnt, err; + + /* worst case; every child is a node */ + cnt = 0; + for_each_child_of_node(tree, node) + cnt++; + + ovinfo = kcalloc(cnt, sizeof(*ovinfo), GFP_KERNEL); + if (ovinfo == NULL) + return -ENOMEM; + + cnt = 0; + for_each_child_of_node(tree, node) { + memset(&ovinfo[cnt], 0, sizeof(*ovinfo)); + err = of_fill_overlay_info(ov, node, &ovinfo[cnt]); + if (err == 0) + cnt++; + } + + /* if nothing filled, return error */ + if (cnt == 0) { + kfree(ovinfo); + return -ENODEV; + } + + ov->count = cnt; + ov->ovinfo_tab = ovinfo; + + return 0; +} + +/** + * of_free_overlay_info() - Free an overlay info array + * @ov Overlay to free the overlay info from + * @ovinfo_tab: Array of overlay_info's to free + * + * Releases the memory of a previously allocated ovinfo array + * by of_build_overlay_info. + * Returns 0, or an error if the arguments are bogus. + */ +static int of_free_overlay_info(struct of_overlay *ov) +{ + struct of_overlay_info *ovinfo; + int i; + + /* do it in reverse */ + for (i = ov->count - 1; i >= 0; i--) { + ovinfo = &ov->ovinfo_tab[i]; + + of_node_put(ovinfo->target); + of_node_put(ovinfo->overlay); + } + kfree(ov->ovinfo_tab); + + return 0; +} + +static LIST_HEAD(ov_list); +static DEFINE_IDR(ov_idr); + +/** + * of_overlay_create() - Create and apply an overlay + * @tree: Device node containing all the overlays + * + * Creates and applies an overlay while also keeping track + * of the overlay in a list. This list can be used to prevent + * illegal overlay removals. + * + * Returns the id of the created overlay, or an negative error number + */ +int of_overlay_create(struct device_node *tree) +{ + struct of_overlay *ov; + int err, id; + + /* allocate the overlay structure */ + ov = kzalloc(sizeof(*ov), GFP_KERNEL); + if (ov == NULL) + return -ENOMEM; + ov->id = -1; + + INIT_LIST_HEAD(&ov->node); + + of_changeset_init(&ov->cset); + + mutex_lock(&of_mutex); + + id = idr_alloc(&ov_idr, ov, 0, 0, GFP_KERNEL); + if (id < 0) { + pr_err("%s: idr_alloc() failed for tree@%s\n", + __func__, tree->full_name); + err = id; + goto err_destroy_trans; + } + ov->id = id; + + /* build the overlay info structures */ + err = of_build_overlay_info(ov, tree); + if (err) { + pr_err("%s: of_build_overlay_info() failed for tree@%s\n", + __func__, tree->full_name); + goto err_free_idr; + } + + /* apply the overlay */ + err = of_overlay_apply(ov); + if (err) { + pr_err("%s: of_overlay_apply() failed for tree@%s\n", + __func__, tree->full_name); + goto err_abort_trans; + } + + /* apply the changeset */ + err = of_changeset_apply(&ov->cset); + if (err) { + pr_err("%s: of_changeset_apply() failed for tree@%s\n", + __func__, tree->full_name); + goto err_revert_overlay; + } + + /* add to the tail of the overlay list */ + list_add_tail(&ov->node, &ov_list); + + mutex_unlock(&of_mutex); + + return id; + +err_revert_overlay: +err_abort_trans: + of_free_overlay_info(ov); +err_free_idr: + idr_remove(&ov_idr, ov->id); +err_destroy_trans: + of_changeset_destroy(&ov->cset); + kfree(ov); + mutex_unlock(&of_mutex); + + return err; +} +EXPORT_SYMBOL_GPL(of_overlay_create); + +/* check whether the given node, lies under the given tree */ +static int overlay_subtree_check(struct device_node *tree, + struct device_node *dn) +{ + struct device_node *child; + + /* match? */ + if (tree == dn) + return 1; + + for_each_child_of_node(tree, child) { + if (overlay_subtree_check(child, dn)) + return 1; + } + + return 0; +} + +/* check whether this overlay is the topmost */ +static int overlay_is_topmost(struct of_overlay *ov, struct device_node *dn) +{ + struct of_overlay *ovt; + struct of_changeset_entry *ce; + + list_for_each_entry_reverse(ovt, &ov_list, node) { + /* if we hit ourselves, we're done */ + if (ovt == ov) + break; + + /* check against each subtree affected by this overlay */ + list_for_each_entry(ce, &ovt->cset.entries, node) { + if (overlay_subtree_check(ce->np, dn)) { + pr_err("%s: #%d clashes #%d @%s\n", + __func__, ov->id, ovt->id, + dn->full_name); + return 0; + } + } + } + + /* overlay is topmost */ + return 1; +} + +/* + * We can safely remove the overlay only if it's the top-most one. + * Newly applied overlays are inserted at the tail of the overlay list, + * so a top most overlay is the one that is closest to the tail. + * + * The topmost check is done by exploiting this property. For each + * affected device node in the log list we check if this overlay is + * the one closest to the tail. If another overlay has affected this + * device node and is closest to the tail, then removal is not permited. + */ +static int overlay_removal_is_ok(struct of_overlay *ov) +{ + struct of_changeset_entry *ce; + + list_for_each_entry(ce, &ov->cset.entries, node) { + if (!overlay_is_topmost(ov, ce->np)) { + pr_err("%s: overlay #%d is not topmost\n", + __func__, ov->id); + return 0; + } + } + + return 1; +} + +/** + * of_overlay_destroy() - Removes an overlay + * @id: Overlay id number returned by a previous call to of_overlay_create + * + * Removes an overlay if it is permissible. + * + * Returns 0 on success, or an negative error number + */ +int of_overlay_destroy(int id) +{ + struct of_overlay *ov; + int err; + + mutex_lock(&of_mutex); + + ov = idr_find(&ov_idr, id); + if (ov == NULL) { + err = -ENODEV; + pr_err("%s: Could not find overlay #%d\n", + __func__, id); + goto out; + } + + /* check whether the overlay is safe to remove */ + if (!overlay_removal_is_ok(ov)) { + err = -EBUSY; + pr_err("%s: removal check failed for overlay #%d\n", + __func__, id); + goto out; + } + + + list_del(&ov->node); + of_changeset_revert(&ov->cset); + of_free_overlay_info(ov); + idr_remove(&ov_idr, id); + of_changeset_destroy(&ov->cset); + kfree(ov); + + err = 0; + +out: + mutex_unlock(&of_mutex); + + return err; +} +EXPORT_SYMBOL_GPL(of_overlay_destroy); + +/** + * of_overlay_destroy_all() - Removes all overlays from the system + * + * Removes all overlays from the system in the correct order. + * + * Returns 0 on success, or an negative error number + */ +int of_overlay_destroy_all(void) +{ + struct of_overlay *ov, *ovn; + + mutex_lock(&of_mutex); + + /* the tail of list is guaranteed to be safe to remove */ + list_for_each_entry_safe_reverse(ov, ovn, &ov_list, node) { + list_del(&ov->node); + of_changeset_revert(&ov->cset); + of_free_overlay_info(ov); + idr_remove(&ov_idr, ov->id); + kfree(ov); + } + + mutex_unlock(&of_mutex); + + return 0; +} +EXPORT_SYMBOL_GPL(of_overlay_destroy_all); diff --git a/drivers/of/pdt.c b/drivers/of/pdt.c index 36b4035881b0..d2acae825af9 100644 --- a/drivers/of/pdt.c +++ b/drivers/of/pdt.c @@ -25,8 +25,7 @@ static struct of_pdt_ops *of_pdt_prom_ops __initdata; -void __initdata (*of_pdt_build_more)(struct device_node *dp, - struct device_node ***nextp); +void __initdata (*of_pdt_build_more)(struct device_node *dp); #if defined(CONFIG_SPARC) unsigned int of_pdt_unique_id __initdata; @@ -192,8 +191,7 @@ static struct device_node * __init of_pdt_create_node(phandle node, } static struct device_node * __init of_pdt_build_tree(struct device_node *parent, - phandle node, - struct device_node ***nextp) + phandle node) { struct device_node *ret = NULL, *prev_sibling = NULL; struct device_node *dp; @@ -210,16 +208,12 @@ static struct device_node * __init of_pdt_build_tree(struct device_node *parent, ret = dp; prev_sibling = dp; - *(*nextp) = dp; - *nextp = &dp->allnext; - dp->full_name = of_pdt_build_full_name(dp); - dp->child = of_pdt_build_tree(dp, - of_pdt_prom_ops->getchild(node), nextp); + dp->child = of_pdt_build_tree(dp, of_pdt_prom_ops->getchild(node)); if (of_pdt_build_more) - of_pdt_build_more(dp, nextp); + of_pdt_build_more(dp); node = of_pdt_prom_ops->getsibling(node); } @@ -234,20 +228,17 @@ static void * __init kernel_tree_alloc(u64 size, u64 align) void __init of_pdt_build_devicetree(phandle root_node, struct of_pdt_ops *ops) { - struct device_node **nextp; - BUG_ON(!ops); of_pdt_prom_ops = ops; - of_allnodes = of_pdt_create_node(root_node, NULL); + of_root = of_pdt_create_node(root_node, NULL); #if defined(CONFIG_SPARC) - of_allnodes->path_component_name = ""; + of_root->path_component_name = ""; #endif - of_allnodes->full_name = "/"; + of_root->full_name = "/"; - nextp = &of_allnodes->allnext; - of_allnodes->child = of_pdt_build_tree(of_allnodes, - of_pdt_prom_ops->getchild(of_allnodes->phandle), &nextp); + of_root->child = of_pdt_build_tree(of_root, + of_pdt_prom_ops->getchild(of_root->phandle)); /* Get pointer to "/chosen" and "/aliases" nodes for use everywhere */ of_alias_scan(kernel_tree_alloc); diff --git a/drivers/of/platform.c b/drivers/of/platform.c index 3b64d0bf5bba..8a002d6151f2 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -25,6 +25,7 @@ const struct of_device_id of_default_bus_match_table[] = { { .compatible = "simple-bus", }, + { .compatible = "simple-mfd", }, #ifdef CONFIG_ARM_AMBA { .compatible = "arm,amba-bus", }, #endif /* CONFIG_ARM_AMBA */ @@ -138,7 +139,7 @@ struct platform_device *of_device_alloc(struct device_node *np, } dev->dev.of_node = of_node_get(np); - dev->dev.parent = parent; + dev->dev.parent = parent ? : &platform_bus; if (bus_id) dev_set_name(&dev->dev, "%s", bus_id); @@ -149,57 +150,9 @@ struct platform_device *of_device_alloc(struct device_node *np, } EXPORT_SYMBOL(of_device_alloc); -/** - * of_dma_configure - Setup DMA configuration - * @dev: Device to apply DMA configuration - * - * Try to get devices's DMA configuration from DT and update it - * accordingly. - * - * In case if platform code need to use own special DMA configuration,it - * can use Platform bus notifier and handle BUS_NOTIFY_ADD_DEVICE event - * to fix up DMA configuration. - */ -static void of_dma_configure(struct device *dev) +static void of_dma_deconfigure(struct device *dev) { - u64 dma_addr, paddr, size; - int ret; - - /* - * Set default dma-mask to 32 bit. Drivers are expected to setup - * the correct supported dma_mask. - */ - dev->coherent_dma_mask = DMA_BIT_MASK(32); - - /* - * Set it to coherent_dma_mask by default if the architecture - * code has not set it. - */ - if (!dev->dma_mask) - dev->dma_mask = &dev->coherent_dma_mask; - - /* - * if dma-coherent property exist, call arch hook to setup - * dma coherent operations. - */ - if (of_dma_is_coherent(dev->of_node)) { - set_arch_dma_coherent_ops(dev); - dev_dbg(dev, "device is dma coherent\n"); - } - - /* - * if dma-ranges property doesn't exist - just return else - * setup the dma offset - */ - ret = of_dma_get_range(dev->of_node, &dma_addr, &paddr, &size); - if (ret < 0) { - dev_dbg(dev, "no dma range information to setup\n"); - return; - } - - /* DMA ranges found. Calculate and set dma_pfn_offset */ - dev->dma_pfn_offset = PFN_DOWN(paddr - dma_addr); - dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", dev->dma_pfn_offset); + arch_teardown_dma_ops(dev); } /** @@ -228,16 +181,13 @@ static struct platform_device *of_platform_device_create_pdata( if (!dev) goto err_clear_flag; - of_dma_configure(&dev->dev); dev->dev.bus = &platform_bus_type; dev->dev.platform_data = platform_data; - - /* We do not fill the DMA ops for platform devices by default. - * This is currently the responsibility of the platform code - * to do such, possibly using a device notifier - */ + of_dma_configure(&dev->dev, dev->dev.of_node); + of_msi_configure(&dev->dev, dev->dev.of_node); if (of_device_add(dev) != 0) { + of_dma_deconfigure(&dev->dev); platform_device_put(dev); goto err_clear_flag; } @@ -291,13 +241,13 @@ static struct amba_device *of_amba_device_create(struct device_node *node, /* setup generic device info */ dev->dev.of_node = of_node_get(node); - dev->dev.parent = parent; + dev->dev.parent = parent ? : &platform_bus; dev->dev.platform_data = platform_data; if (bus_id) dev_set_name(&dev->dev, "%s", bus_id); else of_device_make_bus_id(&dev->dev); - of_dma_configure(&dev->dev); + of_dma_configure(&dev->dev, dev->dev.of_node); /* Allow the HW Peripheral ID to be overridden */ prop = of_get_property(node, "arm,primecell-periphid", NULL); @@ -500,6 +450,7 @@ int of_platform_populate(struct device_node *root, if (rc) break; } + of_node_set_flag(root, OF_POPULATED_BUS); of_node_put(root); return rc; @@ -523,6 +474,7 @@ static int of_platform_device_destroy(struct device *dev, void *data) amba_device_unregister(to_amba_device(dev)); #endif + of_dma_deconfigure(dev); of_node_clear_flag(dev->of_node, OF_POPULATED); of_node_clear_flag(dev->of_node, OF_POPULATED_BUS); return 0; @@ -542,8 +494,75 @@ static int of_platform_device_destroy(struct device *dev, void *data) */ void of_platform_depopulate(struct device *parent) { - device_for_each_child(parent, NULL, of_platform_device_destroy); + if (parent->of_node && of_node_check_flag(parent->of_node, OF_POPULATED_BUS)) { + device_for_each_child(parent, NULL, of_platform_device_destroy); + of_node_clear_flag(parent->of_node, OF_POPULATED_BUS); + } } EXPORT_SYMBOL_GPL(of_platform_depopulate); +#ifdef CONFIG_OF_DYNAMIC +static int of_platform_notify(struct notifier_block *nb, + unsigned long action, void *arg) +{ + struct of_reconfig_data *rd = arg; + struct platform_device *pdev_parent, *pdev; + bool children_left; + + switch (of_reconfig_get_state_change(action, rd)) { + case OF_RECONFIG_CHANGE_ADD: + /* verify that the parent is a bus */ + if (!of_node_check_flag(rd->dn->parent, OF_POPULATED_BUS)) + return NOTIFY_OK; /* not for us */ + + /* already populated? (driver using of_populate manually) */ + if (of_node_check_flag(rd->dn, OF_POPULATED)) + return NOTIFY_OK; + + /* pdev_parent may be NULL when no bus platform device */ + pdev_parent = of_find_device_by_node(rd->dn->parent); + pdev = of_platform_device_create(rd->dn, NULL, + pdev_parent ? &pdev_parent->dev : NULL); + of_dev_put(pdev_parent); + + if (pdev == NULL) { + pr_err("%s: failed to create for '%s'\n", + __func__, rd->dn->full_name); + /* of_platform_device_create tosses the error code */ + return notifier_from_errno(-EINVAL); + } + break; + + case OF_RECONFIG_CHANGE_REMOVE: + + /* already depopulated? */ + if (!of_node_check_flag(rd->dn, OF_POPULATED)) + return NOTIFY_OK; + + /* find our device by node */ + pdev = of_find_device_by_node(rd->dn); + if (pdev == NULL) + return NOTIFY_OK; /* no? not meant for us */ + + /* unregister takes one ref away */ + of_platform_device_destroy(&pdev->dev, &children_left); + + /* and put the reference of the find */ + of_dev_put(pdev); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block platform_of_notifier = { + .notifier_call = of_platform_notify, +}; + +void of_platform_register_reconfig_notifier(void) +{ + WARN_ON(of_reconfig_notifier_register(&platform_of_notifier)); +} +#endif /* CONFIG_OF_DYNAMIC */ + #endif /* CONFIG_OF_ADDRESS */ diff --git a/drivers/of/resolver.c b/drivers/of/resolver.c index aed7959f800d..640eb4cb46e3 100644 --- a/drivers/of/resolver.c +++ b/drivers/of/resolver.c @@ -111,7 +111,8 @@ static void __of_adjust_tree_phandles(struct device_node *node, __of_adjust_tree_phandles(child, phandle_delta); } -static int __of_adjust_phandle_ref(struct device_node *node, struct property *rprop, int value, bool is_delta) +static int __of_adjust_phandle_ref(struct device_node *node, + struct property *rprop, int value) { phandle phandle; struct device_node *refnode; @@ -181,7 +182,7 @@ static int __of_adjust_phandle_ref(struct device_node *node, struct property *rp goto err_fail; } - phandle = is_delta ? be32_to_cpup(sprop->value + offset) + value : value; + phandle = value; *(__be32 *)(sprop->value + offset) = cpu_to_be32(phandle); } @@ -190,36 +191,97 @@ err_fail: return err; } +/* compare nodes taking into account that 'name' strips out the @ part */ +static int __of_node_name_cmp(const struct device_node *dn1, + const struct device_node *dn2) +{ + const char *n1 = strrchr(dn1->full_name, '/') ? : "/"; + const char *n2 = strrchr(dn2->full_name, '/') ? : "/"; + + return of_node_cmp(n1, n2); +} + /* * Adjust the local phandle references by the given phandle delta. - * Assumes the existances of a __local_fixups__ node at the root - * of the tree. Does not take any devtree locks so make sure you - * call this on a tree which is at the detached state. + * Assumes the existances of a __local_fixups__ node at the root. + * Assumes that __of_verify_tree_phandle_references has been called. + * Does not take any devtree locks so make sure you call this on a tree + * which is at the detached state. */ static int __of_adjust_tree_phandle_references(struct device_node *node, - int phandle_delta) + struct device_node *target, int phandle_delta) { - struct device_node *child; - struct property *rprop; - int err; - - /* locate the symbols & fixups nodes on resolve */ - for_each_child_of_node(node, child) - if (of_node_cmp(child->name, "__local_fixups__") == 0) - break; + struct device_node *child, *childtarget; + struct property *rprop, *sprop; + int err, i, count; + unsigned int off; + phandle phandle; - /* no local fixups */ - if (!child) + if (node == NULL) return 0; - /* find the local fixups property */ - for_each_property_of_node(child, rprop) { + for_each_property_of_node(node, rprop) { + /* skip properties added automatically */ - if (of_prop_cmp(rprop->name, "name") == 0) + if (of_prop_cmp(rprop->name, "name") == 0 || + of_prop_cmp(rprop->name, "phandle") == 0 || + of_prop_cmp(rprop->name, "linux,phandle") == 0) continue; - err = __of_adjust_phandle_ref(node, rprop, phandle_delta, true); - if (err) + if ((rprop->length % 4) != 0 || rprop->length == 0) { + pr_err("%s: Illegal property (size) '%s' @%s\n", + __func__, rprop->name, node->full_name); + return -EINVAL; + } + count = rprop->length / sizeof(__be32); + + /* now find the target property */ + for_each_property_of_node(target, sprop) { + if (of_prop_cmp(sprop->name, rprop->name) == 0) + break; + } + + if (sprop == NULL) { + pr_err("%s: Could not find target property '%s' @%s\n", + __func__, rprop->name, node->full_name); + return -EINVAL; + } + + for (i = 0; i < count; i++) { + off = be32_to_cpu(((__be32 *)rprop->value)[i]); + /* make sure the offset doesn't overstep (even wrap) */ + if (off >= sprop->length || + (off + 4) > sprop->length) { + pr_err("%s: Illegal property '%s' @%s\n", + __func__, rprop->name, + node->full_name); + return -EINVAL; + } + + if (phandle_delta) { + /* adjust */ + phandle = be32_to_cpu(*(__be32 *)(sprop->value + off)); + phandle += phandle_delta; + *(__be32 *)(sprop->value + off) = cpu_to_be32(phandle); + } + } + } + + for_each_child_of_node(node, child) { + + for_each_child_of_node(target, childtarget) + if (__of_node_name_cmp(child, childtarget) == 0) + break; + + if (!childtarget) { + pr_err("%s: Could not find target child '%s' @%s\n", + __func__, child->name, node->full_name); + return -EINVAL; + } + + err = __of_adjust_tree_phandle_references(child, childtarget, + phandle_delta); + if (err != 0) return err; } @@ -241,7 +303,7 @@ static int __of_adjust_tree_phandle_references(struct device_node *node, */ int of_resolve_phandles(struct device_node *resolve) { - struct device_node *child, *refnode; + struct device_node *child, *childroot, *refnode; struct device_node *root_sym, *resolve_sym, *resolve_fix; struct property *rprop; const char *refpath; @@ -255,9 +317,23 @@ int of_resolve_phandles(struct device_node *resolve) /* first we need to adjust the phandles */ phandle_delta = of_get_tree_max_phandle() + 1; __of_adjust_tree_phandles(resolve, phandle_delta); - err = __of_adjust_tree_phandle_references(resolve, phandle_delta); - if (err != 0) - return err; + + /* locate the local fixups */ + childroot = NULL; + for_each_child_of_node(resolve, childroot) + if (of_node_cmp(childroot->name, "__local_fixups__") == 0) + break; + + if (childroot != NULL) { + /* resolve root is guaranteed to be the '/' */ + err = __of_adjust_tree_phandle_references(childroot, + resolve, 0); + if (err != 0) + return err; + + BUG_ON(__of_adjust_tree_phandle_references(childroot, + resolve, phandle_delta)); + } root_sym = NULL; resolve_sym = NULL; @@ -322,7 +398,7 @@ int of_resolve_phandles(struct device_node *resolve) pr_debug("%s: %s phandle is 0x%08x\n", __func__, rprop->name, phandle); - err = __of_adjust_phandle_ref(resolve, rprop, phandle, false); + err = __of_adjust_phandle_ref(resolve, rprop, phandle); if (err) break; } diff --git a/drivers/of/testcase-data/testcases.dts b/drivers/of/testcase-data/testcases.dts deleted file mode 100644 index 6994e15c24bf..000000000000 --- a/drivers/of/testcase-data/testcases.dts +++ /dev/null @@ -1,50 +0,0 @@ -/dts-v1/; -/ { - testcase-data { - changeset { - prop-update = "hello"; - prop-remove = "world"; - node-remove { - }; - }; - }; -}; -#include "tests-phandle.dtsi" -#include "tests-interrupts.dtsi" -#include "tests-match.dtsi" -#include "tests-platform.dtsi" - -/* - * phandle fixup data - generated by dtc patches that aren't upstream. - * This data must be regenerated whenever phandle references are modified in - * the testdata tree. - * - * The format of this data may be subject to change. For the time being consider - * this a kernel-internal data format. - */ -/ { __local_fixups__ { - fixup = "/testcase-data/testcase-device2:interrupt-parent:0", - "/testcase-data/testcase-device1:interrupt-parent:0", - "/testcase-data/interrupts/interrupts-extended0:interrupts-extended:60", - "/testcase-data/interrupts/interrupts-extended0:interrupts-extended:52", - "/testcase-data/interrupts/interrupts-extended0:interrupts-extended:44", - "/testcase-data/interrupts/interrupts-extended0:interrupts-extended:36", - "/testcase-data/interrupts/interrupts-extended0:interrupts-extended:24", - "/testcase-data/interrupts/interrupts-extended0:interrupts-extended:8", - "/testcase-data/interrupts/interrupts-extended0:interrupts-extended:0", - "/testcase-data/interrupts/interrupts1:interrupt-parent:0", - "/testcase-data/interrupts/interrupts0:interrupt-parent:0", - "/testcase-data/interrupts/intmap1:interrupt-map:12", - "/testcase-data/interrupts/intmap0:interrupt-map:52", - "/testcase-data/interrupts/intmap0:interrupt-map:36", - "/testcase-data/interrupts/intmap0:interrupt-map:16", - "/testcase-data/interrupts/intmap0:interrupt-map:4", - "/testcase-data/phandle-tests/consumer-a:phandle-list-bad-args:12", - "/testcase-data/phandle-tests/consumer-a:phandle-list-bad-args:0", - "/testcase-data/phandle-tests/consumer-a:phandle-list:56", - "/testcase-data/phandle-tests/consumer-a:phandle-list:52", - "/testcase-data/phandle-tests/consumer-a:phandle-list:40", - "/testcase-data/phandle-tests/consumer-a:phandle-list:24", - "/testcase-data/phandle-tests/consumer-a:phandle-list:8", - "/testcase-data/phandle-tests/consumer-a:phandle-list:0"; -}; }; diff --git a/drivers/of/unittest-data/testcases.dts b/drivers/of/unittest-data/testcases.dts new file mode 100644 index 000000000000..12f7c3d649c8 --- /dev/null +++ b/drivers/of/unittest-data/testcases.dts @@ -0,0 +1,79 @@ +/dts-v1/; +/ { + testcase-data { + changeset { + prop-update = "hello"; + prop-remove = "world"; + node-remove { + }; + }; + }; +}; +#include "tests-phandle.dtsi" +#include "tests-interrupts.dtsi" +#include "tests-match.dtsi" +#include "tests-platform.dtsi" +#include "tests-overlay.dtsi" + +/* + * phandle fixup data - generated by dtc patches that aren't upstream. + * This data must be regenerated whenever phandle references are modified in + * the testdata tree. + * + * The format of this data may be subject to change. For the time being consider + * this a kernel-internal data format. + */ +/ { __local_fixups__ { + testcase-data { + phandle-tests { + consumer-a { + phandle-list = <0x00000000 0x00000008 + 0x00000018 0x00000028 + 0x00000034 0x00000038>; + phandle-list-bad-args = <0x00000000 0x0000000c>; + }; + }; + interrupts { + intmap0 { + interrupt-map = <0x00000004 0x00000010 + 0x00000024 0x00000034>; + }; + intmap1 { + interrupt-map = <0x0000000c>; + }; + interrupts0 { + interrupt-parent = <0x00000000>; + }; + interrupts1 { + interrupt-parent = <0x00000000>; + }; + interrupts-extended0 { + interrupts-extended = <0x00000000 0x00000008 + 0x00000018 0x00000024 + 0x0000002c 0x00000034 + 0x0000003c>; + }; + }; + testcase-device1 { + interrupt-parent = <0x00000000>; + }; + testcase-device2 { + interrupt-parent = <0x00000000>; + }; + overlay2 { + fragment@0 { + target = <0x00000000>; + }; + }; + overlay3 { + fragment@0 { + target = <0x00000000>; + }; + }; + overlay4 { + fragment@0 { + target = <0x00000000>; + }; + }; + }; +}; }; diff --git a/drivers/of/testcase-data/tests-interrupts.dtsi b/drivers/of/unittest-data/tests-interrupts.dtsi index da4695f60351..da4695f60351 100644 --- a/drivers/of/testcase-data/tests-interrupts.dtsi +++ b/drivers/of/unittest-data/tests-interrupts.dtsi diff --git a/drivers/of/testcase-data/tests-match.dtsi b/drivers/of/unittest-data/tests-match.dtsi index c9e541129534..c9e541129534 100644 --- a/drivers/of/testcase-data/tests-match.dtsi +++ b/drivers/of/unittest-data/tests-match.dtsi diff --git a/drivers/of/unittest-data/tests-overlay.dtsi b/drivers/of/unittest-data/tests-overlay.dtsi new file mode 100644 index 000000000000..a2b687d5f324 --- /dev/null +++ b/drivers/of/unittest-data/tests-overlay.dtsi @@ -0,0 +1,235 @@ + +/ { + testcase-data { + overlay-node { + + /* test bus */ + selftestbus: test-bus { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <0>; + + selftest100: test-selftest100 { + compatible = "selftest"; + status = "okay"; + reg = <100>; + }; + + selftest101: test-selftest101 { + compatible = "selftest"; + status = "disabled"; + reg = <101>; + }; + + selftest0: test-selftest0 { + compatible = "selftest"; + status = "disabled"; + reg = <0>; + }; + + selftest1: test-selftest1 { + compatible = "selftest"; + status = "okay"; + reg = <1>; + }; + + selftest2: test-selftest2 { + compatible = "selftest"; + status = "disabled"; + reg = <2>; + }; + + selftest3: test-selftest3 { + compatible = "selftest"; + status = "okay"; + reg = <3>; + }; + + selftest5: test-selftest5 { + compatible = "selftest"; + status = "disabled"; + reg = <5>; + }; + + selftest6: test-selftest6 { + compatible = "selftest"; + status = "disabled"; + reg = <6>; + }; + + selftest7: test-selftest7 { + compatible = "selftest"; + status = "disabled"; + reg = <7>; + }; + + selftest8: test-selftest8 { + compatible = "selftest"; + status = "disabled"; + reg = <8>; + }; + }; + }; + + /* test enable using absolute target path */ + overlay0 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus/test-selftest0"; + __overlay__ { + status = "okay"; + }; + }; + }; + + /* test disable using absolute target path */ + overlay1 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus/test-selftest1"; + __overlay__ { + status = "disabled"; + }; + }; + }; + + /* test enable using label */ + overlay2 { + fragment@0 { + target = <&selftest2>; + __overlay__ { + status = "okay"; + }; + }; + }; + + /* test disable using label */ + overlay3 { + fragment@0 { + target = <&selftest3>; + __overlay__ { + status = "disabled"; + }; + }; + }; + + /* test insertion of a full node */ + overlay4 { + fragment@0 { + target = <&selftestbus>; + __overlay__ { + + /* suppress DTC warning */ + #address-cells = <1>; + #size-cells = <0>; + + test-selftest4 { + compatible = "selftest"; + status = "okay"; + reg = <4>; + }; + }; + }; + }; + + /* test overlay apply revert */ + overlay5 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus/test-selftest5"; + __overlay__ { + status = "okay"; + }; + }; + }; + + /* test overlays application and removal in sequence */ + overlay6 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus/test-selftest6"; + __overlay__ { + status = "okay"; + }; + }; + }; + overlay7 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus/test-selftest7"; + __overlay__ { + status = "okay"; + }; + }; + }; + + /* test overlays application and removal in bad sequence */ + overlay8 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus/test-selftest8"; + __overlay__ { + status = "okay"; + }; + }; + }; + overlay9 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus/test-selftest8"; + __overlay__ { + property-foo = "bar"; + }; + }; + }; + + overlay10 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus"; + __overlay__ { + + /* suppress DTC warning */ + #address-cells = <1>; + #size-cells = <0>; + + test-selftest10 { + compatible = "selftest"; + status = "okay"; + reg = <10>; + + #address-cells = <1>; + #size-cells = <0>; + + test-selftest101 { + compatible = "selftest"; + status = "okay"; + reg = <1>; + }; + + }; + }; + }; + }; + + overlay11 { + fragment@0 { + target-path = "/testcase-data/overlay-node/test-bus"; + __overlay__ { + + /* suppress DTC warning */ + #address-cells = <1>; + #size-cells = <0>; + + test-selftest11 { + compatible = "selftest"; + status = "okay"; + reg = <11>; + + #address-cells = <1>; + #size-cells = <0>; + + test-selftest111 { + compatible = "selftest"; + status = "okay"; + reg = <1>; + }; + + }; + }; + }; + }; + }; +}; diff --git a/drivers/of/testcase-data/tests-phandle.dtsi b/drivers/of/unittest-data/tests-phandle.dtsi index 5b1527e8a7fb..5b1527e8a7fb 100644 --- a/drivers/of/testcase-data/tests-phandle.dtsi +++ b/drivers/of/unittest-data/tests-phandle.dtsi diff --git a/drivers/of/testcase-data/tests-platform.dtsi b/drivers/of/unittest-data/tests-platform.dtsi index eb20eeb2b062..eb20eeb2b062 100644 --- a/drivers/of/testcase-data/tests-platform.dtsi +++ b/drivers/of/unittest-data/tests-platform.dtsi diff --git a/drivers/of/selftest.c b/drivers/of/unittest.c index e2d79afa9dc6..e5fa3e91f82f 100644 --- a/drivers/of/selftest.c +++ b/drivers/of/unittest.c @@ -17,6 +17,8 @@ #include <linux/mutex.h> #include <linux/slab.h> #include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/of_platform.h> #include "of_private.h" @@ -25,24 +27,22 @@ static struct selftest_results { int failed; } selftest_results; -#define NO_OF_NODES 3 -static struct device_node *nodes[NO_OF_NODES]; -static int last_node_index; -static bool selftest_live_tree; - -#define selftest(result, fmt, ...) { \ - if (!(result)) { \ +#define selftest(result, fmt, ...) ({ \ + bool failed = !(result); \ + if (failed) { \ selftest_results.failed++; \ pr_err("FAIL %s():%i " fmt, __func__, __LINE__, ##__VA_ARGS__); \ } else { \ selftest_results.passed++; \ pr_debug("pass %s():%i\n", __func__, __LINE__); \ } \ -} + failed; \ +}) static void __init of_selftest_find_node_by_name(void) { struct device_node *np; + const char *options; np = of_find_node_by_path("/testcase-data"); selftest(np && !strcmp("/testcase-data", np->full_name), @@ -83,6 +83,51 @@ static void __init of_selftest_find_node_by_name(void) np = of_find_node_by_path("testcase-alias/missing-path"); selftest(!np, "non-existent alias with relative path returned node %s\n", np->full_name); of_node_put(np); + + np = of_find_node_opts_by_path("/testcase-data:testoption", &options); + selftest(np && !strcmp("testoption", options), + "option path test failed\n"); + of_node_put(np); + + np = of_find_node_opts_by_path("/testcase-data:test/option", &options); + selftest(np && !strcmp("test/option", options), + "option path test, subcase #1 failed\n"); + of_node_put(np); + + np = of_find_node_opts_by_path("/testcase-data/testcase-device1:test/option", &options); + selftest(np && !strcmp("test/option", options), + "option path test, subcase #2 failed\n"); + of_node_put(np); + + np = of_find_node_opts_by_path("/testcase-data:testoption", NULL); + selftest(np, "NULL option path test failed\n"); + of_node_put(np); + + np = of_find_node_opts_by_path("testcase-alias:testaliasoption", + &options); + selftest(np && !strcmp("testaliasoption", options), + "option alias path test failed\n"); + of_node_put(np); + + np = of_find_node_opts_by_path("testcase-alias:test/alias/option", + &options); + selftest(np && !strcmp("test/alias/option", options), + "option alias path test, subcase #1 failed\n"); + of_node_put(np); + + np = of_find_node_opts_by_path("testcase-alias:testaliasoption", NULL); + selftest(np, "NULL option alias path test failed\n"); + of_node_put(np); + + options = "testoption"; + np = of_find_node_opts_by_path("testcase-alias", &options); + selftest(np && !options, "option clearing test failed\n"); + of_node_put(np); + + options = "testoption"; + np = of_find_node_opts_by_path("/", &options); + selftest(np && !options, "option clearing root node test failed\n"); + of_node_put(np); } static void __init of_selftest_dynamic(void) @@ -148,7 +193,7 @@ static void __init of_selftest_dynamic(void) static int __init of_selftest_check_node_linkage(struct device_node *np) { - struct device_node *child, *allnext_index = np; + struct device_node *child; int count = 0, rc; for_each_child_of_node(np, child) { @@ -158,14 +203,6 @@ static int __init of_selftest_check_node_linkage(struct device_node *np) return -EINVAL; } - while (allnext_index && allnext_index != child) - allnext_index = allnext_index->allnext; - if (allnext_index != child) { - pr_err("Node %s is ordered differently in sibling and allnode lists\n", - child->name); - return -EINVAL; - } - rc = of_selftest_check_node_linkage(child); if (rc < 0) return rc; @@ -180,12 +217,12 @@ static void __init of_selftest_check_tree_linkage(void) struct device_node *np; int allnode_count = 0, child_count; - if (!of_allnodes) + if (!of_root) return; for_each_of_allnodes(np) allnode_count++; - child_count = of_selftest_check_node_linkage(of_allnodes); + child_count = of_selftest_check_node_linkage(of_root); selftest(child_count > 0, "Device node data structure is corrupted\n"); selftest(child_count == allnode_count, "allnodes list size (%i) doesn't match" @@ -354,9 +391,9 @@ static void __init of_selftest_property_string(void) rc = of_property_match_string(np, "phandle-list-names", "first"); selftest(rc == 0, "first expected:0 got:%i\n", rc); rc = of_property_match_string(np, "phandle-list-names", "second"); - selftest(rc == 1, "second expected:0 got:%i\n", rc); + selftest(rc == 1, "second expected:1 got:%i\n", rc); rc = of_property_match_string(np, "phandle-list-names", "third"); - selftest(rc == 2, "third expected:0 got:%i\n", rc); + selftest(rc == 2, "third expected:2 got:%i\n", rc); rc = of_property_match_string(np, "phandle-list-names", "fourth"); selftest(rc == -ENODATA, "unmatched string; rc=%i\n", rc); rc = of_property_match_string(np, "missing-property", "blah"); @@ -451,15 +488,14 @@ static void __init of_selftest_changeset(void) struct property *ppadd, padd = { .name = "prop-add", .length = 0, .value = "" }; struct property *ppupdate, pupdate = { .name = "prop-update", .length = 5, .value = "abcd" }; struct property *ppremove; - struct device_node *n1, *n2, *n21, *nremove, *parent; + struct device_node *n1, *n2, *n21, *nremove, *parent, *np; struct of_changeset chgset; - of_changeset_init(&chgset); - n1 = __of_node_alloc("/testcase-data/changeset/n1", GFP_KERNEL); + n1 = __of_node_dup(NULL, "/testcase-data/changeset/n1"); selftest(n1, "testcase setup failure\n"); - n2 = __of_node_alloc("/testcase-data/changeset/n2", GFP_KERNEL); + n2 = __of_node_dup(NULL, "/testcase-data/changeset/n2"); selftest(n2, "testcase setup failure\n"); - n21 = __of_node_alloc("/testcase-data/changeset/n2/n21", GFP_KERNEL); + n21 = __of_node_dup(NULL, "%s/%s", "/testcase-data/changeset/n2", "n21"); selftest(n21, "testcase setup failure %p\n", n21); nremove = of_find_node_by_path("/testcase-data/changeset/node-remove"); selftest(nremove, "testcase setup failure\n"); @@ -487,6 +523,11 @@ static void __init of_selftest_changeset(void) selftest(!of_changeset_apply(&chgset), "apply failed\n"); mutex_unlock(&of_mutex); + /* Make sure node names are constructed correctly */ + selftest((np = of_find_node_by_path("/testcase-data/changeset/n2/n21")), + "'%s' not added\n", n21->full_name); + of_node_put(np); + mutex_lock(&of_mutex); selftest(!of_changeset_revert(&chgset), "revert failed\n"); mutex_unlock(&of_mutex); @@ -702,10 +743,13 @@ static void __init of_selftest_match_node(void) } } +struct device test_bus = { + .init_name = "unittest-bus", +}; static void __init of_selftest_platform_populate(void) { - int irq; - struct device_node *np, *child; + int irq, rc; + struct device_node *np, *child, *grandchild; struct platform_device *pdev; struct of_device_id match[] = { { .compatible = "test-device", }, @@ -730,20 +774,32 @@ static void __init of_selftest_platform_populate(void) irq = platform_get_irq(pdev, 0); selftest(irq < 0 && irq != -EPROBE_DEFER, "device parsing error failed - %d\n", irq); - np = of_find_node_by_path("/testcase-data/platform-tests"); - if (!np) { - pr_err("No testcase data in device tree\n"); + if (selftest(np = of_find_node_by_path("/testcase-data/platform-tests"), + "No testcase data in device tree\n")); + return; + + if (selftest(!(rc = device_register(&test_bus)), + "testbus registration failed; rc=%i\n", rc)); return; - } for_each_child_of_node(np, child) { - struct device_node *grandchild; - of_platform_populate(child, match, NULL, NULL); + of_platform_populate(child, match, NULL, &test_bus); for_each_child_of_node(child, grandchild) selftest(of_find_device_by_node(grandchild), "Could not create device for node '%s'\n", grandchild->name); } + + of_platform_depopulate(&test_bus); + for_each_child_of_node(np, child) { + for_each_child_of_node(child, grandchild) + selftest(!of_find_device_by_node(grandchild), + "device didn't get destroyed '%s'\n", + grandchild->name); + } + + device_unregister(&test_bus); + of_node_put(np); } /** @@ -775,33 +831,32 @@ static void update_node_properties(struct device_node *np, */ static int attach_node_and_children(struct device_node *np) { - struct device_node *next, *root = np, *dup; + struct device_node *next, *dup, *child; + unsigned long flags; - /* skip root node */ - np = np->child; - /* storing a copy in temporary node */ - dup = np; - - while (dup) { - if (WARN_ON(last_node_index >= NO_OF_NODES)) - return -EINVAL; - nodes[last_node_index++] = dup; - dup = dup->sibling; + dup = of_find_node_by_path(np->full_name); + if (dup) { + update_node_properties(np, dup); + return 0; } - dup = NULL; - while (np) { - next = np->allnext; - dup = of_find_node_by_path(np->full_name); - if (dup) - update_node_properties(np, dup); - else { - np->child = NULL; - if (np->parent == root) - np->parent = of_allnodes; - of_attach_node(np); - } - np = next; + child = np->child; + np->child = NULL; + + mutex_lock(&of_mutex); + raw_spin_lock_irqsave(&devtree_lock, flags); + np->sibling = np->parent->child; + np->parent->child = np; + of_node_clear_flag(np, OF_DETACHED); + raw_spin_unlock_irqrestore(&devtree_lock, flags); + + __of_attach_node_sysfs(np); + mutex_unlock(&of_mutex); + + while (child) { + next = child->sibling; + attach_node_and_children(child); + child = next; } return 0; @@ -846,11 +901,8 @@ static int __init selftest_data_add(void) return -EINVAL; } - if (!of_allnodes) { - /* enabling flag for removing nodes */ - selftest_live_tree = true; - of_allnodes = selftest_data_node; - + if (!of_root) { + of_root = selftest_data_node; for_each_of_allnodes(np) __of_attach_node_sysfs(np); of_aliases = of_find_node_by_path("/aliases"); @@ -859,62 +911,532 @@ static int __init selftest_data_add(void) } /* attach the sub-tree to live tree */ - return attach_node_and_children(selftest_data_node); + np = selftest_data_node->child; + while (np) { + struct device_node *next = np->sibling; + np->parent = of_root; + attach_node_and_children(np); + np = next; + } + return 0; } -/** - * detach_node_and_children - detaches node - * and its children from live tree - * - * @np: Node to detach from live tree - */ -static void detach_node_and_children(struct device_node *np) +#ifdef CONFIG_OF_OVERLAY + +static int selftest_probe(struct platform_device *pdev) { - while (np->child) - detach_node_and_children(np->child); - of_detach_node(np); + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + + if (np == NULL) { + dev_err(dev, "No OF data for device\n"); + return -EINVAL; + + } + + dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name); + + of_platform_populate(np, NULL, NULL, &pdev->dev); + + return 0; } -/** - * selftest_data_remove - removes the selftest data - * nodes from the live tree - */ -static void selftest_data_remove(void) +static int selftest_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + + dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name); + return 0; +} + +static struct of_device_id selftest_match[] = { + { .compatible = "selftest", }, + {}, +}; + +static struct platform_driver selftest_driver = { + .probe = selftest_probe, + .remove = selftest_remove, + .driver = { + .name = "selftest", + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(selftest_match), + }, +}; + +/* get the platform device instantiated at the path */ +static struct platform_device *of_path_to_platform_device(const char *path) { struct device_node *np; - struct property *prop; + struct platform_device *pdev; + + np = of_find_node_by_path(path); + if (np == NULL) + return NULL; + + pdev = of_find_device_by_node(np); + of_node_put(np); + + return pdev; +} + +/* find out if a platform device exists at that path */ +static int of_path_platform_device_exists(const char *path) +{ + struct platform_device *pdev; + + pdev = of_path_to_platform_device(path); + platform_device_put(pdev); + return pdev != NULL; +} + +static const char *selftest_path(int nr) +{ + static char buf[256]; + + snprintf(buf, sizeof(buf) - 1, + "/testcase-data/overlay-node/test-bus/test-selftest%d", nr); + buf[sizeof(buf) - 1] = '\0'; + + return buf; +} + +static const char *overlay_path(int nr) +{ + static char buf[256]; + + snprintf(buf, sizeof(buf) - 1, + "/testcase-data/overlay%d", nr); + buf[sizeof(buf) - 1] = '\0'; + + return buf; +} + +static const char *bus_path = "/testcase-data/overlay-node/test-bus"; + +static int of_selftest_apply_overlay(int selftest_nr, int overlay_nr, + int *overlay_id) +{ + struct device_node *np = NULL; + int ret, id = -1; + + np = of_find_node_by_path(overlay_path(overlay_nr)); + if (np == NULL) { + selftest(0, "could not find overlay node @\"%s\"\n", + overlay_path(overlay_nr)); + ret = -EINVAL; + goto out; + } + + ret = of_overlay_create(np); + if (ret < 0) { + selftest(0, "could not create overlay from \"%s\"\n", + overlay_path(overlay_nr)); + goto out; + } + id = ret; + + ret = 0; + +out: + of_node_put(np); + + if (overlay_id) + *overlay_id = id; - if (selftest_live_tree) { - of_node_put(of_aliases); - of_node_put(of_chosen); - of_aliases = NULL; - of_chosen = NULL; - for_each_child_of_node(of_allnodes, np) - detach_node_and_children(np); - __of_detach_node_sysfs(of_allnodes); - of_allnodes = NULL; + return ret; +} + +/* apply an overlay while checking before and after states */ +static int of_selftest_apply_overlay_check(int overlay_nr, int selftest_nr, + int before, int after) +{ + int ret; + + /* selftest device must not be in before state */ + if (of_path_platform_device_exists(selftest_path(selftest_nr)) + != before) { + selftest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr), + selftest_path(selftest_nr), + !before ? "enabled" : "disabled"); + return -EINVAL; + } + + ret = of_selftest_apply_overlay(overlay_nr, selftest_nr, NULL); + if (ret != 0) { + /* of_selftest_apply_overlay already called selftest() */ + return ret; + } + + /* selftest device must be to set to after state */ + if (of_path_platform_device_exists(selftest_path(selftest_nr)) + != after) { + selftest(0, "overlay @\"%s\" failed to create @\"%s\" %s\n", + overlay_path(overlay_nr), + selftest_path(selftest_nr), + !after ? "enabled" : "disabled"); + return -EINVAL; + } + + return 0; +} + +/* apply an overlay and then revert it while checking before, after states */ +static int of_selftest_apply_revert_overlay_check(int overlay_nr, + int selftest_nr, int before, int after) +{ + int ret, ov_id; + + /* selftest device must be in before state */ + if (of_path_platform_device_exists(selftest_path(selftest_nr)) + != before) { + selftest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr), + selftest_path(selftest_nr), + !before ? "enabled" : "disabled"); + return -EINVAL; + } + + /* apply the overlay */ + ret = of_selftest_apply_overlay(overlay_nr, selftest_nr, &ov_id); + if (ret != 0) { + /* of_selftest_apply_overlay already called selftest() */ + return ret; + } + + /* selftest device must be in after state */ + if (of_path_platform_device_exists(selftest_path(selftest_nr)) + != after) { + selftest(0, "overlay @\"%s\" failed to create @\"%s\" %s\n", + overlay_path(overlay_nr), + selftest_path(selftest_nr), + !after ? "enabled" : "disabled"); + return -EINVAL; + } + + ret = of_overlay_destroy(ov_id); + if (ret != 0) { + selftest(0, "overlay @\"%s\" failed to be destroyed @\"%s\"\n", + overlay_path(overlay_nr), + selftest_path(selftest_nr)); + return ret; + } + + /* selftest device must be again in before state */ + if (of_path_platform_device_exists(selftest_path(selftest_nr)) + != before) { + selftest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr), + selftest_path(selftest_nr), + !before ? "enabled" : "disabled"); + return -EINVAL; + } + + return 0; +} + +/* test activation of device */ +static void of_selftest_overlay_0(void) +{ + int ret; + + /* device should enable */ + ret = of_selftest_apply_overlay_check(0, 0, 0, 1); + if (ret != 0) + return; + + selftest(1, "overlay test %d passed\n", 0); +} + +/* test deactivation of device */ +static void of_selftest_overlay_1(void) +{ + int ret; + + /* device should disable */ + ret = of_selftest_apply_overlay_check(1, 1, 1, 0); + if (ret != 0) + return; + + selftest(1, "overlay test %d passed\n", 1); +} + +/* test activation of device */ +static void of_selftest_overlay_2(void) +{ + int ret; + + /* device should enable */ + ret = of_selftest_apply_overlay_check(2, 2, 0, 1); + if (ret != 0) + return; + + selftest(1, "overlay test %d passed\n", 2); +} + +/* test deactivation of device */ +static void of_selftest_overlay_3(void) +{ + int ret; + + /* device should disable */ + ret = of_selftest_apply_overlay_check(3, 3, 1, 0); + if (ret != 0) + return; + + selftest(1, "overlay test %d passed\n", 3); +} + +/* test activation of a full device node */ +static void of_selftest_overlay_4(void) +{ + int ret; + + /* device should disable */ + ret = of_selftest_apply_overlay_check(4, 4, 0, 1); + if (ret != 0) + return; + + selftest(1, "overlay test %d passed\n", 4); +} + +/* test overlay apply/revert sequence */ +static void of_selftest_overlay_5(void) +{ + int ret; + + /* device should disable */ + ret = of_selftest_apply_revert_overlay_check(5, 5, 0, 1); + if (ret != 0) return; + + selftest(1, "overlay test %d passed\n", 5); +} + +/* test overlay application in sequence */ +static void of_selftest_overlay_6(void) +{ + struct device_node *np; + int ret, i, ov_id[2]; + int overlay_nr = 6, selftest_nr = 6; + int before = 0, after = 1; + + /* selftest device must be in before state */ + for (i = 0; i < 2; i++) { + if (of_path_platform_device_exists( + selftest_path(selftest_nr + i)) + != before) { + selftest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr + i), + selftest_path(selftest_nr + i), + !before ? "enabled" : "disabled"); + return; + } } - while (last_node_index-- > 0) { - if (nodes[last_node_index]) { - np = of_find_node_by_path(nodes[last_node_index]->full_name); - if (np == nodes[last_node_index]) { - if (of_aliases == np) { - of_node_put(of_aliases); - of_aliases = NULL; - } - detach_node_and_children(np); - } else { - for_each_property_of_node(np, prop) { - if (strcmp(prop->name, "testcase-alias") == 0) - of_remove_property(np, prop); - } - } + /* apply the overlays */ + for (i = 0; i < 2; i++) { + + np = of_find_node_by_path(overlay_path(overlay_nr + i)); + if (np == NULL) { + selftest(0, "could not find overlay node @\"%s\"\n", + overlay_path(overlay_nr + i)); + return; + } + + ret = of_overlay_create(np); + if (ret < 0) { + selftest(0, "could not create overlay from \"%s\"\n", + overlay_path(overlay_nr + i)); + return; + } + ov_id[i] = ret; + } + + for (i = 0; i < 2; i++) { + /* selftest device must be in after state */ + if (of_path_platform_device_exists( + selftest_path(selftest_nr + i)) + != after) { + selftest(0, "overlay @\"%s\" failed @\"%s\" %s\n", + overlay_path(overlay_nr + i), + selftest_path(selftest_nr + i), + !after ? "enabled" : "disabled"); + return; } } + + for (i = 1; i >= 0; i--) { + ret = of_overlay_destroy(ov_id[i]); + if (ret != 0) { + selftest(0, "overlay @\"%s\" failed destroy @\"%s\"\n", + overlay_path(overlay_nr + i), + selftest_path(selftest_nr + i)); + return; + } + } + + for (i = 0; i < 2; i++) { + /* selftest device must be again in before state */ + if (of_path_platform_device_exists( + selftest_path(selftest_nr + i)) + != before) { + selftest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr + i), + selftest_path(selftest_nr + i), + !before ? "enabled" : "disabled"); + return; + } + } + + selftest(1, "overlay test %d passed\n", 6); +} + +/* test overlay application in sequence */ +static void of_selftest_overlay_8(void) +{ + struct device_node *np; + int ret, i, ov_id[2]; + int overlay_nr = 8, selftest_nr = 8; + + /* we don't care about device state in this test */ + + /* apply the overlays */ + for (i = 0; i < 2; i++) { + + np = of_find_node_by_path(overlay_path(overlay_nr + i)); + if (np == NULL) { + selftest(0, "could not find overlay node @\"%s\"\n", + overlay_path(overlay_nr + i)); + return; + } + + ret = of_overlay_create(np); + if (ret < 0) { + selftest(0, "could not create overlay from \"%s\"\n", + overlay_path(overlay_nr + i)); + return; + } + ov_id[i] = ret; + } + + /* now try to remove first overlay (it should fail) */ + ret = of_overlay_destroy(ov_id[0]); + if (ret == 0) { + selftest(0, "overlay @\"%s\" was destroyed @\"%s\"\n", + overlay_path(overlay_nr + 0), + selftest_path(selftest_nr)); + return; + } + + /* removing them in order should work */ + for (i = 1; i >= 0; i--) { + ret = of_overlay_destroy(ov_id[i]); + if (ret != 0) { + selftest(0, "overlay @\"%s\" not destroyed @\"%s\"\n", + overlay_path(overlay_nr + i), + selftest_path(selftest_nr)); + return; + } + } + + selftest(1, "overlay test %d passed\n", 8); +} + +/* test insertion of a bus with parent devices */ +static void of_selftest_overlay_10(void) +{ + int ret; + char *child_path; + + /* device should disable */ + ret = of_selftest_apply_overlay_check(10, 10, 0, 1); + if (selftest(ret == 0, "overlay test %d failed; overlay application\n", 10)) + return; + + child_path = kasprintf(GFP_KERNEL, "%s/test-selftest101", + selftest_path(10)); + if (selftest(child_path, "overlay test %d failed; kasprintf\n", 10)) + return; + + ret = of_path_platform_device_exists(child_path); + kfree(child_path); + if (selftest(ret, "overlay test %d failed; no child device\n", 10)) + return; +} + +/* test insertion of a bus with parent devices (and revert) */ +static void of_selftest_overlay_11(void) +{ + int ret; + + /* device should disable */ + ret = of_selftest_apply_revert_overlay_check(11, 11, 0, 1); + if (selftest(ret == 0, "overlay test %d failed; overlay application\n", 11)) + return; +} + +static void __init of_selftest_overlay(void) +{ + struct device_node *bus_np = NULL; + int ret; + + ret = platform_driver_register(&selftest_driver); + if (ret != 0) { + selftest(0, "could not register selftest driver\n"); + goto out; + } + + bus_np = of_find_node_by_path(bus_path); + if (bus_np == NULL) { + selftest(0, "could not find bus_path \"%s\"\n", bus_path); + goto out; + } + + ret = of_platform_populate(bus_np, of_default_bus_match_table, + NULL, NULL); + if (ret != 0) { + selftest(0, "could not populate bus @ \"%s\"\n", bus_path); + goto out; + } + + if (!of_path_platform_device_exists(selftest_path(100))) { + selftest(0, "could not find selftest0 @ \"%s\"\n", + selftest_path(100)); + goto out; + } + + if (of_path_platform_device_exists(selftest_path(101))) { + selftest(0, "selftest1 @ \"%s\" should not exist\n", + selftest_path(101)); + goto out; + } + + selftest(1, "basic infrastructure of overlays passed"); + + /* tests in sequence */ + of_selftest_overlay_0(); + of_selftest_overlay_1(); + of_selftest_overlay_2(); + of_selftest_overlay_3(); + of_selftest_overlay_4(); + of_selftest_overlay_5(); + of_selftest_overlay_6(); + of_selftest_overlay_8(); + + of_selftest_overlay_10(); + of_selftest_overlay_11(); + +out: + of_node_put(bus_np); } +#else +static inline void __init of_selftest_overlay(void) { } +#endif + static int __init of_selftest(void) { struct device_node *np; @@ -947,9 +1469,7 @@ static int __init of_selftest(void) of_selftest_parse_interrupts_extended(); of_selftest_match_node(); of_selftest_platform_populate(); - - /* removing selftest data from live tree */ - selftest_data_remove(); + of_selftest_overlay(); /* Double check linkage after removing testcase data */ of_selftest_check_tree_linkage(); diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 893503fa1782..cced84233ac0 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -4,6 +4,7 @@ config PCI_MSI bool "Message Signaled Interrupts (MSI and MSI-X)" depends on PCI + select GENERIC_MSI_IRQ help This allows device drivers to enable MSI (Message Signaled Interrupts). Message Signaled Interrupts enable a device to @@ -16,6 +17,11 @@ config PCI_MSI If you don't know what to do here, say Y. +config PCI_MSI_IRQ_DOMAIN + bool + depends on PCI_MSI + select GENERIC_MSI_IRQ_DOMAIN + config PCI_DEBUG bool "PCI Debugging" depends on PCI && DEBUG_KERNEL diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index e04fe2d9df3b..e9815acec4a3 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -35,6 +35,7 @@ obj-$(CONFIG_PCI_IOV) += iov.o # obj-$(CONFIG_ALPHA) += setup-irq.o obj-$(CONFIG_ARM) += setup-irq.o +obj-$(CONFIG_ARM64) += setup-irq.o obj-$(CONFIG_UNICORE32) += setup-irq.o obj-$(CONFIG_SUPERH) += setup-irq.o obj-$(CONFIG_MIPS) += setup-irq.o diff --git a/drivers/pci/access.c b/drivers/pci/access.c index 7f249b9ab2ce..b965c12168b7 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -67,6 +67,93 @@ EXPORT_SYMBOL(pci_bus_write_config_byte); EXPORT_SYMBOL(pci_bus_write_config_word); EXPORT_SYMBOL(pci_bus_write_config_dword); +int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + void __iomem *addr; + + addr = bus->ops->map_bus(bus, devfn, where); + if (!addr) { + *val = ~0; + return PCIBIOS_DEVICE_NOT_FOUND; + } + + if (size == 1) + *val = readb(addr); + else if (size == 2) + *val = readw(addr); + else + *val = readl(addr); + + return PCIBIOS_SUCCESSFUL; +} +EXPORT_SYMBOL_GPL(pci_generic_config_read); + +int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + void __iomem *addr; + + addr = bus->ops->map_bus(bus, devfn, where); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (size == 1) + writeb(val, addr); + else if (size == 2) + writew(val, addr); + else + writel(val, addr); + + return PCIBIOS_SUCCESSFUL; +} +EXPORT_SYMBOL_GPL(pci_generic_config_write); + +int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + void __iomem *addr; + + addr = bus->ops->map_bus(bus, devfn, where & ~0x3); + if (!addr) { + *val = ~0; + return PCIBIOS_DEVICE_NOT_FOUND; + } + + *val = readl(addr); + + if (size <= 2) + *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); + + return PCIBIOS_SUCCESSFUL; +} +EXPORT_SYMBOL_GPL(pci_generic_config_read32); + +int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + void __iomem *addr; + u32 mask, tmp; + + addr = bus->ops->map_bus(bus, devfn, where & ~0x3); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (size == 4) { + writel(val, addr); + return PCIBIOS_SUCCESSFUL; + } else { + mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); + } + + tmp = readl(addr) & mask; + tmp |= val << ((where & 0x3) * 8); + writel(tmp, addr); + + return PCIBIOS_SUCCESSFUL; +} +EXPORT_SYMBOL_GPL(pci_generic_config_write32); + /** * pci_bus_set_ops - Set raw operations of pci bus * @bus: pci bus struct diff --git a/drivers/pci/host-bridge.c b/drivers/pci/host-bridge.c index 0e5f3c95af5b..f58e05b0f8d5 100644 --- a/drivers/pci/host-bridge.c +++ b/drivers/pci/host-bridge.c @@ -23,6 +23,20 @@ static struct pci_host_bridge *find_pci_host_bridge(struct pci_bus *bus) return to_pci_host_bridge(root_bus->bridge); } +struct device *pci_get_host_bridge_device(struct pci_dev *dev) +{ + struct pci_bus *root_bus = find_pci_root_bus(dev->bus); + struct device *bridge = root_bus->bridge; + + kobject_get(&bridge->kobj); + return bridge; +} + +void pci_put_host_bridge_device(struct device *dev) +{ + kobject_put(&dev->kobj); +} + void pci_set_host_bridge_release(struct pci_host_bridge *bridge, void (*release_fn)(struct pci_host_bridge *), void *release_data) diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig index 3dc25fad490c..5aacabb59402 100644 --- a/drivers/pci/host/Kconfig +++ b/drivers/pci/host/Kconfig @@ -50,7 +50,7 @@ config PCI_RCAR_GEN2_PCIE config PCI_HOST_GENERIC bool "Generic PCI host controller" - depends on ARM && OF + depends on (ARM || ARM64) && OF help Say Y here if you want to support a simple generic PCI host controller, such as the one emulated by kvmtool. @@ -86,9 +86,18 @@ config PCI_XGENE depends on ARCH_XGENE depends on OF select PCIEPORTBUS + select PCI_MSI_IRQ_DOMAIN if PCI_MSI help Say Y here if you want internal PCI support on APM X-Gene SoC. There are 5 internal PCIe ports available. Each port is GEN3 capable and have varied lanes from x1 to x8. +config PCI_XGENE_MSI + bool "X-Gene v1 PCIe MSI feature" + depends on PCI_XGENE && PCI_MSI + default y + help + Say Y here if you want PCIe MSI support for the APM X-Gene v1 SoC. + This MSI driver supports 5 PCIe ports on the APM X-Gene v1 SoC. + endmenu diff --git a/drivers/pci/host/Makefile b/drivers/pci/host/Makefile index 26b3461d68d7..852ba92f2067 100644 --- a/drivers/pci/host/Makefile +++ b/drivers/pci/host/Makefile @@ -11,3 +11,4 @@ obj-$(CONFIG_PCIE_SPEAR13XX) += pcie-spear13xx.o obj-$(CONFIG_PCI_KEYSTONE) += pci-keystone-dw.o pci-keystone.o obj-$(CONFIG_PCIE_XILINX) += pcie-xilinx.o obj-$(CONFIG_PCI_XGENE) += pci-xgene.o +obj-$(CONFIG_PCI_XGENE_MSI) += pci-xgene-msi.o diff --git a/drivers/pci/host/pci-dra7xx.c b/drivers/pci/host/pci-dra7xx.c index b2d093ea5ff2..6be9485c8245 100644 --- a/drivers/pci/host/pci-dra7xx.c +++ b/drivers/pci/host/pci-dra7xx.c @@ -447,7 +447,6 @@ static struct platform_driver dra7xx_pcie_driver = { .remove = __exit_p(dra7xx_pcie_remove), .driver = { .name = "dra7-pcie", - .owner = THIS_MODULE, .of_match_table = of_dra7xx_pcie_match, }, }; diff --git a/drivers/pci/host/pci-exynos.c b/drivers/pci/host/pci-exynos.c index ab8bcaabaf21..3be96171842f 100644 --- a/drivers/pci/host/pci-exynos.c +++ b/drivers/pci/host/pci-exynos.c @@ -398,7 +398,7 @@ static void exynos_pcie_enable_irq_pulse(struct pcie_port *pp) /* enable INTX interrupt */ val = IRQ_INTA_ASSERT | IRQ_INTB_ASSERT | - IRQ_INTC_ASSERT | IRQ_INTD_ASSERT, + IRQ_INTC_ASSERT | IRQ_INTD_ASSERT; exynos_elb_writel(exynos_pcie, val, PCIE_IRQ_EN_PULSE); return; } @@ -650,7 +650,6 @@ static struct platform_driver exynos_pcie_driver = { .remove = __exit_p(exynos_pcie_remove), .driver = { .name = "exynos-pcie", - .owner = THIS_MODULE, .of_match_table = exynos_pcie_of_match, }, }; diff --git a/drivers/pci/host/pci-host-generic.c b/drivers/pci/host/pci-host-generic.c index 3d2076f59911..7f8b9ebc5453 100644 --- a/drivers/pci/host/pci-host-generic.c +++ b/drivers/pci/host/pci-host-generic.c @@ -32,13 +32,22 @@ struct gen_pci_cfg_bus_ops { struct gen_pci_cfg_windows { struct resource res; - struct resource bus_range; + struct resource *bus_range; void __iomem **win; const struct gen_pci_cfg_bus_ops *ops; }; +/* + * ARM pcibios functions expect the ARM struct pci_sys_data as the PCI + * sysdata. Add pci_sys_data as the first element in struct gen_pci so + * that when we use a gen_pci pointer as sysdata, it is also a pointer to + * a struct pci_sys_data. + */ struct gen_pci { +#ifdef CONFIG_ARM + struct pci_sys_data sys; +#endif struct pci_host_bridge host; struct gen_pci_cfg_windows cfg; struct list_head resources; @@ -48,9 +57,8 @@ static void __iomem *gen_pci_map_cfg_bus_cam(struct pci_bus *bus, unsigned int devfn, int where) { - struct pci_sys_data *sys = bus->sysdata; - struct gen_pci *pci = sys->private_data; - resource_size_t idx = bus->number - pci->cfg.bus_range.start; + struct gen_pci *pci = bus->sysdata; + resource_size_t idx = bus->number - pci->cfg.bus_range->start; return pci->cfg.win[idx] + ((devfn << 8) | where); } @@ -64,9 +72,8 @@ static void __iomem *gen_pci_map_cfg_bus_ecam(struct pci_bus *bus, unsigned int devfn, int where) { - struct pci_sys_data *sys = bus->sysdata; - struct gen_pci *pci = sys->private_data; - resource_size_t idx = bus->number - pci->cfg.bus_range.start; + struct gen_pci *pci = bus->sysdata; + resource_size_t idx = bus->number - pci->cfg.bus_range->start; return pci->cfg.win[idx] + ((devfn << 12) | where); } @@ -76,55 +83,9 @@ static struct gen_pci_cfg_bus_ops gen_pci_cfg_ecam_bus_ops = { .map_bus = gen_pci_map_cfg_bus_ecam, }; -static int gen_pci_config_read(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 *val) -{ - void __iomem *addr; - struct pci_sys_data *sys = bus->sysdata; - struct gen_pci *pci = sys->private_data; - - addr = pci->cfg.ops->map_bus(bus, devfn, where); - - switch (size) { - case 1: - *val = readb(addr); - break; - case 2: - *val = readw(addr); - break; - default: - *val = readl(addr); - } - - return PCIBIOS_SUCCESSFUL; -} - -static int gen_pci_config_write(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - void __iomem *addr; - struct pci_sys_data *sys = bus->sysdata; - struct gen_pci *pci = sys->private_data; - - addr = pci->cfg.ops->map_bus(bus, devfn, where); - - switch (size) { - case 1: - writeb(val, addr); - break; - case 2: - writew(val, addr); - break; - default: - writel(val, addr); - } - - return PCIBIOS_SUCCESSFUL; -} - static struct pci_ops gen_pci_ops = { - .read = gen_pci_config_read, - .write = gen_pci_config_write, + .read = pci_generic_config_read, + .write = pci_generic_config_write, }; static const struct of_device_id gen_pci_of_match[] = { @@ -138,106 +99,50 @@ static const struct of_device_id gen_pci_of_match[] = { }; MODULE_DEVICE_TABLE(of, gen_pci_of_match); -static int gen_pci_calc_io_offset(struct device *dev, - struct of_pci_range *range, - struct resource *res, - resource_size_t *offset) -{ - static atomic_t wins = ATOMIC_INIT(0); - int err, idx, max_win; - unsigned int window; - - if (!PAGE_ALIGNED(range->cpu_addr)) - return -EINVAL; - - max_win = (IO_SPACE_LIMIT + 1) / SZ_64K; - idx = atomic_inc_return(&wins); - if (idx > max_win) - return -ENOSPC; - - window = (idx - 1) * SZ_64K; - err = pci_ioremap_io(window, range->cpu_addr); - if (err) - return err; - - of_pci_range_to_resource(range, dev->of_node, res); - res->start = window; - res->end = res->start + range->size - 1; - *offset = window - range->pci_addr; - return 0; -} - -static int gen_pci_calc_mem_offset(struct device *dev, - struct of_pci_range *range, - struct resource *res, - resource_size_t *offset) -{ - of_pci_range_to_resource(range, dev->of_node, res); - *offset = range->cpu_addr - range->pci_addr; - return 0; -} - static void gen_pci_release_of_pci_ranges(struct gen_pci *pci) { - struct pci_host_bridge_window *win; - - list_for_each_entry(win, &pci->resources, list) - release_resource(win->res); - pci_free_resource_list(&pci->resources); } static int gen_pci_parse_request_of_pci_ranges(struct gen_pci *pci) { - struct of_pci_range range; - struct of_pci_range_parser parser; int err, res_valid = 0; struct device *dev = pci->host.dev.parent; struct device_node *np = dev->of_node; + resource_size_t iobase; + struct pci_host_bridge_window *win; - if (of_pci_range_parser_init(&parser, np)) { - dev_err(dev, "missing \"ranges\" property\n"); - return -EINVAL; - } - - for_each_of_pci_range(&parser, &range) { - struct resource *parent, *res; - resource_size_t offset; - u32 restype = range.flags & IORESOURCE_TYPE_BITS; + err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pci->resources, + &iobase); + if (err) + return err; - res = devm_kmalloc(dev, sizeof(*res), GFP_KERNEL); - if (!res) { - err = -ENOMEM; - goto out_release_res; - } + list_for_each_entry(win, &pci->resources, list) { + struct resource *parent, *res = win->res; - switch (restype) { + switch (resource_type(res)) { case IORESOURCE_IO: parent = &ioport_resource; - err = gen_pci_calc_io_offset(dev, &range, res, &offset); + err = pci_remap_iospace(res, iobase); + if (err) { + dev_warn(dev, "error %d: failed to map resource %pR\n", + err, res); + continue; + } break; case IORESOURCE_MEM: parent = &iomem_resource; - err = gen_pci_calc_mem_offset(dev, &range, res, &offset); - res_valid |= !(res->flags & IORESOURCE_PREFETCH || err); + res_valid |= !(res->flags & IORESOURCE_PREFETCH); break; + case IORESOURCE_BUS: + pci->cfg.bus_range = res; default: - err = -EINVAL; continue; } - if (err) { - dev_warn(dev, - "error %d: failed to add resource [type 0x%x, %lld bytes]\n", - err, restype, range.size); - continue; - } - - err = request_resource(parent, res); + err = devm_request_resource(dev, parent, res); if (err) goto out_release_res; - - pci_add_resource_offset(&pci->resources, res, offset); } if (!res_valid) { @@ -262,38 +167,30 @@ static int gen_pci_parse_map_cfg_windows(struct gen_pci *pci) struct device *dev = pci->host.dev.parent; struct device_node *np = dev->of_node; - if (of_pci_parse_bus_range(np, &pci->cfg.bus_range)) - pci->cfg.bus_range = (struct resource) { - .name = np->name, - .start = 0, - .end = 0xff, - .flags = IORESOURCE_BUS, - }; - err = of_address_to_resource(np, 0, &pci->cfg.res); if (err) { dev_err(dev, "missing \"reg\" property\n"); return err; } - pci->cfg.win = devm_kcalloc(dev, resource_size(&pci->cfg.bus_range), + /* Limit the bus-range to fit within reg */ + bus_max = pci->cfg.bus_range->start + + (resource_size(&pci->cfg.res) >> pci->cfg.ops->bus_shift) - 1; + pci->cfg.bus_range->end = min_t(resource_size_t, + pci->cfg.bus_range->end, bus_max); + + pci->cfg.win = devm_kcalloc(dev, resource_size(pci->cfg.bus_range), sizeof(*pci->cfg.win), GFP_KERNEL); if (!pci->cfg.win) return -ENOMEM; - /* Limit the bus-range to fit within reg */ - bus_max = pci->cfg.bus_range.start + - (resource_size(&pci->cfg.res) >> pci->cfg.ops->bus_shift) - 1; - pci->cfg.bus_range.end = min_t(resource_size_t, pci->cfg.bus_range.end, - bus_max); - /* Map our Configuration Space windows */ if (!devm_request_mem_region(dev, pci->cfg.res.start, resource_size(&pci->cfg.res), "Configuration Space")) return -ENOMEM; - bus_range = &pci->cfg.bus_range; + bus_range = pci->cfg.bus_range; for (busn = bus_range->start; busn <= bus_range->end; ++busn) { u32 idx = busn - bus_range->start; u32 sz = 1 << pci->cfg.ops->bus_shift; @@ -305,18 +202,9 @@ static int gen_pci_parse_map_cfg_windows(struct gen_pci *pci) return -ENOMEM; } - /* Register bus resource */ - pci_add_resource(&pci->resources, bus_range); return 0; } -static int gen_pci_setup(int nr, struct pci_sys_data *sys) -{ - struct gen_pci *pci = sys->private_data; - list_splice_init(&pci->resources, &sys->resources); - return 1; -} - static int gen_pci_probe(struct platform_device *pdev) { int err; @@ -326,13 +214,7 @@ static int gen_pci_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct gen_pci *pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); - struct hw_pci hw = { - .nr_controllers = 1, - .private_data = (void **)&pci, - .setup = gen_pci_setup, - .map_irq = of_irq_parse_and_map_pci, - .ops = &gen_pci_ops, - }; + struct pci_bus *bus, *child; if (!pci) return -ENOMEM; @@ -353,6 +235,7 @@ static int gen_pci_probe(struct platform_device *pdev) of_id = of_match_node(gen_pci_of_match, np); pci->cfg.ops = of_id->data; + gen_pci_ops.map_bus = pci->cfg.ops->map_bus; pci->host.dev.parent = dev; INIT_LIST_HEAD(&pci->host.windows); INIT_LIST_HEAD(&pci->resources); @@ -369,14 +252,33 @@ static int gen_pci_probe(struct platform_device *pdev) return err; } - pci_common_init_dev(dev, &hw); + /* Do not reassign resources if probe only */ + if (!pci_has_flag(PCI_PROBE_ONLY)) + pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS); + + bus = pci_scan_root_bus(dev, 0, &gen_pci_ops, pci, &pci->resources); + if (!bus) { + dev_err(dev, "Scanning rootbus failed"); + return -ENODEV; + } + + pci_fixup_irqs(pci_common_swizzle, of_irq_parse_and_map_pci); + + if (!pci_has_flag(PCI_PROBE_ONLY)) { + pci_bus_size_bridges(bus); + pci_bus_assign_resources(bus); + + list_for_each_entry(child, &bus->children, node) + pcie_bus_configure_settings(child); + } + + pci_bus_add_devices(bus); return 0; } static struct platform_driver gen_pci_driver = { .driver = { .name = "pci-host-generic", - .owner = THIS_MODULE, .of_match_table = gen_pci_of_match, }, .probe = gen_pci_probe, diff --git a/drivers/pci/host/pci-imx6.c b/drivers/pci/host/pci-imx6.c index f1f239f60685..ebcb0ac8512b 100644 --- a/drivers/pci/host/pci-imx6.c +++ b/drivers/pci/host/pci-imx6.c @@ -644,7 +644,6 @@ MODULE_DEVICE_TABLE(of, imx6_pcie_of_match); static struct platform_driver imx6_pcie_driver = { .driver = { .name = "imx6q-pcie", - .owner = THIS_MODULE, .of_match_table = imx6_pcie_of_match, }, .shutdown = imx6_pcie_shutdown, diff --git a/drivers/pci/host/pci-keystone-dw.c b/drivers/pci/host/pci-keystone-dw.c index 34086ce88e8e..1d7b0ba76d1e 100644 --- a/drivers/pci/host/pci-keystone-dw.c +++ b/drivers/pci/host/pci-keystone-dw.c @@ -104,14 +104,13 @@ static void ks_dw_pcie_msi_irq_ack(struct irq_data *d) { u32 offset, reg_offset, bit_pos; struct keystone_pcie *ks_pcie; - unsigned int irq = d->irq; struct msi_desc *msi; struct pcie_port *pp; - msi = irq_get_msi_desc(irq); - pp = sys_to_pcie(msi->dev->bus->sysdata); + msi = irq_data_get_msi_desc(d); + pp = sys_to_pcie(msi_desc_to_pci_sysdata(msi)); ks_pcie = to_keystone_pcie(pp); - offset = irq - irq_linear_revmap(pp->irq_domain, 0); + offset = d->irq - irq_linear_revmap(pp->irq_domain, 0); update_reg_offset_bit_pos(offset, ®_offset, &bit_pos); writel(BIT(bit_pos), @@ -142,20 +141,19 @@ void ks_dw_pcie_msi_clear_irq(struct pcie_port *pp, int irq) static void ks_dw_pcie_msi_irq_mask(struct irq_data *d) { struct keystone_pcie *ks_pcie; - unsigned int irq = d->irq; struct msi_desc *msi; struct pcie_port *pp; u32 offset; - msi = irq_get_msi_desc(irq); - pp = sys_to_pcie(msi->dev->bus->sysdata); + msi = irq_data_get_msi_desc(d); + pp = sys_to_pcie(msi_desc_to_pci_sysdata(msi)); ks_pcie = to_keystone_pcie(pp); - offset = irq - irq_linear_revmap(pp->irq_domain, 0); + offset = d->irq - irq_linear_revmap(pp->irq_domain, 0); /* Mask the end point if PVM implemented */ if (IS_ENABLED(CONFIG_PCI_MSI)) { if (msi->msi_attrib.maskbit) - mask_msi_irq(d); + pci_msi_mask_irq(d); } ks_dw_pcie_msi_clear_irq(pp, offset); @@ -164,20 +162,19 @@ static void ks_dw_pcie_msi_irq_mask(struct irq_data *d) static void ks_dw_pcie_msi_irq_unmask(struct irq_data *d) { struct keystone_pcie *ks_pcie; - unsigned int irq = d->irq; struct msi_desc *msi; struct pcie_port *pp; u32 offset; - msi = irq_get_msi_desc(irq); - pp = sys_to_pcie(msi->dev->bus->sysdata); + msi = irq_data_get_msi_desc(d); + pp = sys_to_pcie(msi_desc_to_pci_sysdata(msi)); ks_pcie = to_keystone_pcie(pp); - offset = irq - irq_linear_revmap(pp->irq_domain, 0); + offset = d->irq - irq_linear_revmap(pp->irq_domain, 0); /* Mask the end point if PVM implemented */ if (IS_ENABLED(CONFIG_PCI_MSI)) { if (msi->msi_attrib.maskbit) - unmask_msi_irq(d); + pci_msi_unmask_irq(d); } ks_dw_pcie_msi_set_irq(pp, offset); @@ -205,7 +202,7 @@ const struct irq_domain_ops ks_dw_pcie_msi_domain_ops = { .map = ks_dw_pcie_msi_map, }; -int ks_dw_pcie_msi_host_init(struct pcie_port *pp, struct msi_chip *chip) +int ks_dw_pcie_msi_host_init(struct pcie_port *pp, struct msi_controller *chip) { struct keystone_pcie *ks_pcie = to_keystone_pcie(pp); int i; diff --git a/drivers/pci/host/pci-keystone.c b/drivers/pci/host/pci-keystone.c index 1e934b7e0139..f27a43558177 100644 --- a/drivers/pci/host/pci-keystone.c +++ b/drivers/pci/host/pci-keystone.c @@ -408,7 +408,6 @@ static struct platform_driver ks_pcie_driver __refdata = { .remove = __exit_p(ks_pcie_remove), .driver = { .name = "keystone-pcie", - .owner = THIS_MODULE, .of_match_table = of_match_ptr(ks_pcie_of_match), }, }; diff --git a/drivers/pci/host/pci-keystone.h b/drivers/pci/host/pci-keystone.h index 1fc1fceede9e..478d932b602d 100644 --- a/drivers/pci/host/pci-keystone.h +++ b/drivers/pci/host/pci-keystone.h @@ -55,4 +55,4 @@ void ks_dw_pcie_msi_set_irq(struct pcie_port *pp, int irq); void ks_dw_pcie_msi_clear_irq(struct pcie_port *pp, int irq); void ks_dw_pcie_v3_65_scan_bus(struct pcie_port *pp); int ks_dw_pcie_msi_host_init(struct pcie_port *pp, - struct msi_chip *chip); + struct msi_controller *chip); diff --git a/drivers/pci/host/pci-mvebu.c b/drivers/pci/host/pci-mvebu.c index b1315e197ffb..b0b181690055 100644 --- a/drivers/pci/host/pci-mvebu.c +++ b/drivers/pci/host/pci-mvebu.c @@ -99,7 +99,7 @@ struct mvebu_pcie_port; struct mvebu_pcie { struct platform_device *pdev; struct mvebu_pcie_port *ports; - struct msi_chip *msi; + struct msi_controller *msi; struct resource io; char io_name[30]; struct resource realio; @@ -774,12 +774,6 @@ static struct pci_bus *mvebu_pcie_scan_bus(int nr, struct pci_sys_data *sys) return bus; } -static void mvebu_pcie_add_bus(struct pci_bus *bus) -{ - struct mvebu_pcie *pcie = sys_to_pcie(bus->sysdata); - bus->msi = pcie->msi; -} - static resource_size_t mvebu_pcie_align_resource(struct pci_dev *dev, const struct resource *res, resource_size_t start, @@ -816,6 +810,10 @@ static void mvebu_pcie_enable(struct mvebu_pcie *pcie) memset(&hw, 0, sizeof(hw)); +#ifdef CONFIG_PCI_MSI + hw.msi_ctrl = pcie->msi; +#endif + hw.nr_controllers = 1; hw.private_data = (void **)&pcie; hw.setup = mvebu_pcie_setup; @@ -823,7 +821,6 @@ static void mvebu_pcie_enable(struct mvebu_pcie *pcie) hw.map_irq = of_irq_parse_and_map_pci; hw.ops = &mvebu_pcie_ops; hw.align_resource = mvebu_pcie_align_resource; - hw.add_bus = mvebu_pcie_add_bus; pci_common_init(&hw); } @@ -1082,7 +1079,6 @@ MODULE_DEVICE_TABLE(of, mvebu_pcie_of_match_table); static struct platform_driver mvebu_pcie_driver = { .driver = { - .owner = THIS_MODULE, .name = "mvebu-pcie", .of_match_table = mvebu_pcie_of_match_table, /* driver unloading/unbinding currently not supported */ diff --git a/drivers/pci/host/pci-rcar-gen2.c b/drivers/pci/host/pci-rcar-gen2.c index 3ef854f5a5b5..d9c042febb1a 100644 --- a/drivers/pci/host/pci-rcar-gen2.c +++ b/drivers/pci/host/pci-rcar-gen2.c @@ -412,7 +412,6 @@ MODULE_DEVICE_TABLE(of, rcar_pci_of_match); static struct platform_driver rcar_pci_driver = { .driver = { .name = "pci-rcar-gen2", - .owner = THIS_MODULE, .suppress_bind_attrs = true, .of_match_table = rcar_pci_of_match, }, diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c index 2898f1597c84..07019ff0ad7c 100644 --- a/drivers/pci/host/pci-tegra.c +++ b/drivers/pci/host/pci-tegra.c @@ -238,7 +238,7 @@ ) struct tegra_msi { - struct msi_chip chip; + struct msi_controller chip; DECLARE_BITMAP(used, INT_PCI_MSI_NR); struct irq_domain *domain; unsigned long pages; @@ -259,7 +259,7 @@ struct tegra_pcie_soc_data { bool has_gen2; }; -static inline struct tegra_msi *to_tegra_msi(struct msi_chip *chip) +static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip) { return container_of(chip, struct tegra_msi, chip); } @@ -692,15 +692,6 @@ static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin) return irq; } -static void tegra_pcie_add_bus(struct pci_bus *bus) -{ - if (IS_ENABLED(CONFIG_PCI_MSI)) { - struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata); - - bus->msi = &pcie->msi.chip; - } -} - static struct pci_bus *tegra_pcie_scan_bus(int nr, struct pci_sys_data *sys) { struct tegra_pcie *pcie = sys_to_pcie(sys); @@ -1280,8 +1271,8 @@ static irqreturn_t tegra_pcie_msi_irq(int irq, void *data) return processed > 0 ? IRQ_HANDLED : IRQ_NONE; } -static int tegra_msi_setup_irq(struct msi_chip *chip, struct pci_dev *pdev, - struct msi_desc *desc) +static int tegra_msi_setup_irq(struct msi_controller *chip, + struct pci_dev *pdev, struct msi_desc *desc) { struct tegra_msi *msi = to_tegra_msi(chip); struct msi_msg msg; @@ -1305,12 +1296,13 @@ static int tegra_msi_setup_irq(struct msi_chip *chip, struct pci_dev *pdev, msg.address_hi = 0; msg.data = hwirq; - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); return 0; } -static void tegra_msi_teardown_irq(struct msi_chip *chip, unsigned int irq) +static void tegra_msi_teardown_irq(struct msi_controller *chip, + unsigned int irq) { struct tegra_msi *msi = to_tegra_msi(chip); struct irq_data *d = irq_get_irq_data(irq); @@ -1322,10 +1314,10 @@ static void tegra_msi_teardown_irq(struct msi_chip *chip, unsigned int irq) static struct irq_chip tegra_msi_irq_chip = { .name = "Tegra PCIe MSI", - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, }; static int tegra_msi_map(struct irq_domain *domain, unsigned int irq, @@ -1893,11 +1885,14 @@ static int tegra_pcie_enable(struct tegra_pcie *pcie) memset(&hw, 0, sizeof(hw)); +#ifdef CONFIG_PCI_MSI + hw.msi_ctrl = &pcie->msi.chip; +#endif + hw.nr_controllers = 1; hw.private_data = (void **)&pcie; hw.setup = tegra_pcie_setup; hw.map_irq = tegra_pcie_map_irq; - hw.add_bus = tegra_pcie_add_bus; hw.scan = tegra_pcie_scan_bus; hw.ops = &tegra_pcie_ops; @@ -2134,7 +2129,6 @@ put_resources: static struct platform_driver tegra_pcie_driver = { .driver = { .name = "tegra-pcie", - .owner = THIS_MODULE, .of_match_table = tegra_pcie_of_match, .suppress_bind_attrs = true, }, diff --git a/drivers/pci/host/pci-versatile.c b/drivers/pci/host/pci-versatile.c new file mode 100644 index 000000000000..e3a2450db2b8 --- /dev/null +++ b/drivers/pci/host/pci-versatile.c @@ -0,0 +1,238 @@ +/* + * Copyright 2004 Koninklijke Philips Electronics NV + * + * Conversion to platform driver and DT: + * Copyright 2014 Linaro Ltd. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * 14/04/2005 Initial version, colin.king@philips.com + */ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/platform_device.h> + +static void __iomem *versatile_pci_base; +static void __iomem *versatile_cfg_base[2]; + +#define PCI_IMAP(m) (versatile_pci_base + ((m) * 4)) +#define PCI_SMAP(m) (versatile_pci_base + 0x14 + ((m) * 4)) +#define PCI_SELFID (versatile_pci_base + 0xc) + +#define VP_PCI_DEVICE_ID 0x030010ee +#define VP_PCI_CLASS_ID 0x0b400000 + +static u32 pci_slot_ignore; + +static int __init versatile_pci_slot_ignore(char *str) +{ + int retval; + int slot; + + while ((retval = get_option(&str, &slot))) { + if ((slot < 0) || (slot > 31)) + pr_err("Illegal slot value: %d\n", slot); + else + pci_slot_ignore |= (1 << slot); + } + return 1; +} +__setup("pci_slot_ignore=", versatile_pci_slot_ignore); + + +static void __iomem *versatile_map_bus(struct pci_bus *bus, + unsigned int devfn, int offset) +{ + unsigned int busnr = bus->number; + + if (pci_slot_ignore & (1 << PCI_SLOT(devfn))) + return NULL; + + return versatile_cfg_base[1] + ((busnr << 16) | (devfn << 8) | offset); +} + +static struct pci_ops pci_versatile_ops = { + .map_bus = versatile_map_bus, + .read = pci_generic_config_read32, + .write = pci_generic_config_write, +}; + +static int versatile_pci_parse_request_of_pci_ranges(struct device *dev, + struct list_head *res) +{ + int err, mem = 1, res_valid = 0; + struct device_node *np = dev->of_node; + resource_size_t iobase; + struct resource_entry *win; + + err = of_pci_get_host_bridge_resources(np, 0, 0xff, res, &iobase); + if (err) + return err; + + resource_list_for_each_entry(win, res, list) { + struct resource *parent, *res = win->res; + + switch (resource_type(res)) { + case IORESOURCE_IO: + parent = &ioport_resource; + err = pci_remap_iospace(res, iobase); + if (err) { + dev_warn(dev, "error %d: failed to map resource %pR\n", + err, res); + continue; + } + break; + case IORESOURCE_MEM: + parent = &iomem_resource; + res_valid |= !(res->flags & IORESOURCE_PREFETCH); + + writel(res->start >> 28, PCI_IMAP(mem)); + writel(PHYS_OFFSET >> 28, PCI_SMAP(mem)); + mem++; + + break; + case IORESOURCE_BUS: + default: + continue; + } + + err = devm_request_resource(dev, parent, res); + if (err) + goto out_release_res; + } + + if (!res_valid) { + dev_err(dev, "non-prefetchable memory resource required\n"); + err = -EINVAL; + goto out_release_res; + } + + return 0; + +out_release_res: + pci_free_resource_list(res); + return err; +} + +/* Unused, temporary to satisfy ARM arch code */ +struct pci_sys_data sys; + +static int versatile_pci_probe(struct platform_device *pdev) +{ + struct resource *res; + int ret, i, myslot = -1; + u32 val; + void __iomem *local_pci_cfg_base; + struct pci_bus *bus; + LIST_HEAD(pci_res); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -ENODEV; + versatile_pci_base = devm_ioremap_resource(&pdev->dev, res); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!res) + return -ENODEV; + versatile_cfg_base[0] = devm_ioremap_resource(&pdev->dev, res); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 2); + if (!res) + return -ENODEV; + versatile_cfg_base[1] = devm_ioremap_resource(&pdev->dev, res); + + ret = versatile_pci_parse_request_of_pci_ranges(&pdev->dev, &pci_res); + if (ret) + return ret; + + /* + * We need to discover the PCI core first to configure itself + * before the main PCI probing is performed + */ + for (i = 0; i < 32; i++) { + if ((readl(versatile_cfg_base[0] + (i << 11) + PCI_VENDOR_ID) == VP_PCI_DEVICE_ID) && + (readl(versatile_cfg_base[0] + (i << 11) + PCI_CLASS_REVISION) == VP_PCI_CLASS_ID)) { + myslot = i; + break; + } + } + if (myslot == -1) { + dev_err(&pdev->dev, "Cannot find PCI core!\n"); + return -EIO; + } + /* + * Do not to map Versatile FPGA PCI device into memory space + */ + pci_slot_ignore |= (1 << myslot); + + dev_info(&pdev->dev, "PCI core found (slot %d)\n", myslot); + + writel(myslot, PCI_SELFID); + local_pci_cfg_base = versatile_cfg_base[1] + (myslot << 11); + + val = readl(local_pci_cfg_base + PCI_COMMAND); + val |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | PCI_COMMAND_INVALIDATE; + writel(val, local_pci_cfg_base + PCI_COMMAND); + + /* + * Configure the PCI inbound memory windows to be 1:1 mapped to SDRAM + */ + writel(PHYS_OFFSET, local_pci_cfg_base + PCI_BASE_ADDRESS_0); + writel(PHYS_OFFSET, local_pci_cfg_base + PCI_BASE_ADDRESS_1); + writel(PHYS_OFFSET, local_pci_cfg_base + PCI_BASE_ADDRESS_2); + + /* + * For many years the kernel and QEMU were symbiotically buggy + * in that they both assumed the same broken IRQ mapping. + * QEMU therefore attempts to auto-detect old broken kernels + * so that they still work on newer QEMU as they did on old + * QEMU. Since we now use the correct (ie matching-hardware) + * IRQ mapping we write a definitely different value to a + * PCI_INTERRUPT_LINE register to tell QEMU that we expect + * real hardware behaviour and it need not be backwards + * compatible for us. This write is harmless on real hardware. + */ + writel(0, versatile_cfg_base[0] + PCI_INTERRUPT_LINE); + + pci_add_flags(PCI_ENABLE_PROC_DOMAINS); + pci_add_flags(PCI_REASSIGN_ALL_BUS | PCI_REASSIGN_ALL_RSRC); + + bus = pci_scan_root_bus(&pdev->dev, 0, &pci_versatile_ops, &sys, &pci_res); + if (!bus) + return -ENOMEM; + + pci_fixup_irqs(pci_common_swizzle, of_irq_parse_and_map_pci); + pci_assign_unassigned_bus_resources(bus); + pci_bus_add_devices(bus); + + return 0; +} + +static const struct of_device_id versatile_pci_of_match[] = { + { .compatible = "arm,versatile-pci", }, + { }, +}; +MODULE_DEVICE_TABLE(of, versatile_pci_of_match); + +static struct platform_driver versatile_pci_driver = { + .driver = { + .name = "versatile-pci", + .of_match_table = versatile_pci_of_match, + }, + .probe = versatile_pci_probe, +}; +module_platform_driver(versatile_pci_driver); + +MODULE_DESCRIPTION("Versatile PCI driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/host/pci-xgene-msi.c b/drivers/pci/host/pci-xgene-msi.c new file mode 100644 index 000000000000..ec5a14b3189b --- /dev/null +++ b/drivers/pci/host/pci-xgene-msi.c @@ -0,0 +1,587 @@ +/* + * APM X-Gene MSI Driver + * + * Copyright (c) 2014, Applied Micro Circuits Corporation + * Author: Tanmay Inamdar <tinamdar@apm.com> + * Duc Dang <dhdang@apm.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_irq.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/of_pci.h> + +#define MSI_IR0 0x000000 +#define MSI_INT0 0x800000 +#define IDX_PER_GROUP 8 +#define IRQS_PER_IDX 16 +#define NR_HW_IRQS 16 +#define NR_MSI_VEC (IDX_PER_GROUP * IRQS_PER_IDX * NR_HW_IRQS) + +struct xgene_msi_group { + struct xgene_msi *msi; + int gic_irq; + u32 msi_grp; +}; + +struct xgene_msi { + struct device_node *node; + struct irq_domain *inner_domain; + struct irq_domain *msi_domain; + u64 msi_addr; + void __iomem *msi_regs; + unsigned long *bitmap; + struct mutex bitmap_lock; + struct xgene_msi_group *msi_groups; + int num_cpus; +}; + +/* Global data */ +static struct xgene_msi xgene_msi_ctrl; + +static struct irq_chip xgene_msi_top_irq_chip = { + .name = "X-Gene1 MSI", + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, +}; + +static struct msi_domain_info xgene_msi_domain_info = { + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSIX), + .chip = &xgene_msi_top_irq_chip, +}; + +/* + * X-Gene v1 has 16 groups of MSI termination registers MSInIRx, where + * n is group number (0..F), x is index of registers in each group (0..7) + * The register layout is as follows: + * MSI0IR0 base_addr + * MSI0IR1 base_addr + 0x10000 + * ... ... + * MSI0IR6 base_addr + 0x60000 + * MSI0IR7 base_addr + 0x70000 + * MSI1IR0 base_addr + 0x80000 + * MSI1IR1 base_addr + 0x90000 + * ... ... + * MSI1IR7 base_addr + 0xF0000 + * MSI2IR0 base_addr + 0x100000 + * ... ... + * MSIFIR0 base_addr + 0x780000 + * MSIFIR1 base_addr + 0x790000 + * ... ... + * MSIFIR7 base_addr + 0x7F0000 + * MSIINT0 base_addr + 0x800000 + * MSIINT1 base_addr + 0x810000 + * ... ... + * MSIINTF base_addr + 0x8F0000 + * + * Each index register supports 16 MSI vectors (0..15) to generate interrupt. + * There are total 16 GIC IRQs assigned for these 16 groups of MSI termination + * registers. + * + * Each MSI termination group has 1 MSIINTn register (n is 0..15) to indicate + * the MSI pending status caused by 1 of its 8 index registers. + */ + +/* MSInIRx read helper */ +static u32 xgene_msi_ir_read(struct xgene_msi *msi, + u32 msi_grp, u32 msir_idx) +{ + return readl_relaxed(msi->msi_regs + MSI_IR0 + + (msi_grp << 19) + (msir_idx << 16)); +} + +/* MSIINTn read helper */ +static u32 xgene_msi_int_read(struct xgene_msi *msi, u32 msi_grp) +{ + return readl_relaxed(msi->msi_regs + MSI_INT0 + (msi_grp << 16)); +} + +/* + * With 2048 MSI vectors supported, the MSI message can be constructed using + * following scheme: + * - Divide into 8 256-vector groups + * Group 0: 0-255 + * Group 1: 256-511 + * Group 2: 512-767 + * ... + * Group 7: 1792-2047 + * - Each 256-vector group is divided into 16 16-vector groups + * As an example: 16 16-vector groups for 256-vector group 0-255 is + * Group 0: 0-15 + * Group 1: 16-32 + * ... + * Group 15: 240-255 + * - The termination address of MSI vector in 256-vector group n and 16-vector + * group x is the address of MSIxIRn + * - The data for MSI vector in 16-vector group x is x + */ +static u32 hwirq_to_reg_set(unsigned long hwirq) +{ + return (hwirq / (NR_HW_IRQS * IRQS_PER_IDX)); +} + +static u32 hwirq_to_group(unsigned long hwirq) +{ + return (hwirq % NR_HW_IRQS); +} + +static u32 hwirq_to_msi_data(unsigned long hwirq) +{ + return ((hwirq / NR_HW_IRQS) % IRQS_PER_IDX); +} + +static void xgene_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct xgene_msi *msi = irq_data_get_irq_chip_data(data); + u32 reg_set = hwirq_to_reg_set(data->hwirq); + u32 group = hwirq_to_group(data->hwirq); + u64 target_addr = msi->msi_addr + (((8 * group) + reg_set) << 16); + + msg->address_hi = upper_32_bits(target_addr); + msg->address_lo = lower_32_bits(target_addr); + msg->data = hwirq_to_msi_data(data->hwirq); +} + +/* + * X-Gene v1 only has 16 MSI GIC IRQs for 2048 MSI vectors. To maintain + * the expected behaviour of .set_affinity for each MSI interrupt, the 16 + * MSI GIC IRQs are statically allocated to 8 X-Gene v1 cores (2 GIC IRQs + * for each core). The MSI vector is moved fom 1 MSI GIC IRQ to another + * MSI GIC IRQ to steer its MSI interrupt to correct X-Gene v1 core. As a + * consequence, the total MSI vectors that X-Gene v1 supports will be + * reduced to 256 (2048/8) vectors. + */ +static int hwirq_to_cpu(unsigned long hwirq) +{ + return (hwirq % xgene_msi_ctrl.num_cpus); +} + +static unsigned long hwirq_to_canonical_hwirq(unsigned long hwirq) +{ + return (hwirq - hwirq_to_cpu(hwirq)); +} + +static int xgene_msi_set_affinity(struct irq_data *irqdata, + const struct cpumask *mask, bool force) +{ + int target_cpu = cpumask_first(mask); + int curr_cpu; + + curr_cpu = hwirq_to_cpu(irqdata->hwirq); + if (curr_cpu == target_cpu) + return IRQ_SET_MASK_OK_DONE; + + /* Update MSI number to target the new CPU */ + irqdata->hwirq = hwirq_to_canonical_hwirq(irqdata->hwirq) + target_cpu; + + return IRQ_SET_MASK_OK; +} + +static struct irq_chip xgene_msi_bottom_irq_chip = { + .name = "MSI", + .irq_set_affinity = xgene_msi_set_affinity, + .irq_compose_msi_msg = xgene_compose_msi_msg, +}; + +static int xgene_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct xgene_msi *msi = domain->host_data; + int msi_irq; + + mutex_lock(&msi->bitmap_lock); + + msi_irq = bitmap_find_next_zero_area(msi->bitmap, NR_MSI_VEC, 0, + msi->num_cpus, 0); + if (msi_irq < NR_MSI_VEC) + bitmap_set(msi->bitmap, msi_irq, msi->num_cpus); + else + msi_irq = -ENOSPC; + + mutex_unlock(&msi->bitmap_lock); + + if (msi_irq < 0) + return msi_irq; + + irq_domain_set_info(domain, virq, msi_irq, + &xgene_msi_bottom_irq_chip, domain->host_data, + handle_simple_irq, NULL, NULL); + set_irq_flags(virq, IRQF_VALID); + + return 0; +} + +static void xgene_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct xgene_msi *msi = irq_data_get_irq_chip_data(d); + u32 hwirq; + + mutex_lock(&msi->bitmap_lock); + + hwirq = hwirq_to_canonical_hwirq(d->hwirq); + bitmap_clear(msi->bitmap, hwirq, msi->num_cpus); + + mutex_unlock(&msi->bitmap_lock); + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = xgene_irq_domain_alloc, + .free = xgene_irq_domain_free, +}; + +static int xgene_allocate_domains(struct xgene_msi *msi) +{ + msi->inner_domain = irq_domain_add_linear(NULL, NR_MSI_VEC, + &msi_domain_ops, msi); + if (!msi->inner_domain) + return -ENOMEM; + + msi->msi_domain = pci_msi_create_irq_domain(msi->node, + &xgene_msi_domain_info, + msi->inner_domain); + + if (!msi->msi_domain) { + irq_domain_remove(msi->inner_domain); + return -ENOMEM; + } + + return 0; +} + +static void xgene_free_domains(struct xgene_msi *msi) +{ + if (msi->msi_domain) + irq_domain_remove(msi->msi_domain); + if (msi->inner_domain) + irq_domain_remove(msi->inner_domain); +} + +static int xgene_msi_init_allocator(struct xgene_msi *xgene_msi) +{ + int size = BITS_TO_LONGS(NR_MSI_VEC) * sizeof(long); + + xgene_msi->bitmap = kzalloc(size, GFP_KERNEL); + if (!xgene_msi->bitmap) + return -ENOMEM; + + mutex_init(&xgene_msi->bitmap_lock); + + xgene_msi->msi_groups = kcalloc(NR_HW_IRQS, + sizeof(struct xgene_msi_group), + GFP_KERNEL); + if (!xgene_msi->msi_groups) + return -ENOMEM; + + return 0; +} + +static void xgene_msi_isr(unsigned int irq, struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct xgene_msi_group *msi_groups; + struct xgene_msi *xgene_msi; + unsigned int virq; + int msir_index, msir_val, hw_irq; + u32 intr_index, grp_select, msi_grp; + + chained_irq_enter(chip, desc); + + msi_groups = irq_desc_get_handler_data(desc); + xgene_msi = msi_groups->msi; + msi_grp = msi_groups->msi_grp; + + /* + * MSIINTn (n is 0..F) indicates if there is a pending MSI interrupt + * If bit x of this register is set (x is 0..7), one or more interupts + * corresponding to MSInIRx is set. + */ + grp_select = xgene_msi_int_read(xgene_msi, msi_grp); + while (grp_select) { + msir_index = ffs(grp_select) - 1; + /* + * Calculate MSInIRx address to read to check for interrupts + * (refer to termination address and data assignment + * described in xgene_compose_msi_msg() ) + */ + msir_val = xgene_msi_ir_read(xgene_msi, msi_grp, msir_index); + while (msir_val) { + intr_index = ffs(msir_val) - 1; + /* + * Calculate MSI vector number (refer to the termination + * address and data assignment described in + * xgene_compose_msi_msg function) + */ + hw_irq = (((msir_index * IRQS_PER_IDX) + intr_index) * + NR_HW_IRQS) + msi_grp; + /* + * As we have multiple hw_irq that maps to single MSI, + * always look up the virq using the hw_irq as seen from + * CPU0 + */ + hw_irq = hwirq_to_canonical_hwirq(hw_irq); + virq = irq_find_mapping(xgene_msi->inner_domain, hw_irq); + WARN_ON(!virq); + if (virq != 0) + generic_handle_irq(virq); + msir_val &= ~(1 << intr_index); + } + grp_select &= ~(1 << msir_index); + + if (!grp_select) { + /* + * We handled all interrupts happened in this group, + * resample this group MSI_INTx register in case + * something else has been made pending in the meantime + */ + grp_select = xgene_msi_int_read(xgene_msi, msi_grp); + } + } + + chained_irq_exit(chip, desc); +} + +static int xgene_msi_remove(struct platform_device *pdev) +{ + int virq, i; + struct xgene_msi *msi = platform_get_drvdata(pdev); + + for (i = 0; i < NR_HW_IRQS; i++) { + virq = msi->msi_groups[i].gic_irq; + if (virq != 0) { + irq_set_chained_handler(virq, NULL); + irq_set_handler_data(virq, NULL); + } + } + kfree(msi->msi_groups); + + kfree(msi->bitmap); + msi->bitmap = NULL; + + xgene_free_domains(msi); + + return 0; +} + +static int xgene_msi_hwirq_alloc(unsigned int cpu) +{ + struct xgene_msi *msi = &xgene_msi_ctrl; + struct xgene_msi_group *msi_group; + cpumask_var_t mask; + int i; + int err; + + for (i = cpu; i < NR_HW_IRQS; i += msi->num_cpus) { + msi_group = &msi->msi_groups[i]; + if (!msi_group->gic_irq) + continue; + + irq_set_chained_handler(msi_group->gic_irq, + xgene_msi_isr); + err = irq_set_handler_data(msi_group->gic_irq, msi_group); + if (err) { + pr_err("failed to register GIC IRQ handler\n"); + return -EINVAL; + } + /* + * Statically allocate MSI GIC IRQs to each CPU core. + * With 8-core X-Gene v1, 2 MSI GIC IRQs are allocated + * to each core. + */ + if (alloc_cpumask_var(&mask, GFP_KERNEL)) { + cpumask_clear(mask); + cpumask_set_cpu(cpu, mask); + err = irq_set_affinity(msi_group->gic_irq, mask); + if (err) + pr_err("failed to set affinity for GIC IRQ"); + free_cpumask_var(mask); + } else { + pr_err("failed to alloc CPU mask for affinity\n"); + err = -EINVAL; + } + + if (err) { + irq_set_chained_handler(msi_group->gic_irq, NULL); + irq_set_handler_data(msi_group->gic_irq, NULL); + return err; + } + } + + return 0; +} + +static void xgene_msi_hwirq_free(unsigned int cpu) +{ + struct xgene_msi *msi = &xgene_msi_ctrl; + struct xgene_msi_group *msi_group; + int i; + + for (i = cpu; i < NR_HW_IRQS; i += msi->num_cpus) { + msi_group = &msi->msi_groups[i]; + if (!msi_group->gic_irq) + continue; + + irq_set_chained_handler(msi_group->gic_irq, NULL); + irq_set_handler_data(msi_group->gic_irq, NULL); + } +} + +static int xgene_msi_cpu_callback(struct notifier_block *nfb, + unsigned long action, void *hcpu) +{ + unsigned cpu = (unsigned long)hcpu; + + switch (action) { + case CPU_ONLINE: + case CPU_ONLINE_FROZEN: + xgene_msi_hwirq_alloc(cpu); + break; + case CPU_DEAD: + case CPU_DEAD_FROZEN: + xgene_msi_hwirq_free(cpu); + break; + default: + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block xgene_msi_cpu_notifier = { + .notifier_call = xgene_msi_cpu_callback, +}; + +static const struct of_device_id xgene_msi_match_table[] = { + {.compatible = "apm,xgene1-msi"}, + {}, +}; + +static int xgene_msi_probe(struct platform_device *pdev) +{ + struct resource *res; + int rc, irq_index; + struct xgene_msi *xgene_msi; + unsigned int cpu; + int virt_msir; + u32 msi_val, msi_idx; + + xgene_msi = &xgene_msi_ctrl; + + platform_set_drvdata(pdev, xgene_msi); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + xgene_msi->msi_regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(xgene_msi->msi_regs)) { + dev_err(&pdev->dev, "no reg space\n"); + rc = -EINVAL; + goto error; + } + xgene_msi->msi_addr = res->start; + xgene_msi->node = pdev->dev.of_node; + xgene_msi->num_cpus = num_possible_cpus(); + + rc = xgene_msi_init_allocator(xgene_msi); + if (rc) { + dev_err(&pdev->dev, "Error allocating MSI bitmap\n"); + goto error; + } + + rc = xgene_allocate_domains(xgene_msi); + if (rc) { + dev_err(&pdev->dev, "Failed to allocate MSI domain\n"); + goto error; + } + + for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) { + virt_msir = platform_get_irq(pdev, irq_index); + if (virt_msir < 0) { + dev_err(&pdev->dev, "Cannot translate IRQ index %d\n", + irq_index); + rc = -EINVAL; + goto error; + } + xgene_msi->msi_groups[irq_index].gic_irq = virt_msir; + xgene_msi->msi_groups[irq_index].msi_grp = irq_index; + xgene_msi->msi_groups[irq_index].msi = xgene_msi; + } + + /* + * MSInIRx registers are read-to-clear; before registering + * interrupt handlers, read all of them to clear spurious + * interrupts that may occur before the driver is probed. + */ + for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) { + for (msi_idx = 0; msi_idx < IDX_PER_GROUP; msi_idx++) + msi_val = xgene_msi_ir_read(xgene_msi, irq_index, + msi_idx); + /* Read MSIINTn to confirm */ + msi_val = xgene_msi_int_read(xgene_msi, irq_index); + if (msi_val) { + dev_err(&pdev->dev, "Failed to clear spurious IRQ\n"); + rc = -EINVAL; + goto error; + } + } + + cpu_notifier_register_begin(); + + for_each_online_cpu(cpu) + if (xgene_msi_hwirq_alloc(cpu)) { + dev_err(&pdev->dev, "failed to register MSI handlers\n"); + cpu_notifier_register_done(); + goto error; + } + + rc = __register_hotcpu_notifier(&xgene_msi_cpu_notifier); + if (rc) { + dev_err(&pdev->dev, "failed to add CPU MSI notifier\n"); + cpu_notifier_register_done(); + goto error; + } + + cpu_notifier_register_done(); + + dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n"); + + return 0; + +error: + xgene_msi_remove(pdev); + return rc; +} + +static struct platform_driver xgene_msi_driver = { + .driver = { + .name = "xgene-msi", + .owner = THIS_MODULE, + .of_match_table = xgene_msi_match_table, + }, + .probe = xgene_msi_probe, + .remove = xgene_msi_remove, +}; + +static int __init xgene_pcie_msi_init(void) +{ + return platform_driver_register(&xgene_msi_driver); +} +subsys_initcall(xgene_pcie_msi_init); diff --git a/drivers/pci/host/pci-xgene.c b/drivers/pci/host/pci-xgene.c index 2988fe136c1e..9d2fd461e9c7 100644 --- a/drivers/pci/host/pci-xgene.c +++ b/drivers/pci/host/pci-xgene.c @@ -600,6 +600,23 @@ static int xgene_pcie_setup(struct xgene_pcie_port *port, return 0; } +static int xgene_pcie_msi_enable(struct pci_bus *bus) +{ + struct device_node *msi_node; + + msi_node = of_parse_phandle(bus->dev.of_node, + "msi-parent", 0); + if (!msi_node) + return -ENODEV; + + bus->msi = of_pci_find_msi_chip_by_node(msi_node); + if (!bus->msi) + return -ENODEV; + + bus->msi->dev = &bus->dev; + return 0; +} + static int xgene_pcie_probe_bridge(struct platform_device *pdev) { struct device_node *dn = pdev->dev.of_node; @@ -636,6 +653,10 @@ static int xgene_pcie_probe_bridge(struct platform_device *pdev) if (!bus) return -ENOMEM; + if (IS_ENABLED(CONFIG_PCI_MSI)) + if (xgene_pcie_msi_enable(bus)) + dev_info(port->dev, "failed to enable MSI\n"); + pci_scan_child_bus(bus); pci_assign_unassigned_bus_resources(bus); pci_bus_add_devices(bus); @@ -652,7 +673,6 @@ static const struct of_device_id xgene_pcie_match_table[] = { static struct platform_driver xgene_pcie_driver = { .driver = { .name = "xgene-pcie", - .owner = THIS_MODULE, .of_match_table = of_match_ptr(xgene_pcie_match_table), }, .probe = xgene_pcie_probe_bridge, diff --git a/drivers/pci/host/pcie-designware.c b/drivers/pci/host/pcie-designware.c index f69b0d0a5ee1..a36a157dd9c3 100644 --- a/drivers/pci/host/pcie-designware.c +++ b/drivers/pci/host/pcie-designware.c @@ -152,10 +152,10 @@ static int dw_pcie_wr_own_conf(struct pcie_port *pp, int where, int size, static struct irq_chip dw_msi_irq_chip = { .name = "PCI-MSI", - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, }; /* MSI int handler */ @@ -238,7 +238,7 @@ static void dw_pcie_msi_set_irq(struct pcie_port *pp, int irq) static int assign_irq(int no_irqs, struct msi_desc *desc, int *pos) { int irq, pos0, i; - struct pcie_port *pp = sys_to_pcie(desc->dev->bus->sysdata); + struct pcie_port *pp = sys_to_pcie(msi_desc_to_pci_sysdata(desc)); pos0 = bitmap_find_free_region(pp->msi_irq_in_use, MAX_MSI_IRQS, order_base_2(no_irqs)); @@ -276,7 +276,7 @@ no_valid_irq: return -ENOSPC; } -static int dw_msi_setup_irq(struct msi_chip *chip, struct pci_dev *pdev, +static int dw_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev, struct msi_desc *desc) { int irq, pos; @@ -301,21 +301,21 @@ static int dw_msi_setup_irq(struct msi_chip *chip, struct pci_dev *pdev, else msg.data = pos; - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); return 0; } -static void dw_msi_teardown_irq(struct msi_chip *chip, unsigned int irq) +static void dw_msi_teardown_irq(struct msi_controller *chip, unsigned int irq) { struct irq_data *data = irq_get_irq_data(irq); - struct msi_desc *msi = irq_data_get_msi(data); - struct pcie_port *pp = sys_to_pcie(msi->dev->bus->sysdata); + struct msi_desc *msi = irq_data_get_msi_desc(data); + struct pcie_port *pp = sys_to_pcie(msi_desc_to_pci_sysdata(msi)); clear_irq_range(pp, irq, 1, data->hwirq); } -static struct msi_chip dw_pcie_msi_chip = { +static struct msi_controller dw_pcie_msi_chip = { .setup_irq = dw_msi_setup_irq, .teardown_irq = dw_msi_teardown_irq, }; @@ -501,6 +501,11 @@ int dw_pcie_host_init(struct pcie_port *pp) val |= PORT_LOGIC_SPEED_CHANGE; dw_pcie_wr_own_conf(pp, PCIE_LINK_WIDTH_SPEED_CONTROL, 4, val); +#ifdef CONFIG_PCI_MSI + dw_pcie_msi_chip.dev = pp->dev; + dw_pci.msi_ctrl = &dw_pcie_msi_chip; +#endif + dw_pci.nr_controllers = 1; dw_pci.private_data = (void **)&pp; @@ -750,21 +755,10 @@ static int dw_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return irq; } -static void dw_pcie_add_bus(struct pci_bus *bus) -{ - if (IS_ENABLED(CONFIG_PCI_MSI)) { - struct pcie_port *pp = sys_to_pcie(bus->sysdata); - - dw_pcie_msi_chip.dev = pp->dev; - bus->msi = &dw_pcie_msi_chip; - } -} - static struct hw_pci dw_pci = { .setup = dw_pcie_setup, .scan = dw_pcie_scan_bus, .map_irq = dw_pcie_map_irq, - .add_bus = dw_pcie_add_bus, }; void dw_pcie_setup_rc(struct pcie_port *pp) diff --git a/drivers/pci/host/pcie-designware.h b/drivers/pci/host/pcie-designware.h index c6256751daff..d0bbd276840d 100644 --- a/drivers/pci/host/pcie-designware.h +++ b/drivers/pci/host/pcie-designware.h @@ -73,7 +73,7 @@ struct pcie_host_ops { u32 (*get_msi_addr)(struct pcie_port *pp); u32 (*get_msi_data)(struct pcie_port *pp, int pos); void (*scan_bus)(struct pcie_port *pp); - int (*msi_host_init)(struct pcie_port *pp, struct msi_chip *chip); + int (*msi_host_init)(struct pcie_port *pp, struct msi_controller *chip); }; int dw_pcie_cfg_read(void __iomem *addr, int where, int size, u32 *val); diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c index bfab844f772d..28b387e6fc03 100644 --- a/drivers/pci/host/pcie-rcar.c +++ b/drivers/pci/host/pcie-rcar.c @@ -111,14 +111,14 @@ struct rcar_msi { DECLARE_BITMAP(used, INT_PCI_MSI_NR); struct irq_domain *domain; - struct msi_chip chip; + struct msi_controller chip; unsigned long pages; struct mutex lock; int irq1; int irq2; }; -static inline struct rcar_msi *to_rcar_msi(struct msi_chip *chip) +static inline struct rcar_msi *to_rcar_msi(struct msi_controller *chip) { return container_of(chip, struct rcar_msi, chip); } @@ -380,20 +380,10 @@ static int rcar_pcie_setup(int nr, struct pci_sys_data *sys) return 1; } -static void rcar_pcie_add_bus(struct pci_bus *bus) -{ - if (IS_ENABLED(CONFIG_PCI_MSI)) { - struct rcar_pcie *pcie = sys_to_pcie(bus->sysdata); - - bus->msi = &pcie->msi.chip; - } -} - struct hw_pci rcar_pci = { .setup = rcar_pcie_setup, .map_irq = of_irq_parse_and_map_pci, .ops = &rcar_pcie_ops, - .add_bus = rcar_pcie_add_bus, }; static void rcar_pcie_enable(struct rcar_pcie *pcie) @@ -402,6 +392,9 @@ static void rcar_pcie_enable(struct rcar_pcie *pcie) rcar_pci.nr_controllers = 1; rcar_pci.private_data = (void **)&pcie; +#ifdef CONFIG_PCI_MSI + rcar_pci.msi_ctrl = &pcie->msi.chip; +#endif pci_common_init_dev(&pdev->dev, &rcar_pci); #ifdef CONFIG_PCI_DOMAINS @@ -622,7 +615,7 @@ static irqreturn_t rcar_pcie_msi_irq(int irq, void *data) return IRQ_HANDLED; } -static int rcar_msi_setup_irq(struct msi_chip *chip, struct pci_dev *pdev, +static int rcar_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev, struct msi_desc *desc) { struct rcar_msi *msi = to_rcar_msi(chip); @@ -647,12 +640,12 @@ static int rcar_msi_setup_irq(struct msi_chip *chip, struct pci_dev *pdev, msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR); msg.data = hwirq; - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); return 0; } -static void rcar_msi_teardown_irq(struct msi_chip *chip, unsigned int irq) +static void rcar_msi_teardown_irq(struct msi_controller *chip, unsigned int irq) { struct rcar_msi *msi = to_rcar_msi(chip); struct irq_data *d = irq_get_irq_data(irq); @@ -662,10 +655,10 @@ static void rcar_msi_teardown_irq(struct msi_chip *chip, unsigned int irq) static struct irq_chip rcar_msi_irq_chip = { .name = "R-Car PCIe MSI", - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, }; static int rcar_msi_map(struct irq_domain *domain, unsigned int irq, @@ -990,7 +983,6 @@ static int rcar_pcie_probe(struct platform_device *pdev) static struct platform_driver rcar_pcie_driver = { .driver = { .name = DRV_NAME, - .owner = THIS_MODULE, .of_match_table = rcar_pcie_of_match, .suppress_bind_attrs = true, }, diff --git a/drivers/pci/host/pcie-spear13xx.c b/drivers/pci/host/pcie-spear13xx.c index 925cfbad5be6..109dfadd314c 100644 --- a/drivers/pci/host/pcie-spear13xx.c +++ b/drivers/pci/host/pcie-spear13xx.c @@ -376,7 +376,6 @@ static struct platform_driver spear13xx_pcie_driver = { .probe = spear13xx_pcie_probe, .driver = { .name = "spear-pcie", - .owner = THIS_MODULE, .of_match_table = of_match_ptr(spear13xx_pcie_of_match), }, }; diff --git a/drivers/pci/host/pcie-xilinx.c b/drivers/pci/host/pcie-xilinx.c index 514d16b22619..c177977ef957 100644 --- a/drivers/pci/host/pcie-xilinx.c +++ b/drivers/pci/host/pcie-xilinx.c @@ -297,18 +297,16 @@ static struct pci_ops xilinx_pcie_ops = { */ static void xilinx_pcie_destroy_msi(unsigned int irq) { - struct irq_desc *desc; struct msi_desc *msi; struct xilinx_pcie_port *port; - desc = irq_to_desc(irq); - msi = irq_desc_get_msi_desc(desc); - port = sys_to_pcie(msi->dev->bus->sysdata); - - if (!test_bit(irq, msi_irq_in_use)) + if (!test_bit(irq, msi_irq_in_use)) { + msi = irq_get_msi_desc(irq); + port = sys_to_pcie(msi_desc_to_pci_sys_data(msi)); dev_err(port->dev, "Trying to free unused MSI#%d\n", irq); - else + } else { clear_bit(irq, msi_irq_in_use); + } } /** @@ -335,7 +333,8 @@ static int xilinx_pcie_assign_msi(struct xilinx_pcie_port *port) * @chip: MSI Chip descriptor * @irq: MSI IRQ to destroy */ -static void xilinx_msi_teardown_irq(struct msi_chip *chip, unsigned int irq) +static void xilinx_msi_teardown_irq(struct msi_controller *chip, + unsigned int irq) { xilinx_pcie_destroy_msi(irq); } @@ -348,7 +347,7 @@ static void xilinx_msi_teardown_irq(struct msi_chip *chip, unsigned int irq) * * Return: '0' on success and error value on failure */ -static int xilinx_pcie_msi_setup_irq(struct msi_chip *chip, +static int xilinx_pcie_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev, struct msi_desc *desc) { @@ -374,13 +373,13 @@ static int xilinx_pcie_msi_setup_irq(struct msi_chip *chip, msg.address_lo = msg_addr; msg.data = irq; - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); return 0; } /* MSI Chip Descriptor */ -static struct msi_chip xilinx_pcie_msi_chip = { +static struct msi_controller xilinx_pcie_msi_chip = { .setup_irq = xilinx_pcie_msi_setup_irq, .teardown_irq = xilinx_msi_teardown_irq, }; @@ -388,10 +387,10 @@ static struct msi_chip xilinx_pcie_msi_chip = { /* HW Interrupt Chip Descriptor */ static struct irq_chip xilinx_msi_irq_chip = { .name = "Xilinx PCIe MSI", - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, }; /** @@ -431,20 +430,6 @@ static void xilinx_pcie_enable_msi(struct xilinx_pcie_port *port) pcie_write(port, msg_addr, XILINX_PCIE_REG_MSIBASE2); } -/** - * xilinx_pcie_add_bus - Add MSI chip info to PCIe bus - * @bus: PCIe bus - */ -static void xilinx_pcie_add_bus(struct pci_bus *bus) -{ - if (IS_ENABLED(CONFIG_PCI_MSI)) { - struct xilinx_pcie_port *port = sys_to_pcie(bus->sysdata); - - xilinx_pcie_msi_chip.dev = port->dev; - bus->msi = &xilinx_pcie_msi_chip; - } -} - /* INTx Functions */ /** @@ -925,10 +910,14 @@ static int xilinx_pcie_probe(struct platform_device *pdev) .private_data = (void **)&port, .setup = xilinx_pcie_setup, .map_irq = of_irq_parse_and_map_pci, - .add_bus = xilinx_pcie_add_bus, .scan = xilinx_pcie_scan_bus, .ops = &xilinx_pcie_ops, }; + +#ifdef CONFIG_PCI_MSI + xilinx_pcie_msi_chip.dev = port->dev; + hw.msi_ctrl = &xilinx_pcie_msi_chip; +#endif pci_common_init_dev(dev, &hw); return 0; @@ -957,7 +946,6 @@ static struct of_device_id xilinx_pcie_of_match[] = { static struct platform_driver xilinx_pcie_driver = { .driver = { .name = "xilinx-pcie", - .owner = THIS_MODULE, .of_match_table = xilinx_pcie_of_match, .suppress_bind_attrs = true, }, diff --git a/drivers/pci/hotplug/ibmphp_core.c b/drivers/pci/hotplug/ibmphp_core.c index 3efaf4c38528..1d5b646e3db8 100644 --- a/drivers/pci/hotplug/ibmphp_core.c +++ b/drivers/pci/hotplug/ibmphp_core.c @@ -740,7 +740,7 @@ static void ibm_unconfigure_device(struct pci_func *func) */ static u8 bus_structure_fixup(u8 busno) { - struct pci_bus *bus; + struct pci_bus *bus, *b; struct pci_dev *dev; u16 l; @@ -767,7 +767,11 @@ static u8 bus_structure_fixup(u8 busno) (l != 0x0000) && (l != 0xffff)) { debug("%s - Inside bus_structure_fixup()\n", __func__); - pci_scan_bus(busno, ibmphp_pci_bus->ops, NULL); + b = pci_scan_bus(busno, ibmphp_pci_bus->ops, NULL); + if (!b) + continue; + + pci_bus_add_devices(b); break; } } diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index 084587d7cd13..1c7f6a8e69d1 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -19,19 +19,81 @@ #include <linux/errno.h> #include <linux/io.h> #include <linux/slab.h> +#include <linux/irqdomain.h> #include "pci.h" static int pci_msi_enable = 1; +int pci_msi_ignore_mask; #define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) +#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN +static struct irq_domain *pci_msi_default_domain; +static DEFINE_MUTEX(pci_msi_domain_lock); + +struct irq_domain * __weak arch_get_pci_msi_domain(struct pci_dev *dev) +{ + return pci_msi_default_domain; +} + +static struct irq_domain *pci_msi_get_domain(struct pci_dev *dev) +{ + struct irq_domain *domain; + + domain = dev_get_msi_domain(&dev->dev); + if (domain) + return domain; + + return arch_get_pci_msi_domain(dev); +} + +static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + struct irq_domain *domain; + + domain = pci_msi_get_domain(dev); + if (domain) + return pci_msi_domain_alloc_irqs(domain, dev, nvec, type); + + return arch_setup_msi_irqs(dev, nvec, type); +} + +static void pci_msi_teardown_msi_irqs(struct pci_dev *dev) +{ + struct irq_domain *domain; + + domain = pci_msi_get_domain(dev); + if (domain) + pci_msi_domain_free_irqs(domain, dev); + else + arch_teardown_msi_irqs(dev); +} +#else +#define pci_msi_setup_msi_irqs arch_setup_msi_irqs +#define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs +#endif /* Arch hooks */ +struct msi_controller * __weak pcibios_msi_controller(struct pci_dev *dev) +{ + return NULL; +} + +static struct msi_controller *pci_msi_controller(struct pci_dev *dev) +{ + struct msi_controller *msi_ctrl = dev->bus->msi; + + if (msi_ctrl) + return msi_ctrl; + + return pcibios_msi_controller(dev); +} + int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) { - struct msi_chip *chip = dev->bus->msi; + struct msi_controller *chip = pci_msi_controller(dev); int err; if (!chip || !chip->setup_irq) @@ -48,7 +110,7 @@ int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) void __weak arch_teardown_msi_irq(unsigned int irq) { - struct msi_chip *chip = irq_get_chip_data(irq); + struct msi_controller *chip = irq_get_chip_data(irq); if (!chip || !chip->teardown_irq) return; @@ -85,19 +147,13 @@ int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) */ void default_teardown_msi_irqs(struct pci_dev *dev) { + int i; struct msi_desc *entry; - list_for_each_entry(entry, &dev->msi_list, list) { - int i, nvec; - if (entry->irq == 0) - continue; - if (entry->nvec_used) - nvec = entry->nvec_used; - else - nvec = 1 << entry->msi_attrib.multiple; - for (i = 0; i < nvec; i++) - arch_teardown_msi_irq(entry->irq + i); - } + list_for_each_entry(entry, &dev->msi_list, list) + if (entry->irq) + for (i = 0; i < entry->nvec_used; i++) + arch_teardown_msi_irq(entry->irq + i); } void __weak arch_teardown_msi_irqs(struct pci_dev *dev) @@ -120,7 +176,7 @@ static void default_restore_msi_irq(struct pci_dev *dev, int irq) } if (entry) - __write_msi_msg(entry, &entry->msg); + __pci_write_msi_msg(entry, &entry->msg); } void __weak arch_restore_msi_irqs(struct pci_dev *dev) @@ -128,27 +184,6 @@ void __weak arch_restore_msi_irqs(struct pci_dev *dev) return default_restore_msi_irqs(dev); } -static void msi_set_enable(struct pci_dev *dev, int enable) -{ - u16 control; - - pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); - control &= ~PCI_MSI_FLAGS_ENABLE; - if (enable) - control |= PCI_MSI_FLAGS_ENABLE; - pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); -} - -static void msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set) -{ - u16 ctrl; - - pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl); - ctrl &= ~clear; - ctrl |= set; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl); -} - static inline __attribute_const__ u32 msi_mask(unsigned x) { /* Don't shift by >= width of type */ @@ -163,28 +198,24 @@ static inline __attribute_const__ u32 msi_mask(unsigned x) * reliably as devices without an INTx disable bit will then generate a * level IRQ which will never be cleared. */ -u32 default_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) +u32 __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) { u32 mask_bits = desc->masked; - if (!desc->msi_attrib.maskbit) + if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit) return 0; mask_bits &= ~mask; mask_bits |= flag; - pci_write_config_dword(desc->dev, desc->mask_pos, mask_bits); + pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos, + mask_bits); return mask_bits; } -__weak u32 arch_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) -{ - return default_msi_mask_irq(desc, mask, flag); -} - static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) { - desc->masked = arch_msi_mask_irq(desc, mask, flag); + desc->masked = __pci_msi_desc_mask_irq(desc, mask, flag); } /* @@ -194,11 +225,15 @@ static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) * file. This saves a few milliseconds when initialising devices with lots * of MSI-X interrupts. */ -u32 default_msix_mask_irq(struct msi_desc *desc, u32 flag) +u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag) { u32 mask_bits = desc->masked; unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL; + + if (pci_msi_ignore_mask) + return 0; + mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; if (flag) mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT; @@ -207,19 +242,14 @@ u32 default_msix_mask_irq(struct msi_desc *desc, u32 flag) return mask_bits; } -__weak u32 arch_msix_mask_irq(struct msi_desc *desc, u32 flag) -{ - return default_msix_mask_irq(desc, flag); -} - static void msix_mask_irq(struct msi_desc *desc, u32 flag) { - desc->masked = arch_msix_mask_irq(desc, flag); + desc->masked = __pci_msix_desc_mask_irq(desc, flag); } static void msi_set_mask_bit(struct irq_data *data, u32 flag) { - struct msi_desc *desc = irq_data_get_msi(data); + struct msi_desc *desc = irq_data_get_msi_desc(data); if (desc->msi_attrib.is_msix) { msix_mask_irq(desc, flag); @@ -230,12 +260,20 @@ static void msi_set_mask_bit(struct irq_data *data, u32 flag) } } -void mask_msi_irq(struct irq_data *data) +/** + * pci_msi_mask_irq - Generic irq chip callback to mask PCI/MSI interrupts + * @data: pointer to irqdata associated to that interrupt + */ +void pci_msi_mask_irq(struct irq_data *data) { msi_set_mask_bit(data, 1); } -void unmask_msi_irq(struct irq_data *data) +/** + * pci_msi_unmask_irq - Generic irq chip callback to unmask PCI/MSI interrupts + * @data: pointer to irqdata associated to that interrupt + */ +void pci_msi_unmask_irq(struct irq_data *data) { msi_set_mask_bit(data, 0); } @@ -244,14 +282,15 @@ void default_restore_msi_irqs(struct pci_dev *dev) { struct msi_desc *entry; - list_for_each_entry(entry, &dev->msi_list, list) { + list_for_each_entry(entry, &dev->msi_list, list) default_restore_msi_irq(dev, entry->irq); - } } -void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) +void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { - BUG_ON(entry->dev->current_state != PCI_D0); + struct pci_dev *dev = msi_desc_to_pci_dev(entry); + + BUG_ON(dev->current_state != PCI_D0); if (entry->msi_attrib.is_msix) { void __iomem *base = entry->mask_base + @@ -261,7 +300,6 @@ void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); msg->data = readl(base + PCI_MSIX_ENTRY_DATA); } else { - struct pci_dev *dev = entry->dev; int pos = dev->msi_cap; u16 data; @@ -279,34 +317,11 @@ void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) } } -void read_msi_msg(unsigned int irq, struct msi_msg *msg) -{ - struct msi_desc *entry = irq_get_msi_desc(irq); - - __read_msi_msg(entry, msg); -} - -void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) +void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { - /* Assert that the cache is valid, assuming that - * valid messages are not all-zeroes. */ - BUG_ON(!(entry->msg.address_hi | entry->msg.address_lo | - entry->msg.data)); + struct pci_dev *dev = msi_desc_to_pci_dev(entry); - *msg = entry->msg; -} - -void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) -{ - struct msi_desc *entry = irq_get_msi_desc(irq); - - __get_cached_msi_msg(entry, msg); -} -EXPORT_SYMBOL_GPL(get_cached_msi_msg); - -void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) -{ - if (entry->dev->current_state != PCI_D0) { + if (dev->current_state != PCI_D0) { /* Don't touch the hardware now */ } else if (entry->msi_attrib.is_msix) { void __iomem *base; @@ -317,7 +332,6 @@ void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); writel(msg->data, base + PCI_MSIX_ENTRY_DATA); } else { - struct pci_dev *dev = entry->dev; int pos = dev->msi_cap; u16 msgctl; @@ -341,34 +355,27 @@ void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) entry->msg = *msg; } -void write_msi_msg(unsigned int irq, struct msi_msg *msg) +void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) { struct msi_desc *entry = irq_get_msi_desc(irq); - __write_msi_msg(entry, msg); + __pci_write_msi_msg(entry, msg); } -EXPORT_SYMBOL_GPL(write_msi_msg); +EXPORT_SYMBOL_GPL(pci_write_msi_msg); static void free_msi_irqs(struct pci_dev *dev) { struct msi_desc *entry, *tmp; struct attribute **msi_attrs; struct device_attribute *dev_attr; - int count = 0; + int i, count = 0; - list_for_each_entry(entry, &dev->msi_list, list) { - int i, nvec; - if (!entry->irq) - continue; - if (entry->nvec_used) - nvec = entry->nvec_used; - else - nvec = 1 << entry->msi_attrib.multiple; - for (i = 0; i < nvec; i++) - BUG_ON(irq_has_action(entry->irq + i)); - } + list_for_each_entry(entry, &dev->msi_list, list) + if (entry->irq) + for (i = 0; i < entry->nvec_used; i++) + BUG_ON(irq_has_action(entry->irq + i)); - arch_teardown_msi_irqs(dev); + pci_msi_teardown_msi_irqs(dev); list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) { if (entry->msi_attrib.is_msix) { @@ -426,7 +433,7 @@ static void __pci_restore_msi_state(struct pci_dev *dev) entry = irq_get_msi_desc(dev->irq); pci_intx_for_msi(dev, 0); - msi_set_enable(dev, 0); + pci_msi_set_enable(dev, 0); arch_restore_msi_irqs(dev); pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); @@ -447,15 +454,14 @@ static void __pci_restore_msix_state(struct pci_dev *dev) /* route the table */ pci_intx_for_msi(dev, 0); - msix_clear_and_set_ctrl(dev, 0, + pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); arch_restore_msi_irqs(dev); - list_for_each_entry(entry, &dev->msi_list, list) { + list_for_each_entry(entry, &dev->msi_list, list) msix_mask_irq(entry, entry->masked); - } - msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } void pci_restore_msi_state(struct pci_dev *dev) @@ -497,9 +503,8 @@ static int populate_msi_sysfs(struct pci_dev *pdev) int count = 0; /* Determine how many msi entries we have */ - list_for_each_entry(entry, &pdev->msi_list, list) { + list_for_each_entry(entry, &pdev->msi_list, list) ++num_msi; - } if (!num_msi) return 0; @@ -559,7 +564,7 @@ error_attrs: return ret; } -static struct msi_desc *msi_setup_entry(struct pci_dev *dev) +static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec) { u16 control; struct msi_desc *entry; @@ -577,6 +582,8 @@ static struct msi_desc *msi_setup_entry(struct pci_dev *dev) entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT); entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; + entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); + entry->nvec_used = nvec; if (control & PCI_MSI_FLAGS_64BIT) entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64; @@ -621,9 +628,9 @@ static int msi_capability_init(struct pci_dev *dev, int nvec) int ret; unsigned mask; - msi_set_enable(dev, 0); /* Disable MSI during set up */ + pci_msi_set_enable(dev, 0); /* Disable MSI during set up */ - entry = msi_setup_entry(dev); + entry = msi_setup_entry(dev, nvec); if (!entry) return -ENOMEM; @@ -634,7 +641,7 @@ static int msi_capability_init(struct pci_dev *dev, int nvec) list_add_tail(&entry->list, &dev->msi_list); /* Configure MSI capability structure */ - ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); + ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); if (ret) { msi_mask_irq(entry, mask, ~mask); free_msi_irqs(dev); @@ -657,7 +664,7 @@ static int msi_capability_init(struct pci_dev *dev, int nvec) /* Set MSI enabled bits */ pci_intx_for_msi(dev, 0); - msi_set_enable(dev, 1); + pci_msi_set_enable(dev, 1); dev->msi_enabled = 1; dev->irq = entry->irq; @@ -701,6 +708,7 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, entry->msi_attrib.entry_nr = entries[i].entry; entry->msi_attrib.default_irq = dev->irq; entry->mask_base = base; + entry->nvec_used = 1; list_add_tail(&entry->list, &dev->msi_list); } @@ -719,7 +727,6 @@ static void msix_program_entries(struct pci_dev *dev, PCI_MSIX_ENTRY_VECTOR_CTRL; entries[i].vector = entry->irq; - irq_set_msi_desc(entry->irq, entry); entry->masked = readl(entry->mask_base + offset); msix_mask_irq(entry, 1); i++; @@ -744,7 +751,7 @@ static int msix_capability_init(struct pci_dev *dev, void __iomem *base; /* Ensure MSI-X is disabled while it is set up */ - msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); /* Request & Map MSI-X table region */ @@ -756,7 +763,7 @@ static int msix_capability_init(struct pci_dev *dev, if (ret) return ret; - ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); + ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); if (ret) goto out_avail; @@ -770,7 +777,7 @@ static int msix_capability_init(struct pci_dev *dev, * MSI-X registers. We need to mask all the vectors to prevent * interrupts coming in before they're fully set up. */ - msix_clear_and_set_ctrl(dev, 0, + pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE); msix_program_entries(dev, entries); @@ -783,7 +790,7 @@ static int msix_capability_init(struct pci_dev *dev, pci_intx_for_msi(dev, 0); dev->msix_enabled = 1; - msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); return 0; @@ -888,14 +895,14 @@ void pci_msi_shutdown(struct pci_dev *dev) BUG_ON(list_empty(&dev->msi_list)); desc = list_first_entry(&dev->msi_list, struct msi_desc, list); - msi_set_enable(dev, 0); + pci_msi_set_enable(dev, 0); pci_intx_for_msi(dev, 1); dev->msi_enabled = 0; /* Return the device with MSI unmasked as initial states */ mask = msi_mask(desc->msi_attrib.multi_cap); /* Keep cached state to be restored */ - arch_msi_mask_irq(desc, mask, ~mask); + __pci_msi_desc_mask_irq(desc, mask, ~mask); /* Restore dev->irq to its default pin-assertion irq */ dev->irq = desc->msi_attrib.default_irq; @@ -993,10 +1000,10 @@ void pci_msix_shutdown(struct pci_dev *dev) /* Return the device with MSI-X masked as initial states */ list_for_each_entry(entry, &dev->msi_list, list) { /* Keep cached states to be restored */ - arch_msix_mask_irq(entry, 1); + __pci_msix_desc_mask_irq(entry, 1); } - msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); pci_intx_for_msi(dev, 1); dev->msix_enabled = 0; } @@ -1031,18 +1038,6 @@ EXPORT_SYMBOL(pci_msi_enabled); void pci_msi_init_pci_dev(struct pci_dev *dev) { INIT_LIST_HEAD(&dev->msi_list); - - /* Disable the msi hardware to avoid screaming interrupts - * during boot. This is the power on reset default so - * usually this should be a noop. - */ - dev->msi_cap = pci_find_capability(dev, PCI_CAP_ID_MSI); - if (dev->msi_cap) - msi_set_enable(dev, 0); - - dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); - if (dev->msix_cap) - msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); } /** @@ -1138,3 +1133,212 @@ int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, return nvec; } EXPORT_SYMBOL(pci_enable_msix_range); + +void *msi_desc_to_pci_sysdata(struct msi_desc *desc) +{ + struct pci_dev *dev = msi_desc_to_pci_dev(desc); + + return dev->bus->sysdata; +} +EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata); + +#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN +/** + * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space + * @irq_data: Pointer to interrupt data of the MSI interrupt + * @msg: Pointer to the message + */ +void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) +{ + struct msi_desc *desc = irq_data->msi_desc; + + /* + * For MSI-X desc->irq is always equal to irq_data->irq. For + * MSI only the first interrupt of MULTI MSI passes the test. + */ + if (desc->irq == irq_data->irq) + __pci_write_msi_msg(desc, msg); +} + +/** + * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source + * @dev: Pointer to the PCI device + * @desc: Pointer to the msi descriptor + * + * The ID number is only used within the irqdomain. + */ +irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev, + struct msi_desc *desc) +{ + return (irq_hw_number_t)desc->msi_attrib.entry_nr | + PCI_DEVID(dev->bus->number, dev->devfn) << 11 | + (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27; +} + +static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc) +{ + return !desc->msi_attrib.is_msix && desc->nvec_used > 1; +} + +/** + * pci_msi_domain_check_cap - Verify that @domain supports the capabilities for @dev + * @domain: The interrupt domain to check + * @info: The domain info for verification + * @dev: The device to check + * + * Returns: + * 0 if the functionality is supported + * 1 if Multi MSI is requested, but the domain does not support it + * -ENOTSUPP otherwise + */ +int pci_msi_domain_check_cap(struct irq_domain *domain, + struct msi_domain_info *info, struct device *dev) +{ + struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev)); + + /* Special handling to support pci_enable_msi_range() */ + if (pci_msi_desc_is_multi_msi(desc) && + !(info->flags & MSI_FLAG_MULTI_PCI_MSI)) + return 1; + else if (desc->msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX)) + return -ENOTSUPP; + + return 0; +} + +static int pci_msi_domain_handle_error(struct irq_domain *domain, + struct msi_desc *desc, int error) +{ + /* Special handling to support pci_enable_msi_range() */ + if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC) + return 1; + + return error; +} + +#ifdef GENERIC_MSI_DOMAIN_OPS +static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, + struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc), + desc); +} +#else +#define pci_msi_domain_set_desc NULL +#endif + +static struct msi_domain_ops pci_msi_domain_ops_default = { + .set_desc = pci_msi_domain_set_desc, + .msi_check = pci_msi_domain_check_cap, + .handle_error = pci_msi_domain_handle_error, +}; + +static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) +{ + struct msi_domain_ops *ops = info->ops; + + if (ops == NULL) { + info->ops = &pci_msi_domain_ops_default; + } else { + if (ops->set_desc == NULL) + ops->set_desc = pci_msi_domain_set_desc; + if (ops->msi_check == NULL) + ops->msi_check = pci_msi_domain_check_cap; + if (ops->handle_error == NULL) + ops->handle_error = pci_msi_domain_handle_error; + } +} + +static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) +{ + struct irq_chip *chip = info->chip; + + BUG_ON(!chip); + if (!chip->irq_write_msi_msg) + chip->irq_write_msi_msg = pci_msi_domain_write_msg; +} + +/** + * pci_msi_create_irq_domain - Creat a MSI interrupt domain + * @node: Optional device-tree node of the interrupt controller + * @info: MSI domain info + * @parent: Parent irq domain + * + * Updates the domain and chip ops and creates a MSI interrupt domain. + * + * Returns: + * A domain pointer or NULL in case of failure. + */ +struct irq_domain *pci_msi_create_irq_domain(struct device_node *node, + struct msi_domain_info *info, + struct irq_domain *parent) +{ + struct irq_domain *domain; + + if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) + pci_msi_domain_update_dom_ops(info); + if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) + pci_msi_domain_update_chip_ops(info); + + domain = msi_create_irq_domain(node, info, parent); + if (!domain) + return NULL; + + domain->bus_token = DOMAIN_BUS_PCI_MSI; + return domain; +} + +/** + * pci_msi_domain_alloc_irqs - Allocate interrupts for @dev in @domain + * @domain: The interrupt domain to allocate from + * @dev: The device for which to allocate + * @nvec: The number of interrupts to allocate + * @type: Unused to allow simpler migration from the arch_XXX interfaces + * + * Returns: + * A virtual interrupt number or an error code in case of failure + */ +int pci_msi_domain_alloc_irqs(struct irq_domain *domain, struct pci_dev *dev, + int nvec, int type) +{ + return msi_domain_alloc_irqs(domain, &dev->dev, nvec); +} + +/** + * pci_msi_domain_free_irqs - Free interrupts for @dev in @domain + * @domain: The interrupt domain + * @dev: The device for which to free interrupts + */ +void pci_msi_domain_free_irqs(struct irq_domain *domain, struct pci_dev *dev) +{ + msi_domain_free_irqs(domain, &dev->dev); +} + +/** + * pci_msi_create_default_irq_domain - Create a default MSI interrupt domain + * @node: Optional device-tree node of the interrupt controller + * @info: MSI domain info + * @parent: Parent irq domain + * + * Returns: A domain pointer or NULL in case of failure. If successful + * the default PCI/MSI irqdomain pointer is updated. + */ +struct irq_domain *pci_msi_create_default_irq_domain(struct device_node *node, + struct msi_domain_info *info, struct irq_domain *parent) +{ + struct irq_domain *domain; + + mutex_lock(&pci_msi_domain_lock); + if (pci_msi_default_domain) { + pr_err("PCI: default irq domain for PCI MSI has already been created.\n"); + domain = NULL; + } else { + domain = pci_msi_create_irq_domain(node, info, parent); + pci_msi_default_domain = domain; + } + mutex_unlock(&pci_msi_domain_lock); + + return domain; +} +#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ diff --git a/drivers/pci/of.c b/drivers/pci/of.c index f0929934bb7a..2e99a500cb83 100644 --- a/drivers/pci/of.c +++ b/drivers/pci/of.c @@ -9,6 +9,7 @@ * 2 of the License, or (at your option) any later version. */ +#include <linux/irqdomain.h> #include <linux/kernel.h> #include <linux/pci.h> #include <linux/of.h> @@ -59,3 +60,32 @@ struct device_node * __weak pcibios_get_phb_of_node(struct pci_bus *bus) return of_node_get(bus->bridge->parent->of_node); return NULL; } + +struct irq_domain *pci_host_bridge_of_msi_domain(struct pci_bus *bus) +{ +#ifdef CONFIG_IRQ_DOMAIN + struct device_node *np; + struct irq_domain *d; + + if (!bus->dev.of_node) + return NULL; + + /* Start looking for a phandle to an MSI controller. */ + np = of_parse_phandle(bus->dev.of_node, "msi-parent", 0); + + /* + * If we don't have an msi-parent property, look for a domain + * directly attached to the host bridge. + */ + if (!np) + np = bus->dev.of_node; + + d = irq_find_matching_host(np, DOMAIN_BUS_PCI_MSI); + if (d) + return d; + + return irq_find_host(np); +#else + return NULL; +#endif +} diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 1563cfadeaef..f0164e5e0d41 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -126,15 +126,16 @@ EXPORT_SYMBOL_GPL(pci_bus_max_busnr); #ifdef CONFIG_HAS_IOMEM void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) { + struct resource *res = &pdev->resource[bar]; + /* * Make sure the BAR is actually a memory resource, not an IO resource */ - if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) { - WARN_ON(1); + if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) { + dev_warn(&pdev->dev, "can't ioremap BAR %d: %pR\n", bar, res); return NULL; } - return ioremap_nocache(pci_resource_start(pdev, bar), - pci_resource_len(pdev, bar)); + return ioremap_nocache(res->start, resource_size(res)); } EXPORT_SYMBOL_GPL(pci_ioremap_bar); #endif diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index b5defca86795..df2169edd23b 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -140,6 +140,27 @@ static inline void pci_no_msi(void) { } static inline void pci_msi_init_pci_dev(struct pci_dev *dev) { } #endif +static inline void pci_msi_set_enable(struct pci_dev *dev, int enable) +{ + u16 control; + + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); + control &= ~PCI_MSI_FLAGS_ENABLE; + if (enable) + control |= PCI_MSI_FLAGS_ENABLE; + pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); +} + +static inline void pci_msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set) +{ + u16 ctrl; + + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl); + ctrl &= ~clear; + ctrl |= set; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl); +} + void pci_realloc_get_opt(char *); static inline int pci_no_d1d2(struct pci_dev *dev) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index c43300ced57d..111a7c1832bf 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -6,6 +6,7 @@ #include <linux/delay.h> #include <linux/init.h> #include <linux/pci.h> +#include <linux/of_pci.h> #include <linux/pci_hotplug.h> #include <linux/slab.h> #include <linux/module.h> @@ -676,6 +677,35 @@ static void pci_set_bus_speed(struct pci_bus *bus) } } +static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus) +{ + struct irq_domain *d; + + /* + * Any firmware interface that can resolve the msi_domain + * should be called from here. + */ + d = pci_host_bridge_of_msi_domain(bus); + + return d; +} + +static void pci_set_bus_msi_domain(struct pci_bus *bus) +{ + struct irq_domain *d; + + /* + * Either bus is the root, and we must obtain it from the + * firmware, or we inherit it from the bridge device. + */ + if (pci_is_root_bus(bus)) + d = pci_host_bridge_msi_domain(bus); + else + d = dev_get_msi_domain(&bus->self->dev); + + dev_set_msi_domain(&bus->dev, d); +} + static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent, struct pci_dev *bridge, int busnr) { @@ -729,6 +759,7 @@ static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent, bridge->subordinate = child; add_dev: + pci_set_bus_msi_domain(child); ret = device_register(&child->dev); WARN_ON(ret < 0); @@ -1118,6 +1149,22 @@ int pci_cfg_space_size(struct pci_dev *dev) #define LEGACY_IO_RESOURCE (IORESOURCE_IO | IORESOURCE_PCI_FIXED) +static void pci_msi_setup_pci_dev(struct pci_dev *dev) +{ + /* + * Disable the MSI hardware to avoid screaming interrupts + * during boot. This is the power on reset default so + * usually this should be a noop. + */ + dev->msi_cap = pci_find_capability(dev, PCI_CAP_ID_MSI); + if (dev->msi_cap) + pci_msi_set_enable(dev, 0); + + dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); + if (dev->msix_cap) + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); +} + /** * pci_setup_device - fill in class and map information of a device * @dev: the device structure to fill @@ -1174,6 +1221,8 @@ int pci_setup_device(struct pci_dev *dev) /* "Unknown power state" */ dev->current_state = PCI_UNKNOWN; + pci_msi_setup_pci_dev(dev); + /* Early fixups, before probing the BARs */ pci_fixup_device(pci_fixup_early, dev); /* device class may be changed after fixup */ @@ -1563,6 +1612,17 @@ static void pci_init_capabilities(struct pci_dev *dev) pci_enable_acs(dev); } +static void pci_set_msi_domain(struct pci_dev *dev) +{ + /* + * If no domain has been set through the pcibios_add_device + * callback, inherit the default from the bus device. + */ + if (!dev_get_msi_domain(&dev->dev)) + dev_set_msi_domain(&dev->dev, + dev_get_msi_domain(&dev->bus->dev)); +} + void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) { int ret; @@ -1576,6 +1636,7 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) dev->dev.dma_mask = &dev->dma_mask; dev->dev.dma_parms = &dev->dma_parms; dev->dev.coherent_dma_mask = 0xffffffffull; + of_pci_dma_configure(dev); pci_set_dma_max_seg_size(dev, 65536); pci_set_dma_seg_boundary(dev, 0xffffffff); @@ -1603,6 +1664,9 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) ret = pcibios_add_device(dev); WARN_ON(ret < 0); + /* Setup MSI irq domain */ + pci_set_msi_domain(dev); + /* Notifier could use PCI capabilities */ dev->match_driver = false; ret = device_add(&dev->dev); @@ -1993,6 +2057,7 @@ struct pci_bus *pci_create_root_bus(struct device *parent, int bus, b->bridge = get_device(&bridge->dev); device_enable_async_suspend(b->bridge); pci_set_bus_of_node(b); + pci_set_bus_msi_domain(b); if (!parent) set_dev_node(b->bridge, pcibus_to_node(b)); @@ -2143,7 +2208,6 @@ struct pci_bus *pci_scan_root_bus(struct device *parent, int bus, if (!found) pci_bus_update_busn_res_end(b, max); - pci_bus_add_devices(b); return b; } EXPORT_SYMBOL(pci_scan_root_bus); @@ -2179,7 +2243,6 @@ struct pci_bus *pci_scan_bus(int bus, struct pci_ops *ops, b = pci_create_root_bus(NULL, bus, ops, sysdata, &resources); if (b) { pci_scan_child_bus(b); - pci_bus_add_devices(b); } else { pci_free_resource_list(&resources); } diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index b7c3a5ea1fca..232f9254c11a 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -120,6 +120,7 @@ int pci_claim_resource(struct pci_dev *dev, int resource) if (!root) { dev_info(&dev->dev, "can't claim BAR %d %pR: no compatible bridge window\n", resource, res); + res->flags |= IORESOURCE_UNSET; return -EINVAL; } @@ -127,6 +128,7 @@ int pci_claim_resource(struct pci_dev *dev, int resource) if (conflict) { dev_info(&dev->dev, "can't claim BAR %d %pR: address conflict with %s %pR\n", resource, res, conflict->name, conflict); + res->flags |= IORESOURCE_UNSET; return -EBUSY; } diff --git a/drivers/platform/x86/acerhdf.c b/drivers/platform/x86/acerhdf.c index f94467c05225..7f21f75baa59 100644 --- a/drivers/platform/x86/acerhdf.c +++ b/drivers/platform/x86/acerhdf.c @@ -330,7 +330,8 @@ static int acerhdf_bind(struct thermal_zone_device *thermal, return 0; if (thermal_zone_bind_cooling_device(thermal, 0, cdev, - THERMAL_NO_LIMIT, THERMAL_NO_LIMIT)) { + THERMAL_NO_LIMIT, THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT)) { pr_err("error binding cooling dev\n"); return -EINVAL; } diff --git a/drivers/pnp/quirks.c b/drivers/pnp/quirks.c index ebf0d6710b5a..943c1cb9566c 100644 --- a/drivers/pnp/quirks.c +++ b/drivers/pnp/quirks.c @@ -246,13 +246,16 @@ static void quirk_system_pci_resources(struct pnp_dev *dev) */ for_each_pci_dev(pdev) { for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { - unsigned long type; + unsigned long flags, type; - type = pci_resource_flags(pdev, i) & - (IORESOURCE_IO | IORESOURCE_MEM); + flags = pci_resource_flags(pdev, i); + type = flags & (IORESOURCE_IO | IORESOURCE_MEM); if (!type || pci_resource_len(pdev, i) == 0) continue; + if (flags & IORESOURCE_UNSET) + continue; + pci_start = pci_resource_start(pdev, i); pci_end = pci_resource_end(pdev, i); for (j = 0; diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c index f277ac9c5467..a6e62bcc88ac 100644 --- a/drivers/s390/scsi/zfcp_erp.c +++ b/drivers/s390/scsi/zfcp_erp.c @@ -742,11 +742,11 @@ static int zfcp_erp_adapter_strategy_open_fsf(struct zfcp_erp_action *act) return ZFCP_ERP_FAILED; if (mempool_resize(act->adapter->pool.sr_data, - act->adapter->stat_read_buf_num, GFP_KERNEL)) + act->adapter->stat_read_buf_num)) return ZFCP_ERP_FAILED; if (mempool_resize(act->adapter->pool.status_read_req, - act->adapter->stat_read_buf_num, GFP_KERNEL)) + act->adapter->stat_read_buf_num)) return ZFCP_ERP_FAILED; atomic_set(&act->adapter->stat_miss, act->adapter->stat_read_buf_num); diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index f554d25b4399..8b7d47f2f3aa 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -42,6 +42,17 @@ config THERMAL_OF Say 'Y' here if you need to build thermal infrastructure based on device tree. +config THERMAL_WRITABLE_TRIPS + bool "Enable writable trip points" + help + This option allows the system integrator to choose whether + trip temperatures can be changed from userspace. The + writable trips need to be specified when setting up the + thermal zone but the choice here takes precedence. + + Say 'Y' here if you would like to allow userspace tools to + change trip temperatures. + choice prompt "Default Thermal governor" default THERMAL_DEFAULT_GOV_STEP_WISE @@ -71,6 +82,14 @@ config THERMAL_DEFAULT_GOV_USER_SPACE Select this if you want to let the user space manage the platform thermals. +config THERMAL_DEFAULT_GOV_POWER_ALLOCATOR + bool "power_allocator" + select THERMAL_GOV_POWER_ALLOCATOR + help + Select this if you want to control temperature based on + system and device power allocation. This governor can only + operate on cooling devices that implement the power API. + endchoice config THERMAL_GOV_FAIR_SHARE @@ -99,6 +118,13 @@ config THERMAL_GOV_USER_SPACE help Enable this to let the user space manage the platform thermals. +config THERMAL_GOV_POWER_ALLOCATOR + bool "Power allocator thermal governor" + select THERMAL_POWER_ACTOR + help + Enable this to manage platform thermals by dynamically + allocating and limiting power to devices. + config CPU_THERMAL bool "generic cpu cooling support" depends on CPU_FREQ @@ -112,6 +138,18 @@ config CPU_THERMAL If you want this support, you should say Y here. +config CLOCK_THERMAL + bool "Generic clock cooling support" + depends on COMMON_CLK + depends on PM_OPP + help + This entry implements the generic clock cooling mechanism through + frequency clipping. Typically used to cool off co-processors. The + device that is configured to use this cooling mechanism will be + controlled to reduce clock frequency whenever temperature is high. + + If you want this support, you should say Y here. + config THERMAL_EMULATION bool "Thermal emulation mode support" help @@ -185,6 +223,16 @@ config ARMADA_THERMAL Enable this option if you want to have support for thermal management controller present in Armada 370 and Armada XP SoC. +config TEGRA_SOCTHERM + tristate "Tegra SOCTHERM thermal management" + depends on ARCH_TEGRA + help + Enable this option for integrated thermal management support on NVIDIA + Tegra124 systems-on-chip. The driver supports four thermal zones + (CPU, GPU, MEM, PLLX). Cooling devices can be bound to the thermal + zones to manage temperatures. This option is also required for the + emergency thermal reset (thermtrip) feature to function. + config DB8500_CPUFREQ_COOLING tristate "DB8500 cpufreq cooling" depends on ARCH_U8500 diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index 39c4fe87da2f..dae37c089bb4 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -14,10 +14,14 @@ thermal_sys-$(CONFIG_THERMAL_GOV_FAIR_SHARE) += fair_share.o thermal_sys-$(CONFIG_THERMAL_GOV_BANG_BANG) += gov_bang_bang.o thermal_sys-$(CONFIG_THERMAL_GOV_STEP_WISE) += step_wise.o thermal_sys-$(CONFIG_THERMAL_GOV_USER_SPACE) += user_space.o +thermal_sys-$(CONFIG_THERMAL_GOV_POWER_ALLOCATOR) += power_allocator.o # cpufreq cooling thermal_sys-$(CONFIG_CPU_THERMAL) += cpu_cooling.o +# clock cooling +thermal_sys-$(CONFIG_CLOCK_THERMAL) += clock_cooling.o + # platform thermal drivers obj-$(CONFIG_SPEAR_THERMAL) += spear_thermal.o obj-$(CONFIG_RCAR_THERMAL) += rcar_thermal.o @@ -34,3 +38,4 @@ obj-$(CONFIG_INTEL_SOC_DTS_THERMAL) += intel_soc_dts_thermal.o obj-$(CONFIG_TI_SOC_THERMAL) += ti-soc-thermal/ obj-$(CONFIG_INT340X_THERMAL) += int340x_thermal/ obj-$(CONFIG_ST_THERMAL) += st/ +obj-$(CONFIG_TEGRA_SOCTHERM) += tegra_soctherm.o diff --git a/drivers/thermal/clock_cooling.c b/drivers/thermal/clock_cooling.c new file mode 100644 index 000000000000..1b4ff0f4c716 --- /dev/null +++ b/drivers/thermal/clock_cooling.c @@ -0,0 +1,485 @@ +/* + * drivers/thermal/clock_cooling.c + * + * Copyright (C) 2014 Eduardo Valentin <edubezval@gmail.com> + * + * Copyright (C) 2013 Texas Instruments Inc. + * Contact: Eduardo Valentin <eduardo.valentin@ti.com> + * + * Highly based on cpu_cooling.c. + * Copyright (C) 2012 Samsung Electronics Co., Ltd(http://www.samsung.com) + * Copyright (C) 2012 Amit Daniel <amit.kachhap@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include <linux/clk.h> +#include <linux/cpufreq.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/idr.h> +#include <linux/mutex.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> +#include <linux/thermal.h> +#include <linux/clock_cooling.h> + +/** + * struct clock_cooling_device - data for cooling device with clock + * @id: unique integer value corresponding to each clock_cooling_device + * registered. + * @dev: struct device pointer to the device being used to cool off using + * clock frequencies. + * @cdev: thermal_cooling_device pointer to keep track of the + * registered cooling device. + * @clk_rate_change_nb: reference to notifier block used to receive clock + * rate changes. + * @freq_table: frequency table used to keep track of available frequencies. + * @clock_state: integer value representing the current state of clock + * cooling devices. + * @clock_val: integer value representing the absolute value of the clipped + * frequency. + * @clk: struct clk reference used to enforce clock limits. + * @lock: mutex lock to protect this struct. + * + * This structure is required for keeping information of each + * clock_cooling_device registered. In order to prevent corruption of this a + * mutex @lock is used. + */ +struct clock_cooling_device { + int id; + struct device *dev; + struct thermal_cooling_device *cdev; + struct notifier_block clk_rate_change_nb; + struct cpufreq_frequency_table *freq_table; + unsigned long clock_state; + unsigned long clock_val; + struct clk *clk; + struct mutex lock; /* lock to protect the content of this struct */ +}; +#define to_clock_cooling_device(x) \ + container_of(x, struct clock_cooling_device, clk_rate_change_nb) +static DEFINE_IDR(clock_idr); +static DEFINE_MUTEX(cooling_clock_lock); + +/** + * clock_cooling_get_idr - function to get an unique id. + * @id: int * value generated by this function. + * + * This function will populate @id with an unique + * id, using the idr API. + * + * Return: 0 on success, an error code on failure. + */ +static int clock_cooling_get_idr(int *id) +{ + int ret; + + mutex_lock(&cooling_clock_lock); + ret = idr_alloc(&clock_idr, NULL, 0, 0, GFP_KERNEL); + mutex_unlock(&cooling_clock_lock); + if (unlikely(ret < 0)) + return ret; + *id = ret; + + return 0; +} + +/** + * release_idr - function to free the unique id. + * @id: int value representing the unique id. + */ +static void release_idr(int id) +{ + mutex_lock(&cooling_clock_lock); + idr_remove(&clock_idr, id); + mutex_unlock(&cooling_clock_lock); +} + +/* Below code defines functions to be used for clock as cooling device */ + +enum clock_cooling_property { + GET_LEVEL, + GET_FREQ, + GET_MAXL, +}; + +/** + * clock_cooling_get_property - fetch a property of interest for a give cpu. + * @ccdev: clock cooling device reference + * @input: query parameter + * @output: query return + * @property: type of query (frequency, level, max level) + * + * This is the common function to + * 1. get maximum clock cooling states + * 2. translate frequency to cooling state + * 3. translate cooling state to frequency + * Note that the code may be not in good shape + * but it is written in this way in order to: + * a) reduce duplicate code as most of the code can be shared. + * b) make sure the logic is consistent when translating between + * cooling states and frequencies. + * + * Return: 0 on success, -EINVAL when invalid parameters are passed. + */ +static int clock_cooling_get_property(struct clock_cooling_device *ccdev, + unsigned long input, + unsigned long *output, + enum clock_cooling_property property) +{ + int i; + unsigned long max_level = 0, level = 0; + unsigned int freq = CPUFREQ_ENTRY_INVALID; + int descend = -1; + struct cpufreq_frequency_table *pos, *table = ccdev->freq_table; + + if (!output) + return -EINVAL; + + if (!table) + return -EINVAL; + + cpufreq_for_each_valid_entry(pos, table) { + /* ignore duplicate entry */ + if (freq == pos->frequency) + continue; + + /* get the frequency order */ + if (freq != CPUFREQ_ENTRY_INVALID && descend == -1) + descend = freq > pos->frequency; + + freq = pos->frequency; + max_level++; + } + + /* No valid cpu frequency entry */ + if (max_level == 0) + return -EINVAL; + + /* max_level is an index, not a counter */ + max_level--; + + /* get max level */ + if (property == GET_MAXL) { + *output = max_level; + return 0; + } + + if (property == GET_FREQ) + level = descend ? input : (max_level - input); + + i = 0; + cpufreq_for_each_valid_entry(pos, table) { + /* ignore duplicate entry */ + if (freq == pos->frequency) + continue; + + /* now we have a valid frequency entry */ + freq = pos->frequency; + + if (property == GET_LEVEL && (unsigned int)input == freq) { + /* get level by frequency */ + *output = descend ? i : (max_level - i); + return 0; + } + if (property == GET_FREQ && level == i) { + /* get frequency by level */ + *output = freq; + return 0; + } + i++; + } + + return -EINVAL; +} + +/** + * clock_cooling_get_level - return the cooling level of given clock cooling. + * @cdev: reference of a thermal cooling device of used as clock cooling device + * @freq: the frequency of interest + * + * This function will match the cooling level corresponding to the + * requested @freq and return it. + * + * Return: The matched cooling level on success or THERMAL_CSTATE_INVALID + * otherwise. + */ +unsigned long clock_cooling_get_level(struct thermal_cooling_device *cdev, + unsigned long freq) +{ + struct clock_cooling_device *ccdev = cdev->devdata; + unsigned long val; + + if (clock_cooling_get_property(ccdev, (unsigned long)freq, &val, + GET_LEVEL)) + return THERMAL_CSTATE_INVALID; + + return val; +} +EXPORT_SYMBOL_GPL(clock_cooling_get_level); + +/** + * clock_cooling_get_frequency - get the absolute value of frequency from level. + * @ccdev: clock cooling device reference + * @level: cooling level + * + * This function matches cooling level with frequency. Based on a cooling level + * of frequency, equals cooling state of cpu cooling device, it will return + * the corresponding frequency. + * e.g level=0 --> 1st MAX FREQ, level=1 ---> 2nd MAX FREQ, .... etc + * + * Return: 0 on error, the corresponding frequency otherwise. + */ +static unsigned long +clock_cooling_get_frequency(struct clock_cooling_device *ccdev, + unsigned long level) +{ + int ret = 0; + unsigned long freq; + + ret = clock_cooling_get_property(ccdev, level, &freq, GET_FREQ); + if (ret) + return 0; + + return freq; +} + +/** + * clock_cooling_apply - function to apply frequency clipping. + * @ccdev: clock_cooling_device pointer containing frequency clipping data. + * @cooling_state: value of the cooling state. + * + * Function used to make sure the clock layer is aware of current thermal + * limits. The limits are applied by updating the clock rate in case it is + * higher than the corresponding frequency based on the requested cooling_state. + * + * Return: 0 on success, an error code otherwise (-EINVAL in case wrong + * cooling state). + */ +static int clock_cooling_apply(struct clock_cooling_device *ccdev, + unsigned long cooling_state) +{ + unsigned long clip_freq, cur_freq; + int ret = 0; + + /* Here we write the clipping */ + /* Check if the old cooling action is same as new cooling action */ + if (ccdev->clock_state == cooling_state) + return 0; + + clip_freq = clock_cooling_get_frequency(ccdev, cooling_state); + if (!clip_freq) + return -EINVAL; + + cur_freq = clk_get_rate(ccdev->clk); + + mutex_lock(&ccdev->lock); + ccdev->clock_state = cooling_state; + ccdev->clock_val = clip_freq; + /* enforce clock level */ + if (cur_freq > clip_freq) + ret = clk_set_rate(ccdev->clk, clip_freq); + mutex_unlock(&ccdev->lock); + + return ret; +} + +/** + * clock_cooling_clock_notifier - notifier callback on clock rate changes. + * @nb: struct notifier_block * with callback info. + * @event: value showing clock event for which this function invoked. + * @data: callback-specific data + * + * Callback to hijack the notification on clock transition. + * Every time there is a clock change, we intercept all pre change events + * and block the transition in case the new rate infringes thermal limits. + * + * Return: NOTIFY_DONE (success) or NOTIFY_BAD (new_rate > thermal limit). + */ +static int clock_cooling_clock_notifier(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct clk_notifier_data *ndata = data; + struct clock_cooling_device *ccdev = to_clock_cooling_device(nb); + + switch (event) { + case PRE_RATE_CHANGE: + /* + * checks on current state + * TODO: current method is not best we can find as it + * allows possibly voltage transitions, in case DVFS + * layer is also hijacking clock pre notifications. + */ + if (ndata->new_rate > ccdev->clock_val) + return NOTIFY_BAD; + /* fall through */ + case POST_RATE_CHANGE: + case ABORT_RATE_CHANGE: + default: + return NOTIFY_DONE; + } +} + +/* clock cooling device thermal callback functions are defined below */ + +/** + * clock_cooling_get_max_state - callback function to get the max cooling state. + * @cdev: thermal cooling device pointer. + * @state: fill this variable with the max cooling state. + * + * Callback for the thermal cooling device to return the clock + * max cooling state. + * + * Return: 0 on success, an error code otherwise. + */ +static int clock_cooling_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct clock_cooling_device *ccdev = cdev->devdata; + unsigned long count = 0; + int ret; + + ret = clock_cooling_get_property(ccdev, 0, &count, GET_MAXL); + if (!ret) + *state = count; + + return ret; +} + +/** + * clock_cooling_get_cur_state - function to get the current cooling state. + * @cdev: thermal cooling device pointer. + * @state: fill this variable with the current cooling state. + * + * Callback for the thermal cooling device to return the clock + * current cooling state. + * + * Return: 0 (success) + */ +static int clock_cooling_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct clock_cooling_device *ccdev = cdev->devdata; + + *state = ccdev->clock_state; + + return 0; +} + +/** + * clock_cooling_set_cur_state - function to set the current cooling state. + * @cdev: thermal cooling device pointer. + * @state: set this variable to the current cooling state. + * + * Callback for the thermal cooling device to change the clock cooling + * current cooling state. + * + * Return: 0 on success, an error code otherwise. + */ +static int clock_cooling_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state) +{ + struct clock_cooling_device *clock_device = cdev->devdata; + + return clock_cooling_apply(clock_device, state); +} + +/* Bind clock callbacks to thermal cooling device ops */ +static struct thermal_cooling_device_ops const clock_cooling_ops = { + .get_max_state = clock_cooling_get_max_state, + .get_cur_state = clock_cooling_get_cur_state, + .set_cur_state = clock_cooling_set_cur_state, +}; + +/** + * clock_cooling_register - function to create clock cooling device. + * @dev: struct device pointer to the device used as clock cooling device. + * @clock_name: string containing the clock used as cooling mechanism. + * + * This interface function registers the clock cooling device with the name + * "thermal-clock-%x". The cooling device is based on clock frequencies. + * The struct device is assumed to be capable of DVFS transitions. + * The OPP layer is used to fetch and fill the available frequencies for + * the referred device. The ordered frequency table is used to control + * the clock cooling device cooling states and to limit clock transitions + * based on the cooling state requested by the thermal framework. + * + * Return: a valid struct thermal_cooling_device pointer on success, + * on failure, it returns a corresponding ERR_PTR(). + */ +struct thermal_cooling_device * +clock_cooling_register(struct device *dev, const char *clock_name) +{ + struct thermal_cooling_device *cdev; + struct clock_cooling_device *ccdev = NULL; + char dev_name[THERMAL_NAME_LENGTH]; + int ret = 0; + + ccdev = devm_kzalloc(dev, sizeof(*ccdev), GFP_KERNEL); + if (!ccdev) + return ERR_PTR(-ENOMEM); + + ccdev->dev = dev; + ccdev->clk = devm_clk_get(dev, clock_name); + if (IS_ERR(ccdev->clk)) + return ERR_CAST(ccdev->clk); + + ret = clock_cooling_get_idr(&ccdev->id); + if (ret) + return ERR_PTR(-EINVAL); + + snprintf(dev_name, sizeof(dev_name), "thermal-clock-%d", ccdev->id); + + cdev = thermal_cooling_device_register(dev_name, ccdev, + &clock_cooling_ops); + if (IS_ERR(cdev)) { + release_idr(ccdev->id); + return ERR_PTR(-EINVAL); + } + ccdev->cdev = cdev; + ccdev->clk_rate_change_nb.notifier_call = clock_cooling_clock_notifier; + + /* Assuming someone has already filled the opp table for this device */ + ret = dev_pm_opp_init_cpufreq_table(dev, &ccdev->freq_table); + if (ret) { + release_idr(ccdev->id); + return ERR_PTR(ret); + } + ccdev->clock_state = 0; + ccdev->clock_val = clock_cooling_get_frequency(ccdev, 0); + + clk_notifier_register(ccdev->clk, &ccdev->clk_rate_change_nb); + + return cdev; +} +EXPORT_SYMBOL_GPL(clock_cooling_register); + +/** + * clock_cooling_unregister - function to remove clock cooling device. + * @cdev: thermal cooling device pointer. + * + * This interface function unregisters the "thermal-clock-%x" cooling device. + */ +void clock_cooling_unregister(struct thermal_cooling_device *cdev) +{ + struct clock_cooling_device *ccdev; + + if (!cdev) + return; + + ccdev = cdev->devdata; + + clk_notifier_unregister(ccdev->clk, &ccdev->clk_rate_change_nb); + dev_pm_opp_free_cpufreq_table(ccdev->dev, &ccdev->freq_table); + + thermal_cooling_device_unregister(ccdev->cdev); + release_idr(ccdev->id); +} +EXPORT_SYMBOL_GPL(clock_cooling_unregister); diff --git a/drivers/thermal/cpu_cooling.c b/drivers/thermal/cpu_cooling.c index ad09e51ffae4..6509c61b9648 100644 --- a/drivers/thermal/cpu_cooling.c +++ b/drivers/thermal/cpu_cooling.c @@ -4,6 +4,8 @@ * Copyright (C) 2012 Samsung Electronics Co., Ltd(http://www.samsung.com) * Copyright (C) 2012 Amit Daniel <amit.kachhap@linaro.org> * + * Copyright (C) 2014 Viresh Kumar <viresh.kumar@linaro.org> + * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,10 +26,40 @@ #include <linux/thermal.h> #include <linux/cpufreq.h> #include <linux/err.h> +#include <linux/pm_opp.h> #include <linux/slab.h> #include <linux/cpu.h> #include <linux/cpu_cooling.h> +#include <trace/events/thermal.h> + +/* + * Cooling state <-> CPUFreq frequency + * + * Cooling states are translated to frequencies throughout this driver and this + * is the relation between them. + * + * Highest cooling state corresponds to lowest possible frequency. + * + * i.e. + * level 0 --> 1st Max Freq + * level 1 --> 2nd Max Freq + * ... + */ + +/** + * struct power_table - frequency to power conversion + * @frequency: frequency in KHz + * @power: power in mW + * + * This structure is built when the cooling device registers and helps + * in translating frequency to power and viceversa. + */ +struct power_table { + u32 frequency; + u32 power; +}; + /** * struct cpufreq_cooling_device - data for cooling device with cpufreq * @id: unique integer value corresponding to each cpufreq_cooling_device @@ -38,25 +70,43 @@ * cooling devices. * @cpufreq_val: integer value representing the absolute value of the clipped * frequency. + * @max_level: maximum cooling level. One less than total number of valid + * cpufreq frequencies. * @allowed_cpus: all the cpus involved for this cpufreq_cooling_device. + * @node: list_head to link all cpufreq_cooling_device together. + * @last_load: load measured by the latest call to cpufreq_get_actual_power() + * @time_in_idle: previous reading of the absolute time that this cpu was idle + * @time_in_idle_timestamp: wall time of the last invocation of + * get_cpu_idle_time_us() + * @dyn_power_table: array of struct power_table for frequency to power + * conversion, sorted in ascending order. + * @dyn_power_table_entries: number of entries in the @dyn_power_table array + * @cpu_dev: the first cpu_device from @allowed_cpus that has OPPs registered + * @plat_get_static_power: callback to calculate the static power * - * This structure is required for keeping information of each - * cpufreq_cooling_device registered. In order to prevent corruption of this a - * mutex lock cooling_cpufreq_lock is used. + * This structure is required for keeping information of each registered + * cpufreq_cooling_device. */ struct cpufreq_cooling_device { int id; struct thermal_cooling_device *cool_dev; unsigned int cpufreq_state; unsigned int cpufreq_val; + unsigned int max_level; + unsigned int *freq_table; /* In descending order */ struct cpumask allowed_cpus; struct list_head node; + u32 last_load; + u64 *time_in_idle; + u64 *time_in_idle_timestamp; + struct power_table *dyn_power_table; + int dyn_power_table_entries; + struct device *cpu_dev; + get_static_t plat_get_static_power; }; static DEFINE_IDR(cpufreq_idr); static DEFINE_MUTEX(cooling_cpufreq_lock); -static unsigned int cpufreq_dev_count; - static LIST_HEAD(cpufreq_dev_list); /** @@ -98,120 +148,30 @@ static void release_idr(struct idr *idr, int id) /* Below code defines functions to be used for cpufreq as cooling device */ /** - * is_cpufreq_valid - function to check frequency transitioning capability. - * @cpu: cpu for which check is needed. + * get_level: Find the level for a particular frequency + * @cpufreq_dev: cpufreq_dev for which the property is required + * @freq: Frequency * - * This function will check the current state of the system if - * it is capable of changing the frequency for a given @cpu. - * - * Return: 0 if the system is not currently capable of changing - * the frequency of given cpu. !0 in case the frequency is changeable. - */ -static int is_cpufreq_valid(int cpu) -{ - struct cpufreq_policy policy; - - return !cpufreq_get_policy(&policy, cpu); -} - -enum cpufreq_cooling_property { - GET_LEVEL, - GET_FREQ, - GET_MAXL, -}; - -/** - * get_property - fetch a property of interest for a give cpu. - * @cpu: cpu for which the property is required - * @input: query parameter - * @output: query return - * @property: type of query (frequency, level, max level) - * - * This is the common function to - * 1. get maximum cpu cooling states - * 2. translate frequency to cooling state - * 3. translate cooling state to frequency - * Note that the code may be not in good shape - * but it is written in this way in order to: - * a) reduce duplicate code as most of the code can be shared. - * b) make sure the logic is consistent when translating between - * cooling states and frequencies. - * - * Return: 0 on success, -EINVAL when invalid parameters are passed. + * Return: level on success, THERMAL_CSTATE_INVALID on error. */ -static int get_property(unsigned int cpu, unsigned long input, - unsigned int *output, - enum cpufreq_cooling_property property) +static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_dev, + unsigned int freq) { - int i; - unsigned long max_level = 0, level = 0; - unsigned int freq = CPUFREQ_ENTRY_INVALID; - int descend = -1; - struct cpufreq_frequency_table *pos, *table = - cpufreq_frequency_get_table(cpu); - - if (!output) - return -EINVAL; - - if (!table) - return -EINVAL; - - cpufreq_for_each_valid_entry(pos, table) { - /* ignore duplicate entry */ - if (freq == pos->frequency) - continue; - - /* get the frequency order */ - if (freq != CPUFREQ_ENTRY_INVALID && descend == -1) - descend = freq > pos->frequency; - - freq = pos->frequency; - max_level++; - } - - /* No valid cpu frequency entry */ - if (max_level == 0) - return -EINVAL; + unsigned long level; - /* max_level is an index, not a counter */ - max_level--; - - /* get max level */ - if (property == GET_MAXL) { - *output = (unsigned int)max_level; - return 0; - } - - if (property == GET_FREQ) - level = descend ? input : (max_level - input); - - i = 0; - cpufreq_for_each_valid_entry(pos, table) { - /* ignore duplicate entry */ - if (freq == pos->frequency) - continue; - - /* now we have a valid frequency entry */ - freq = pos->frequency; + for (level = 0; level <= cpufreq_dev->max_level; level++) { + if (freq == cpufreq_dev->freq_table[level]) + return level; - if (property == GET_LEVEL && (unsigned int)input == freq) { - /* get level by frequency */ - *output = descend ? i : (max_level - i); - return 0; - } - if (property == GET_FREQ && level == i) { - /* get frequency by level */ - *output = freq; - return 0; - } - i++; + if (freq > cpufreq_dev->freq_table[level]) + break; } - return -EINVAL; + return THERMAL_CSTATE_INVALID; } /** - * cpufreq_cooling_get_level - for a give cpu, return the cooling level. + * cpufreq_cooling_get_level - for a given cpu, return the cooling level. * @cpu: cpu for which the level is required * @freq: the frequency of interest * @@ -223,119 +183,272 @@ static int get_property(unsigned int cpu, unsigned long input, */ unsigned long cpufreq_cooling_get_level(unsigned int cpu, unsigned int freq) { - unsigned int val; + struct cpufreq_cooling_device *cpufreq_dev; - if (get_property(cpu, (unsigned long)freq, &val, GET_LEVEL)) - return THERMAL_CSTATE_INVALID; + mutex_lock(&cooling_cpufreq_lock); + list_for_each_entry(cpufreq_dev, &cpufreq_dev_list, node) { + if (cpumask_test_cpu(cpu, &cpufreq_dev->allowed_cpus)) { + mutex_unlock(&cooling_cpufreq_lock); + return get_level(cpufreq_dev, freq); + } + } + mutex_unlock(&cooling_cpufreq_lock); - return (unsigned long)val; + pr_err("%s: cpu:%d not part of any cooling device\n", __func__, cpu); + return THERMAL_CSTATE_INVALID; } EXPORT_SYMBOL_GPL(cpufreq_cooling_get_level); /** - * get_cpu_frequency - get the absolute value of frequency from level. - * @cpu: cpu for which frequency is fetched. - * @level: cooling level + * cpufreq_thermal_notifier - notifier callback for cpufreq policy change. + * @nb: struct notifier_block * with callback info. + * @event: value showing cpufreq event for which this function invoked. + * @data: callback-specific data * - * This function matches cooling level with frequency. Based on a cooling level - * of frequency, equals cooling state of cpu cooling device, it will return - * the corresponding frequency. - * e.g level=0 --> 1st MAX FREQ, level=1 ---> 2nd MAX FREQ, .... etc + * Callback to hijack the notification on cpufreq policy transition. + * Every time there is a change in policy, we will intercept and + * update the cpufreq policy with thermal constraints. * - * Return: 0 on error, the corresponding frequency otherwise. + * Return: 0 (success) */ -static unsigned int get_cpu_frequency(unsigned int cpu, unsigned long level) +static int cpufreq_thermal_notifier(struct notifier_block *nb, + unsigned long event, void *data) { - int ret = 0; - unsigned int freq; + struct cpufreq_policy *policy = data; + unsigned long max_freq = 0; + struct cpufreq_cooling_device *cpufreq_dev; - ret = get_property(cpu, level, &freq, GET_FREQ); - if (ret) - return 0; + switch (event) { + + case CPUFREQ_ADJUST: + mutex_lock(&cooling_cpufreq_lock); + list_for_each_entry(cpufreq_dev, &cpufreq_dev_list, node) { + if (!cpumask_test_cpu(policy->cpu, + &cpufreq_dev->allowed_cpus)) + continue; - return freq; + max_freq = cpufreq_dev->cpufreq_val; + + if (policy->max != max_freq) + cpufreq_verify_within_limits(policy, 0, + max_freq); + } + mutex_unlock(&cooling_cpufreq_lock); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; } /** - * cpufreq_apply_cooling - function to apply frequency clipping. - * @cpufreq_device: cpufreq_cooling_device pointer containing frequency - * clipping data. - * @cooling_state: value of the cooling state. + * build_dyn_power_table() - create a dynamic power to frequency table + * @cpufreq_device: the cpufreq cooling device in which to store the table + * @capacitance: dynamic power coefficient for these cpus * - * Function used to make sure the cpufreq layer is aware of current thermal - * limits. The limits are applied by updating the cpufreq policy. + * Build a dynamic power to frequency table for this cpu and store it + * in @cpufreq_device. This table will be used in cpu_power_to_freq() and + * cpu_freq_to_power() to convert between power and frequency + * efficiently. Power is stored in mW, frequency in KHz. The + * resulting table is in ascending order. * - * Return: 0 on success, an error code otherwise (-EINVAL in case wrong - * cooling state). + * Return: 0 on success, -E* on error. */ -static int cpufreq_apply_cooling(struct cpufreq_cooling_device *cpufreq_device, - unsigned long cooling_state) +static int build_dyn_power_table(struct cpufreq_cooling_device *cpufreq_device, + u32 capacitance) { - unsigned int cpuid, clip_freq; - struct cpumask *mask = &cpufreq_device->allowed_cpus; - unsigned int cpu = cpumask_any(mask); + struct power_table *power_table; + struct dev_pm_opp *opp; + struct device *dev = NULL; + int num_opps = 0, cpu, i, ret = 0; + unsigned long freq; + + rcu_read_lock(); + + for_each_cpu(cpu, &cpufreq_device->allowed_cpus) { + dev = get_cpu_device(cpu); + if (!dev) { + dev_warn(&cpufreq_device->cool_dev->device, + "No cpu device for cpu %d\n", cpu); + continue; + } + num_opps = dev_pm_opp_get_opp_count(dev); + if (num_opps > 0) { + break; + } else if (num_opps < 0) { + ret = num_opps; + goto unlock; + } + } - /* Check if the old cooling action is same as new cooling action */ - if (cpufreq_device->cpufreq_state == cooling_state) - return 0; + if (num_opps == 0) { + ret = -EINVAL; + goto unlock; + } - clip_freq = get_cpu_frequency(cpu, cooling_state); - if (!clip_freq) - return -EINVAL; + power_table = kcalloc(num_opps, sizeof(*power_table), GFP_KERNEL); + if (!power_table) { + ret = -ENOMEM; + goto unlock; + } - cpufreq_device->cpufreq_state = cooling_state; - cpufreq_device->cpufreq_val = clip_freq; + for (freq = 0, i = 0; + opp = dev_pm_opp_find_freq_ceil(dev, &freq), !IS_ERR(opp); + freq++, i++) { + u32 freq_mhz, voltage_mv; + u64 power; + + freq_mhz = freq / 1000000; + voltage_mv = dev_pm_opp_get_voltage(opp) / 1000; + + /* + * Do the multiplication with MHz and millivolt so as + * to not overflow. + */ + power = (u64)capacitance * freq_mhz * voltage_mv * voltage_mv; + do_div(power, 1000000000); - for_each_cpu(cpuid, mask) { - if (is_cpufreq_valid(cpuid)) - cpufreq_update_policy(cpuid); + /* frequency is stored in power_table in KHz */ + power_table[i].frequency = freq / 1000; + + /* power is stored in mW */ + power_table[i].power = power; } - return 0; + if (i == 0) { + ret = PTR_ERR(opp); + goto unlock; + } + + cpufreq_device->cpu_dev = dev; + cpufreq_device->dyn_power_table = power_table; + cpufreq_device->dyn_power_table_entries = i; + +unlock: + rcu_read_unlock(); + return ret; +} + +static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_device, + u32 freq) +{ + int i; + struct power_table *pt = cpufreq_device->dyn_power_table; + + for (i = 1; i < cpufreq_device->dyn_power_table_entries; i++) + if (freq < pt[i].frequency) + break; + + return pt[i - 1].power; +} + +static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_device, + u32 power) +{ + int i; + struct power_table *pt = cpufreq_device->dyn_power_table; + + for (i = 1; i < cpufreq_device->dyn_power_table_entries; i++) + if (power < pt[i].power) + break; + + return pt[i - 1].frequency; } /** - * cpufreq_thermal_notifier - notifier callback for cpufreq policy change. - * @nb: struct notifier_block * with callback info. - * @event: value showing cpufreq event for which this function invoked. - * @data: callback-specific data + * get_load() - get load for a cpu since last updated + * @cpufreq_device: &struct cpufreq_cooling_device for this cpu + * @cpu: cpu number * - * Callback to hijack the notification on cpufreq policy transition. - * Every time there is a change in policy, we will intercept and - * update the cpufreq policy with thermal constraints. - * - * Return: 0 (success) + * Return: The average load of cpu @cpu in percentage since this + * function was last called. */ -static int cpufreq_thermal_notifier(struct notifier_block *nb, - unsigned long event, void *data) +static u32 get_load(struct cpufreq_cooling_device *cpufreq_device, int cpu) { - struct cpufreq_policy *policy = data; - unsigned long max_freq = 0; - struct cpufreq_cooling_device *cpufreq_dev; + u32 load; + u64 now, now_idle, delta_time, delta_idle; + + now_idle = get_cpu_idle_time(cpu, &now, 0); + delta_idle = now_idle - cpufreq_device->time_in_idle[cpu]; + delta_time = now - cpufreq_device->time_in_idle_timestamp[cpu]; + + if (delta_time <= delta_idle) + load = 0; + else + load = div64_u64(100 * (delta_time - delta_idle), delta_time); - if (event != CPUFREQ_ADJUST) + cpufreq_device->time_in_idle[cpu] = now_idle; + cpufreq_device->time_in_idle_timestamp[cpu] = now; + + return load; +} + +/** + * get_static_power() - calculate the static power consumed by the cpus + * @cpufreq_device: struct &cpufreq_cooling_device for this cpu cdev + * @tz: thermal zone device in which we're operating + * @freq: frequency in KHz + * @power: pointer in which to store the calculated static power + * + * Calculate the static power consumed by the cpus described by + * @cpu_actor running at frequency @freq. This function relies on a + * platform specific function that should have been provided when the + * actor was registered. If it wasn't, the static power is assumed to + * be negligible. The calculated static power is stored in @power. + * + * Return: 0 on success, -E* on failure. + */ +static int get_static_power(struct cpufreq_cooling_device *cpufreq_device, + struct thermal_zone_device *tz, unsigned long freq, + u32 *power) +{ + struct dev_pm_opp *opp; + unsigned long voltage; + struct cpumask *cpumask = &cpufreq_device->allowed_cpus; + unsigned long freq_hz = freq * 1000; + + if (!cpufreq_device->plat_get_static_power || + !cpufreq_device->cpu_dev) { + *power = 0; return 0; + } - mutex_lock(&cooling_cpufreq_lock); - list_for_each_entry(cpufreq_dev, &cpufreq_dev_list, node) { - if (!cpumask_test_cpu(policy->cpu, - &cpufreq_dev->allowed_cpus)) - continue; + rcu_read_lock(); - if (!cpufreq_dev->cpufreq_val) - cpufreq_dev->cpufreq_val = get_cpu_frequency( - cpumask_any(&cpufreq_dev->allowed_cpus), - cpufreq_dev->cpufreq_state); + opp = dev_pm_opp_find_freq_exact(cpufreq_device->cpu_dev, freq_hz, + true); + voltage = dev_pm_opp_get_voltage(opp); - max_freq = cpufreq_dev->cpufreq_val; + rcu_read_unlock(); - if (policy->max != max_freq) - cpufreq_verify_within_limits(policy, 0, max_freq); + if (voltage == 0) { + dev_warn_ratelimited(cpufreq_device->cpu_dev, + "Failed to get voltage for frequency %lu: %ld\n", + freq_hz, IS_ERR(opp) ? PTR_ERR(opp) : 0); + return -EINVAL; } - mutex_unlock(&cooling_cpufreq_lock); - return 0; + return cpufreq_device->plat_get_static_power(cpumask, tz->passive_delay, + voltage, power); +} + +/** + * get_dynamic_power() - calculate the dynamic power + * @cpufreq_device: &cpufreq_cooling_device for this cdev + * @freq: current frequency + * + * Return: the dynamic power consumed by the cpus described by + * @cpufreq_device. + */ +static u32 get_dynamic_power(struct cpufreq_cooling_device *cpufreq_device, + unsigned long freq) +{ + u32 raw_cpu_power; + + raw_cpu_power = cpu_freq_to_power(cpufreq_device, freq); + return (raw_cpu_power * cpufreq_device->last_load) / 100; } /* cpufreq cooling device callback functions are defined below */ @@ -354,19 +467,9 @@ static int cpufreq_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state) { struct cpufreq_cooling_device *cpufreq_device = cdev->devdata; - struct cpumask *mask = &cpufreq_device->allowed_cpus; - unsigned int cpu; - unsigned int count = 0; - int ret; - cpu = cpumask_any(mask); - - ret = get_property(cpu, 0, &count, GET_MAXL); - - if (count > 0) - *state = count; - - return ret; + *state = cpufreq_device->max_level; + return 0; } /** @@ -403,12 +506,225 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) { struct cpufreq_cooling_device *cpufreq_device = cdev->devdata; + unsigned int cpu = cpumask_any(&cpufreq_device->allowed_cpus); + unsigned int clip_freq; + + /* Request state should be less than max_level */ + if (WARN_ON(state > cpufreq_device->max_level)) + return -EINVAL; + + /* Check if the old cooling action is same as new cooling action */ + if (cpufreq_device->cpufreq_state == state) + return 0; + + clip_freq = cpufreq_device->freq_table[state]; + cpufreq_device->cpufreq_state = state; + cpufreq_device->cpufreq_val = clip_freq; + + cpufreq_update_policy(cpu); + + return 0; +} + +/** + * cpufreq_get_requested_power() - get the current power + * @cdev: &thermal_cooling_device pointer + * @tz: a valid thermal zone device pointer + * @power: pointer in which to store the resulting power + * + * Calculate the current power consumption of the cpus in milliwatts + * and store it in @power. This function should actually calculate + * the requested power, but it's hard to get the frequency that + * cpufreq would have assigned if there were no thermal limits. + * Instead, we calculate the current power on the assumption that the + * immediate future will look like the immediate past. + * + * We use the current frequency and the average load since this + * function was last called. In reality, there could have been + * multiple opps since this function was last called and that affects + * the load calculation. While it's not perfectly accurate, this + * simplification is good enough and works. REVISIT this, as more + * complex code may be needed if experiments show that it's not + * accurate enough. + * + * Return: 0 on success, -E* if getting the static power failed. + */ +static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, + u32 *power) +{ + unsigned long freq; + int i = 0, cpu, ret; + u32 static_power, dynamic_power, total_load = 0; + struct cpufreq_cooling_device *cpufreq_device = cdev->devdata; + u32 *load_cpu = NULL; + + cpu = cpumask_any_and(&cpufreq_device->allowed_cpus, cpu_online_mask); + + /* + * All the CPUs are offline, thus the requested power by + * the cdev is 0 + */ + if (cpu >= nr_cpu_ids) { + *power = 0; + return 0; + } + + freq = cpufreq_quick_get(cpu); + + if (trace_thermal_power_cpu_get_power_enabled()) { + u32 ncpus = cpumask_weight(&cpufreq_device->allowed_cpus); + + load_cpu = devm_kcalloc(&cdev->device, ncpus, sizeof(*load_cpu), + GFP_KERNEL); + } + + for_each_cpu(cpu, &cpufreq_device->allowed_cpus) { + u32 load; + + if (cpu_online(cpu)) + load = get_load(cpufreq_device, cpu); + else + load = 0; + + total_load += load; + if (trace_thermal_power_cpu_limit_enabled() && load_cpu) + load_cpu[i] = load; + + i++; + } + + cpufreq_device->last_load = total_load; + + dynamic_power = get_dynamic_power(cpufreq_device, freq); + ret = get_static_power(cpufreq_device, tz, freq, &static_power); + if (ret) { + if (load_cpu) + devm_kfree(&cdev->device, load_cpu); + return ret; + } + + if (load_cpu) { + trace_thermal_power_cpu_get_power( + &cpufreq_device->allowed_cpus, + freq, load_cpu, i, dynamic_power, static_power); + + devm_kfree(&cdev->device, load_cpu); + } + + *power = static_power + dynamic_power; + return 0; +} + +/** + * cpufreq_state2power() - convert a cpu cdev state to power consumed + * @cdev: &thermal_cooling_device pointer + * @tz: a valid thermal zone device pointer + * @state: cooling device state to be converted + * @power: pointer in which to store the resulting power + * + * Convert cooling device state @state into power consumption in + * milliwatts assuming 100% load. Store the calculated power in + * @power. + * + * Return: 0 on success, -EINVAL if the cooling device state could not + * be converted into a frequency or other -E* if there was an error + * when calculating the static power. + */ +static int cpufreq_state2power(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, + unsigned long state, u32 *power) +{ + unsigned int freq, num_cpus; + cpumask_t cpumask; + u32 static_power, dynamic_power; + int ret; + struct cpufreq_cooling_device *cpufreq_device = cdev->devdata; + + cpumask_and(&cpumask, &cpufreq_device->allowed_cpus, cpu_online_mask); + num_cpus = cpumask_weight(&cpumask); + + /* None of our cpus are online, so no power */ + if (num_cpus == 0) { + *power = 0; + return 0; + } + + freq = cpufreq_device->freq_table[state]; + if (!freq) + return -EINVAL; - return cpufreq_apply_cooling(cpufreq_device, state); + dynamic_power = cpu_freq_to_power(cpufreq_device, freq) * num_cpus; + ret = get_static_power(cpufreq_device, tz, freq, &static_power); + if (ret) + return ret; + + *power = static_power + dynamic_power; + return 0; +} + +/** + * cpufreq_power2state() - convert power to a cooling device state + * @cdev: &thermal_cooling_device pointer + * @tz: a valid thermal zone device pointer + * @power: power in milliwatts to be converted + * @state: pointer in which to store the resulting state + * + * Calculate a cooling device state for the cpus described by @cdev + * that would allow them to consume at most @power mW and store it in + * @state. Note that this calculation depends on external factors + * such as the cpu load or the current static power. Calling this + * function with the same power as input can yield different cooling + * device states depending on those external factors. + * + * Return: 0 on success, -ENODEV if no cpus are online or -EINVAL if + * the calculated frequency could not be converted to a valid state. + * The latter should not happen unless the frequencies available to + * cpufreq have changed since the initialization of the cpu cooling + * device. + */ +static int cpufreq_power2state(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, u32 power, + unsigned long *state) +{ + unsigned int cpu, cur_freq, target_freq; + int ret; + s32 dyn_power; + u32 last_load, normalised_power, static_power; + struct cpufreq_cooling_device *cpufreq_device = cdev->devdata; + + cpu = cpumask_any_and(&cpufreq_device->allowed_cpus, cpu_online_mask); + + /* None of our cpus are online */ + if (cpu >= nr_cpu_ids) + return -ENODEV; + + cur_freq = cpufreq_quick_get(cpu); + ret = get_static_power(cpufreq_device, tz, cur_freq, &static_power); + if (ret) + return ret; + + dyn_power = power - static_power; + dyn_power = dyn_power > 0 ? dyn_power : 0; + last_load = cpufreq_device->last_load ?: 1; + normalised_power = (dyn_power * 100) / last_load; + target_freq = cpu_power_to_freq(cpufreq_device, normalised_power); + + *state = cpufreq_cooling_get_level(cpu, target_freq); + if (*state == THERMAL_CSTATE_INVALID) { + dev_warn_ratelimited(&cdev->device, + "Failed to convert %dKHz for cpu %d into a cdev state\n", + target_freq, cpu); + return -EINVAL; + } + + trace_thermal_power_cpu_limit(&cpufreq_device->allowed_cpus, + target_freq, *state, power); + return 0; } /* Bind cpufreq callbacks to thermal cooling device ops */ -static struct thermal_cooling_device_ops const cpufreq_cooling_ops = { +static struct thermal_cooling_device_ops cpufreq_cooling_ops = { .get_max_state = cpufreq_get_max_state, .get_cur_state = cpufreq_get_cur_state, .set_cur_state = cpufreq_set_cur_state, @@ -419,10 +735,28 @@ static struct notifier_block thermal_cpufreq_notifier_block = { .notifier_call = cpufreq_thermal_notifier, }; +static unsigned int find_next_max(struct cpufreq_frequency_table *table, + unsigned int prev_max) +{ + struct cpufreq_frequency_table *pos; + unsigned int max = 0; + + cpufreq_for_each_valid_entry(pos, table) { + if (pos->frequency > max && pos->frequency < prev_max) + max = pos->frequency; + } + + return max; +} + /** * __cpufreq_cooling_register - helper function to create cpufreq cooling device * @np: a valid struct device_node to the cooling device device tree node * @clip_cpus: cpumask of cpus where the frequency constraints will happen. + * Normally this should be same as cpufreq policy->related_cpus. + * @capacitance: dynamic power coefficient for these cpus + * @plat_static_func: function to calculate the static power consumed by these + * cpus (optional) * * This interface function registers the cpufreq cooling device with the name * "thermal-cpufreq-%x". This api can support multiple instances of cpufreq @@ -434,40 +768,77 @@ static struct notifier_block thermal_cpufreq_notifier_block = { */ static struct thermal_cooling_device * __cpufreq_cooling_register(struct device_node *np, - const struct cpumask *clip_cpus) + const struct cpumask *clip_cpus, u32 capacitance, + get_static_t plat_static_func) { struct thermal_cooling_device *cool_dev; - struct cpufreq_cooling_device *cpufreq_dev = NULL; - unsigned int min = 0, max = 0; + struct cpufreq_cooling_device *cpufreq_dev; char dev_name[THERMAL_NAME_LENGTH]; - int ret = 0, i; - struct cpufreq_policy policy; + struct cpufreq_frequency_table *pos, *table; + unsigned int freq, i, num_cpus; + int ret; - /* Verify that all the clip cpus have same freq_min, freq_max limit */ - for_each_cpu(i, clip_cpus) { - /* continue if cpufreq policy not found and not return error */ - if (!cpufreq_get_policy(&policy, i)) - continue; - if (min == 0 && max == 0) { - min = policy.cpuinfo.min_freq; - max = policy.cpuinfo.max_freq; - } else { - if (min != policy.cpuinfo.min_freq || - max != policy.cpuinfo.max_freq) - return ERR_PTR(-EINVAL); - } + table = cpufreq_frequency_get_table(cpumask_first(clip_cpus)); + if (!table) { + pr_debug("%s: CPUFreq table not found\n", __func__); + return ERR_PTR(-EPROBE_DEFER); } - cpufreq_dev = kzalloc(sizeof(struct cpufreq_cooling_device), - GFP_KERNEL); + + cpufreq_dev = kzalloc(sizeof(*cpufreq_dev), GFP_KERNEL); if (!cpufreq_dev) return ERR_PTR(-ENOMEM); + num_cpus = cpumask_weight(clip_cpus); + cpufreq_dev->time_in_idle = kcalloc(num_cpus, + sizeof(*cpufreq_dev->time_in_idle), + GFP_KERNEL); + if (!cpufreq_dev->time_in_idle) { + cool_dev = ERR_PTR(-ENOMEM); + goto free_cdev; + } + + cpufreq_dev->time_in_idle_timestamp = + kcalloc(num_cpus, sizeof(*cpufreq_dev->time_in_idle_timestamp), + GFP_KERNEL); + if (!cpufreq_dev->time_in_idle_timestamp) { + cool_dev = ERR_PTR(-ENOMEM); + goto free_time_in_idle; + } + + /* Find max levels */ + cpufreq_for_each_valid_entry(pos, table) + cpufreq_dev->max_level++; + + cpufreq_dev->freq_table = kmalloc(sizeof(*cpufreq_dev->freq_table) * + cpufreq_dev->max_level, GFP_KERNEL); + if (!cpufreq_dev->freq_table) { + cool_dev = ERR_PTR(-ENOMEM); + goto free_time_in_idle_timestamp; + } + + /* max_level is an index, not a counter */ + cpufreq_dev->max_level--; + cpumask_copy(&cpufreq_dev->allowed_cpus, clip_cpus); + if (capacitance) { + cpufreq_cooling_ops.get_requested_power = + cpufreq_get_requested_power; + cpufreq_cooling_ops.state2power = cpufreq_state2power; + cpufreq_cooling_ops.power2state = cpufreq_power2state; + cpufreq_dev->plat_get_static_power = plat_static_func; + + ret = build_dyn_power_table(cpufreq_dev, capacitance); + if (ret) { + cool_dev = ERR_PTR(ret); + goto free_table; + } + } + ret = get_idr(&cpufreq_idr, &cpufreq_dev->id); if (ret) { - kfree(cpufreq_dev); - return ERR_PTR(-EINVAL); + cool_dev = ERR_PTR(ret); + goto free_table; } snprintf(dev_name, sizeof(dev_name), "thermal-cpufreq-%d", @@ -475,25 +846,48 @@ __cpufreq_cooling_register(struct device_node *np, cool_dev = thermal_of_cooling_device_register(np, dev_name, cpufreq_dev, &cpufreq_cooling_ops); - if (IS_ERR(cool_dev)) { - release_idr(&cpufreq_idr, cpufreq_dev->id); - kfree(cpufreq_dev); - return cool_dev; + if (IS_ERR(cool_dev)) + goto remove_idr; + + /* Fill freq-table in descending order of frequencies */ + for (i = 0, freq = -1; i <= cpufreq_dev->max_level; i++) { + freq = find_next_max(table, freq); + cpufreq_dev->freq_table[i] = freq; + + /* Warn for duplicate entries */ + if (!freq) + pr_warn("%s: table has duplicate entries\n", __func__); + else + pr_debug("%s: freq:%u KHz\n", __func__, freq); } + + cpufreq_dev->cpufreq_val = cpufreq_dev->freq_table[0]; cpufreq_dev->cool_dev = cool_dev; - cpufreq_dev->cpufreq_state = 0; + mutex_lock(&cooling_cpufreq_lock); /* Register the notifier for first cpufreq cooling device */ - if (cpufreq_dev_count == 0) + if (list_empty(&cpufreq_dev_list)) cpufreq_register_notifier(&thermal_cpufreq_notifier_block, CPUFREQ_POLICY_NOTIFIER); - cpufreq_dev_count++; list_add(&cpufreq_dev->node, &cpufreq_dev_list); mutex_unlock(&cooling_cpufreq_lock); return cool_dev; + +remove_idr: + release_idr(&cpufreq_idr, cpufreq_dev->id); +free_table: + kfree(cpufreq_dev->freq_table); +free_time_in_idle_timestamp: + kfree(cpufreq_dev->time_in_idle_timestamp); +free_time_in_idle: + kfree(cpufreq_dev->time_in_idle); +free_cdev: + kfree(cpufreq_dev); + + return cool_dev; } /** @@ -510,7 +904,7 @@ __cpufreq_cooling_register(struct device_node *np, struct thermal_cooling_device * cpufreq_cooling_register(const struct cpumask *clip_cpus) { - return __cpufreq_cooling_register(NULL, clip_cpus); + return __cpufreq_cooling_register(NULL, clip_cpus, 0, NULL); } EXPORT_SYMBOL_GPL(cpufreq_cooling_register); @@ -534,11 +928,78 @@ of_cpufreq_cooling_register(struct device_node *np, if (!np) return ERR_PTR(-EINVAL); - return __cpufreq_cooling_register(np, clip_cpus); + return __cpufreq_cooling_register(np, clip_cpus, 0, NULL); } EXPORT_SYMBOL_GPL(of_cpufreq_cooling_register); /** + * cpufreq_power_cooling_register() - create cpufreq cooling device with power extensions + * @clip_cpus: cpumask of cpus where the frequency constraints will happen + * @capacitance: dynamic power coefficient for these cpus + * @plat_static_func: function to calculate the static power consumed by these + * cpus (optional) + * + * This interface function registers the cpufreq cooling device with + * the name "thermal-cpufreq-%x". This api can support multiple + * instances of cpufreq cooling devices. Using this function, the + * cooling device will implement the power extensions by using a + * simple cpu power model. The cpus must have registered their OPPs + * using the OPP library. + * + * An optional @plat_static_func may be provided to calculate the + * static power consumed by these cpus. If the platform's static + * power consumption is unknown or negligible, make it NULL. + * + * Return: a valid struct thermal_cooling_device pointer on success, + * on failure, it returns a corresponding ERR_PTR(). + */ +struct thermal_cooling_device * +cpufreq_power_cooling_register(const struct cpumask *clip_cpus, u32 capacitance, + get_static_t plat_static_func) +{ + return __cpufreq_cooling_register(NULL, clip_cpus, capacitance, + plat_static_func); +} +EXPORT_SYMBOL(cpufreq_power_cooling_register); + +/** + * of_cpufreq_power_cooling_register() - create cpufreq cooling device with power extensions + * @np: a valid struct device_node to the cooling device device tree node + * @clip_cpus: cpumask of cpus where the frequency constraints will happen + * @capacitance: dynamic power coefficient for these cpus + * @plat_static_func: function to calculate the static power consumed by these + * cpus (optional) + * + * This interface function registers the cpufreq cooling device with + * the name "thermal-cpufreq-%x". This api can support multiple + * instances of cpufreq cooling devices. Using this API, the cpufreq + * cooling device will be linked to the device tree node provided. + * Using this function, the cooling device will implement the power + * extensions by using a simple cpu power model. The cpus must have + * registered their OPPs using the OPP library. + * + * An optional @plat_static_func may be provided to calculate the + * static power consumed by these cpus. If the platform's static + * power consumption is unknown or negligible, make it NULL. + * + * Return: a valid struct thermal_cooling_device pointer on success, + * on failure, it returns a corresponding ERR_PTR(). + */ +struct thermal_cooling_device * +of_cpufreq_power_cooling_register(struct device_node *np, + const struct cpumask *clip_cpus, + u32 capacitance, + get_static_t plat_static_func) +{ + if (!np) + return ERR_PTR(-EINVAL); + + return __cpufreq_cooling_register(np, clip_cpus, capacitance, + plat_static_func); +} +EXPORT_SYMBOL(of_cpufreq_power_cooling_register); + +/** * cpufreq_cooling_unregister - function to remove cpufreq cooling device. * @cdev: thermal cooling device pointer. * @@ -554,16 +1015,18 @@ void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev) cpufreq_dev = cdev->devdata; mutex_lock(&cooling_cpufreq_lock); list_del(&cpufreq_dev->node); - cpufreq_dev_count--; /* Unregister the notifier for the last cpufreq cooling device */ - if (cpufreq_dev_count == 0) + if (list_empty(&cpufreq_dev_list)) cpufreq_unregister_notifier(&thermal_cpufreq_notifier_block, CPUFREQ_POLICY_NOTIFIER); mutex_unlock(&cooling_cpufreq_lock); thermal_cooling_device_unregister(cpufreq_dev->cool_dev); release_idr(&cpufreq_idr, cpufreq_dev->id); + kfree(cpufreq_dev->time_in_idle_timestamp); + kfree(cpufreq_dev->time_in_idle); + kfree(cpufreq_dev->freq_table); kfree(cpufreq_dev); } EXPORT_SYMBOL_GPL(cpufreq_cooling_unregister); diff --git a/drivers/thermal/db8500_cpufreq_cooling.c b/drivers/thermal/db8500_cpufreq_cooling.c index 786d19263ab0..1ac7ec651c3f 100644 --- a/drivers/thermal/db8500_cpufreq_cooling.c +++ b/drivers/thermal/db8500_cpufreq_cooling.c @@ -18,7 +18,6 @@ */ #include <linux/cpu_cooling.h> -#include <linux/cpufreq.h> #include <linux/err.h> #include <linux/module.h> #include <linux/of.h> @@ -30,10 +29,6 @@ static int db8500_cpufreq_cooling_probe(struct platform_device *pdev) struct thermal_cooling_device *cdev; struct cpumask mask_val; - /* make sure cpufreq driver has been initialized */ - if (!cpufreq_frequency_get_table(0)) - return -EPROBE_DEFER; - cpumask_set_cpu(0, &mask_val); cdev = cpufreq_cooling_register(&mask_val); diff --git a/drivers/thermal/db8500_thermal.c b/drivers/thermal/db8500_thermal.c index 1e3b3bf9f993..e3ccc2218eb3 100644 --- a/drivers/thermal/db8500_thermal.c +++ b/drivers/thermal/db8500_thermal.c @@ -76,7 +76,7 @@ static int db8500_cdev_bind(struct thermal_zone_device *thermal, upper = lower = i > max_state ? max_state : i; ret = thermal_zone_bind_cooling_device(thermal, i, cdev, - upper, lower); + upper, lower, THERMAL_WEIGHT_DEFAULT); dev_info(&cdev->device, "%s bind to %d: %d-%s\n", cdev->type, i, ret, ret ? "fail" : "succeed"); diff --git a/drivers/thermal/fair_share.c b/drivers/thermal/fair_share.c index 6e0a3fbfae86..8c50b8d6afb7 100644 --- a/drivers/thermal/fair_share.c +++ b/drivers/thermal/fair_share.c @@ -59,13 +59,13 @@ static int get_trip_level(struct thermal_zone_device *tz) } static long get_target_state(struct thermal_zone_device *tz, - struct thermal_cooling_device *cdev, int weight, int level) + struct thermal_cooling_device *cdev, int percentage, int level) { unsigned long max_state; cdev->ops->get_max_state(cdev, &max_state); - return (long)(weight * level * max_state) / (100 * tz->trips); + return (long)(percentage * level * max_state) / (100 * tz->trips); } /** @@ -77,7 +77,7 @@ static long get_target_state(struct thermal_zone_device *tz, * * Parameters used for Throttling: * P1. max_state: Maximum throttle state exposed by the cooling device. - * P2. weight[i]/100: + * P2. percentage[i]/100: * How 'effective' the 'i'th device is, in cooling the given zone. * P3. cur_trip_level/max_no_of_trips: * This describes the extent to which the devices should be throttled. @@ -88,28 +88,33 @@ static long get_target_state(struct thermal_zone_device *tz, */ static int fair_share_throttle(struct thermal_zone_device *tz, int trip) { - const struct thermal_zone_params *tzp; - struct thermal_cooling_device *cdev; struct thermal_instance *instance; - int i; + int total_weight = 0; + int total_instance = 0; int cur_trip_level = get_trip_level(tz); - if (!tz->tzp || !tz->tzp->tbp) - return -EINVAL; + list_for_each_entry(instance, &tz->thermal_instances, tz_node) { + if (instance->trip != trip) + continue; + + total_weight += instance->weight; + total_instance++; + } - tzp = tz->tzp; + list_for_each_entry(instance, &tz->thermal_instances, tz_node) { + int percentage; + struct thermal_cooling_device *cdev = instance->cdev; - for (i = 0; i < tzp->num_tbps; i++) { - if (!tzp->tbp[i].cdev) + if (instance->trip != trip) continue; - cdev = tzp->tbp[i].cdev; - instance = get_thermal_instance(tz, cdev, trip); - if (!instance) - continue; + if (!total_weight) + percentage = 100 / total_instance; + else + percentage = (instance->weight * 100) / total_weight; - instance->target = get_target_state(tz, cdev, - tzp->tbp[i].weight, cur_trip_level); + instance->target = get_target_state(tz, cdev, percentage, + cur_trip_level); instance->cdev->updated = false; thermal_cdev_update(cdev); diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 5a1f1070b702..31ada06c7dc6 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -9,7 +9,6 @@ #include <linux/clk.h> #include <linux/cpu_cooling.h> -#include <linux/cpufreq.h> #include <linux/delay.h> #include <linux/device.h> #include <linux/init.h> @@ -307,7 +306,8 @@ static int imx_bind(struct thermal_zone_device *tz, ret = thermal_zone_bind_cooling_device(tz, IMX_TRIP_PASSIVE, cdev, THERMAL_NO_LIMIT, - THERMAL_NO_LIMIT); + THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT); if (ret) { dev_err(&tz->device, "binding zone %s with cdev %s failed:%d\n", @@ -459,10 +459,6 @@ static int imx_thermal_probe(struct platform_device *pdev) int measure_freq; int ret; - if (!cpufreq_get_current_driver()) { - dev_dbg(&pdev->dev, "no cpufreq driver!"); - return -EPROBE_DEFER; - } data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; diff --git a/drivers/thermal/int340x_thermal/int3403_thermal.c b/drivers/thermal/int340x_thermal/int3403_thermal.c index 6e9fb62eb817..76910d35eecb 100644 --- a/drivers/thermal/int340x_thermal/int3403_thermal.c +++ b/drivers/thermal/int340x_thermal/int3403_thermal.c @@ -471,7 +471,6 @@ static struct platform_driver int3403_driver = { .remove = int3403_remove, .driver = { .name = "int3403 thermal", - .owner = THIS_MODULE, .acpi_match_table = int3403_device_ids, }, }; diff --git a/drivers/thermal/of-thermal.c b/drivers/thermal/of-thermal.c index 62143ba31001..b295b2b6c191 100644 --- a/drivers/thermal/of-thermal.c +++ b/drivers/thermal/of-thermal.c @@ -30,27 +30,13 @@ #include <linux/err.h> #include <linux/export.h> #include <linux/string.h> +#include <linux/thermal.h> #include "thermal_core.h" /*** Private data structures to represent thermal device tree data ***/ /** - * struct __thermal_trip - representation of a point in temperature domain - * @np: pointer to struct device_node that this trip point was created from - * @temperature: temperature value in miliCelsius - * @hysteresis: relative hysteresis in miliCelsius - * @type: trip point type - */ - -struct __thermal_trip { - struct device_node *np; - unsigned long int temperature; - unsigned long int hysteresis; - enum thermal_trip_type type; -}; - -/** * struct __thermal_bind_param - a match between trip and cooling device * @cooling_device: a pointer to identify the referred cooling device * @trip_id: the trip point index @@ -72,23 +58,26 @@ struct __thermal_bind_params { * @mode: current thermal zone device mode (enabled/disabled) * @passive_delay: polling interval while passive cooling is activated * @polling_delay: zone polling interval + * @slope: slope of the temperature adjustment curve + * @offset: offset of the temperature adjustment curve * @ntrips: number of trip points * @trips: an array of trip points (0..ntrips - 1) * @num_tbps: number of thermal bind params * @tbps: an array of thermal bind params (0..num_tbps - 1) * @sensor_data: sensor private data used while reading temperature and trend - * @get_temp: sensor callback to read temperature - * @get_trend: sensor callback to read temperature trend + * @ops: set of callbacks to handle the thermal zone based on DT */ struct __thermal_zone { enum thermal_device_mode mode; int passive_delay; int polling_delay; + int slope; + int offset; /* trip data */ int ntrips; - struct __thermal_trip *trips; + struct thermal_trip *trips; /* cooling binding data */ int num_tbps; @@ -96,8 +85,7 @@ struct __thermal_zone { /* sensor interface */ void *sensor_data; - int (*get_temp)(void *, long *); - int (*get_trend)(void *, long *); + const struct thermal_zone_of_device_ops *ops; }; /*** DT thermal zone device callbacks ***/ @@ -107,10 +95,96 @@ static int of_thermal_get_temp(struct thermal_zone_device *tz, { struct __thermal_zone *data = tz->devdata; - if (!data->get_temp) + if (!data->ops->get_temp) return -EINVAL; - return data->get_temp(data->sensor_data, temp); + return data->ops->get_temp(data->sensor_data, temp); +} + +/** + * of_thermal_get_ntrips - function to export number of available trip + * points. + * @tz: pointer to a thermal zone + * + * This function is a globally visible wrapper to get number of trip points + * stored in the local struct __thermal_zone + * + * Return: number of available trip points, -ENODEV when data not available + */ +int of_thermal_get_ntrips(struct thermal_zone_device *tz) +{ + struct __thermal_zone *data = tz->devdata; + + if (!data || IS_ERR(data)) + return -ENODEV; + + return data->ntrips; +} +EXPORT_SYMBOL_GPL(of_thermal_get_ntrips); + +/** + * of_thermal_is_trip_valid - function to check if trip point is valid + * + * @tz: pointer to a thermal zone + * @trip: trip point to evaluate + * + * This function is responsible for checking if passed trip point is valid + * + * Return: true if trip point is valid, false otherwise + */ +bool of_thermal_is_trip_valid(struct thermal_zone_device *tz, int trip) +{ + struct __thermal_zone *data = tz->devdata; + + if (!data || trip >= data->ntrips || trip < 0) + return false; + + return true; +} +EXPORT_SYMBOL_GPL(of_thermal_is_trip_valid); + +/** + * of_thermal_get_trip_points - function to get access to a globally exported + * trip points + * + * @tz: pointer to a thermal zone + * + * This function provides a pointer to trip points table + * + * Return: pointer to trip points table, NULL otherwise + */ +const struct thermal_trip * +of_thermal_get_trip_points(struct thermal_zone_device *tz) +{ + struct __thermal_zone *data = tz->devdata; + + if (!data) + return NULL; + + return data->trips; +} +EXPORT_SYMBOL_GPL(of_thermal_get_trip_points); + +/** + * of_thermal_set_emul_temp - function to set emulated temperature + * + * @tz: pointer to a thermal zone + * @temp: temperature to set + * + * This function gives the ability to set emulated value of temperature, + * which is handy for debugging + * + * Return: zero on success, error code otherwise + */ +static int of_thermal_set_emul_temp(struct thermal_zone_device *tz, + unsigned long temp) +{ + struct __thermal_zone *data = tz->devdata; + + if (!data->ops || !data->ops->set_emul_temp) + return -EINVAL; + + return data->ops->set_emul_temp(data->sensor_data, temp); } static int of_thermal_get_trend(struct thermal_zone_device *tz, int trip, @@ -120,10 +194,10 @@ static int of_thermal_get_trend(struct thermal_zone_device *tz, int trip, long dev_trend; int r; - if (!data->get_trend) + if (!data->ops->get_trend) return -EINVAL; - r = data->get_trend(data->sensor_data, &dev_trend); + r = data->ops->get_trend(data->sensor_data, &dev_trend); if (r) return r; @@ -157,7 +231,8 @@ static int of_thermal_bind(struct thermal_zone_device *thermal, ret = thermal_zone_bind_cooling_device(thermal, tbp->trip_id, cdev, tbp->max, - tbp->min); + tbp->min, + tbp->usage); if (ret) return ret; } @@ -324,8 +399,7 @@ static struct thermal_zone_device_ops of_thermal_ops = { static struct thermal_zone_device * thermal_zone_of_add_sensor(struct device_node *zone, struct device_node *sensor, void *data, - int (*get_temp)(void *, long *), - int (*get_trend)(void *, long *)) + const struct thermal_zone_of_device_ops *ops) { struct thermal_zone_device *tzd; struct __thermal_zone *tz; @@ -336,13 +410,16 @@ thermal_zone_of_add_sensor(struct device_node *zone, tz = tzd->devdata; + if (!ops) + return ERR_PTR(-EINVAL); + mutex_lock(&tzd->lock); - tz->get_temp = get_temp; - tz->get_trend = get_trend; + tz->ops = ops; tz->sensor_data = data; tzd->ops->get_temp = of_thermal_get_temp; tzd->ops->get_trend = of_thermal_get_trend; + tzd->ops->set_emul_temp = of_thermal_set_emul_temp; mutex_unlock(&tzd->lock); return tzd; @@ -356,8 +433,7 @@ thermal_zone_of_add_sensor(struct device_node *zone, * than one sensors * @data: a private pointer (owned by the caller) that will be passed * back, when a temperature reading is needed. - * @get_temp: a pointer to a function that reads the sensor temperature. - * @get_trend: a pointer to a function that reads the sensor temperature trend. + * @ops: struct thermal_zone_of_device_ops *. Must contain at least .get_temp. * * This function will search the list of thermal zones described in device * tree and look for the zone that refer to the sensor device pointed by @@ -382,9 +458,8 @@ thermal_zone_of_add_sensor(struct device_node *zone, * check the return value with help of IS_ERR() helper. */ struct thermal_zone_device * -thermal_zone_of_sensor_register(struct device *dev, int sensor_id, - void *data, int (*get_temp)(void *, long *), - int (*get_trend)(void *, long *)) +thermal_zone_of_sensor_register(struct device *dev, int sensor_id, void *data, + const struct thermal_zone_of_device_ops *ops) { struct device_node *np, *child, *sensor_np; struct thermal_zone_device *tzd = ERR_PTR(-ENODEV); @@ -426,9 +501,10 @@ thermal_zone_of_sensor_register(struct device *dev, int sensor_id, if (sensor_specs.np == sensor_np && id == sensor_id) { tzd = thermal_zone_of_add_sensor(child, sensor_np, - data, - get_temp, - get_trend); + data, ops); + if (!IS_ERR(tzd)) + tzd->ops->set_mode(tzd, THERMAL_DEVICE_ENABLED); + of_node_put(sensor_specs.np); of_node_put(child); goto exit; @@ -475,9 +551,9 @@ void thermal_zone_of_sensor_unregister(struct device *dev, mutex_lock(&tzd->lock); tzd->ops->get_temp = NULL; tzd->ops->get_trend = NULL; + tzd->ops->set_emul_temp = NULL; - tz->get_temp = NULL; - tz->get_trend = NULL; + tz->ops = NULL; tz->sensor_data = NULL; mutex_unlock(&tzd->lock); } @@ -501,7 +577,7 @@ EXPORT_SYMBOL_GPL(thermal_zone_of_sensor_unregister); */ static int thermal_of_populate_bind_params(struct device_node *np, struct __thermal_bind_params *__tbp, - struct __thermal_trip *trips, + struct thermal_trip *trips, int ntrips) { struct of_phandle_args cooling_spec; @@ -510,7 +586,7 @@ static int thermal_of_populate_bind_params(struct device_node *np, u32 prop; /* Default weight. Usage is optional */ - __tbp->usage = 0; + __tbp->usage = THERMAL_WEIGHT_DEFAULT; ret = of_property_read_u32(np, "contribution", &prop); if (ret == 0) __tbp->usage = prop; @@ -604,7 +680,7 @@ static int thermal_of_get_trip_type(struct device_node *np, * Return: 0 on success, proper error code otherwise */ static int thermal_of_populate_trip(struct device_node *np, - struct __thermal_trip *trip) + struct thermal_trip *trip) { int prop; int ret; @@ -644,7 +720,7 @@ static int thermal_of_populate_trip(struct device_node *np, * @np parameter and fills the read data into a __thermal_zone data structure * and return this pointer. * - * TODO: Missing properties to parse: thermal-sensor-names and coefficients + * TODO: Missing properties to parse: thermal-sensor-names * * Return: On success returns a valid struct __thermal_zone, * otherwise, it returns a corresponding ERR_PTR(). Caller must @@ -656,7 +732,7 @@ thermal_of_build_thermal_zone(struct device_node *np) struct device_node *child = NULL, *gchild; struct __thermal_zone *tz; int ret, i; - u32 prop; + u32 prop, coef[2]; if (!np) { pr_err("no thermal zone np\n"); @@ -681,6 +757,20 @@ thermal_of_build_thermal_zone(struct device_node *np) } tz->polling_delay = prop; + /* + * REVIST: for now, the thermal framework supports only + * one sensor per thermal zone. Thus, we are considering + * only the first two values as slope and offset. + */ + ret = of_property_read_u32_array(np, "coefficients", coef, 2); + if (ret == 0) { + tz->slope = coef[0]; + tz->offset = coef[1]; + } else { + tz->slope = 1; + tz->offset = 0; + } + /* trips */ child = of_get_child_by_name(np, "trips"); @@ -794,6 +884,8 @@ int __init of_parse_thermal_zones(void) for_each_child_of_node(np, child) { struct thermal_zone_device *zone; struct thermal_zone_params *tzp; + int i, mask = 0; + u32 prop; /* Check whether child is enabled or not */ if (!of_device_is_available(child)) @@ -820,8 +912,18 @@ int __init of_parse_thermal_zones(void) /* No hwmon because there might be hwmon drivers registering */ tzp->no_hwmon = true; + if (!of_property_read_u32(child, "sustainable-power", &prop)) + tzp->sustainable_power = prop; + + for (i = 0; i < tz->ntrips; i++) + mask |= 1 << i; + + /* these two are left for temperature drivers to use */ + tzp->slope = tz->slope; + tzp->offset = tz->offset; + zone = thermal_zone_device_register(child->name, tz->ntrips, - 0, tz, + mask, tz, ops, tzp, tz->passive_delay, tz->polling_delay); diff --git a/drivers/thermal/power_allocator.c b/drivers/thermal/power_allocator.c new file mode 100644 index 000000000000..251676902869 --- /dev/null +++ b/drivers/thermal/power_allocator.c @@ -0,0 +1,544 @@ +/* + * A power allocator to manage temperature + * + * Copyright (C) 2014 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#define pr_fmt(fmt) "Power allocator: " fmt + +#include <linux/rculist.h> +#include <linux/slab.h> +#include <linux/thermal.h> + +#define CREATE_TRACE_POINTS +#include <trace/events/thermal_power_allocator.h> + +#include "thermal_core.h" + +#define FRAC_BITS 10 +#define int_to_frac(x) ((x) << FRAC_BITS) +#define frac_to_int(x) ((x) >> FRAC_BITS) + +/** + * mul_frac() - multiply two fixed-point numbers + * @x: first multiplicand + * @y: second multiplicand + * + * Return: the result of multiplying two fixed-point numbers. The + * result is also a fixed-point number. + */ +static inline s64 mul_frac(s64 x, s64 y) +{ + return (x * y) >> FRAC_BITS; +} + +/** + * div_frac() - divide two fixed-point numbers + * @x: the dividend + * @y: the divisor + * + * Return: the result of dividing two fixed-point numbers. The + * result is also a fixed-point number. + */ +static inline s64 div_frac(s64 x, s64 y) +{ + return div_s64(x << FRAC_BITS, y); +} + +/** + * struct power_allocator_params - parameters for the power allocator governor + * @err_integral: accumulated error in the PID controller. + * @prev_err: error in the previous iteration of the PID controller. + * Used to calculate the derivative term. + * @trip_switch_on: first passive trip point of the thermal zone. The + * governor switches on when this trip point is crossed. + * @trip_max_desired_temperature: last passive trip point of the thermal + * zone. The temperature we are + * controlling for. + */ +struct power_allocator_params { + s64 err_integral; + s32 prev_err; + int trip_switch_on; + int trip_max_desired_temperature; +}; + +/** + * pid_controller() - PID controller + * @tz: thermal zone we are operating in + * @current_temp: the current temperature in millicelsius + * @control_temp: the target temperature in millicelsius + * @max_allocatable_power: maximum allocatable power for this thermal zone + * + * This PID controller increases the available power budget so that the + * temperature of the thermal zone gets as close as possible to + * @control_temp and limits the power if it exceeds it. k_po is the + * proportional term when we are overshooting, k_pu is the + * proportional term when we are undershooting. integral_cutoff is a + * threshold below which we stop accumulating the error. The + * accumulated error is only valid if the requested power will make + * the system warmer. If the system is mostly idle, there's no point + * in accumulating positive error. + * + * Return: The power budget for the next period. + */ +static u32 pid_controller(struct thermal_zone_device *tz, + unsigned long current_temp, + unsigned long control_temp, + u32 max_allocatable_power) +{ + s64 p, i, d, power_range; + s32 err, max_power_frac; + struct power_allocator_params *params = tz->governor_data; + + max_power_frac = int_to_frac(max_allocatable_power); + + err = ((s32)control_temp - (s32)current_temp); + err = int_to_frac(err); + + /* Calculate the proportional term */ + p = mul_frac(err < 0 ? tz->tzp->k_po : tz->tzp->k_pu, err); + + /* + * Calculate the integral term + * + * if the error is less than cut off allow integration (but + * the integral is limited to max power) + */ + i = mul_frac(tz->tzp->k_i, params->err_integral); + + if (err < int_to_frac(tz->tzp->integral_cutoff)) { + s64 i_next = i + mul_frac(tz->tzp->k_i, err); + + if (abs64(i_next) < max_power_frac) { + i = i_next; + params->err_integral += err; + } + } + + /* + * Calculate the derivative term + * + * We do err - prev_err, so with a positive k_d, a decreasing + * error (i.e. driving closer to the line) results in less + * power being applied, slowing down the controller) + */ + d = mul_frac(tz->tzp->k_d, err - params->prev_err); + d = div_frac(d, tz->passive_delay); + params->prev_err = err; + + power_range = p + i + d; + + /* feed-forward the known sustainable dissipatable power */ + power_range = tz->tzp->sustainable_power + frac_to_int(power_range); + + power_range = clamp(power_range, (s64)0, (s64)max_allocatable_power); + + trace_thermal_power_allocator_pid(tz, frac_to_int(err), + frac_to_int(params->err_integral), + frac_to_int(p), frac_to_int(i), + frac_to_int(d), power_range); + + return power_range; +} + +/** + * divvy_up_power() - divvy the allocated power between the actors + * @req_power: each actor's requested power + * @max_power: each actor's maximum available power + * @num_actors: size of the @req_power, @max_power and @granted_power's array + * @total_req_power: sum of @req_power + * @power_range: total allocated power + * @granted_power: output array: each actor's granted power + * @extra_actor_power: an appropriately sized array to be used in the + * function as temporary storage of the extra power given + * to the actors + * + * This function divides the total allocated power (@power_range) + * fairly between the actors. It first tries to give each actor a + * share of the @power_range according to how much power it requested + * compared to the rest of the actors. For example, if only one actor + * requests power, then it receives all the @power_range. If + * three actors each requests 1mW, each receives a third of the + * @power_range. + * + * If any actor received more than their maximum power, then that + * surplus is re-divvied among the actors based on how far they are + * from their respective maximums. + * + * Granted power for each actor is written to @granted_power, which + * should've been allocated by the calling function. + */ +static void divvy_up_power(u32 *req_power, u32 *max_power, int num_actors, + u32 total_req_power, u32 power_range, + u32 *granted_power, u32 *extra_actor_power) +{ + u32 extra_power, capped_extra_power; + int i; + + /* + * Prevent division by 0 if none of the actors request power. + */ + if (!total_req_power) + total_req_power = 1; + + capped_extra_power = 0; + extra_power = 0; + for (i = 0; i < num_actors; i++) { + u64 req_range = req_power[i] * power_range; + + granted_power[i] = DIV_ROUND_CLOSEST_ULL(req_range, + total_req_power); + + if (granted_power[i] > max_power[i]) { + extra_power += granted_power[i] - max_power[i]; + granted_power[i] = max_power[i]; + } + + extra_actor_power[i] = max_power[i] - granted_power[i]; + capped_extra_power += extra_actor_power[i]; + } + + if (!extra_power) + return; + + /* + * Re-divvy the reclaimed extra among actors based on + * how far they are from the max + */ + extra_power = min(extra_power, capped_extra_power); + if (capped_extra_power > 0) + for (i = 0; i < num_actors; i++) + granted_power[i] += (extra_actor_power[i] * + extra_power) / capped_extra_power; +} + +static int allocate_power(struct thermal_zone_device *tz, + unsigned long current_temp, + unsigned long control_temp) +{ + struct thermal_instance *instance; + struct power_allocator_params *params = tz->governor_data; + u32 *req_power, *max_power, *granted_power, *extra_actor_power; + u32 *weighted_req_power; + u32 total_req_power, max_allocatable_power, total_weighted_req_power; + u32 total_granted_power, power_range; + int i, num_actors, total_weight, ret = 0; + int trip_max_desired_temperature = params->trip_max_desired_temperature; + + mutex_lock(&tz->lock); + + num_actors = 0; + total_weight = 0; + list_for_each_entry(instance, &tz->thermal_instances, tz_node) { + if ((instance->trip == trip_max_desired_temperature) && + cdev_is_power_actor(instance->cdev)) { + num_actors++; + total_weight += instance->weight; + } + } + + /* + * We need to allocate five arrays of the same size: + * req_power, max_power, granted_power, extra_actor_power and + * weighted_req_power. They are going to be needed until this + * function returns. Allocate them all in one go to simplify + * the allocation and deallocation logic. + */ + BUILD_BUG_ON(sizeof(*req_power) != sizeof(*max_power)); + BUILD_BUG_ON(sizeof(*req_power) != sizeof(*granted_power)); + BUILD_BUG_ON(sizeof(*req_power) != sizeof(*extra_actor_power)); + BUILD_BUG_ON(sizeof(*req_power) != sizeof(*weighted_req_power)); + req_power = kcalloc(num_actors * 5, sizeof(*req_power), GFP_KERNEL); + if (!req_power) { + ret = -ENOMEM; + goto unlock; + } + + max_power = &req_power[num_actors]; + granted_power = &req_power[2 * num_actors]; + extra_actor_power = &req_power[3 * num_actors]; + weighted_req_power = &req_power[4 * num_actors]; + + i = 0; + total_weighted_req_power = 0; + total_req_power = 0; + max_allocatable_power = 0; + + list_for_each_entry(instance, &tz->thermal_instances, tz_node) { + int weight; + struct thermal_cooling_device *cdev = instance->cdev; + + if (instance->trip != trip_max_desired_temperature) + continue; + + if (!cdev_is_power_actor(cdev)) + continue; + + if (cdev->ops->get_requested_power(cdev, tz, &req_power[i])) + continue; + + if (!total_weight) + weight = 1 << FRAC_BITS; + else + weight = instance->weight; + + weighted_req_power[i] = frac_to_int(weight * req_power[i]); + + if (power_actor_get_max_power(cdev, tz, &max_power[i])) + continue; + + total_req_power += req_power[i]; + max_allocatable_power += max_power[i]; + total_weighted_req_power += weighted_req_power[i]; + + i++; + } + + power_range = pid_controller(tz, current_temp, control_temp, + max_allocatable_power); + + divvy_up_power(weighted_req_power, max_power, num_actors, + total_weighted_req_power, power_range, granted_power, + extra_actor_power); + + total_granted_power = 0; + i = 0; + list_for_each_entry(instance, &tz->thermal_instances, tz_node) { + if (instance->trip != trip_max_desired_temperature) + continue; + + if (!cdev_is_power_actor(instance->cdev)) + continue; + + power_actor_set_power(instance->cdev, instance, + granted_power[i]); + total_granted_power += granted_power[i]; + + i++; + } + + trace_thermal_power_allocator(tz, req_power, total_req_power, + granted_power, total_granted_power, + num_actors, power_range, + max_allocatable_power, current_temp, + (s32)control_temp - (s32)current_temp); + + kfree(req_power); +unlock: + mutex_unlock(&tz->lock); + + return ret; +} + +static int get_governor_trips(struct thermal_zone_device *tz, + struct power_allocator_params *params) +{ + int i, ret, last_passive; + bool found_first_passive; + + found_first_passive = false; + last_passive = -1; + ret = -EINVAL; + + for (i = 0; i < tz->trips; i++) { + enum thermal_trip_type type; + + ret = tz->ops->get_trip_type(tz, i, &type); + if (ret) + return ret; + + if (!found_first_passive) { + if (type == THERMAL_TRIP_PASSIVE) { + params->trip_switch_on = i; + found_first_passive = true; + } + } else if (type == THERMAL_TRIP_PASSIVE) { + last_passive = i; + } else { + break; + } + } + + if (last_passive != -1) { + params->trip_max_desired_temperature = last_passive; + ret = 0; + } else { + ret = -EINVAL; + } + + return ret; +} + +static void reset_pid_controller(struct power_allocator_params *params) +{ + params->err_integral = 0; + params->prev_err = 0; +} + +static void allow_maximum_power(struct thermal_zone_device *tz) +{ + struct thermal_instance *instance; + struct power_allocator_params *params = tz->governor_data; + + list_for_each_entry(instance, &tz->thermal_instances, tz_node) { + if ((instance->trip != params->trip_max_desired_temperature) || + (!cdev_is_power_actor(instance->cdev))) + continue; + + instance->target = 0; + instance->cdev->updated = false; + thermal_cdev_update(instance->cdev); + } +} + +/** + * power_allocator_bind() - bind the power_allocator governor to a thermal zone + * @tz: thermal zone to bind it to + * + * Check that the thermal zone is valid for this governor, that is, it + * has two thermal trips. If so, initialize the PID controller + * parameters and bind it to the thermal zone. + * + * Return: 0 on success, -EINVAL if the trips were invalid or -ENOMEM + * if we ran out of memory. + */ +static int power_allocator_bind(struct thermal_zone_device *tz) +{ + int ret; + struct power_allocator_params *params; + unsigned long switch_on_temp, control_temp; + u32 temperature_threshold; + + if (!tz->tzp || !tz->tzp->sustainable_power) { + dev_err(&tz->device, + "power_allocator: missing sustainable_power\n"); + return -EINVAL; + } + + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) + return -ENOMEM; + + ret = get_governor_trips(tz, params); + if (ret) { + dev_err(&tz->device, + "thermal zone %s has wrong trip setup for power allocator\n", + tz->type); + goto free; + } + + ret = tz->ops->get_trip_temp(tz, params->trip_switch_on, + &switch_on_temp); + if (ret) + goto free; + + ret = tz->ops->get_trip_temp(tz, params->trip_max_desired_temperature, + &control_temp); + if (ret) + goto free; + + temperature_threshold = control_temp - switch_on_temp; + + tz->tzp->k_po = tz->tzp->k_po ?: + int_to_frac(tz->tzp->sustainable_power) / temperature_threshold; + tz->tzp->k_pu = tz->tzp->k_pu ?: + int_to_frac(2 * tz->tzp->sustainable_power) / + temperature_threshold; + tz->tzp->k_i = tz->tzp->k_i ?: int_to_frac(10) / 1000; + /* + * The default for k_d and integral_cutoff is 0, so we can + * leave them as they are. + */ + + reset_pid_controller(params); + + tz->governor_data = params; + + return 0; + +free: + kfree(params); + return ret; +} + +static void power_allocator_unbind(struct thermal_zone_device *tz) +{ + dev_dbg(&tz->device, "Unbinding from thermal zone %d\n", tz->id); + kfree(tz->governor_data); + tz->governor_data = NULL; +} + +static int power_allocator_throttle(struct thermal_zone_device *tz, int trip) +{ + int ret; + unsigned long switch_on_temp, control_temp, current_temp; + struct power_allocator_params *params = tz->governor_data; + + /* + * We get called for every trip point but we only need to do + * our calculations once + */ + if (trip != params->trip_max_desired_temperature) + return 0; + + ret = thermal_zone_get_temp(tz, ¤t_temp); + if (ret) { + dev_warn(&tz->device, "Failed to get temperature: %d\n", ret); + return ret; + } + + ret = tz->ops->get_trip_temp(tz, params->trip_switch_on, + &switch_on_temp); + if (ret) { + dev_warn(&tz->device, + "Failed to get switch on temperature: %d\n", ret); + return ret; + } + + if (current_temp < switch_on_temp) { + tz->passive = 0; + reset_pid_controller(params); + allow_maximum_power(tz); + return 0; + } + + tz->passive = 1; + + ret = tz->ops->get_trip_temp(tz, params->trip_max_desired_temperature, + &control_temp); + if (ret) { + dev_warn(&tz->device, + "Failed to get the maximum desired temperature: %d\n", + ret); + return ret; + } + + return allocate_power(tz, current_temp, control_temp); +} + +static struct thermal_governor thermal_gov_power_allocator = { + .name = "power_allocator", + .bind_to_tz = power_allocator_bind, + .unbind_from_tz = power_allocator_unbind, + .throttle = power_allocator_throttle, +}; + +int thermal_gov_power_allocator_register(void) +{ + return thermal_register_governor(&thermal_gov_power_allocator); +} + +void thermal_gov_power_allocator_unregister(void) +{ + thermal_unregister_governor(&thermal_gov_power_allocator); +} diff --git a/drivers/thermal/samsung/Kconfig b/drivers/thermal/samsung/Kconfig index f760389a204c..c43306ecc0ab 100644 --- a/drivers/thermal/samsung/Kconfig +++ b/drivers/thermal/samsung/Kconfig @@ -1,6 +1,6 @@ config EXYNOS_THERMAL tristate "Exynos thermal management unit driver" - depends on ARCH_HAS_BANDGAP && OF + depends on OF help If you say yes here you get support for the TMU (Thermal Management Unit) driver for SAMSUNG EXYNOS series of SoCs. This driver initialises diff --git a/drivers/thermal/samsung/exynos_thermal_common.c b/drivers/thermal/samsung/exynos_thermal_common.c index b6be572704a4..7c97db1cf829 100644 --- a/drivers/thermal/samsung/exynos_thermal_common.c +++ b/drivers/thermal/samsung/exynos_thermal_common.c @@ -163,7 +163,7 @@ static int exynos_bind(struct thermal_zone_device *thermal, case MONITOR_ZONE: case WARN_ZONE: if (thermal_zone_bind_cooling_device(thermal, i, cdev, - level, 0)) { + level, 0, THERMAL_WEIGHT_DEFAULT)) { dev_err(data->dev, "error unbinding cdev inst=%d\n", i); ret = -EINVAL; @@ -371,9 +371,11 @@ int exynos_register_thermal(struct thermal_sensor_conf *sensor_conf) th_zone->cool_dev[th_zone->cool_dev_size] = cpufreq_cooling_register(&mask_val); if (IS_ERR(th_zone->cool_dev[th_zone->cool_dev_size])) { - dev_err(sensor_conf->dev, - "Failed to register cpufreq cooling device\n"); - ret = -EINVAL; + ret = PTR_ERR(th_zone->cool_dev[th_zone->cool_dev_size]); + if (ret != -EPROBE_DEFER) + dev_err(sensor_conf->dev, + "Failed to register cpufreq cooling device: %d\n", + ret); goto err_unregister; } th_zone->cool_dev_size++; diff --git a/drivers/thermal/samsung/exynos_tmu.c b/drivers/thermal/samsung/exynos_tmu.c index 49c09243fd38..2afca9bf40d5 100644 --- a/drivers/thermal/samsung/exynos_tmu.c +++ b/drivers/thermal/samsung/exynos_tmu.c @@ -683,7 +683,10 @@ static int exynos_tmu_probe(struct platform_device *pdev) /* Register the sensor with thermal management interface */ ret = exynos_register_thermal(sensor_conf); if (ret) { - dev_err(&pdev->dev, "Failed to register thermal interface\n"); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, + "Failed to register thermal interface: %d\n", + ret); goto err_clk; } data->reg_conf = sensor_conf; diff --git a/drivers/thermal/tegra_soctherm.c b/drivers/thermal/tegra_soctherm.c new file mode 100644 index 000000000000..9197fc05c5cc --- /dev/null +++ b/drivers/thermal/tegra_soctherm.c @@ -0,0 +1,476 @@ +/* + * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved. + * + * Author: + * Mikko Perttunen <mperttunen@nvidia.com> + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/thermal.h> + +#include <soc/tegra/fuse.h> + +#define SENSOR_CONFIG0 0 +#define SENSOR_CONFIG0_STOP BIT(0) +#define SENSOR_CONFIG0_TALL_SHIFT 8 +#define SENSOR_CONFIG0_TCALC_OVER BIT(4) +#define SENSOR_CONFIG0_OVER BIT(3) +#define SENSOR_CONFIG0_CPTR_OVER BIT(2) + +#define SENSOR_CONFIG1 4 +#define SENSOR_CONFIG1_TSAMPLE_SHIFT 0 +#define SENSOR_CONFIG1_TIDDQ_EN_SHIFT 15 +#define SENSOR_CONFIG1_TEN_COUNT_SHIFT 24 +#define SENSOR_CONFIG1_TEMP_ENABLE BIT(31) + +#define SENSOR_CONFIG2 8 +#define SENSOR_CONFIG2_THERMA_SHIFT 16 +#define SENSOR_CONFIG2_THERMB_SHIFT 0 + +#define SENSOR_PDIV 0x1c0 +#define SENSOR_PDIV_T124 0x8888 +#define SENSOR_HOTSPOT_OFF 0x1c4 +#define SENSOR_HOTSPOT_OFF_T124 0x00060600 +#define SENSOR_TEMP1 0x1c8 +#define SENSOR_TEMP2 0x1cc + +#define SENSOR_TEMP_MASK 0xffff +#define READBACK_VALUE_MASK 0xff00 +#define READBACK_VALUE_SHIFT 8 +#define READBACK_ADD_HALF BIT(7) +#define READBACK_NEGATE BIT(1) + +#define FUSE_TSENSOR8_CALIB 0x180 +#define FUSE_SPARE_REALIGNMENT_REG_0 0x1fc + +#define FUSE_TSENSOR_CALIB_CP_TS_BASE_MASK 0x1fff +#define FUSE_TSENSOR_CALIB_FT_TS_BASE_MASK (0x1fff << 13) +#define FUSE_TSENSOR_CALIB_FT_TS_BASE_SHIFT 13 + +#define FUSE_TSENSOR8_CALIB_CP_TS_BASE_MASK 0x3ff +#define FUSE_TSENSOR8_CALIB_FT_TS_BASE_MASK (0x7ff << 10) +#define FUSE_TSENSOR8_CALIB_FT_TS_BASE_SHIFT 10 + +#define FUSE_SPARE_REALIGNMENT_REG_SHIFT_CP_MASK 0x3f +#define FUSE_SPARE_REALIGNMENT_REG_SHIFT_FT_MASK (0x1f << 21) +#define FUSE_SPARE_REALIGNMENT_REG_SHIFT_FT_SHIFT 21 + +#define NOMINAL_CALIB_FT_T124 105 +#define NOMINAL_CALIB_CP_T124 25 + +struct tegra_tsensor_configuration { + u32 tall, tsample, tiddq_en, ten_count, pdiv, tsample_ate, pdiv_ate; +}; + +struct tegra_tsensor { + const struct tegra_tsensor_configuration *config; + u32 base, calib_fuse_offset; + /* Correction values used to modify values read from calibration fuses */ + s32 fuse_corr_alpha, fuse_corr_beta; +}; + +struct tegra_thermctl_zone { + void __iomem *reg; + unsigned int shift; +}; + +static const struct tegra_tsensor_configuration t124_tsensor_config = { + .tall = 16300, + .tsample = 120, + .tiddq_en = 1, + .ten_count = 1, + .pdiv = 8, + .tsample_ate = 480, + .pdiv_ate = 8 +}; + +static const struct tegra_tsensor t124_tsensors[] = { + { + .config = &t124_tsensor_config, + .base = 0xc0, + .calib_fuse_offset = 0x098, + .fuse_corr_alpha = 1135400, + .fuse_corr_beta = -6266900, + }, + { + .config = &t124_tsensor_config, + .base = 0xe0, + .calib_fuse_offset = 0x084, + .fuse_corr_alpha = 1122220, + .fuse_corr_beta = -5700700, + }, + { + .config = &t124_tsensor_config, + .base = 0x100, + .calib_fuse_offset = 0x088, + .fuse_corr_alpha = 1127000, + .fuse_corr_beta = -6768200, + }, + { + .config = &t124_tsensor_config, + .base = 0x120, + .calib_fuse_offset = 0x12c, + .fuse_corr_alpha = 1110900, + .fuse_corr_beta = -6232000, + }, + { + .config = &t124_tsensor_config, + .base = 0x140, + .calib_fuse_offset = 0x158, + .fuse_corr_alpha = 1122300, + .fuse_corr_beta = -5936400, + }, + { + .config = &t124_tsensor_config, + .base = 0x160, + .calib_fuse_offset = 0x15c, + .fuse_corr_alpha = 1145700, + .fuse_corr_beta = -7124600, + }, + { + .config = &t124_tsensor_config, + .base = 0x180, + .calib_fuse_offset = 0x154, + .fuse_corr_alpha = 1120100, + .fuse_corr_beta = -6000500, + }, + { + .config = &t124_tsensor_config, + .base = 0x1a0, + .calib_fuse_offset = 0x160, + .fuse_corr_alpha = 1106500, + .fuse_corr_beta = -6729300, + }, +}; + +struct tegra_soctherm { + struct reset_control *reset; + struct clk *clock_tsensor; + struct clk *clock_soctherm; + void __iomem *regs; + + struct thermal_zone_device *thermctl_tzs[4]; +}; + +struct tsensor_shared_calibration { + u32 base_cp, base_ft; + u32 actual_temp_cp, actual_temp_ft; +}; + +static int calculate_shared_calibration(struct tsensor_shared_calibration *r) +{ + u32 val, shifted_cp, shifted_ft; + int err; + + err = tegra_fuse_readl(FUSE_TSENSOR8_CALIB, &val); + if (err) + return err; + r->base_cp = val & FUSE_TSENSOR8_CALIB_CP_TS_BASE_MASK; + r->base_ft = (val & FUSE_TSENSOR8_CALIB_FT_TS_BASE_MASK) + >> FUSE_TSENSOR8_CALIB_FT_TS_BASE_SHIFT; + val = ((val & FUSE_SPARE_REALIGNMENT_REG_SHIFT_FT_MASK) + >> FUSE_SPARE_REALIGNMENT_REG_SHIFT_FT_SHIFT); + shifted_ft = sign_extend32(val, 4); + + err = tegra_fuse_readl(FUSE_SPARE_REALIGNMENT_REG_0, &val); + if (err) + return err; + shifted_cp = sign_extend32(val, 5); + + r->actual_temp_cp = 2 * NOMINAL_CALIB_CP_T124 + shifted_cp; + r->actual_temp_ft = 2 * NOMINAL_CALIB_FT_T124 + shifted_ft; + + return 0; +} + +static s64 div64_s64_precise(s64 a, s64 b) +{ + s64 r, al; + + /* Scale up for increased precision division */ + al = a << 16; + + r = div64_s64(al * 2 + 1, 2 * b); + return r >> 16; +} + +static int +calculate_tsensor_calibration(const struct tegra_tsensor *sensor, + const struct tsensor_shared_calibration *shared, + u32 *calib) +{ + u32 val; + s32 actual_tsensor_ft, actual_tsensor_cp, delta_sens, delta_temp, + mult, div; + s16 therma, thermb; + s64 tmp; + int err; + + err = tegra_fuse_readl(sensor->calib_fuse_offset, &val); + if (err) + return err; + + actual_tsensor_cp = (shared->base_cp * 64) + sign_extend32(val, 12); + val = (val & FUSE_TSENSOR_CALIB_FT_TS_BASE_MASK) + >> FUSE_TSENSOR_CALIB_FT_TS_BASE_SHIFT; + actual_tsensor_ft = (shared->base_ft * 32) + sign_extend32(val, 12); + + delta_sens = actual_tsensor_ft - actual_tsensor_cp; + delta_temp = shared->actual_temp_ft - shared->actual_temp_cp; + + mult = sensor->config->pdiv * sensor->config->tsample_ate; + div = sensor->config->tsample * sensor->config->pdiv_ate; + + therma = div64_s64_precise((s64) delta_temp * (1LL << 13) * mult, + (s64) delta_sens * div); + + tmp = (s64)actual_tsensor_ft * shared->actual_temp_cp - + (s64)actual_tsensor_cp * shared->actual_temp_ft; + thermb = div64_s64_precise(tmp, (s64)delta_sens); + + therma = div64_s64_precise((s64)therma * sensor->fuse_corr_alpha, + (s64)1000000LL); + thermb = div64_s64_precise((s64)thermb * sensor->fuse_corr_alpha + + sensor->fuse_corr_beta, (s64)1000000LL); + + *calib = ((u16)therma << SENSOR_CONFIG2_THERMA_SHIFT) | + ((u16)thermb << SENSOR_CONFIG2_THERMB_SHIFT); + + return 0; +} + +static int enable_tsensor(struct tegra_soctherm *tegra, + const struct tegra_tsensor *sensor, + const struct tsensor_shared_calibration *shared) +{ + void __iomem *base = tegra->regs + sensor->base; + unsigned int val; + u32 calib; + int err; + + err = calculate_tsensor_calibration(sensor, shared, &calib); + if (err) + return err; + + val = sensor->config->tall << SENSOR_CONFIG0_TALL_SHIFT; + writel(val, base + SENSOR_CONFIG0); + + val = (sensor->config->tsample - 1) << SENSOR_CONFIG1_TSAMPLE_SHIFT; + val |= sensor->config->tiddq_en << SENSOR_CONFIG1_TIDDQ_EN_SHIFT; + val |= sensor->config->ten_count << SENSOR_CONFIG1_TEN_COUNT_SHIFT; + val |= SENSOR_CONFIG1_TEMP_ENABLE; + writel(val, base + SENSOR_CONFIG1); + + writel(calib, base + SENSOR_CONFIG2); + + return 0; +} + +/* + * Translate from soctherm readback format to millicelsius. + * The soctherm readback format in bits is as follows: + * TTTTTTTT H______N + * where T's contain the temperature in Celsius, + * H denotes an addition of 0.5 Celsius and N denotes negation + * of the final value. + */ +static long translate_temp(u16 val) +{ + long t; + + t = ((val & READBACK_VALUE_MASK) >> READBACK_VALUE_SHIFT) * 1000; + if (val & READBACK_ADD_HALF) + t += 500; + if (val & READBACK_NEGATE) + t *= -1; + + return t; +} + +static int tegra_thermctl_get_temp(void *data, long *out_temp) +{ + struct tegra_thermctl_zone *zone = data; + u32 val; + + val = (readl(zone->reg) >> zone->shift) & SENSOR_TEMP_MASK; + *out_temp = translate_temp(val); + + return 0; +} + +static const struct thermal_zone_of_device_ops tegra_of_thermal_ops = { + .get_temp = tegra_thermctl_get_temp, +}; + +static const struct of_device_id tegra_soctherm_of_match[] = { + { .compatible = "nvidia,tegra124-soctherm" }, + { }, +}; +MODULE_DEVICE_TABLE(of, tegra_soctherm_of_match); + +struct thermctl_zone_desc { + unsigned int offset; + unsigned int shift; +}; + +static const struct thermctl_zone_desc t124_thermctl_temp_zones[] = { + { SENSOR_TEMP1, 16 }, + { SENSOR_TEMP2, 16 }, + { SENSOR_TEMP1, 0 }, + { SENSOR_TEMP2, 0 } +}; + +static int tegra_soctherm_probe(struct platform_device *pdev) +{ + struct tegra_soctherm *tegra; + struct thermal_zone_device *tz; + struct tsensor_shared_calibration shared_calib; + struct resource *res; + unsigned int i; + int err; + + const struct tegra_tsensor *tsensors = t124_tsensors; + + tegra = devm_kzalloc(&pdev->dev, sizeof(*tegra), GFP_KERNEL); + if (!tegra) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + tegra->regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(tegra->regs)) + return PTR_ERR(tegra->regs); + + tegra->reset = devm_reset_control_get(&pdev->dev, "soctherm"); + if (IS_ERR(tegra->reset)) { + dev_err(&pdev->dev, "can't get soctherm reset\n"); + return PTR_ERR(tegra->reset); + } + + tegra->clock_tsensor = devm_clk_get(&pdev->dev, "tsensor"); + if (IS_ERR(tegra->clock_tsensor)) { + dev_err(&pdev->dev, "can't get tsensor clock\n"); + return PTR_ERR(tegra->clock_tsensor); + } + + tegra->clock_soctherm = devm_clk_get(&pdev->dev, "soctherm"); + if (IS_ERR(tegra->clock_soctherm)) { + dev_err(&pdev->dev, "can't get soctherm clock\n"); + return PTR_ERR(tegra->clock_soctherm); + } + + reset_control_assert(tegra->reset); + + err = clk_prepare_enable(tegra->clock_soctherm); + if (err) + return err; + + err = clk_prepare_enable(tegra->clock_tsensor); + if (err) { + clk_disable_unprepare(tegra->clock_soctherm); + return err; + } + + reset_control_deassert(tegra->reset); + + /* Initialize raw sensors */ + + err = calculate_shared_calibration(&shared_calib); + if (err) + goto disable_clocks; + + for (i = 0; i < ARRAY_SIZE(t124_tsensors); ++i) { + err = enable_tsensor(tegra, tsensors + i, &shared_calib); + if (err) + goto disable_clocks; + } + + writel(SENSOR_PDIV_T124, tegra->regs + SENSOR_PDIV); + writel(SENSOR_HOTSPOT_OFF_T124, tegra->regs + SENSOR_HOTSPOT_OFF); + + /* Initialize thermctl sensors */ + + for (i = 0; i < ARRAY_SIZE(tegra->thermctl_tzs); ++i) { + struct tegra_thermctl_zone *zone = + devm_kzalloc(&pdev->dev, sizeof(*zone), GFP_KERNEL); + if (!zone) { + err = -ENOMEM; + goto unregister_tzs; + } + + zone->reg = tegra->regs + t124_thermctl_temp_zones[i].offset; + zone->shift = t124_thermctl_temp_zones[i].shift; + + tz = thermal_zone_of_sensor_register(&pdev->dev, i, zone, + &tegra_of_thermal_ops); + if (IS_ERR(tz)) { + err = PTR_ERR(tz); + dev_err(&pdev->dev, "failed to register sensor: %d\n", + err); + goto unregister_tzs; + } + + tegra->thermctl_tzs[i] = tz; + } + + return 0; + +unregister_tzs: + while (i--) + thermal_zone_of_sensor_unregister(&pdev->dev, + tegra->thermctl_tzs[i]); + +disable_clocks: + clk_disable_unprepare(tegra->clock_tsensor); + clk_disable_unprepare(tegra->clock_soctherm); + + return err; +} + +static int tegra_soctherm_remove(struct platform_device *pdev) +{ + struct tegra_soctherm *tegra = platform_get_drvdata(pdev); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(tegra->thermctl_tzs); ++i) { + thermal_zone_of_sensor_unregister(&pdev->dev, + tegra->thermctl_tzs[i]); + } + + clk_disable_unprepare(tegra->clock_tsensor); + clk_disable_unprepare(tegra->clock_soctherm); + + return 0; +} + +static struct platform_driver tegra_soctherm_driver = { + .probe = tegra_soctherm_probe, + .remove = tegra_soctherm_remove, + .driver = { + .name = "tegra-soctherm", + .of_match_table = tegra_soctherm_of_match, + }, +}; +module_platform_driver(tegra_soctherm_driver); + +MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra SOCTHERM thermal management driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 885adc345f57..97633ca4b0a7 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -78,6 +78,58 @@ static struct thermal_governor *__find_governor(const char *name) return NULL; } +/** + * bind_previous_governor() - bind the previous governor of the thermal zone + * @tz: a valid pointer to a struct thermal_zone_device + * @failed_gov_name: the name of the governor that failed to register + * + * Register the previous governor of the thermal zone after a new + * governor has failed to be bound. + */ +static void bind_previous_governor(struct thermal_zone_device *tz, + const char *failed_gov_name) +{ + if (tz->governor && tz->governor->bind_to_tz) { + if (tz->governor->bind_to_tz(tz)) { + dev_err(&tz->device, + "governor %s failed to bind and the previous one (%s) failed to bind again, thermal zone %s has no governor\n", + failed_gov_name, tz->governor->name, tz->type); + tz->governor = NULL; + } + } +} + +/** + * thermal_set_governor() - Switch to another governor + * @tz: a valid pointer to a struct thermal_zone_device + * @new_gov: pointer to the new governor + * + * Change the governor of thermal zone @tz. + * + * Return: 0 on success, an error if the new governor's bind_to_tz() failed. + */ +static int thermal_set_governor(struct thermal_zone_device *tz, + struct thermal_governor *new_gov) +{ + int ret = 0; + + if (tz->governor && tz->governor->unbind_from_tz) + tz->governor->unbind_from_tz(tz); + + if (new_gov && new_gov->bind_to_tz) { + ret = new_gov->bind_to_tz(tz); + if (ret) { + bind_previous_governor(tz, new_gov->name); + + return ret; + } + } + + tz->governor = new_gov; + + return ret; +} + int thermal_register_governor(struct thermal_governor *governor) { int err; @@ -110,8 +162,15 @@ int thermal_register_governor(struct thermal_governor *governor) name = pos->tzp->governor_name; - if (!strncasecmp(name, governor->name, THERMAL_NAME_LENGTH)) - pos->governor = governor; + if (!strncasecmp(name, governor->name, THERMAL_NAME_LENGTH)) { + int ret; + + ret = thermal_set_governor(pos, governor); + if (ret) + dev_err(&pos->device, + "Failed to set governor %s for thermal zone %s: %d\n", + governor->name, pos->type, ret); + } } mutex_unlock(&thermal_list_lock); @@ -137,7 +196,7 @@ void thermal_unregister_governor(struct thermal_governor *governor) list_for_each_entry(pos, &thermal_tz_list, node) { if (!strncasecmp(pos->governor->name, governor->name, THERMAL_NAME_LENGTH)) - pos->governor = NULL; + thermal_set_governor(pos, NULL); } mutex_unlock(&thermal_list_lock); @@ -221,7 +280,8 @@ static void print_bind_err_msg(struct thermal_zone_device *tz, static void __bind(struct thermal_zone_device *tz, int mask, struct thermal_cooling_device *cdev, - unsigned long *limits) + unsigned long *limits, + unsigned int weight) { int i, ret; @@ -236,7 +296,8 @@ static void __bind(struct thermal_zone_device *tz, int mask, upper = limits[i * 2 + 1]; } ret = thermal_zone_bind_cooling_device(tz, i, cdev, - upper, lower); + upper, lower, + weight); if (ret) print_bind_err_msg(tz, cdev, ret); } @@ -283,7 +344,8 @@ static void bind_cdev(struct thermal_cooling_device *cdev) continue; tzp->tbp[i].cdev = cdev; __bind(pos, tzp->tbp[i].trip_mask, cdev, - tzp->tbp[i].binding_limits); + tzp->tbp[i].binding_limits, + tzp->tbp[i].weight); } } @@ -322,7 +384,8 @@ static void bind_tz(struct thermal_zone_device *tz) continue; tzp->tbp[i].cdev = pos; __bind(tz, tzp->tbp[i].trip_mask, pos, - tzp->tbp[i].binding_limits); + tzp->tbp[i].binding_limits, + tzp->tbp[i].weight); } } exit: @@ -371,7 +434,7 @@ static void handle_critical_trips(struct thermal_zone_device *tz, tz->ops->get_trip_temp(tz, trip, &trip_temp); /* If we have not crossed the trip_temp, we do not care. */ - if (tz->temperature < trip_temp) + if (trip_temp <= 0 || tz->temperature < trip_temp) return; trace_thermal_zone_trip(tz, trip, trip_type); @@ -735,7 +798,8 @@ passive_store(struct device *dev, struct device_attribute *attr, thermal_zone_bind_cooling_device(tz, THERMAL_TRIPS_NONE, cdev, THERMAL_NO_LIMIT, - THERMAL_NO_LIMIT); + THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT); } mutex_unlock(&thermal_list_lock); if (!tz->passive_delay) @@ -781,15 +845,18 @@ policy_store(struct device *dev, struct device_attribute *attr, snprintf(name, sizeof(name), "%s", buf); mutex_lock(&thermal_governor_lock); + mutex_lock(&tz->lock); gov = __find_governor(strim(name)); if (!gov) goto exit; - tz->governor = gov; - ret = count; + ret = thermal_set_governor(tz, gov); + if (!ret) + ret = count; exit: + mutex_unlock(&tz->lock); mutex_unlock(&thermal_governor_lock); return ret; } @@ -830,6 +897,158 @@ emul_temp_store(struct device *dev, struct device_attribute *attr, static DEVICE_ATTR(emul_temp, S_IWUSR, NULL, emul_temp_store); #endif/*CONFIG_THERMAL_EMULATION*/ +static ssize_t +sustainable_power_show(struct device *dev, struct device_attribute *devattr, + char *buf) +{ + struct thermal_zone_device *tz = to_thermal_zone(dev); + + if (tz->tzp) + return sprintf(buf, "%u\n", tz->tzp->sustainable_power); + else + return -EIO; +} + +static ssize_t +sustainable_power_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct thermal_zone_device *tz = to_thermal_zone(dev); + u32 sustainable_power; + + if (!tz->tzp) + return -EIO; + + if (kstrtou32(buf, 10, &sustainable_power)) + return -EINVAL; + + tz->tzp->sustainable_power = sustainable_power; + + return count; +} +static DEVICE_ATTR(sustainable_power, S_IWUSR | S_IRUGO, sustainable_power_show, + sustainable_power_store); + +#define create_s32_tzp_attr(name) \ + static ssize_t \ + name##_show(struct device *dev, struct device_attribute *devattr, \ + char *buf) \ + { \ + struct thermal_zone_device *tz = to_thermal_zone(dev); \ + \ + if (tz->tzp) \ + return sprintf(buf, "%u\n", tz->tzp->name); \ + else \ + return -EIO; \ + } \ + \ + static ssize_t \ + name##_store(struct device *dev, struct device_attribute *devattr, \ + const char *buf, size_t count) \ + { \ + struct thermal_zone_device *tz = to_thermal_zone(dev); \ + s32 value; \ + \ + if (!tz->tzp) \ + return -EIO; \ + \ + if (kstrtos32(buf, 10, &value)) \ + return -EINVAL; \ + \ + tz->tzp->name = value; \ + \ + return count; \ + } \ + static DEVICE_ATTR(name, S_IWUSR | S_IRUGO, name##_show, name##_store) + +create_s32_tzp_attr(k_po); +create_s32_tzp_attr(k_pu); +create_s32_tzp_attr(k_i); +create_s32_tzp_attr(k_d); +create_s32_tzp_attr(integral_cutoff); +create_s32_tzp_attr(slope); +create_s32_tzp_attr(offset); +#undef create_s32_tzp_attr + +static struct device_attribute *dev_tzp_attrs[] = { + &dev_attr_sustainable_power, + &dev_attr_k_po, + &dev_attr_k_pu, + &dev_attr_k_i, + &dev_attr_k_d, + &dev_attr_integral_cutoff, + &dev_attr_slope, + &dev_attr_offset, +}; + +static int create_tzp_attrs(struct device *dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(dev_tzp_attrs); i++) { + int ret; + struct device_attribute *dev_attr = dev_tzp_attrs[i]; + + ret = device_create_file(dev, dev_attr); + if (ret) + return ret; + } + + return 0; +} + +/** + * power_actor_get_max_power() - get the maximum power that a cdev can consume + * @cdev: pointer to &thermal_cooling_device + * @tz: a valid thermal zone device pointer + * @max_power: pointer in which to store the maximum power + * + * Calculate the maximum power consumption in milliwats that the + * cooling device can currently consume and store it in @max_power. + * + * Return: 0 on success, -EINVAL if @cdev doesn't support the + * power_actor API or -E* on other error. + */ +int power_actor_get_max_power(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, u32 *max_power) +{ + if (!cdev_is_power_actor(cdev)) + return -EINVAL; + + return cdev->ops->state2power(cdev, tz, 0, max_power); +} + +/** + * power_actor_set_power() - limit the maximum power that a cooling device can consume + * @cdev: pointer to &thermal_cooling_device + * @instance: thermal instance to update + * @power: the power in milliwatts + * + * Set the cooling device to consume at most @power milliwatts. + * + * Return: 0 on success, -EINVAL if the cooling device does not + * implement the power actor API or -E* for other failures. + */ +int power_actor_set_power(struct thermal_cooling_device *cdev, + struct thermal_instance *instance, u32 power) +{ + unsigned long state; + int ret; + + if (!cdev_is_power_actor(cdev)) + return -EINVAL; + + ret = cdev->ops->power2state(cdev, instance->tz, power, &state); + if (ret) + return ret; + + instance->target = state; + cdev->updated = false; + thermal_cdev_update(cdev); + + return 0; +} + static DEVICE_ATTR(type, 0444, type_show, NULL); static DEVICE_ATTR(temp, 0444, temp_show, NULL); static DEVICE_ATTR(mode, 0644, mode_show, mode_store); @@ -921,6 +1140,50 @@ thermal_cooling_device_trip_point_show(struct device *dev, return sprintf(buf, "%d\n", instance->trip); } +static struct attribute *cooling_device_attrs[] = { + &dev_attr_cdev_type.attr, + &dev_attr_max_state.attr, + &dev_attr_cur_state.attr, + NULL, +}; + +static const struct attribute_group cooling_device_attr_group = { + .attrs = cooling_device_attrs, +}; + +static const struct attribute_group *cooling_device_attr_groups[] = { + &cooling_device_attr_group, + NULL, +}; + +static ssize_t +thermal_cooling_device_weight_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct thermal_instance *instance; + + instance = container_of(attr, struct thermal_instance, weight_attr); + + return sprintf(buf, "%d\n", instance->weight); +} + +static ssize_t +thermal_cooling_device_weight_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct thermal_instance *instance; + int ret, weight; + + ret = kstrtoint(buf, 0, &weight); + if (ret) + return ret; + + instance = container_of(attr, struct thermal_instance, weight_attr); + instance->weight = weight; + + return count; +} /* Device management */ /** @@ -935,6 +1198,9 @@ thermal_cooling_device_trip_point_show(struct device *dev, * @lower: the Minimum cooling state can be used for this trip point. * THERMAL_NO_LIMIT means no lower limit, * and the cooling device can be in cooling state 0. + * @weight: The weight of the cooling device to be bound to the + * thermal zone. Use THERMAL_WEIGHT_DEFAULT for the + * default value * * This interface function bind a thermal cooling device to the certain trip * point of a thermal zone device. @@ -945,7 +1211,8 @@ thermal_cooling_device_trip_point_show(struct device *dev, int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz, int trip, struct thermal_cooling_device *cdev, - unsigned long upper, unsigned long lower) + unsigned long upper, unsigned long lower, + unsigned int weight) { struct thermal_instance *dev; struct thermal_instance *pos; @@ -988,6 +1255,7 @@ int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz, dev->upper = upper; dev->lower = lower; dev->target = THERMAL_NO_TARGET; + dev->weight = weight; result = get_idr(&tz->idr, &tz->lock, &dev->id); if (result) @@ -1008,6 +1276,16 @@ int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz, if (result) goto remove_symbol_link; + sprintf(dev->weight_attr_name, "cdev%d_weight", dev->id); + sysfs_attr_init(&dev->weight_attr.attr); + dev->weight_attr.attr.name = dev->weight_attr_name; + dev->weight_attr.attr.mode = S_IWUSR | S_IRUGO; + dev->weight_attr.show = thermal_cooling_device_weight_show; + dev->weight_attr.store = thermal_cooling_device_weight_store; + result = device_create_file(&tz->device, &dev->weight_attr); + if (result) + goto remove_trip_file; + mutex_lock(&tz->lock); mutex_lock(&cdev->lock); list_for_each_entry(pos, &tz->thermal_instances, tz_node) @@ -1026,6 +1304,8 @@ int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz, if (!result) return 0; + device_remove_file(&tz->device, &dev->weight_attr); +remove_trip_file: device_remove_file(&tz->device, &dev->attr); remove_symbol_link: sysfs_remove_link(&tz->device.kobj, dev->name); @@ -1152,6 +1432,7 @@ __thermal_cooling_device_register(struct device_node *np, cdev->ops = ops; cdev->updated = false; cdev->device.class = &thermal_class; + cdev->device.groups = cooling_device_attr_groups; cdev->devdata = devdata; dev_set_name(&cdev->device, "cooling_device%d", cdev->id); result = device_register(&cdev->device); @@ -1161,21 +1442,6 @@ __thermal_cooling_device_register(struct device_node *np, return ERR_PTR(result); } - /* sys I/F */ - if (type) { - result = device_create_file(&cdev->device, &dev_attr_cdev_type); - if (result) - goto unregister; - } - - result = device_create_file(&cdev->device, &dev_attr_max_state); - if (result) - goto unregister; - - result = device_create_file(&cdev->device, &dev_attr_cur_state); - if (result) - goto unregister; - /* Add 'this' new cdev to the global cdev list */ mutex_lock(&thermal_list_lock); list_add(&cdev->node, &thermal_cdev_list); @@ -1191,11 +1457,6 @@ __thermal_cooling_device_register(struct device_node *np, mutex_unlock(&thermal_list_lock); return cdev; - -unregister: - release_idr(&thermal_cdev_idr, &thermal_idr_lock, cdev->id); - device_unregister(&cdev->device); - return ERR_PTR(result); } /** @@ -1406,7 +1667,8 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_temp_attrs[indx].name; tz->trip_temp_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_temp_attrs[indx].attr.show = trip_point_temp_show; - if (mask & (1 << indx)) { + if (IS_ENABLED(CONFIG_THERMAL_WRITABLE_TRIPS) && + mask & (1 << indx)) { tz->trip_temp_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_temp_attrs[indx].attr.store = trip_point_temp_store; @@ -1483,7 +1745,7 @@ static void remove_trip_attrs(struct thermal_zone_device *tz) struct thermal_zone_device *thermal_zone_device_register(const char *type, int trips, int mask, void *devdata, struct thermal_zone_device_ops *ops, - const struct thermal_zone_params *tzp, + struct thermal_zone_params *tzp, int passive_delay, int polling_delay) { struct thermal_zone_device *tz; @@ -1492,6 +1754,7 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type, int result; int count; int passive = 0; + struct thermal_governor *governor; if (type && strlen(type) >= THERMAL_NAME_LENGTH) return ERR_PTR(-EINVAL); @@ -1586,13 +1849,24 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type, if (result) goto unregister; + /* Add thermal zone params */ + result = create_tzp_attrs(&tz->device); + if (result) + goto unregister; + /* Update 'this' zone's governor information */ mutex_lock(&thermal_governor_lock); if (tz->tzp) - tz->governor = __find_governor(tz->tzp->governor_name); + governor = __find_governor(tz->tzp->governor_name); else - tz->governor = def_governor; + governor = def_governor; + + result = thermal_set_governor(tz, governor); + if (result) { + mutex_unlock(&thermal_governor_lock); + goto unregister; + } mutex_unlock(&thermal_governor_lock); @@ -1684,7 +1958,7 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz) device_remove_file(&tz->device, &dev_attr_mode); device_remove_file(&tz->device, &dev_attr_policy); remove_trip_attrs(tz); - tz->governor = NULL; + thermal_set_governor(tz, NULL); thermal_remove_hwmon_sysfs(tz); release_idr(&thermal_tz_idr, &thermal_idr_lock, tz->id); @@ -1844,7 +2118,11 @@ static int __init thermal_register_governors(void) if (result) return result; - return thermal_gov_user_space_register(); + result = thermal_gov_user_space_register(); + if (result) + return result; + + return thermal_gov_power_allocator_register(); } static void thermal_unregister_governors(void) @@ -1853,6 +2131,7 @@ static void thermal_unregister_governors(void) thermal_gov_fair_share_unregister(); thermal_gov_bang_bang_unregister(); thermal_gov_user_space_unregister(); + thermal_gov_power_allocator_unregister(); } static int thermal_pm_notify(struct notifier_block *nb, diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index 64ddc2f0ec5c..9e20e4dbedd4 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -47,8 +47,11 @@ struct thermal_instance { unsigned long target; /* expected cooling state */ char attr_name[THERMAL_NAME_LENGTH]; struct device_attribute attr; + char weight_attr_name[THERMAL_NAME_LENGTH]; + struct device_attribute weight_attr; struct list_head tz_node; /* node in tz->thermal_instances */ struct list_head cdev_node; /* node in cdev->thermal_instances */ + unsigned int weight; /* The weight of the cooling device */ }; int thermal_register_governor(struct thermal_governor *); @@ -86,13 +89,39 @@ static inline int thermal_gov_user_space_register(void) { return 0; } static inline void thermal_gov_user_space_unregister(void) {} #endif /* CONFIG_THERMAL_GOV_USER_SPACE */ +#ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR +int thermal_gov_power_allocator_register(void); +void thermal_gov_power_allocator_unregister(void); +#else +static inline int thermal_gov_power_allocator_register(void) { return 0; } +static inline void thermal_gov_power_allocator_unregister(void) {} +#endif /* CONFIG_THERMAL_GOV_POWER_ALLOCATOR */ + /* device tree support */ #ifdef CONFIG_THERMAL_OF int of_parse_thermal_zones(void); void of_thermal_destroy_zones(void); +int of_thermal_get_ntrips(struct thermal_zone_device *); +bool of_thermal_is_trip_valid(struct thermal_zone_device *, int); +const struct thermal_trip * +of_thermal_get_trip_points(struct thermal_zone_device *); #else static inline int of_parse_thermal_zones(void) { return 0; } static inline void of_thermal_destroy_zones(void) { } +static inline int of_thermal_get_ntrips(struct thermal_zone_device *tz) +{ + return 0; +} +static inline bool of_thermal_is_trip_valid(struct thermal_zone_device *tz, + int trip) +{ + return 0; +} +static inline const struct thermal_trip * +of_thermal_get_trip_points(struct thermal_zone_device *tz) +{ + return NULL; +} #endif #endif /* __THERMAL_CORE_H__ */ diff --git a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c index 9eec26dc0448..68f53fcb8fb1 100644 --- a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c +++ b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c @@ -28,7 +28,6 @@ #include <linux/kernel.h> #include <linux/workqueue.h> #include <linux/thermal.h> -#include <linux/cpufreq.h> #include <linux/cpumask.h> #include <linux/cpu_cooling.h> #include <linux/of.h> @@ -147,7 +146,8 @@ static int ti_thermal_bind(struct thermal_zone_device *thermal, return thermal_zone_bind_cooling_device(thermal, 0, cdev, /* bind with min and max states defined by cpu_cooling */ THERMAL_NO_LIMIT, - THERMAL_NO_LIMIT); + THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT); } /* Unbind callback functions for thermal zone */ @@ -286,6 +286,11 @@ static int ti_thermal_get_crit_temp(struct thermal_zone_device *thermal, return ti_thermal_get_trip_temp(thermal, OMAP_TRIP_NUMBER - 1, temp); } +static const struct thermal_zone_of_device_ops ti_of_thermal_ops = { + .get_temp = __ti_thermal_get_temp, + .get_trend = __ti_thermal_get_trend, +}; + static struct thermal_zone_device_ops ti_thermal_ops = { .get_temp = ti_thermal_get_temp, .get_trend = ti_thermal_get_trend, @@ -333,8 +338,7 @@ int ti_thermal_expose_sensor(struct ti_bandgap *bgp, int id, /* in case this is specified by DT */ data->ti_thermal = thermal_zone_of_sensor_register(bgp->dev, id, - data, __ti_thermal_get_temp, - __ti_thermal_get_trend); + data, &ti_of_thermal_ops); if (IS_ERR(data->ti_thermal)) { /* Create thermal zone */ data->ti_thermal = thermal_zone_device_register(domain, @@ -403,11 +407,6 @@ int ti_thermal_register_cpu_cooling(struct ti_bandgap *bgp, int id) if (!data) return -EINVAL; - if (!cpufreq_get_current_driver()) { - dev_dbg(bgp->dev, "no cpufreq driver yet\n"); - return -EPROBE_DEFER; - } - /* Register cooling device */ data->cool_dev = cpufreq_cooling_register(cpu_present_mask); if (IS_ERR(data->cool_dev)) { diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index c5417cb68ea1..bc579195c50c 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -560,7 +560,7 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, struct msi_msg msg; get_cached_msi_msg(irq, &msg); - write_msi_msg(irq, &msg); + pci_write_msi_msg(irq, &msg); } ret = request_irq(irq, vfio_msihandler, 0, diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 7d4c2bf2fea2..05db4c641a3e 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -790,8 +790,7 @@ static void clean_demultiplex_info(struct TCP_Server_Info *server) length = atomic_dec_return(&tcpSesAllocCount); if (length > 0) - mempool_resize(cifs_req_poolp, length + cifs_min_rcv, - GFP_KERNEL); + mempool_resize(cifs_req_poolp, length + cifs_min_rcv); } static int @@ -872,8 +871,7 @@ cifs_demultiplex_thread(void *p) length = atomic_inc_return(&tcpSesAllocCount); if (length > 1) - mempool_resize(cifs_req_poolp, length + cifs_min_rcv, - GFP_KERNEL); + mempool_resize(cifs_req_poolp, length + cifs_min_rcv); set_freezable(); while (server->tcpStatus != CifsExiting) { diff --git a/fs/dcache.c b/fs/dcache.c index c064eea24c4a..b07e4a1117db 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -38,6 +38,8 @@ #include <linux/prefetch.h> #include <linux/ratelimit.h> #include <linux/list_lru.h> +#include <linux/kasan.h> + #include "internal.h" #include "mount.h" @@ -1552,6 +1554,9 @@ struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) } atomic_set(&p->u.count, 1); dname = p->name; + if (IS_ENABLED(CONFIG_DCACHE_WORD_ACCESS)) + kasan_unpoison_shadow(dname, + round_up(name->len + 1, sizeof(unsigned long))); } else { dname = dentry->d_iname; } diff --git a/include/asm-generic/early_ioremap.h b/include/asm-generic/early_ioremap.h index a5de55c04fb2..e539f27ec51b 100644 --- a/include/asm-generic/early_ioremap.h +++ b/include/asm-generic/early_ioremap.h @@ -33,6 +33,12 @@ extern void early_ioremap_setup(void); */ extern void early_ioremap_reset(void); +/* + * Early copy from unmapped memory to kernel mapped memory. + */ +extern void copy_from_early_mem(void *dest, phys_addr_t src, + unsigned long size); + #else static inline void early_ioremap_init(void) { } static inline void early_ioremap_setup(void) { } diff --git a/include/asm-generic/msi.h b/include/asm-generic/msi.h new file mode 100644 index 000000000000..61c58d8878ce --- /dev/null +++ b/include/asm-generic/msi.h @@ -0,0 +1,32 @@ +#ifndef __ASM_GENERIC_MSI_H +#define __ASM_GENERIC_MSI_H + +#include <linux/types.h> + +#ifndef NUM_MSI_ALLOC_SCRATCHPAD_REGS +# define NUM_MSI_ALLOC_SCRATCHPAD_REGS 2 +#endif + +struct msi_desc; + +/** + * struct msi_alloc_info - Default structure for MSI interrupt allocation. + * @desc: Pointer to msi descriptor + * @hwirq: Associated hw interrupt number in the domain + * @scratchpad: Storage for implementation specific scratch data + * + * Architectures can provide their own implementation by not including + * asm-generic/msi.h into their arch specific header file. + */ +typedef struct msi_alloc_info { + struct msi_desc *desc; + irq_hw_number_t hwirq; + union { + unsigned long ul; + void *ptr; + } scratchpad[NUM_MSI_ALLOC_SCRATCHPAD_REGS]; +} msi_alloc_info_t; + +#define GENERIC_MSI_DOMAIN_OPS 1 + +#endif diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index aa70cbda327c..ac78910d7416 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -164,6 +164,7 @@ #define CLKSRC_OF_TABLES() OF_TABLE(CONFIG_CLKSRC_OF, clksrc) #define IRQCHIP_OF_MATCH_TABLE() OF_TABLE(CONFIG_IRQCHIP, irqchip) #define CLK_OF_TABLES() OF_TABLE(CONFIG_COMMON_CLK, clk) +#define IOMMU_OF_TABLES() OF_TABLE(CONFIG_OF_IOMMU, iommu) #define RESERVEDMEM_OF_TABLES() OF_TABLE(CONFIG_OF_RESERVED_MEM, reservedmem) #define CPU_METHOD_OF_TABLES() OF_TABLE(CONFIG_SMP, cpu_method) #define EARLYCON_OF_TABLES() OF_TABLE(CONFIG_SERIAL_EARLYCON, earlycon) @@ -477,6 +478,7 @@ #define KERNEL_CTORS() . = ALIGN(8); \ VMLINUX_SYMBOL(__ctors_start) = .; \ *(.ctors) \ + *(SORT(.init_array.*)) \ *(.init_array) \ VMLINUX_SYMBOL(__ctors_end) = .; #else @@ -497,6 +499,7 @@ CLK_OF_TABLES() \ RESERVEDMEM_OF_TABLES() \ CLKSRC_OF_TABLES() \ + IOMMU_OF_TABLES() \ CPU_METHOD_OF_TABLES() \ KERNEL_DTB() \ IRQCHIP_OF_MATCH_TABLE() \ diff --git a/include/dt-bindings/thermal/thermal.h b/include/dt-bindings/thermal/thermal.h index 59822a995858..b5e6b0069ac7 100644 --- a/include/dt-bindings/thermal/thermal.h +++ b/include/dt-bindings/thermal/thermal.h @@ -11,7 +11,7 @@ #define _DT_BINDINGS_THERMAL_THERMAL_H /* On cooling devices upper and lower limits */ -#define THERMAL_NO_LIMIT (-1UL) +#define THERMAL_NO_LIMIT (~0) #endif diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index c324f5700d1a..d024bd9c9d9b 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -23,6 +23,7 @@ #define AMBA_NR_IRQS 9 #define AMBA_CID 0xb105f00d +#define CORESIGHT_CID 0xb105900d struct clk; diff --git a/include/linux/clock_cooling.h b/include/linux/clock_cooling.h new file mode 100644 index 000000000000..4d1019d56f7f --- /dev/null +++ b/include/linux/clock_cooling.h @@ -0,0 +1,65 @@ +/* + * linux/include/linux/clock_cooling.h + * + * Copyright (C) 2014 Eduardo Valentin <edubezval@gmail.com> + * + * Copyright (C) 2013 Texas Instruments Inc. + * Contact: Eduardo Valentin <eduardo.valentin@ti.com> + * + * Highly based on cpu_cooling.c. + * Copyright (C) 2012 Samsung Electronics Co., Ltd(http://www.samsung.com) + * Copyright (C) 2012 Amit Daniel <amit.kachhap@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#ifndef __CPU_COOLING_H__ +#define __CPU_COOLING_H__ + +#include <linux/of.h> +#include <linux/thermal.h> +#include <linux/cpumask.h> + +#ifdef CONFIG_CLOCK_THERMAL +/** + * clock_cooling_register - function to create clock cooling device. + * @dev: struct device pointer to the device used as clock cooling device. + * @clock_name: string containing the clock used as cooling mechanism. + */ +struct thermal_cooling_device * +clock_cooling_register(struct device *dev, const char *clock_name); + +/** + * clock_cooling_unregister - function to remove clock cooling device. + * @cdev: thermal cooling device pointer. + */ +void clock_cooling_unregister(struct thermal_cooling_device *cdev); + +unsigned long clock_cooling_get_level(struct thermal_cooling_device *cdev, + unsigned long freq); +#else /* !CONFIG_CLOCK_THERMAL */ +static inline struct thermal_cooling_device * +clock_cooling_register(struct device *dev, const char *clock_name) +{ + return NULL; +} +static inline +void clock_cooling_unregister(struct thermal_cooling_device *cdev) +{ +} +static inline +unsigned long clock_cooling_get_level(struct thermal_cooling_device *cdev, + unsigned long freq) +{ + return THERMAL_CSTATE_INVALID; +} +#endif /* CONFIG_CLOCK_THERMAL */ + +#endif /* __CPU_COOLING_H__ */ diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index 633716ef19b0..d26a858a4c0d 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h @@ -5,9 +5,9 @@ /* * Common definitions for all gcc versions go here. */ -#define GCC_VERSION (__GNUC__ * 10000 \ - + __GNUC_MINOR__ * 100 \ - + __GNUC_PATCHLEVEL__) +#define GCC_VERSION (__GNUC__ * 10000 \ + + __GNUC_MINOR__ * 100 \ + + __GNUC_PATCHLEVEL__) /* Optimization barrier */ @@ -32,54 +32,63 @@ * the inline assembly constraint from =g to =r, in this particular * case either is valid. */ -#define RELOC_HIDE(ptr, off) \ - ({ unsigned long __ptr; \ - __asm__ ("" : "=r"(__ptr) : "0"(ptr)); \ - (typeof(ptr)) (__ptr + (off)); }) +#define RELOC_HIDE(ptr, off) \ +({ \ + unsigned long __ptr; \ + __asm__ ("" : "=r"(__ptr) : "0"(ptr)); \ + (typeof(ptr)) (__ptr + (off)); \ +}) /* Make the optimizer believe the variable can be manipulated arbitrarily. */ -#define OPTIMIZER_HIDE_VAR(var) __asm__ ("" : "=r" (var) : "0" (var)) +#define OPTIMIZER_HIDE_VAR(var) \ + __asm__ ("" : "=r" (var) : "0" (var)) #ifdef __CHECKER__ -#define __must_be_array(arr) 0 +#define __must_be_array(a) 0 #else /* &a[0] degrades to a pointer: a different type from an array */ -#define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0])) +#define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0])) #endif /* * Force always-inline if the user requests it so via the .config, * or if gcc is too old: */ -#if !defined(CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING) || \ +#if !defined(CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING) || \ !defined(CONFIG_OPTIMIZE_INLINING) || (__GNUC__ < 4) -# define inline inline __attribute__((always_inline)) notrace -# define __inline__ __inline__ __attribute__((always_inline)) notrace -# define __inline __inline __attribute__((always_inline)) notrace +#define inline inline __attribute__((always_inline)) notrace +#define __inline__ __inline__ __attribute__((always_inline)) notrace +#define __inline __inline __attribute__((always_inline)) notrace #else /* A lot of inline functions can cause havoc with function tracing */ -# define inline inline notrace -# define __inline__ __inline__ notrace -# define __inline __inline notrace +#define inline inline notrace +#define __inline__ __inline__ notrace +#define __inline __inline notrace #endif -#define __deprecated __attribute__((deprecated)) -#define __packed __attribute__((packed)) -#define __weak __attribute__((weak)) +#define __always_inline inline __attribute__((always_inline)) +#define noinline __attribute__((noinline)) + +#define __deprecated __attribute__((deprecated)) +#define __packed __attribute__((packed)) +#define __weak __attribute__((weak)) +#define __alias(symbol) __attribute__((alias(#symbol))) /* - * it doesn't make sense on ARM (currently the only user of __naked) to trace - * naked functions because then mcount is called without stack and frame pointer - * being set up and there is no chance to restore the lr register to the value - * before mcount was called. + * it doesn't make sense on ARM (currently the only user of __naked) + * to trace naked functions because then mcount is called without + * stack and frame pointer being set up and there is no chance to + * restore the lr register to the value before mcount was called. + * + * The asm() bodies of naked functions often depend on standard calling + * conventions, therefore they must be noinline and noclone. * - * The asm() bodies of naked functions often depend on standard calling conventions, - * therefore they must be noinline and noclone. GCC 4.[56] currently fail to enforce - * this, so we must do so ourselves. See GCC PR44290. + * GCC 4.[56] currently fail to enforce this, so we must do so ourselves. + * See GCC PR44290. */ -#define __naked __attribute__((naked)) noinline __noclone notrace +#define __naked __attribute__((naked)) noinline __noclone notrace -#define __noreturn __attribute__((noreturn)) +#define __noreturn __attribute__((noreturn)) /* * From the GCC manual: @@ -215,16 +224,27 @@ #define KASAN_ABI_VERSION 3 #endif +#if GCC_VERSION >= 40902 +/* + * Tell the compiler that address safety instrumentation (KASAN) + * should not be applied to that function. + * Conflicts with inlining: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368 + */ +#define __no_sanitize_address __attribute__((no_sanitize_address)) +#endif + #endif /* gcc version >= 40000 specific checks */ #if !defined(__noclone) #define __noclone /* not needed */ #endif +#if !defined(__no_sanitize_address) +#define __no_sanitize_address +#endif + /* * A trick to suppress uninitialized variable warning without generating any * code */ #define uninitialized_var(x) x = x - -#define __always_inline inline __attribute__((always_inline)) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 1a6f01df1496..4d1995792303 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -188,32 +188,45 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); #include <uapi/linux/types.h> -static __always_inline void data_access_exceeds_word_size(void) -#ifdef __compiletime_warning -__compiletime_warning("data access exceeds word size and won't be atomic") -#endif -; +#define __READ_ONCE_SIZE \ +({ \ + switch (size) { \ + case 1: *(__u8 *)res = *(volatile __u8 *)p; break; \ + case 2: *(__u16 *)res = *(volatile __u16 *)p; break; \ + case 4: *(__u32 *)res = *(volatile __u32 *)p; break; \ + case 8: *(__u64 *)res = *(volatile __u64 *)p; break; \ + default: \ + barrier(); \ + __builtin_memcpy((void *)res, (const void *)p, size); \ + barrier(); \ + } \ +}) -static __always_inline void data_access_exceeds_word_size(void) +static __always_inline +void __read_once_size(const volatile void *p, void *res, int size) { + __READ_ONCE_SIZE; } -static __always_inline void __read_once_size(const volatile void *p, void *res, int size) +#ifdef CONFIG_KASAN +/* + * This function is not 'inline' because __no_sanitize_address confilcts + * with inlining. Attempt to inline it may cause a build failure. + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368 + * '__maybe_unused' allows us to avoid defined-but-not-used warnings. + */ +static __no_sanitize_address __maybe_unused +void __read_once_size_nocheck(const volatile void *p, void *res, int size) { - switch (size) { - case 1: *(__u8 *)res = *(volatile __u8 *)p; break; - case 2: *(__u16 *)res = *(volatile __u16 *)p; break; - case 4: *(__u32 *)res = *(volatile __u32 *)p; break; -#ifdef CONFIG_64BIT - case 8: *(__u64 *)res = *(volatile __u64 *)p; break; -#endif - default: - barrier(); - __builtin_memcpy((void *)res, (const void *)p, size); - data_access_exceeds_word_size(); - barrier(); - } + __READ_ONCE_SIZE; } +#else +static __always_inline +void __read_once_size_nocheck(const volatile void *p, void *res, int size) +{ + __READ_ONCE_SIZE; +} +#endif static __always_inline void __write_once_size(volatile void *p, void *res, int size) { @@ -221,13 +234,10 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s case 1: *(volatile __u8 *)p = *(__u8 *)res; break; case 2: *(volatile __u16 *)p = *(__u16 *)res; break; case 4: *(volatile __u32 *)p = *(__u32 *)res; break; -#ifdef CONFIG_64BIT case 8: *(volatile __u64 *)p = *(__u64 *)res; break; -#endif default: barrier(); __builtin_memcpy((void *)p, (const void *)res, size); - data_access_exceeds_word_size(); barrier(); } } @@ -254,8 +264,22 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s * required ordering. */ -#define READ_ONCE(x) \ - ({ union { typeof(x) __val; char __c[1]; } __u; __read_once_size(&(x), __u.__c, sizeof(x)); __u.__val; }) +#define __READ_ONCE(x, check) \ +({ \ + union { typeof(x) __val; char __c[1]; } __u; \ + if (check) \ + __read_once_size(&(x), __u.__c, sizeof(x)); \ + else \ + __read_once_size_nocheck(&(x), __u.__c, sizeof(x)); \ + __u.__val; \ +}) +#define READ_ONCE(x) __READ_ONCE(x, 1) + +/* + * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need + * to hide memory access from KASAN. + */ +#define READ_ONCE_NOCHECK(x) __READ_ONCE(x, 0) #define WRITE_ONCE(x, val) \ ({ typeof(x) __val = (val); __write_once_size(&(x), &__val, sizeof(__val)); __val; }) diff --git a/include/linux/coresight.h b/include/linux/coresight.h new file mode 100644 index 000000000000..44c1597a738a --- /dev/null +++ b/include/linux/coresight.h @@ -0,0 +1,251 @@ +/* Copyright (c) 2012, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _LINUX_CORESIGHT_H +#define _LINUX_CORESIGHT_H + +#include <linux/device.h> + +/* Peripheral id registers (0xFD0-0xFEC) */ +#define CORESIGHT_PERIPHIDR4 0xfd0 +#define CORESIGHT_PERIPHIDR5 0xfd4 +#define CORESIGHT_PERIPHIDR6 0xfd8 +#define CORESIGHT_PERIPHIDR7 0xfdC +#define CORESIGHT_PERIPHIDR0 0xfe0 +#define CORESIGHT_PERIPHIDR1 0xfe4 +#define CORESIGHT_PERIPHIDR2 0xfe8 +#define CORESIGHT_PERIPHIDR3 0xfeC +/* Component id registers (0xFF0-0xFFC) */ +#define CORESIGHT_COMPIDR0 0xff0 +#define CORESIGHT_COMPIDR1 0xff4 +#define CORESIGHT_COMPIDR2 0xff8 +#define CORESIGHT_COMPIDR3 0xffC + +#define ETM_ARCH_V3_3 0x23 +#define ETM_ARCH_V3_5 0x25 +#define PFT_ARCH_V1_0 0x30 +#define PFT_ARCH_V1_1 0x31 + +#define CORESIGHT_UNLOCK 0xc5acce55 + +extern struct bus_type coresight_bustype; + +enum coresight_dev_type { + CORESIGHT_DEV_TYPE_NONE, + CORESIGHT_DEV_TYPE_SINK, + CORESIGHT_DEV_TYPE_LINK, + CORESIGHT_DEV_TYPE_LINKSINK, + CORESIGHT_DEV_TYPE_SOURCE, +}; + +enum coresight_dev_subtype_sink { + CORESIGHT_DEV_SUBTYPE_SINK_NONE, + CORESIGHT_DEV_SUBTYPE_SINK_PORT, + CORESIGHT_DEV_SUBTYPE_SINK_BUFFER, +}; + +enum coresight_dev_subtype_link { + CORESIGHT_DEV_SUBTYPE_LINK_NONE, + CORESIGHT_DEV_SUBTYPE_LINK_MERG, + CORESIGHT_DEV_SUBTYPE_LINK_SPLIT, + CORESIGHT_DEV_SUBTYPE_LINK_FIFO, +}; + +enum coresight_dev_subtype_source { + CORESIGHT_DEV_SUBTYPE_SOURCE_NONE, + CORESIGHT_DEV_SUBTYPE_SOURCE_PROC, + CORESIGHT_DEV_SUBTYPE_SOURCE_BUS, + CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE, +}; + +/** + * struct coresight_dev_subtype - further characterisation of a type + * @sink_subtype: type of sink this component is, as defined + by @coresight_dev_subtype_sink. + * @link_subtype: type of link this component is, as defined + by @coresight_dev_subtype_link. + * @source_subtype: type of source this component is, as defined + by @coresight_dev_subtype_source. + */ +struct coresight_dev_subtype { + enum coresight_dev_subtype_sink sink_subtype; + enum coresight_dev_subtype_link link_subtype; + enum coresight_dev_subtype_source source_subtype; +}; + +/** + * struct coresight_platform_data - data harvested from the DT specification + * @cpu: the CPU a source belongs to. Only applicable for ETM/PTMs. + * @name: name of the component as shown under sysfs. + * @nr_inport: number of input ports for this component. + * @outports: list of remote enpoint port number. + * @child_names:name of all child components connected to this device. + * @child_ports:child component port number the current component is + connected to. + * @nr_outport: number of output ports for this component. + * @clk: The clock this component is associated to. + */ +struct coresight_platform_data { + int cpu; + const char *name; + int nr_inport; + int *outports; + const char **child_names; + int *child_ports; + int nr_outport; + struct clk *clk; +}; + +/** + * struct coresight_desc - description of a component required from drivers + * @type: as defined by @coresight_dev_type. + * @subtype: as defined by @coresight_dev_subtype. + * @ops: generic operations for this component, as defined + by @coresight_ops. + * @pdata: platform data collected from DT. + * @dev: The device entity associated to this component. + * @groups :operations specific to this component. These will end up + in the component's sysfs sub-directory. + */ +struct coresight_desc { + enum coresight_dev_type type; + struct coresight_dev_subtype subtype; + const struct coresight_ops *ops; + struct coresight_platform_data *pdata; + struct device *dev; + const struct attribute_group **groups; +}; + +/** + * struct coresight_connection - representation of a single connection + * @ref_count: keeping count a port' references. + * @outport: a connection's output port number. + * @chid_name: remote component's name. + * @child_port: remote component's port number @output is connected to. + * @child_dev: a @coresight_device representation of the component + connected to @outport. + */ +struct coresight_connection { + int outport; + const char *child_name; + int child_port; + struct coresight_device *child_dev; +}; + +/** + * struct coresight_device - representation of a device as used by the framework + * @nr_inport: number of input port associated to this component. + * @nr_outport: number of output port associated to this component. + * @type: as defined by @coresight_dev_type. + * @subtype: as defined by @coresight_dev_subtype. + * @ops: generic operations for this component, as defined + by @coresight_ops. + * @dev: The device entity associated to this component. + * @refcnt: keep track of what is in use. + * @path_link: link of current component into the path being enabled. + * @orphan: true if the component has connections that haven't been linked. + * @enable: 'true' if component is currently part of an active path. + * @activated: 'true' only if a _sink_ has been activated. A sink can be + activated but not yet enabled. Enabling for a _sink_ + happens when a source has been selected for that it. + */ +struct coresight_device { + struct coresight_connection *conns; + int nr_inport; + int nr_outport; + enum coresight_dev_type type; + struct coresight_dev_subtype subtype; + const struct coresight_ops *ops; + struct device dev; + atomic_t *refcnt; + struct list_head path_link; + bool orphan; + bool enable; /* true only if configured as part of a path */ + bool activated; /* true only if a sink is part of a path */ +}; + +#define to_coresight_device(d) container_of(d, struct coresight_device, dev) + +#define source_ops(csdev) csdev->ops->source_ops +#define sink_ops(csdev) csdev->ops->sink_ops +#define link_ops(csdev) csdev->ops->link_ops + +/** + * struct coresight_ops_sink - basic operations for a sink + * Operations available for sinks + * @enable: enables the sink. + * @disable: disables the sink. + */ +struct coresight_ops_sink { + int (*enable)(struct coresight_device *csdev); + void (*disable)(struct coresight_device *csdev); +}; + +/** + * struct coresight_ops_link - basic operations for a link + * Operations available for links. + * @enable: enables flow between iport and oport. + * @disable: disables flow between iport and oport. + */ +struct coresight_ops_link { + int (*enable)(struct coresight_device *csdev, int iport, int oport); + void (*disable)(struct coresight_device *csdev, int iport, int oport); +}; + +/** + * struct coresight_ops_source - basic operations for a source + * Operations available for sources. + * @trace_id: returns the value of the component's trace ID as known + to the HW. + * @enable: enables tracing from a source. + * @disable: disables tracing for a source. + */ +struct coresight_ops_source { + int (*trace_id)(struct coresight_device *csdev); + int (*enable)(struct coresight_device *csdev); + void (*disable)(struct coresight_device *csdev); +}; + +struct coresight_ops { + const struct coresight_ops_sink *sink_ops; + const struct coresight_ops_link *link_ops; + const struct coresight_ops_source *source_ops; +}; + +#ifdef CONFIG_CORESIGHT +extern struct coresight_device * +coresight_register(struct coresight_desc *desc); +extern void coresight_unregister(struct coresight_device *csdev); +extern int coresight_enable(struct coresight_device *csdev); +extern void coresight_disable(struct coresight_device *csdev); +extern int coresight_timeout(void __iomem *addr, u32 offset, + int position, int value); +#else +static inline struct coresight_device * +coresight_register(struct coresight_desc *desc) { return NULL; } +static inline void coresight_unregister(struct coresight_device *csdev) {} +static inline int +coresight_enable(struct coresight_device *csdev) { return -ENOSYS; } +static inline void coresight_disable(struct coresight_device *csdev) {} +static inline int coresight_timeout(void __iomem *addr, u32 offset, + int position, int value) { return 1; } +#endif + +#ifdef CONFIG_OF +extern struct coresight_platform_data *of_get_coresight_platform_data( + struct device *dev, struct device_node *node); +#else +static inline struct coresight_platform_data *of_get_coresight_platform_data( + struct device *dev, struct device_node *node) { return NULL; } +#endif + +#endif diff --git a/include/linux/cpu_cooling.h b/include/linux/cpu_cooling.h index c303d383def1..8ac033182419 100644 --- a/include/linux/cpu_cooling.h +++ b/include/linux/cpu_cooling.h @@ -28,6 +28,9 @@ #include <linux/thermal.h> #include <linux/cpumask.h> +typedef int (*get_static_t)(cpumask_t *cpumask, int interval, + unsigned long voltage, u32 *power); + #ifdef CONFIG_CPU_THERMAL /** * cpufreq_cooling_register - function to create cpufreq cooling device. @@ -36,6 +39,10 @@ struct thermal_cooling_device * cpufreq_cooling_register(const struct cpumask *clip_cpus); +struct thermal_cooling_device * +cpufreq_power_cooling_register(const struct cpumask *clip_cpus, + u32 capacitance, get_static_t plat_static_func); + /** * of_cpufreq_cooling_register - create cpufreq cooling device based on DT. * @np: a valid struct device_node to the cooling device device tree node. @@ -45,6 +52,12 @@ cpufreq_cooling_register(const struct cpumask *clip_cpus); struct thermal_cooling_device * of_cpufreq_cooling_register(struct device_node *np, const struct cpumask *clip_cpus); + +struct thermal_cooling_device * +of_cpufreq_power_cooling_register(struct device_node *np, + const struct cpumask *clip_cpus, + u32 capacitance, + get_static_t plat_static_func); #else static inline struct thermal_cooling_device * of_cpufreq_cooling_register(struct device_node *np, @@ -52,6 +65,15 @@ of_cpufreq_cooling_register(struct device_node *np, { return NULL; } + +static inline struct thermal_cooling_device * +of_cpufreq_power_cooling_register(struct device_node *np, + const struct cpumask *clip_cpus, + u32 capacitance, + get_static_t plat_static_func) +{ + return NULL; +} #endif /** @@ -68,11 +90,28 @@ cpufreq_cooling_register(const struct cpumask *clip_cpus) return NULL; } static inline struct thermal_cooling_device * +cpufreq_power_cooling_register(const struct cpumask *clip_cpus, + u32 capacitance, get_static_t plat_static_func) +{ + return NULL; +} + +static inline struct thermal_cooling_device * of_cpufreq_cooling_register(struct device_node *np, const struct cpumask *clip_cpus) { return NULL; } + +static inline struct thermal_cooling_device * +of_cpufreq_power_cooling_register(struct device_node *np, + const struct cpumask *clip_cpus, + u32 capacitance, + get_static_t plat_static_func) +{ + return NULL; +} + static inline void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev) { diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 503b085b7832..6734c9150ebb 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -217,26 +217,26 @@ __ATTR(_name, 0644, show_##_name, store_##_name) struct cpufreq_driver { - char name[CPUFREQ_NAME_LEN]; - u8 flags; - void *driver_data; + char name[CPUFREQ_NAME_LEN]; + u8 flags; + void *driver_data; /* needed by all drivers */ - int (*init) (struct cpufreq_policy *policy); - int (*verify) (struct cpufreq_policy *policy); + int (*init)(struct cpufreq_policy *policy); + int (*verify)(struct cpufreq_policy *policy); /* define one out of two */ - int (*setpolicy) (struct cpufreq_policy *policy); + int (*setpolicy)(struct cpufreq_policy *policy); /* * On failure, should always restore frequency to policy->restore_freq * (i.e. old freq). */ - int (*target) (struct cpufreq_policy *policy, /* Deprecated */ - unsigned int target_freq, - unsigned int relation); - int (*target_index) (struct cpufreq_policy *policy, - unsigned int index); + int (*target)(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation); /* Deprecated */ + int (*target_index)(struct cpufreq_policy *policy, + unsigned int index); /* * Only for drivers with target_index() and CPUFREQ_ASYNC_NOTIFICATION * unset. @@ -252,27 +252,31 @@ struct cpufreq_driver { * wish to switch to intermediate frequency for some target frequency. * In that case core will directly call ->target_index(). */ - unsigned int (*get_intermediate)(struct cpufreq_policy *policy, - unsigned int index); - int (*target_intermediate)(struct cpufreq_policy *policy, - unsigned int index); + unsigned int (*get_intermediate)(struct cpufreq_policy *policy, + unsigned int index); + int (*target_intermediate)(struct cpufreq_policy *policy, + unsigned int index); /* should be defined, if possible */ - unsigned int (*get) (unsigned int cpu); + unsigned int (*get)(unsigned int cpu); /* optional */ - int (*bios_limit) (int cpu, unsigned int *limit); + int (*bios_limit)(int cpu, unsigned int *limit); + + int (*exit)(struct cpufreq_policy *policy); + void (*stop_cpu)(struct cpufreq_policy *policy); + int (*suspend)(struct cpufreq_policy *policy); + int (*resume)(struct cpufreq_policy *policy); - int (*exit) (struct cpufreq_policy *policy); - void (*stop_cpu) (struct cpufreq_policy *policy); - int (*suspend) (struct cpufreq_policy *policy); - int (*resume) (struct cpufreq_policy *policy); - struct freq_attr **attr; + /* Will be called after the driver is fully initialized */ + void (*ready)(struct cpufreq_policy *policy); + + struct freq_attr **attr; /* platform specific boost support code */ - bool boost_supported; - bool boost_enabled; - int (*set_boost) (int state); + bool boost_supported; + bool boost_enabled; + int (*set_boost)(int state); }; /* flags */ @@ -570,6 +574,8 @@ ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf); int cpufreq_boost_trigger_state(int state); int cpufreq_boost_supported(void); int cpufreq_boost_enabled(void); +int cpufreq_enable_boost_support(void); +bool policy_has_boost_freq(struct cpufreq_policy *policy); #else static inline int cpufreq_boost_trigger_state(int state) { @@ -583,12 +589,23 @@ static inline int cpufreq_boost_enabled(void) { return 0; } + +static inline int cpufreq_enable_boost_support(void) +{ + return -EINVAL; +} + +static inline bool policy_has_boost_freq(struct cpufreq_policy *policy) +{ + return false; +} #endif /* the following funtion is for cpufreq core use only */ struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu); /* the following are really really optional */ extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs; +extern struct freq_attr cpufreq_freq_attr_scaling_boost_freqs; extern struct freq_attr *cpufreq_generic_attr[]; int cpufreq_table_validate_and_show(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table); diff --git a/include/linux/device.h b/include/linux/device.h index ce1f21608b16..61a62c201264 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -690,6 +690,7 @@ struct acpi_dev_node { * along with subsystem-level and driver-level callbacks. * @pins: For device pin management. * See Documentation/pinctrl.txt for details. + * @msi_domain: The generic MSI domain this device is using. * @numa_node: NUMA node this device is close to. * @dma_mask: Dma mask (if dma'ble device). * @coherent_dma_mask: Like dma_mask, but for alloc_coherent mapping as not all @@ -750,6 +751,9 @@ struct device { struct dev_pm_info power; struct dev_pm_domain *pm_domain; +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN + struct irq_domain *msi_domain; +#endif #ifdef CONFIG_PINCTRL struct dev_pin_info *pins; #endif @@ -837,6 +841,22 @@ static inline void set_dev_node(struct device *dev, int node) } #endif +static inline struct irq_domain *dev_get_msi_domain(const struct device *dev) +{ +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN + return dev->msi_domain; +#else + return NULL; +#endif +} + +static inline void dev_set_msi_domain(struct device *dev, struct irq_domain *d) +{ +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN + dev->msi_domain = d; +#endif +} + static inline void *dev_get_drvdata(const struct device *dev) { return dev->driver_data; diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index d5d388160f42..c3007cb4bfa6 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -129,11 +129,14 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask) extern u64 dma_get_required_mask(struct device *dev); -#ifndef set_arch_dma_coherent_ops -static inline int set_arch_dma_coherent_ops(struct device *dev) -{ - return 0; -} +#ifndef arch_setup_dma_ops +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, + u64 size, struct iommu_ops *iommu, + bool coherent) { } +#endif + +#ifndef arch_teardown_dma_ops +static inline void arch_teardown_dma_ops(struct device *dev) { } #endif static inline unsigned int dma_get_max_seg_size(struct device *dev) diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h index 3d0320675e12..0d7b5840cd64 100644 --- a/include/linux/ftrace_event.h +++ b/include/linux/ftrace_event.h @@ -44,6 +44,10 @@ const char *ftrace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr, const char *ftrace_print_hex_seq(struct trace_seq *p, const unsigned char *buf, int len); +const char *ftrace_print_array_seq(struct trace_seq *p, + const void *buf, int buf_len, + size_t el_size); + struct trace_iterator; struct trace_event; diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 77fc43f8fb72..a041f64cf29a 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -166,6 +166,13 @@ extern struct task_group root_task_group; # define INIT_RT_MUTEXES(tsk) #endif +#ifdef CONFIG_KASAN +# define INIT_KASAN(tsk) \ + .kasan_depth = 1, +#else +# define INIT_KASAN(tsk) +#endif + /* * INIT_TASK is used to set up the first task table, touch at * your own risk!. Base=0, limit=0x1fffff (=2MB) @@ -237,6 +244,7 @@ extern struct task_group root_task_group; INIT_CPUSET_SEQ(tsk) \ INIT_RT_MUTEXES(tsk) \ INIT_VTIME(tsk) \ + INIT_KASAN(tsk) \ } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e6a7c9ff72f2..415c7613d02c 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -21,6 +21,7 @@ #include <linux/errno.h> #include <linux/err.h> +#include <linux/of.h> #include <linux/types.h> #include <trace/events/iommu.h> @@ -102,7 +103,9 @@ enum iommu_attr { * @remove_device: remove device from iommu grouping * @domain_get_attr: Query domain attributes * @domain_set_attr: Change domain attributes + * @of_xlate: add OF master IDs to iommu grouping * @pgsize_bitmap: bitmap of supported page sizes + * @priv: per-instance data private to the iommu driver */ struct iommu_ops { bool (*capable)(enum iommu_cap); @@ -132,7 +135,12 @@ struct iommu_ops { /* Get the numer of window per domain */ u32 (*domain_get_windows)(struct iommu_domain *domain); +#ifdef CONFIG_OF_IOMMU + int (*of_xlate)(struct device *dev, struct of_phandle_args *args); +#endif + unsigned long pgsize_bitmap; + void *priv; }; #define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */ diff --git a/include/linux/irq.h b/include/linux/irq.h index c91ce60746f7..2fd2b6d442b8 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -15,6 +15,7 @@ #include <linux/spinlock.h> #include <linux/cpumask.h> #include <linux/gfp.h> +#include <linux/irqhandler.h> #include <linux/irqreturn.h> #include <linux/irqnr.h> #include <linux/errno.h> @@ -28,11 +29,7 @@ struct seq_file; struct module; -struct irq_desc; -struct irq_data; -typedef void (*irq_flow_handler_t)(unsigned int irq, - struct irq_desc *desc); -typedef void (*irq_preflow_handler_t)(struct irq_data *data); +struct msi_msg; /* * IRQ line status. @@ -114,10 +111,14 @@ enum { * * IRQ_SET_MASK_OK - OK, core updates irq_data.affinity * IRQ_SET_MASK_NOCPY - OK, chip did update irq_data.affinity + * IRQ_SET_MASK_OK_DONE - Same as IRQ_SET_MASK_OK for core. Special code to + * support stacked irqchips, which indicates skipping + * all descendent irqchips. */ enum { IRQ_SET_MASK_OK = 0, IRQ_SET_MASK_OK_NOCOPY, + IRQ_SET_MASK_OK_DONE, }; struct msi_desc; @@ -134,6 +135,8 @@ struct irq_domain; * @chip: low level interrupt hardware access * @domain: Interrupt translation domain; responsible for mapping * between hwirq number and linux irq number. + * @parent_data: pointer to parent struct irq_data to support hierarchy + * irq_domain * @handler_data: per-IRQ data for the irq_chip methods * @chip_data: platform-specific per-chip private data for the chip * methods, to allow shared chip implementations @@ -152,6 +155,9 @@ struct irq_data { unsigned int state_use_accessors; struct irq_chip *chip; struct irq_domain *domain; +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY + struct irq_data *parent_data; +#endif void *handler_data; void *chip_data; struct msi_desc *msi_desc; @@ -316,6 +322,8 @@ static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d) * any other callback related to this irq * @irq_release_resources: optional to release resources acquired with * irq_request_resources + * @irq_compose_msi_msg: optional to compose message content for MSI + * @irq_write_msi_msg: optional to write message content for MSI * @flags: chip specific flags */ struct irq_chip { @@ -352,6 +360,9 @@ struct irq_chip { int (*irq_request_resources)(struct irq_data *data); void (*irq_release_resources)(struct irq_data *data); + void (*irq_compose_msi_msg)(struct irq_data *data, struct msi_msg *msg); + void (*irq_write_msi_msg)(struct irq_data *data, struct msi_msg *msg); + unsigned long flags; }; @@ -439,6 +450,18 @@ extern void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc); extern void handle_bad_irq(unsigned int irq, struct irq_desc *desc); extern void handle_nested_irq(unsigned int irq); +extern int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg); +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY +extern void irq_chip_ack_parent(struct irq_data *data); +extern int irq_chip_retrigger_hierarchy(struct irq_data *data); +extern void irq_chip_mask_parent(struct irq_data *data); +extern void irq_chip_unmask_parent(struct irq_data *data); +extern void irq_chip_eoi_parent(struct irq_data *data); +extern int irq_chip_set_affinity_parent(struct irq_data *data, + const struct cpumask *dest, + bool force); +#endif + /* Handling of unhandled and spurious interrupts: */ extern void note_interrupt(unsigned int irq, struct irq_desc *desc, irqreturn_t action_ret); @@ -583,7 +606,7 @@ static inline struct msi_desc *irq_get_msi_desc(unsigned int irq) return d ? d->msi_desc : NULL; } -static inline struct msi_desc *irq_data_get_msi(struct irq_data *d) +static inline struct msi_desc *irq_data_get_msi_desc(struct irq_data *d) { return d->msi_desc; } @@ -686,6 +709,8 @@ struct irq_chip_type { * struct irq_chip_generic - Generic irq chip data structure * @lock: Lock to protect register and cache data access * @reg_base: Register base address (virtual) + * @reg_readl: Alternate I/O accessor (defaults to readl if NULL) + * @reg_writel: Alternate I/O accessor (defaults to writel if NULL) * @irq_base: Interrupt base nr for this chip * @irq_cnt: Number of interrupts handled by this chip * @mask_cache: Cached mask register shared between all chip types @@ -710,6 +735,8 @@ struct irq_chip_type { struct irq_chip_generic { raw_spinlock_t lock; void __iomem *reg_base; + u32 (*reg_readl)(void __iomem *addr); + void (*reg_writel)(u32 val, void __iomem *addr); unsigned int irq_base; unsigned int irq_cnt; u32 mask_cache; @@ -734,12 +761,14 @@ struct irq_chip_generic { * the parent irq. Usually GPIO implementations * @IRQ_GC_MASK_CACHE_PER_TYPE: Mask cache is chip type private * @IRQ_GC_NO_MASK: Do not calculate irq_data->mask + * @IRQ_GC_BE_IO: Use big-endian register accesses (default: LE) */ enum irq_gc_flags { IRQ_GC_INIT_MASK_CACHE = 1 << 0, IRQ_GC_INIT_NESTED_LOCK = 1 << 1, IRQ_GC_MASK_CACHE_PER_TYPE = 1 << 2, IRQ_GC_NO_MASK = 1 << 3, + IRQ_GC_BE_IO = 1 << 4, }; /* @@ -815,6 +844,13 @@ static inline void irq_gc_lock(struct irq_chip_generic *gc) { } static inline void irq_gc_unlock(struct irq_chip_generic *gc) { } #endif +static inline void irq_reg_writel(struct irq_chip_generic *gc, + u32 val, int reg_offset) +{ + if (gc->reg_writel) + gc->reg_writel(val, gc->reg_base + reg_offset); + else + writel(val, gc->reg_base + reg_offset); /* * The irqsave variants are for usage in non interrupt code. Do not use * them in irq_chip callbacks. Use irq_gc_lock() instead. @@ -825,16 +861,15 @@ static inline void irq_gc_unlock(struct irq_chip_generic *gc) { } #define irq_gc_unlock_irqrestore(gc, flags) \ raw_spin_unlock_irqrestore(&(gc)->lock, flags) -static inline void irq_reg_writel(struct irq_chip_generic *gc, - u32 val, int reg_offset) -{ - writel(val, gc->reg_base + reg_offset); } static inline u32 irq_reg_readl(struct irq_chip_generic *gc, int reg_offset) { - return readl(gc->reg_base + reg_offset); + if (gc->reg_readl) + return gc->reg_readl(gc->reg_base + reg_offset); + else + return readl(gc->reg_base + reg_offset); } #endif /* _LINUX_IRQ_H */ diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index e93d2c5549db..f0b4c487a60e 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -49,6 +49,10 @@ #define GICD_CTLR_ENABLE_G1A (1U << 1) #define GICD_CTLR_ENABLE_G1 (1U << 0) +#define GICD_TYPER_ID_BITS(typer) ((((typer) >> 19) & 0x1f) + 1) +#define GICD_TYPER_IRQS(typer) ((((typer) & 0x1f) + 1) * 32) +#define GICD_TYPER_LPIS (1U << 17) + #define GICD_IROUTER_SPI_MODE_ONE (0U << 31) #define GICD_IROUTER_SPI_MODE_ANY (1U << 31) @@ -76,9 +80,42 @@ #define GICR_MOVALLR 0x0110 #define GICR_PIDR2 GICD_PIDR2 +#define GICR_CTLR_ENABLE_LPIS (1UL << 0) + +#define GICR_TYPER_CPU_NUMBER(r) (((r) >> 8) & 0xffff) + #define GICR_WAKER_ProcessorSleep (1U << 1) #define GICR_WAKER_ChildrenAsleep (1U << 2) +#define GICR_PROPBASER_NonShareable (0U << 10) +#define GICR_PROPBASER_InnerShareable (1U << 10) +#define GICR_PROPBASER_OuterShareable (2U << 10) +#define GICR_PROPBASER_SHAREABILITY_MASK (3UL << 10) +#define GICR_PROPBASER_nCnB (0U << 7) +#define GICR_PROPBASER_nC (1U << 7) +#define GICR_PROPBASER_RaWt (2U << 7) +#define GICR_PROPBASER_RaWb (3U << 7) +#define GICR_PROPBASER_WaWt (4U << 7) +#define GICR_PROPBASER_WaWb (5U << 7) +#define GICR_PROPBASER_RaWaWt (6U << 7) +#define GICR_PROPBASER_RaWaWb (7U << 7) +#define GICR_PROPBASER_CACHEABILITY_MASK (7U << 7) +#define GICR_PROPBASER_IDBITS_MASK (0x1f) + +#define GICR_PENDBASER_NonShareable (0U << 10) +#define GICR_PENDBASER_InnerShareable (1U << 10) +#define GICR_PENDBASER_OuterShareable (2U << 10) +#define GICR_PENDBASER_SHAREABILITY_MASK (3UL << 10) +#define GICR_PENDBASER_nCnB (0U << 7) +#define GICR_PENDBASER_nC (1U << 7) +#define GICR_PENDBASER_RaWt (2U << 7) +#define GICR_PENDBASER_RaWb (3U << 7) +#define GICR_PENDBASER_WaWt (4U << 7) +#define GICR_PENDBASER_WaWb (5U << 7) +#define GICR_PENDBASER_RaWaWt (6U << 7) +#define GICR_PENDBASER_RaWaWb (7U << 7) +#define GICR_PENDBASER_CACHEABILITY_MASK (7U << 7) + /* * Re-Distributor registers, offsets from SGI_base */ @@ -91,9 +128,100 @@ #define GICR_IPRIORITYR0 GICD_IPRIORITYR #define GICR_ICFGR0 GICD_ICFGR +#define GICR_TYPER_PLPIS (1U << 0) #define GICR_TYPER_VLPIS (1U << 1) #define GICR_TYPER_LAST (1U << 4) +#define LPI_PROP_GROUP1 (1 << 1) +#define LPI_PROP_ENABLED (1 << 0) + +/* + * ITS registers, offsets from ITS_base + */ +#define GITS_CTLR 0x0000 +#define GITS_IIDR 0x0004 +#define GITS_TYPER 0x0008 +#define GITS_CBASER 0x0080 +#define GITS_CWRITER 0x0088 +#define GITS_CREADR 0x0090 +#define GITS_BASER 0x0100 +#define GITS_PIDR2 GICR_PIDR2 + +#define GITS_TRANSLATER 0x10040 + +#define GITS_CTLR_ENABLE (1U << 0) +#define GITS_CTLR_QUIESCENT (1U << 31) + +#define GITS_TYPER_DEVBITS_SHIFT 13 +#define GITS_TYPER_DEVBITS(r) ((((r) >> GITS_TYPER_DEVBITS_SHIFT) & 0x1f) + 1) +#define GITS_TYPER_PTA (1UL << 19) + +#define GITS_CBASER_VALID (1UL << 63) +#define GITS_CBASER_nCnB (0UL << 59) +#define GITS_CBASER_nC (1UL << 59) +#define GITS_CBASER_RaWt (2UL << 59) +#define GITS_CBASER_RaWb (3UL << 59) +#define GITS_CBASER_WaWt (4UL << 59) +#define GITS_CBASER_WaWb (5UL << 59) +#define GITS_CBASER_RaWaWt (6UL << 59) +#define GITS_CBASER_RaWaWb (7UL << 59) +#define GITS_CBASER_CACHEABILITY_MASK (7UL << 59) +#define GITS_CBASER_NonShareable (0UL << 10) +#define GITS_CBASER_InnerShareable (1UL << 10) +#define GITS_CBASER_OuterShareable (2UL << 10) +#define GITS_CBASER_SHAREABILITY_MASK (3UL << 10) + +#define GITS_BASER_NR_REGS 8 + +#define GITS_BASER_VALID (1UL << 63) +#define GITS_BASER_nCnB (0UL << 59) +#define GITS_BASER_nC (1UL << 59) +#define GITS_BASER_RaWt (2UL << 59) +#define GITS_BASER_RaWb (3UL << 59) +#define GITS_BASER_WaWt (4UL << 59) +#define GITS_BASER_WaWb (5UL << 59) +#define GITS_BASER_RaWaWt (6UL << 59) +#define GITS_BASER_RaWaWb (7UL << 59) +#define GITS_BASER_CACHEABILITY_MASK (7UL << 59) +#define GITS_BASER_TYPE_SHIFT (56) +#define GITS_BASER_TYPE(r) (((r) >> GITS_BASER_TYPE_SHIFT) & 7) +#define GITS_BASER_ENTRY_SIZE_SHIFT (48) +#define GITS_BASER_ENTRY_SIZE(r) ((((r) >> GITS_BASER_ENTRY_SIZE_SHIFT) & 0xff) + 1) +#define GITS_BASER_NonShareable (0UL << 10) +#define GITS_BASER_InnerShareable (1UL << 10) +#define GITS_BASER_OuterShareable (2UL << 10) +#define GITS_BASER_SHAREABILITY_SHIFT (10) +#define GITS_BASER_SHAREABILITY_MASK (3UL << GITS_BASER_SHAREABILITY_SHIFT) +#define GITS_BASER_PAGE_SIZE_SHIFT (8) +#define GITS_BASER_PAGE_SIZE_4K (0UL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_16K (1UL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_64K (2UL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_MASK (3UL << GITS_BASER_PAGE_SIZE_SHIFT) + +#define GITS_BASER_TYPE_NONE 0 +#define GITS_BASER_TYPE_DEVICE 1 +#define GITS_BASER_TYPE_VCPU 2 +#define GITS_BASER_TYPE_CPU 3 +#define GITS_BASER_TYPE_COLLECTION 4 +#define GITS_BASER_TYPE_RESERVED5 5 +#define GITS_BASER_TYPE_RESERVED6 6 +#define GITS_BASER_TYPE_RESERVED7 7 + +/* + * ITS commands + */ +#define GITS_CMD_MAPD 0x08 +#define GITS_CMD_MAPC 0x09 +#define GITS_CMD_MAPVI 0x0a +#define GITS_CMD_MOVI 0x01 +#define GITS_CMD_DISCARD 0x0f +#define GITS_CMD_INV 0x0c +#define GITS_CMD_MOVALL 0x0e +#define GITS_CMD_INVALL 0x0d +#define GITS_CMD_INT 0x03 +#define GITS_CMD_CLEAR 0x04 +#define GITS_CMD_SYNC 0x05 + /* * CPU interface registers */ @@ -200,6 +328,24 @@ #ifndef __ASSEMBLY__ #include <linux/stringify.h> +#include <asm/msi.h> + +/* + * We need a value to serve as a irq-type for LPIs. Choose one that will + * hopefully pique the interest of the reviewer. + */ +#define GIC_IRQ_TYPE_LPI 0xa110c8ed + +struct rdists { + struct { + void __iomem *rd_base; + struct page *pend_page; + phys_addr_t phys_base; + } __percpu *rdist; + struct page *prop_page; + int id_bits; + u64 flags; +}; static inline void gic_write_eoir(u64 irq) { @@ -207,6 +353,11 @@ static inline void gic_write_eoir(u64 irq) isb(); } +struct irq_domain; +int its_cpu_init(void); +int its_init(struct device_node *node, struct rdists *rdists, + struct irq_domain *domain); + #endif #endif diff --git a/include/linux/irqchip/arm-gic.h b/include/linux/irqchip/arm-gic.h index 13eed92c7d24..71d706d5f169 100644 --- a/include/linux/irqchip/arm-gic.h +++ b/include/linux/irqchip/arm-gic.h @@ -91,6 +91,8 @@ #ifndef __ASSEMBLY__ +#include <linux/irqdomain.h> + struct device_node; extern struct irq_chip gic_arch_extn; @@ -106,6 +108,8 @@ static inline void gic_init(unsigned int nr, int start, gic_init_bases(nr, start, dist, cpu, 0, NULL); } +int gicv2m_of_init(struct device_node *node, struct irq_domain *parent); + void gic_send_sgi(unsigned int cpu_id, unsigned int irq); int gic_get_cpu_id(unsigned int cpu); void gic_migrate_target(unsigned int new_cpu_id); diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index b0f9d16e48f6..3c5ca459ac98 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -33,15 +33,32 @@ #define _LINUX_IRQDOMAIN_H #include <linux/types.h> +#include <linux/irqhandler.h> #include <linux/radix-tree.h> struct device_node; struct irq_domain; struct of_device_id; +struct irq_chip; +struct irq_data; /* Number of irqs reserved for a legacy isa controller */ #define NUM_ISA_INTERRUPTS 16 +/* + * Should several domains have the same device node, but serve + * different purposes (for example one domain is for PCI/MSI, and the + * other for wired IRQs), they can be distinguished using a + * bus-specific token. Most domains are expected to only carry + * DOMAIN_BUS_ANY. + */ +enum irq_domain_bus_token { + DOMAIN_BUS_ANY = 0, + DOMAIN_BUS_PCI_MSI, + DOMAIN_BUS_PLATFORM_MSI, + DOMAIN_BUS_NEXUS, +}; + /** * struct irq_domain_ops - Methods for irq_domain objects * @match: Match an interrupt controller device node to a host, returns @@ -58,12 +75,23 @@ struct of_device_id; * to setup the irq_desc when returning from map(). */ struct irq_domain_ops { - int (*match)(struct irq_domain *d, struct device_node *node); + int (*match)(struct irq_domain *d, struct device_node *node, + enum irq_domain_bus_token bus_token); int (*map)(struct irq_domain *d, unsigned int virq, irq_hw_number_t hw); void (*unmap)(struct irq_domain *d, unsigned int virq); int (*xlate)(struct irq_domain *d, struct device_node *node, const u32 *intspec, unsigned int intsize, unsigned long *out_hwirq, unsigned int *out_type); + +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY + /* extended V2 interfaces to support hierarchy irq_domains */ + int (*alloc)(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *arg); + void (*free)(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs); + void (*activate)(struct irq_domain *d, struct irq_data *irq_data); + void (*deactivate)(struct irq_domain *d, struct irq_data *irq_data); +#endif }; extern struct irq_domain_ops irq_generic_chip_ops; @@ -77,6 +105,7 @@ struct irq_domain_chip_generic; * @ops: pointer to irq_domain methods * @host_data: private data pointer for use by owner. Not touched by irq_domain * core code. + * @flags: host per irq_domain flags * * Optional elements * @of_node: Pointer to device tree nodes associated with the irq_domain. Used @@ -84,6 +113,7 @@ struct irq_domain_chip_generic; * @gc: Pointer to a list of generic chips. There is a helper function for * setting up one or more generic chips for interrupt controllers * drivers using the generic chip library which uses this pointer. + * @parent: Pointer to parent irq_domain to support hierarchy irq_domains * * Revmap data, used internally by irq_domain * @revmap_direct_max_irq: The largest hwirq that can be set for controllers that @@ -97,10 +127,15 @@ struct irq_domain { const char *name; const struct irq_domain_ops *ops; void *host_data; + unsigned int flags; /* Optional data */ struct device_node *of_node; + enum irq_domain_bus_token bus_token; struct irq_domain_chip_generic *gc; +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY + struct irq_domain *parent; +#endif /* reverse map data. The linear map gets appended to the irq_domain */ irq_hw_number_t hwirq_max; @@ -110,6 +145,22 @@ struct irq_domain { unsigned int linear_revmap[]; }; +/* Irq domain flags */ +enum { + /* Irq domain is hierarchical */ + IRQ_DOMAIN_FLAG_HIERARCHY = (1 << 0), + + /* Core calls alloc/free recursive through the domain hierarchy. */ + IRQ_DOMAIN_FLAG_AUTO_RECURSIVE = (1 << 1), + + /* + * Flags starting from IRQ_DOMAIN_FLAG_NONCORE are reserved + * for implementation specific purposes and ignored by the + * core code. + */ + IRQ_DOMAIN_FLAG_NONCORE = (1 << 16), +}; + #ifdef CONFIG_IRQ_DOMAIN struct irq_domain *__irq_domain_add(struct device_node *of_node, int size, irq_hw_number_t hwirq_max, int direct_max, @@ -126,9 +177,15 @@ struct irq_domain *irq_domain_add_legacy(struct device_node *of_node, irq_hw_number_t first_hwirq, const struct irq_domain_ops *ops, void *host_data); -extern struct irq_domain *irq_find_host(struct device_node *node); +extern struct irq_domain *irq_find_matching_host(struct device_node *node, + enum irq_domain_bus_token bus_token); extern void irq_set_default_host(struct irq_domain *host); +static inline struct irq_domain *irq_find_host(struct device_node *node) +{ + return irq_find_matching_host(node, DOMAIN_BUS_ANY); +} + /** * irq_domain_add_linear() - Allocate and register a linear revmap irq_domain. * @of_node: pointer to interrupt controller's device tree node. @@ -220,8 +277,74 @@ int irq_domain_xlate_onetwocell(struct irq_domain *d, struct device_node *ctrlr, const u32 *intspec, unsigned int intsize, irq_hw_number_t *out_hwirq, unsigned int *out_type); +/* V2 interfaces to support hierarchy IRQ domains. */ +extern struct irq_data *irq_domain_get_irq_data(struct irq_domain *domain, + unsigned int virq); +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY +extern struct irq_domain *irq_domain_add_hierarchy(struct irq_domain *parent, + unsigned int flags, unsigned int size, + struct device_node *node, + const struct irq_domain_ops *ops, void *host_data); +extern int __irq_domain_alloc_irqs(struct irq_domain *domain, int irq_base, + unsigned int nr_irqs, int node, void *arg, + bool realloc); +extern void irq_domain_free_irqs(unsigned int virq, unsigned int nr_irqs); +extern void irq_domain_activate_irq(struct irq_data *irq_data); +extern void irq_domain_deactivate_irq(struct irq_data *irq_data); + +static inline int irq_domain_alloc_irqs(struct irq_domain *domain, + unsigned int nr_irqs, int node, void *arg) +{ + return __irq_domain_alloc_irqs(domain, -1, nr_irqs, node, arg, false); +} + +extern int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, + unsigned int virq, + irq_hw_number_t hwirq, + struct irq_chip *chip, + void *chip_data); +extern void irq_domain_set_info(struct irq_domain *domain, unsigned int virq, + irq_hw_number_t hwirq, struct irq_chip *chip, + void *chip_data, irq_flow_handler_t handler, + void *handler_data, const char *handler_name); +extern void irq_domain_reset_irq_data(struct irq_data *irq_data); +extern void irq_domain_free_irqs_common(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs); +extern void irq_domain_free_irqs_top(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs); + +extern int irq_domain_alloc_irqs_parent(struct irq_domain *domain, + unsigned int irq_base, + unsigned int nr_irqs, void *arg); + +extern void irq_domain_free_irqs_parent(struct irq_domain *domain, + unsigned int irq_base, + unsigned int nr_irqs); + +static inline bool irq_domain_is_hierarchy(struct irq_domain *domain) +{ + return domain->flags & IRQ_DOMAIN_FLAG_HIERARCHY; +} +#else /* CONFIG_IRQ_DOMAIN_HIERARCHY */ +static inline void irq_domain_activate_irq(struct irq_data *data) { } +static inline void irq_domain_deactivate_irq(struct irq_data *data) { } +static inline int irq_domain_alloc_irqs(struct irq_domain *domain, + unsigned int nr_irqs, int node, void *arg) +{ + return -1; +} + +static inline bool irq_domain_is_hierarchy(struct irq_domain *domain) +{ + return false; +} +#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */ + #else /* CONFIG_IRQ_DOMAIN */ static inline void irq_dispose_mapping(unsigned int virq) { } +static inline void irq_domain_activate_irq(struct irq_data *data) { } +static inline void irq_domain_deactivate_irq(struct irq_data *data) { } #endif /* !CONFIG_IRQ_DOMAIN */ #endif /* _LINUX_IRQDOMAIN_H */ diff --git a/include/linux/irqhandler.h b/include/linux/irqhandler.h new file mode 100644 index 000000000000..62d543004197 --- /dev/null +++ b/include/linux/irqhandler.h @@ -0,0 +1,14 @@ +#ifndef _LINUX_IRQHANDLER_H +#define _LINUX_IRQHANDLER_H + +/* + * Interrupt flow handler typedefs are defined here to avoid circular + * include dependencies. + */ + +struct irq_desc; +struct irq_data; +typedef void (*irq_flow_handler_t)(unsigned int irq, struct irq_desc *desc); +typedef void (*irq_preflow_handler_t)(struct irq_data *data); + +#endif diff --git a/include/linux/kasan.h b/include/linux/kasan.h new file mode 100644 index 000000000000..0fdc798e3ff7 --- /dev/null +++ b/include/linux/kasan.h @@ -0,0 +1,100 @@ +#ifndef _LINUX_KASAN_H +#define _LINUX_KASAN_H + +#include <linux/sched.h> +#include <linux/types.h> + +struct kmem_cache; +struct page; +struct vm_struct; + +#ifdef CONFIG_KASAN + +#define KASAN_SHADOW_SCALE_SHIFT 3 + +#include <asm/kasan.h> +#include <asm/pgtable.h> + +extern unsigned char kasan_zero_page[PAGE_SIZE]; +extern pte_t kasan_zero_pte[PTRS_PER_PTE]; +extern pmd_t kasan_zero_pmd[PTRS_PER_PMD]; +extern pud_t kasan_zero_pud[PTRS_PER_PUD]; + +void kasan_populate_zero_shadow(const void *shadow_start, + const void *shadow_end); + +static inline void *kasan_mem_to_shadow(const void *addr) +{ + return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT) + + KASAN_SHADOW_OFFSET; +} + +/* Enable reporting bugs after kasan_disable_current() */ +static inline void kasan_enable_current(void) +{ + current->kasan_depth++; +} + +/* Disable reporting bugs for current task */ +static inline void kasan_disable_current(void) +{ + current->kasan_depth--; +} + +void kasan_unpoison_shadow(const void *address, size_t size); + +void kasan_unpoison_task_stack(struct task_struct *task); + +void kasan_alloc_pages(struct page *page, unsigned int order); +void kasan_free_pages(struct page *page, unsigned int order); + +void kasan_poison_slab(struct page *page); +void kasan_unpoison_object_data(struct kmem_cache *cache, void *object); +void kasan_poison_object_data(struct kmem_cache *cache, void *object); + +void kasan_kmalloc_large(const void *ptr, size_t size); +void kasan_kfree_large(const void *ptr); +void kasan_kfree(void *ptr); +void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size); +void kasan_krealloc(const void *object, size_t new_size); + +void kasan_slab_alloc(struct kmem_cache *s, void *object); +void kasan_slab_free(struct kmem_cache *s, void *object); + +int kasan_module_alloc(void *addr, size_t size); +void kasan_free_shadow(const struct vm_struct *vm); + +#else /* CONFIG_KASAN */ + +static inline void kasan_unpoison_shadow(const void *address, size_t size) {} + +static inline void kasan_unpoison_task_stack(struct task_struct *task) {} + +static inline void kasan_enable_current(void) {} +static inline void kasan_disable_current(void) {} + +static inline void kasan_alloc_pages(struct page *page, unsigned int order) {} +static inline void kasan_free_pages(struct page *page, unsigned int order) {} + +static inline void kasan_poison_slab(struct page *page) {} +static inline void kasan_unpoison_object_data(struct kmem_cache *cache, + void *object) {} +static inline void kasan_poison_object_data(struct kmem_cache *cache, + void *object) {} + +static inline void kasan_kmalloc_large(void *ptr, size_t size) {} +static inline void kasan_kfree_large(const void *ptr) {} +static inline void kasan_kfree(void *ptr) {} +static inline void kasan_kmalloc(struct kmem_cache *s, const void *object, + size_t size) {} +static inline void kasan_krealloc(const void *object, size_t new_size) {} + +static inline void kasan_slab_alloc(struct kmem_cache *s, void *object) {} +static inline void kasan_slab_free(struct kmem_cache *s, void *object) {} + +static inline int kasan_module_alloc(void *addr, size_t size) { return 0; } +static inline void kasan_free_shadow(const struct vm_struct *vm) {} + +#endif /* CONFIG_KASAN */ + +#endif /* LINUX_KASAN_H */ diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 0fe0cb8a5862..5c1fcdf0415f 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -103,6 +103,18 @@ (((__x) - ((__d) / 2)) / (__d)); \ } \ ) +/* + * Same as above but for u64 dividends. divisor must be a 32-bit + * number. + */ +#define DIV_ROUND_CLOSEST_ULL(x, divisor)( \ +{ \ + typeof(divisor) __d = divisor; \ + unsigned long long _tmp = (x) + (__d) / 2; \ + do_div(_tmp, __d); \ + _tmp; \ +} \ +) /* * Multiplies an integer by a fraction, while avoiding unnecessary diff --git a/include/linux/kmemleak.h b/include/linux/kmemleak.h index 19acfdc3828f..d0a1f99e24e3 100644 --- a/include/linux/kmemleak.h +++ b/include/linux/kmemleak.h @@ -21,6 +21,8 @@ #ifndef __KMEMLEAK_H #define __KMEMLEAK_H +#include <linux/slab.h> + #ifdef CONFIG_DEBUG_KMEMLEAK extern void kmemleak_init(void) __ref; diff --git a/include/linux/mempool.h b/include/linux/mempool.h index 39ed62ab5b8a..69b6951e8fd2 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -29,14 +29,15 @@ extern mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, void *pool_data, gfp_t gfp_mask, int nid); -extern int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask); +extern int mempool_resize(mempool_t *pool, int new_min_nr); extern void mempool_destroy(mempool_t *pool); extern void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask); extern void mempool_free(void *element, mempool_t *pool); /* * A mempool_alloc_t and mempool_free_t that get the memory from - * a slab that is passed in through pool_data. + * a slab cache that is passed in through pool_data. + * Note: the slab cache may not have a ctor function. */ void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data); void mempool_free_slab(void *element, void *pool_data); diff --git a/include/linux/mfd/syscon/imx6q-iomuxc-gpr.h b/include/linux/mfd/syscon/imx6q-iomuxc-gpr.h index ff44374a1a4e..c877cad61a13 100644 --- a/include/linux/mfd/syscon/imx6q-iomuxc-gpr.h +++ b/include/linux/mfd/syscon/imx6q-iomuxc-gpr.h @@ -395,4 +395,43 @@ #define IMX6SL_GPR1_FEC_CLOCK_MUX1_SEL_MASK (0x3 << 17) #define IMX6SL_GPR1_FEC_CLOCK_MUX2_SEL_MASK (0x1 << 14) +/* For imx6sx iomux gpr register field define */ +#define IMX6SX_GPR1_VDEC_SW_RST_MASK (0x1 << 20) +#define IMX6SX_GPR1_VDEC_SW_RST_RESET (0x1 << 20) +#define IMX6SX_GPR1_VDEC_SW_RST_RELEASE (0x0 << 20) +#define IMX6SX_GPR1_VADC_SW_RST_MASK (0x1 << 19) +#define IMX6SX_GPR1_VADC_SW_RST_RESET (0x1 << 19) +#define IMX6SX_GPR1_VADC_SW_RST_RELEASE (0x0 << 19) +#define IMX6SX_GPR1_FEC_CLOCK_MUX_SEL_MASK (0x3 << 13) +#define IMX6SX_GPR1_FEC_CLOCK_PAD_DIR_MASK (0x3 << 17) +#define IMX6SX_GPR1_FEC_CLOCK_MUX_SEL_EXT (0x3 << 13) + +#define IMX6SX_GPR4_FEC_ENET1_STOP_REQ (0x1 << 3) +#define IMX6SX_GPR4_FEC_ENET2_STOP_REQ (0x1 << 4) + +#define IMX6SX_GPR5_DISP_MUX_LDB_CTRL_MASK (0x1 << 3) +#define IMX6SX_GPR5_DISP_MUX_LDB_CTRL_LCDIF1 (0x0 << 3) +#define IMX6SX_GPR5_DISP_MUX_LDB_CTRL_LCDIF2 (0x1 << 3) + +#define IMX6SX_GPR5_CSI2_MUX_CTRL_MASK (0x3 << 27) +#define IMX6SX_GPR5_CSI2_MUX_CTRL_EXT_PIN (0x0 << 27) +#define IMX6SX_GPR5_CSI2_MUX_CTRL_CVD (0x1 << 27) +#define IMX6SX_GPR5_CSI2_MUX_CTRL_VDAC_TO_CSI (0x2 << 27) +#define IMX6SX_GPR5_CSI2_MUX_CTRL_GND (0x3 << 27) +#define IMX6SX_GPR5_VADC_TO_CSI_CAPTURE_EN_MASK (0x1 << 26) +#define IMX6SX_GPR5_VADC_TO_CSI_CAPTURE_EN_ENABLE (0x1 << 26) +#define IMX6SX_GPR5_VADC_TO_CSI_CAPTURE_EN_DISABLE (0x0 << 26) +#define IMX6SX_GPR5_CSI1_MUX_CTRL_MASK (0x3 << 4) +#define IMX6SX_GPR5_CSI1_MUX_CTRL_EXT_PIN (0x0 << 4) +#define IMX6SX_GPR5_CSI1_MUX_CTRL_CVD (0x1 << 4) +#define IMX6SX_GPR5_CSI1_MUX_CTRL_VDAC_TO_CSI (0x2 << 4) +#define IMX6SX_GPR5_CSI1_MUX_CTRL_GND (0x3 << 4) + +#define IMX6SX_GPR5_DISP_MUX_DCIC2_LCDIF2 (0x0 << 2) +#define IMX6SX_GPR5_DISP_MUX_DCIC2_LVDS (0x1 << 2) +#define IMX6SX_GPR5_DISP_MUX_DCIC2_MASK (0x1 << 2) +#define IMX6SX_GPR5_DISP_MUX_DCIC1_LCDIF1 (0x0 << 1) +#define IMX6SX_GPR5_DISP_MUX_DCIC1_LVDS (0x1 << 1) +#define IMX6SX_GPR5_DISP_MUX_DCIC1_MASK (0x1 << 1) + #endif /* __LINUX_IMX6Q_IOMUXC_GPR_H */ diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h index 7eeb9bbfb816..94f268e0ff2c 100644 --- a/include/linux/moduleloader.h +++ b/include/linux/moduleloader.h @@ -82,4 +82,11 @@ int module_finalize(const Elf_Ehdr *hdr, /* Any cleanup needed when module leaves. */ void module_arch_cleanup(struct module *mod); +#ifdef CONFIG_KASAN +#include <linux/kasan.h> +#define MODULE_ALIGN (PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT) +#else +#define MODULE_ALIGN PAGE_SIZE +#endif + #endif diff --git a/include/linux/msi.h b/include/linux/msi.h index 44f4746d033b..5b612c460b51 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -10,17 +10,12 @@ struct msi_msg { u32 data; /* 16 bits of msi message data */ }; +extern int pci_msi_ignore_mask; /* Helper functions */ struct irq_data; struct msi_desc; -void mask_msi_irq(struct irq_data *data); -void unmask_msi_irq(struct irq_data *data); -void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg); void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg); -void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg); -void read_msi_msg(unsigned int irq, struct msi_msg *msg); void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg); -void write_msi_msg(unsigned int irq, struct msi_msg *msg); struct msi_desc { struct { @@ -48,6 +43,59 @@ struct msi_desc { struct msi_msg msg; }; +/* Helpers to hide struct msi_desc implementation details */ +#define msi_desc_to_dev(desc) (&(desc)->dev.dev) +#define dev_to_msi_list(dev) (&to_pci_dev((dev))->msi_list) +#define first_msi_entry(dev) \ + list_first_entry(dev_to_msi_list((dev)), struct msi_desc, list) +#define for_each_msi_entry(desc, dev) \ + list_for_each_entry((desc), dev_to_msi_list((dev)), list) + +#ifdef CONFIG_PCI_MSI +#define first_pci_msi_entry(pdev) first_msi_entry(&(pdev)->dev) +#define for_each_pci_msi_entry(desc, pdev) \ + for_each_msi_entry((desc), &(pdev)->dev) + +static inline struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc) +{ + return desc->dev; +} + +void *msi_desc_to_pci_sysdata(struct msi_desc *desc); +#else /* CONFIG_PCI_MSI */ +static inline void *msi_desc_to_pci_sysdata(struct msi_desc *desc) +{ + return NULL; +} +#endif /* CONFIG_PCI_MSI */ + +void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg); +void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg); +void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg); + +u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag); +u32 __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag); +void pci_msi_mask_irq(struct irq_data *data); +void pci_msi_unmask_irq(struct irq_data *data); + +/* Conversion helpers. Should be removed after merging */ +static inline void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) +{ + __pci_write_msi_msg(entry, msg); +} +static inline void write_msi_msg(int irq, struct msi_msg *msg) +{ + pci_write_msi_msg(irq, msg); +} +static inline void mask_msi_irq(struct irq_data *data) +{ + pci_msi_mask_irq(data); +} +static inline void unmask_msi_irq(struct irq_data *data) +{ + pci_msi_unmask_irq(data); +} + /* * The arch hooks to setup up msi irqs. Those functions are * implemented as weak symbols so that they /can/ be overriden by @@ -61,18 +109,141 @@ void arch_restore_msi_irqs(struct pci_dev *dev); void default_teardown_msi_irqs(struct pci_dev *dev); void default_restore_msi_irqs(struct pci_dev *dev); -u32 default_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag); -u32 default_msix_mask_irq(struct msi_desc *desc, u32 flag); +#define default_msi_mask_irq __msi_mask_irq +#define default_msix_mask_irq __msix_mask_irq -struct msi_chip { +struct msi_controller { struct module *owner; struct device *dev; struct device_node *of_node; struct list_head list; - int (*setup_irq)(struct msi_chip *chip, struct pci_dev *dev, + int (*setup_irq)(struct msi_controller *chip, struct pci_dev *dev, struct msi_desc *desc); - void (*teardown_irq)(struct msi_chip *chip, unsigned int irq); + void (*teardown_irq)(struct msi_controller *chip, unsigned int irq); }; +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN + +#include <linux/irqhandler.h> +#include <asm/msi.h> + +struct irq_domain; +struct irq_chip; +struct device_node; +struct msi_domain_info; + +/** + * struct msi_domain_ops - MSI interrupt domain callbacks + * @get_hwirq: Retrieve the resulting hw irq number + * @msi_init: Domain specific init function for MSI interrupts + * @msi_free: Domain specific function to free a MSI interrupts + * @msi_check: Callback for verification of the domain/info/dev data + * @msi_prepare: Prepare the allocation of the interrupts in the domain + * @msi_finish: Optional callbacl to finalize the allocation + * @set_desc: Set the msi descriptor for an interrupt + * @handle_error: Optional error handler if the allocation fails + * + * @get_hwirq, @msi_init and @msi_free are callbacks used by + * msi_create_irq_domain() and related interfaces + * + * @msi_check, @msi_prepare, @msi_finish, @set_desc and @handle_error + * are callbacks used by msi_irq_domain_alloc_irqs() and related + * interfaces which are based on msi_desc. + */ +struct msi_domain_ops { + irq_hw_number_t (*get_hwirq)(struct msi_domain_info *info, + msi_alloc_info_t *arg); + int (*msi_init)(struct irq_domain *domain, + struct msi_domain_info *info, + unsigned int virq, irq_hw_number_t hwirq, + msi_alloc_info_t *arg); + void (*msi_free)(struct irq_domain *domain, + struct msi_domain_info *info, + unsigned int virq); + int (*msi_check)(struct irq_domain *domain, + struct msi_domain_info *info, + struct device *dev); + int (*msi_prepare)(struct irq_domain *domain, + struct device *dev, int nvec, + msi_alloc_info_t *arg); + void (*msi_finish)(msi_alloc_info_t *arg, int retval); + void (*set_desc)(msi_alloc_info_t *arg, + struct msi_desc *desc); + int (*handle_error)(struct irq_domain *domain, + struct msi_desc *desc, int error); +}; + +/** + * struct msi_domain_info - MSI interrupt domain data + * @flags: Flags to decribe features and capabilities + * @ops: The callback data structure + * @chip: Optional: associated interrupt chip + * @chip_data: Optional: associated interrupt chip data + * @handler: Optional: associated interrupt flow handler + * @handler_data: Optional: associated interrupt flow handler data + * @handler_name: Optional: associated interrupt flow handler name + * @data: Optional: domain specific data + */ +struct msi_domain_info { + u32 flags; + struct msi_domain_ops *ops; + struct irq_chip *chip; + void *chip_data; + irq_flow_handler_t handler; + void *handler_data; + const char *handler_name; + void *data; +}; + +/* Flags for msi_domain_info */ +enum { + /* + * Init non implemented ops callbacks with default MSI domain + * callbacks. + */ + MSI_FLAG_USE_DEF_DOM_OPS = (1 << 0), + /* + * Init non implemented chip callbacks with default MSI chip + * callbacks. + */ + MSI_FLAG_USE_DEF_CHIP_OPS = (1 << 1), + /* Build identity map between hwirq and irq */ + MSI_FLAG_IDENTITY_MAP = (1 << 2), + /* Support multiple PCI MSI interrupts */ + MSI_FLAG_MULTI_PCI_MSI = (1 << 3), + /* Support PCI MSIX interrupts */ + MSI_FLAG_PCI_MSIX = (1 << 4), +}; + +int msi_domain_set_affinity(struct irq_data *data, const struct cpumask *mask, + bool force); + +struct irq_domain *msi_create_irq_domain(struct device_node *of_node, + struct msi_domain_info *info, + struct irq_domain *parent); +int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, + int nvec); +void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev); +struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain); + +#endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */ + +#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN +void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg); +struct irq_domain *pci_msi_create_irq_domain(struct device_node *node, + struct msi_domain_info *info, + struct irq_domain *parent); +int pci_msi_domain_alloc_irqs(struct irq_domain *domain, struct pci_dev *dev, + int nvec, int type); +void pci_msi_domain_free_irqs(struct irq_domain *domain, struct pci_dev *dev); +struct irq_domain *pci_msi_create_default_irq_domain(struct device_node *node, + struct msi_domain_info *info, struct irq_domain *parent); + +irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev, + struct msi_desc *desc); +int pci_msi_domain_check_cap(struct irq_domain *domain, + struct msi_domain_info *info, struct device *dev); +#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ + #endif /* LINUX_MSI_H */ diff --git a/include/linux/of.h b/include/linux/of.h index 4a6a489a7506..45447c6f21f7 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -24,6 +24,7 @@ #include <linux/topology.h> #include <linux/notifier.h> #include <linux/property.h> +#include <linux/list.h> #include <asm/byteorder.h> #include <asm/errno.h> @@ -56,8 +57,6 @@ struct device_node { struct device_node *parent; struct device_node *child; struct device_node *sibling; - struct device_node *next; /* next device of same type */ - struct device_node *allnext; /* next in list of all nodes */ struct kobject kobj; unsigned long _flags; void *data; @@ -75,6 +74,12 @@ struct of_phandle_args { uint32_t args[MAX_PHANDLE_ARGS]; }; +struct of_reconfig_data { + struct device_node *dn; + struct property *prop; + struct property *old_prop; +}; + /* initialize a node */ extern struct kobj_type of_node_ktype; static inline void of_node_init(struct device_node *node) @@ -106,18 +111,17 @@ static inline struct device_node *of_node_get(struct device_node *node) static inline void of_node_put(struct device_node *node) { } #endif /* !CONFIG_OF_DYNAMIC */ -#ifdef CONFIG_OF - /* Pointer for first entry in chain of all nodes. */ -extern struct device_node *of_allnodes; +extern struct device_node *of_root; extern struct device_node *of_chosen; extern struct device_node *of_aliases; extern struct device_node *of_stdout; extern raw_spinlock_t devtree_lock; +#ifdef CONFIG_OF static inline bool of_have_populated_dt(void) { - return of_allnodes != NULL; + return of_root != NULL; } static inline bool of_node_is_root(const struct device_node *node) @@ -161,6 +165,7 @@ static inline void of_property_clear_flag(struct property *p, unsigned long flag clear_bit(flag, &p->_flags); } +extern struct device_node *__of_find_all_nodes(struct device_node *prev); extern struct device_node *of_find_all_nodes(struct device_node *prev); /* @@ -216,8 +221,9 @@ static inline const char *of_node_full_name(const struct device_node *np) return np ? np->full_name : "<no-node>"; } -#define for_each_of_allnodes(dn) \ - for (dn = of_allnodes; dn; dn = dn->allnext) +#define for_each_of_allnodes_from(from, dn) \ + for (dn = __of_find_all_nodes(from); dn; dn = __of_find_all_nodes(dn)) +#define for_each_of_allnodes(dn) for_each_of_allnodes_from(NULL, dn) extern struct device_node *of_find_node_by_name(struct device_node *from, const char *name); extern struct device_node *of_find_node_by_type(struct device_node *from, @@ -229,7 +235,13 @@ extern struct device_node *of_find_matching_node_and_match( const struct of_device_id *matches, const struct of_device_id **match); -extern struct device_node *of_find_node_by_path(const char *path); +extern struct device_node *of_find_node_opts_by_path(const char *path, + const char **opts); +static inline struct device_node *of_find_node_by_path(const char *path) +{ + return of_find_node_opts_by_path(path, NULL); +} + extern struct device_node *of_find_node_by_phandle(phandle handle); extern struct device_node *of_get_parent(const struct device_node *node); extern struct device_node *of_get_next_parent(struct device_node *node); @@ -280,7 +292,7 @@ extern int of_property_read_string_helper(struct device_node *np, const char **out_strs, size_t sz, int index); extern int of_device_is_compatible(const struct device_node *device, const char *); -extern int of_device_is_available(const struct device_node *device); +extern bool of_device_is_available(const struct device_node *device); extern const void *of_get_property(const struct device_node *node, const char *name, int *lenp); @@ -322,16 +334,6 @@ extern int of_update_property(struct device_node *np, struct property *newprop); #define OF_RECONFIG_REMOVE_PROPERTY 0x0004 #define OF_RECONFIG_UPDATE_PROPERTY 0x0005 -struct of_prop_reconfig { - struct device_node *dn; - struct property *prop; - struct property *old_prop; -}; - -extern int of_reconfig_notifier_register(struct notifier_block *); -extern int of_reconfig_notifier_unregister(struct notifier_block *); -extern int of_reconfig_notify(unsigned long, void *); - extern int of_attach_node(struct device_node *); extern int of_detach_node(struct device_node *); @@ -390,6 +392,12 @@ static inline struct device_node *of_find_node_by_path(const char *path) return NULL; } +static inline struct device_node *of_find_node_opts_by_path(const char *path, + const char **opts) +{ + return NULL; +} + static inline struct device_node *of_get_parent(const struct device_node *node) { return NULL; @@ -431,9 +439,9 @@ static inline int of_device_is_compatible(const struct device_node *device, return 0; } -static inline int of_device_is_available(const struct device_node *device) +static inline bool of_device_is_available(const struct device_node *device) { - return 0; + return false; } static inline struct property *of_find_property(const struct device_node *np, @@ -775,6 +783,13 @@ static inline int of_property_read_u32(const struct device_node *np, return of_property_read_u32_array(np, propname, out_value, 1); } +static inline int of_property_read_s32(const struct device_node *np, + const char *propname, + s32 *out_value) +{ + return of_property_read_u32(np, propname, (u32*) out_value); +} + #define of_property_for_each_u32(np, propname, prop, p, u) \ for (prop = of_find_property(np, propname, NULL), \ p = of_prop_next_u32(prop, NULL, &u); \ @@ -843,7 +858,7 @@ static inline int of_get_available_child_count(const struct device_node *np) = { .compatible = compat, \ .data = (fn == (fn_type)NULL) ? fn : fn } #else -#define _OF_DECLARE(table, name, compat, fn, fn_type) \ +#define _OF_DECLARE(table, name, compat, fn, fn_type) \ static const struct of_device_id __of_table_##name \ __attribute__((unused)) \ = { .compatible = compat, \ @@ -894,7 +909,19 @@ struct of_changeset { struct list_head entries; }; +enum of_reconfig_change { + OF_RECONFIG_NO_CHANGE = 0, + OF_RECONFIG_CHANGE_ADD, + OF_RECONFIG_CHANGE_REMOVE, +}; + #ifdef CONFIG_OF_DYNAMIC +extern int of_reconfig_notifier_register(struct notifier_block *); +extern int of_reconfig_notifier_unregister(struct notifier_block *); +extern int of_reconfig_notify(unsigned long, struct of_reconfig_data *rd); +extern int of_reconfig_get_state_change(unsigned long action, + struct of_reconfig_data *arg); + extern void of_changeset_init(struct of_changeset *ocs); extern void of_changeset_destroy(struct of_changeset *ocs); extern int of_changeset_apply(struct of_changeset *ocs); @@ -932,9 +959,58 @@ static inline int of_changeset_update_property(struct of_changeset *ocs, { return of_changeset_action(ocs, OF_RECONFIG_UPDATE_PROPERTY, np, prop); } -#endif +#else /* CONFIG_OF_DYNAMIC */ +static inline int of_reconfig_notifier_register(struct notifier_block *nb) +{ + return -EINVAL; +} +static inline int of_reconfig_notifier_unregister(struct notifier_block *nb) +{ + return -EINVAL; +} +static inline int of_reconfig_notify(unsigned long action, + struct of_reconfig_data *arg) +{ + return -EINVAL; +} +static inline int of_reconfig_get_state_change(unsigned long action, + struct of_reconfig_data *arg) +{ + return -EINVAL; +} +#endif /* CONFIG_OF_DYNAMIC */ /* CONFIG_OF_RESOLVE api */ extern int of_resolve_phandles(struct device_node *tree); +/** + * Overlay support + */ + +#ifdef CONFIG_OF_OVERLAY + +/* ID based overlays; the API for external users */ +int of_overlay_create(struct device_node *tree); +int of_overlay_destroy(int id); +int of_overlay_destroy_all(void); + +#else + +static inline int of_overlay_create(struct device_node *tree) +{ + return -ENOTSUPP; +} + +static inline int of_overlay_destroy(int id) +{ + return -ENOTSUPP; +} + +static inline int of_overlay_destroy_all(void) +{ + return -ENOTSUPP; +} + +#endif + #endif /* _LINUX_OF_H */ diff --git a/include/linux/of_address.h b/include/linux/of_address.h index 8cb14eb393d6..d88e81be6368 100644 --- a/include/linux/of_address.h +++ b/include/linux/of_address.h @@ -106,7 +106,7 @@ extern int of_address_to_resource(struct device_node *dev, int index, struct resource *r); void __iomem *of_iomap(struct device_node *node, int index); void __iomem *of_io_request_and_map(struct device_node *device, - int index, char *name); + int index, const char *name); #else #include <linux/io.h> @@ -123,7 +123,7 @@ static inline void __iomem *of_iomap(struct device_node *device, int index) } static inline void __iomem *of_io_request_and_map(struct device_node *device, - int index, char *name) + int index, const char *name) { return IOMEM_ERR_PTR(-EINVAL); } diff --git a/include/linux/of_device.h b/include/linux/of_device.h index ef370210ffb2..22801b10cef5 100644 --- a/include/linux/of_device.h +++ b/include/linux/of_device.h @@ -53,6 +53,7 @@ static inline struct device_node *of_cpu_device_node_get(int cpu) return of_node_get(cpu_dev->of_node); } +void of_dma_configure(struct device *dev, struct device_node *np); #else /* CONFIG_OF */ static inline int of_driver_match_device(struct device *dev, @@ -90,6 +91,8 @@ static inline struct device_node *of_cpu_device_node_get(int cpu) { return NULL; } +static inline void of_dma_configure(struct device *dev, struct device_node *np) +{} #endif /* CONFIG_OF */ #endif /* _LINUX_OF_DEVICE_H */ diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h index 51a560f34bca..ffbe4707d4aa 100644 --- a/include/linux/of_iommu.h +++ b/include/linux/of_iommu.h @@ -1,12 +1,20 @@ #ifndef __OF_IOMMU_H #define __OF_IOMMU_H +#include <linux/device.h> +#include <linux/iommu.h> +#include <linux/of.h> + #ifdef CONFIG_OF_IOMMU extern int of_get_dma_window(struct device_node *dn, const char *prefix, int index, unsigned long *busno, dma_addr_t *addr, size_t *size); +extern void of_iommu_init(void); +extern struct iommu_ops *of_iommu_configure(struct device *dev, + struct device_node *master_np); + #else static inline int of_get_dma_window(struct device_node *dn, const char *prefix, @@ -16,6 +24,23 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix, return -EINVAL; } +static inline void of_iommu_init(void) { } +static inline struct iommu_ops *of_iommu_configure(struct device *dev, + struct device_node *master_np) +{ + return NULL; +} + #endif /* CONFIG_OF_IOMMU */ +void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops); +struct iommu_ops *of_iommu_get_ops(struct device_node *np); + +extern struct of_device_id __iommu_of_table; + +typedef int (*of_iommu_init_fn)(struct device_node *); + +#define IOMMU_OF_DECLARE(name, compat, fn) \ + _OF_DECLARE(iommu, name, compat, fn, of_iommu_init_fn) + #endif /* __OF_IOMMU_H */ diff --git a/include/linux/of_irq.h b/include/linux/of_irq.h index bfec136a6d1e..563ad28684c6 100644 --- a/include/linux/of_irq.h +++ b/include/linux/of_irq.h @@ -69,6 +69,7 @@ static inline int of_irq_get_byname(struct device_node *dev, const char *name) */ extern unsigned int irq_of_parse_and_map(struct device_node *node, int index); extern struct device_node *of_irq_find_parent(struct device_node *child); +extern void of_msi_configure(struct device *dev, struct device_node *np); #else /* !CONFIG_OF */ static inline unsigned int irq_of_parse_and_map(struct device_node *dev, diff --git a/include/linux/of_pci.h b/include/linux/of_pci.h index 1fd207e7a847..29fd3fe1c035 100644 --- a/include/linux/of_pci.h +++ b/include/linux/of_pci.h @@ -16,6 +16,7 @@ int of_pci_get_devfn(struct device_node *np); int of_irq_parse_and_map_pci(const struct pci_dev *dev, u8 slot, u8 pin); int of_pci_parse_bus_range(struct device_node *node, struct resource *res); int of_get_pci_domain_nr(struct device_node *node); +void of_pci_dma_configure(struct pci_dev *pci_dev); #else static inline int of_irq_parse_pci(const struct pci_dev *pdev, struct of_phandle_args *out_irq) { @@ -50,6 +51,8 @@ of_get_pci_domain_nr(struct device_node *node) { return -1; } + +static inline void of_pci_dma_configure(struct pci_dev *pci_dev) { } #endif #if defined(CONFIG_OF_ADDRESS) @@ -59,13 +62,13 @@ int of_pci_get_host_bridge_resources(struct device_node *dev, #endif #if defined(CONFIG_OF) && defined(CONFIG_PCI_MSI) -int of_pci_msi_chip_add(struct msi_chip *chip); -void of_pci_msi_chip_remove(struct msi_chip *chip); -struct msi_chip *of_pci_find_msi_chip_by_node(struct device_node *of_node); +int of_pci_msi_chip_add(struct msi_controller *chip); +void of_pci_msi_chip_remove(struct msi_controller *chip); +struct msi_controller *of_pci_find_msi_chip_by_node(struct device_node *of_node); #else -static inline int of_pci_msi_chip_add(struct msi_chip *chip) { return -EINVAL; } -static inline void of_pci_msi_chip_remove(struct msi_chip *chip) { } -static inline struct msi_chip * +static inline int of_pci_msi_chip_add(struct msi_controller *chip) { return -EINVAL; } +static inline void of_pci_msi_chip_remove(struct msi_controller *chip) { } +static inline struct msi_controller * of_pci_find_msi_chip_by_node(struct device_node *of_node) { return NULL; } #endif diff --git a/include/linux/of_pdt.h b/include/linux/of_pdt.h index c65a18a0cfdf..7e09244bb679 100644 --- a/include/linux/of_pdt.h +++ b/include/linux/of_pdt.h @@ -39,7 +39,6 @@ extern void *prom_early_alloc(unsigned long size); /* for building the device tree */ extern void of_pdt_build_devicetree(phandle root_node, struct of_pdt_ops *ops); -extern void (*of_pdt_build_more)(struct device_node *dp, - struct device_node ***nextp); +extern void (*of_pdt_build_more)(struct device_node *dp); #endif /* _LINUX_OF_PDT_H */ diff --git a/include/linux/of_platform.h b/include/linux/of_platform.h index c2b0627a2317..8a860f096c35 100644 --- a/include/linux/of_platform.h +++ b/include/linux/of_platform.h @@ -84,4 +84,10 @@ static inline int of_platform_populate(struct device_node *root, static inline void of_platform_depopulate(struct device *parent) { } #endif +#ifdef CONFIG_OF_DYNAMIC +extern void of_platform_register_reconfig_notifier(void); +#else +static inline void of_platform_register_reconfig_notifier(void) { } +#endif + #endif /* _LINUX_OF_PLATFORM_H */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 88f1faf3de43..cde0e8a24694 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -459,7 +459,7 @@ struct pci_bus { struct resource busn_res; /* bus numbers routed to this bus */ struct pci_ops *ops; /* configuration access functions */ - struct msi_chip *msi; /* MSI controller */ + struct msi_controller *msi; /* MSI controller */ void *sysdata; /* hook for sys-specific extension */ struct proc_dir_entry *procdir; /* directory entry in /proc/bus/pci */ @@ -519,6 +519,9 @@ static inline struct pci_dev *pci_upstream_bridge(struct pci_dev *dev) return dev->bus->self; } +struct device *pci_get_host_bridge_device(struct pci_dev *dev); +void pci_put_host_bridge_device(struct device *dev); + #ifdef CONFIG_PCI_MSI static inline bool pci_dev_msi_enabled(struct pci_dev *pci_dev) { @@ -568,6 +571,7 @@ static inline int pcibios_err_to_errno(int err) /* Low-level architecture-dependent routines */ struct pci_ops { + void __iomem *(*map_bus)(struct pci_bus *bus, unsigned int devfn, int where); int (*read)(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val); int (*write)(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val); }; @@ -865,6 +869,16 @@ int pci_bus_write_config_word(struct pci_bus *bus, unsigned int devfn, int where, u16 val); int pci_bus_write_config_dword(struct pci_bus *bus, unsigned int devfn, int where, u32 val); + +int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val); +int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val); +int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val); +int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val); + struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops); static inline int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val) @@ -1826,10 +1840,12 @@ int pci_vpd_find_info_keyword(const u8 *buf, unsigned int off, /* PCI <-> OF binding helpers */ #ifdef CONFIG_OF struct device_node; +struct irq_domain; void pci_set_of_node(struct pci_dev *dev); void pci_release_of_node(struct pci_dev *dev); void pci_set_bus_of_node(struct pci_bus *bus); void pci_release_bus_of_node(struct pci_bus *bus); +struct irq_domain *pci_host_bridge_of_msi_domain(struct pci_bus *bus); /* Arch may override this (weak) */ struct device_node *pcibios_get_phb_of_node(struct pci_bus *bus); @@ -1850,6 +1866,10 @@ static inline void pci_set_of_node(struct pci_dev *dev) { } static inline void pci_release_of_node(struct pci_dev *dev) { } static inline void pci_set_bus_of_node(struct pci_bus *bus) { } static inline void pci_release_bus_of_node(struct pci_bus *bus) { } +static inline struct device_node * +pci_device_to_OF_node(const struct pci_dev *pdev) { return NULL; } +static inline struct irq_domain * +pci_host_bridge_of_msi_domain(struct pci_bus *bus) { return NULL; } #endif /* CONFIG_OF */ #ifdef CONFIG_EEH diff --git a/include/linux/pm.h b/include/linux/pm.h index 383fd68aaee1..03f3ae96842e 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -610,7 +610,7 @@ struct dev_pm_info { extern void update_pm_runtime_accounting(struct device *dev); extern int dev_pm_get_subsys_data(struct device *dev); -extern int dev_pm_put_subsys_data(struct device *dev); +extern void dev_pm_put_subsys_data(struct device *dev); /* * Power domains provide callbacks that are executed during system suspend, diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 0330217abfad..cccaf4a29e9f 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -21,7 +21,7 @@ struct dev_pm_opp; struct device; enum dev_pm_opp_event { - OPP_EVENT_ADD, OPP_EVENT_ENABLE, OPP_EVENT_DISABLE, + OPP_EVENT_ADD, OPP_EVENT_REMOVE, OPP_EVENT_ENABLE, OPP_EVENT_DISABLE, }; #if defined(CONFIG_PM_OPP) @@ -30,7 +30,13 @@ unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp); unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp); +bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp); + int dev_pm_opp_get_opp_count(struct device *dev); +unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev); +unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev); +unsigned long dev_pm_opp_get_max_transition_latency(struct device *dev); +struct dev_pm_opp *dev_pm_opp_get_suspend_opp(struct device *dev); struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev, unsigned long freq, @@ -44,12 +50,21 @@ struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev, int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt); +void dev_pm_opp_remove(struct device *dev, unsigned long freq); int dev_pm_opp_enable(struct device *dev, unsigned long freq); int dev_pm_opp_disable(struct device *dev, unsigned long freq); struct srcu_notifier_head *dev_pm_opp_get_notifier(struct device *dev); +int dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, + unsigned int count); +void dev_pm_opp_put_supported_hw(struct device *dev); +int dev_pm_opp_set_prop_name(struct device *dev, const char *name); +void dev_pm_opp_put_prop_name(struct device *dev); +int dev_pm_opp_set_regulator(struct device *dev, const char *name); +void dev_pm_opp_put_regulator(struct device *dev); +int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); #else static inline unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp) { @@ -61,11 +76,36 @@ static inline unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp) return 0; } +static inline bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp) +{ + return false; +} + static inline int dev_pm_opp_get_opp_count(struct device *dev) { return 0; } +static inline unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev) +{ + return 0; +} + +static inline unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev) +{ + return 0; +} + +static inline unsigned long dev_pm_opp_get_max_transition_latency(struct device *dev) +{ + return 0; +} + +static inline struct dev_pm_opp *dev_pm_opp_get_suspend_opp(struct device *dev) +{ + return NULL; +} + static inline struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev, unsigned long freq, bool available) { @@ -90,6 +130,10 @@ static inline int dev_pm_opp_add(struct device *dev, unsigned long freq, return -EINVAL; } +static inline void dev_pm_opp_remove(struct device *dev, unsigned long freq) +{ +} + static inline int dev_pm_opp_enable(struct device *dev, unsigned long freq) { return 0; @@ -105,15 +149,72 @@ static inline struct srcu_notifier_head *dev_pm_opp_get_notifier( { return ERR_PTR(-EINVAL); } + +static inline int dev_pm_opp_set_supported_hw(struct device *dev, + const u32 *versions, + unsigned int count) +{ + return -EINVAL; +} + +static inline void dev_pm_opp_put_supported_hw(struct device *dev) {} + +static inline int dev_pm_opp_set_prop_name(struct device *dev, const char *name) +{ + return -EINVAL; +} + +static inline void dev_pm_opp_put_prop_name(struct device *dev) {} + +static inline int dev_pm_opp_set_regulator(struct device *dev, const char *name) +{ + return -EINVAL; +} + +static inline void dev_pm_opp_put_regulator(struct device *dev) {} + +static inline int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) +{ + return -EINVAL; +} + #endif /* CONFIG_PM_OPP */ #if defined(CONFIG_PM_OPP) && defined(CONFIG_OF) -int of_init_opp_table(struct device *dev); +int dev_pm_opp_of_add_table(struct device *dev); +void dev_pm_opp_of_remove_table(struct device *dev); +int dev_pm_opp_of_cpumask_add_table(cpumask_var_t cpumask); +void dev_pm_opp_of_cpumask_remove_table(cpumask_var_t cpumask); +int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, cpumask_var_t cpumask); +int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, cpumask_var_t cpumask); #else -static inline int of_init_opp_table(struct device *dev) +static inline int dev_pm_opp_of_add_table(struct device *dev) { return -EINVAL; } + +static inline void dev_pm_opp_of_remove_table(struct device *dev) +{ +} + +static inline int dev_pm_opp_of_cpumask_add_table(cpumask_var_t cpumask) +{ + return -ENOSYS; +} + +static inline void dev_pm_opp_of_cpumask_remove_table(cpumask_var_t cpumask) +{ +} + +static inline int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, cpumask_var_t cpumask) +{ + return -ENOSYS; +} + +static inline int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, cpumask_var_t cpumask) +{ + return -ENOSYS; +} #endif #endif /* __LINUX_OPP_H__ */ diff --git a/include/linux/regulator/consumer.h b/include/linux/regulator/consumer.h index f540b1496e2f..7f96b1b8f7bb 100644 --- a/include/linux/regulator/consumer.h +++ b/include/linux/regulator/consumer.h @@ -533,6 +533,16 @@ static inline int regulator_count_voltages(struct regulator *regulator) } #endif +static inline int regulator_set_voltage_triplet(struct regulator *regulator, + int min_uV, int target_uV, + int max_uV) +{ + if (regulator_set_voltage(regulator, target_uV, max_uV) == 0) + return 0; + + return regulator_set_voltage(regulator, min_uV, max_uV); +} + static inline int regulator_set_voltage_tol(struct regulator *regulator, int new_uV, int tol_uV) { diff --git a/include/linux/sched.h b/include/linux/sched.h index ab10455e5b02..697478551043 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1636,6 +1636,9 @@ struct task_struct { unsigned long timer_slack_ns; unsigned long default_timer_slack_ns; +#ifdef CONFIG_KASAN + unsigned int kasan_depth; +#endif #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* Index of current stored address in ret_stack */ int curr_ret_stack; diff --git a/include/linux/slab.h b/include/linux/slab.h index 5474847bee86..ec2e894119ab 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -104,6 +104,7 @@ (unsigned long)ZERO_SIZE_PTR) #include <linux/kmemleak.h> +#include <linux/kasan.h> struct mem_cgroup; /* @@ -326,7 +327,10 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) { - return kmem_cache_alloc(s, flags); + void *ret = kmem_cache_alloc(s, flags); + + kasan_kmalloc(s, ret, size); + return ret; } static __always_inline void * @@ -334,7 +338,10 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) { - return kmem_cache_alloc_node(s, gfpflags, node); + void *ret = kmem_cache_alloc_node(s, gfpflags, node); + + kasan_kmalloc(s, ret, size); + return ret; } #endif /* CONFIG_TRACING */ diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index d82abd40a3c0..e565d7c10c9c 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -110,4 +110,23 @@ static inline void sysfs_slab_remove(struct kmem_cache *s) } #endif + +/** + * virt_to_obj - returns address of the beginning of object. + * @s: object's kmem_cache + * @slab_page: address of slab page + * @x: address within object memory range + * + * Returns address of the beginning of object + */ +static inline void *virt_to_obj(struct kmem_cache *s, + const void *slab_page, + const void *x) +{ + return (void *)x - ((x - slab_page) % s->size); +} + +void object_err(struct kmem_cache *s, struct page *page, + u8 *object, char *reason); + #endif /* _LINUX_SLUB_DEF_H */ diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 96c305167a1e..fbb64253a27b 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -29,16 +29,19 @@ #include <linux/idr.h> #include <linux/device.h> #include <linux/workqueue.h> +#include <uapi/linux/thermal.h> #define THERMAL_TRIPS_NONE -1 #define THERMAL_MAX_TRIPS 12 -#define THERMAL_NAME_LENGTH 20 /* invalid cooling state */ #define THERMAL_CSTATE_INVALID -1UL /* No upper/lower limit requirement */ -#define THERMAL_NO_LIMIT THERMAL_CSTATE_INVALID +#define THERMAL_NO_LIMIT ((u32)~0) + +/* Default weight of a bound cooling device */ +#define THERMAL_WEIGHT_DEFAULT 0 /* use value, which < 0K, to indicate an invalid/uninitialized temperature */ #define THERMAL_TEMP_INVALID -274000 @@ -52,11 +55,6 @@ #define MILLICELSIUS_TO_DECI_KELVIN_WITH_OFFSET(t, off) (((t) / 100) + (off)) #define MILLICELSIUS_TO_DECI_KELVIN(t) MILLICELSIUS_TO_DECI_KELVIN_WITH_OFFSET(t, 2732) -/* Adding event notification support elements */ -#define THERMAL_GENL_FAMILY_NAME "thermal_event" -#define THERMAL_GENL_VERSION 0x01 -#define THERMAL_GENL_MCAST_GROUP_NAME "thermal_mc_grp" - /* Default Thermal Governor */ #if defined(CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE) #define DEFAULT_THERMAL_GOVERNOR "step_wise" @@ -64,10 +62,13 @@ #define DEFAULT_THERMAL_GOVERNOR "fair_share" #elif defined(CONFIG_THERMAL_DEFAULT_GOV_USER_SPACE) #define DEFAULT_THERMAL_GOVERNOR "user_space" +#elif defined(CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR) +#define DEFAULT_THERMAL_GOVERNOR "power_allocator" #endif struct thermal_zone_device; struct thermal_cooling_device; +struct thermal_instance; enum thermal_device_mode { THERMAL_DEVICE_DISABLED = 0, @@ -89,30 +90,6 @@ enum thermal_trend { THERMAL_TREND_DROP_FULL, /* apply lowest cooling action */ }; -/* Events supported by Thermal Netlink */ -enum events { - THERMAL_AUX0, - THERMAL_AUX1, - THERMAL_CRITICAL, - THERMAL_DEV_FAULT, -}; - -/* attributes of thermal_genl_family */ -enum { - THERMAL_GENL_ATTR_UNSPEC, - THERMAL_GENL_ATTR_EVENT, - __THERMAL_GENL_ATTR_MAX, -}; -#define THERMAL_GENL_ATTR_MAX (__THERMAL_GENL_ATTR_MAX - 1) - -/* commands supported by the thermal_genl_family */ -enum { - THERMAL_GENL_CMD_UNSPEC, - THERMAL_GENL_CMD_EVENT, - __THERMAL_GENL_CMD_MAX, -}; -#define THERMAL_GENL_CMD_MAX (__THERMAL_GENL_CMD_MAX - 1) - struct thermal_zone_device_ops { int (*bind) (struct thermal_zone_device *, struct thermal_cooling_device *); @@ -145,6 +122,12 @@ struct thermal_cooling_device_ops { int (*get_max_state) (struct thermal_cooling_device *, unsigned long *); int (*get_cur_state) (struct thermal_cooling_device *, unsigned long *); int (*set_cur_state) (struct thermal_cooling_device *, unsigned long); + int (*get_requested_power)(struct thermal_cooling_device *, + struct thermal_zone_device *, u32 *); + int (*state2power)(struct thermal_cooling_device *, + struct thermal_zone_device *, unsigned long, u32 *); + int (*power2state)(struct thermal_cooling_device *, + struct thermal_zone_device *, u32, unsigned long *); }; struct thermal_cooling_device { @@ -177,8 +160,7 @@ struct thermal_attr { * @trips: number of trip points the thermal zone supports * @trips_disabled; bitmap for disabled trips * @passive_delay: number of milliseconds to wait between polls when - * performing passive cooling. Currenty only used by the - * step-wise governor + * performing passive cooling. * @polling_delay: number of milliseconds to wait between polls when * checking whether trip points have been crossed (0 for * interrupt driven systems) @@ -188,7 +170,6 @@ struct thermal_attr { * @last_temperature: previous temperature read * @emul_temperature: emulated temperature when using CONFIG_THERMAL_EMULATION * @passive: 1 if you've crossed a passive trip point, 0 otherwise. - * Currenty only used by the step-wise governor. * @forced_passive: If > 0, temperature at which to switch on all ACPI * processor cooling devices. Currently only used by the * step-wise governor. @@ -196,6 +177,7 @@ struct thermal_attr { * @ops: operations this &thermal_zone_device supports * @tzp: thermal zone parameters * @governor: pointer to the governor for this thermal zone + * @governor_data: private pointer for governor data * @thermal_instances: list of &struct thermal_instance of this thermal zone * @idr: &struct idr to generate unique id for this zone's cooling * devices @@ -222,8 +204,9 @@ struct thermal_zone_device { unsigned int forced_passive; atomic_t need_update; struct thermal_zone_device_ops *ops; - const struct thermal_zone_params *tzp; + struct thermal_zone_params *tzp; struct thermal_governor *governor; + void *governor_data; struct list_head thermal_instances; struct idr idr; struct mutex lock; @@ -234,12 +217,19 @@ struct thermal_zone_device { /** * struct thermal_governor - structure that holds thermal governor information * @name: name of the governor + * @bind_to_tz: callback called when binding to a thermal zone. If it + * returns 0, the governor is bound to the thermal zone, + * otherwise it fails. + * @unbind_from_tz: callback called when a governor is unbound from a + * thermal zone. * @throttle: callback called for every trip point even if temperature is * below the trip point temperature * @governor_list: node in thermal_governor_list (in thermal_core.c) */ struct thermal_governor { char name[THERMAL_NAME_LENGTH]; + int (*bind_to_tz)(struct thermal_zone_device *tz); + void (*unbind_from_tz)(struct thermal_zone_device *tz); int (*throttle)(struct thermal_zone_device *tz, int trip); struct list_head governor_list; }; @@ -250,9 +240,12 @@ struct thermal_bind_params { /* * This is a measure of 'how effectively these devices can - * cool 'this' thermal zone. The shall be determined by platform - * characterization. This is on a 'percentage' scale. - * See Documentation/thermal/sysfs-api.txt for more information. + * cool 'this' thermal zone. It shall be determined by + * platform characterization. This value is relative to the + * rest of the weights so a cooling device whose weight is + * double that of another cooling device is twice as + * effective. See Documentation/thermal/sysfs-api.txt for more + * information. */ int weight; @@ -289,6 +282,44 @@ struct thermal_zone_params { int num_tbps; /* Number of tbp entries */ struct thermal_bind_params *tbp; + + /* + * Sustainable power (heat) that this thermal zone can dissipate in + * mW + */ + u32 sustainable_power; + + /* + * Proportional parameter of the PID controller when + * overshooting (i.e., when temperature is below the target) + */ + s32 k_po; + + /* + * Proportional parameter of the PID controller when + * undershooting + */ + s32 k_pu; + + /* Integral parameter of the PID controller */ + s32 k_i; + + /* Derivative parameter of the PID controller */ + s32 k_d; + + /* threshold below which the error is no longer accumulated */ + s32 integral_cutoff; + + /* + * @slope: slope of a linear temperature adjustment curve. + * Used by thermal zone drivers. + */ + int slope; + /* + * @offset: offset of a linear temperature adjustment curve. + * Used by thermal zone drivers (default 0). + */ + int offset; }; struct thermal_genl_event { @@ -296,19 +327,49 @@ struct thermal_genl_event { enum events event; }; +/** + * struct thermal_zone_of_device_ops - scallbacks for handling DT based zones + * + * Mandatory: + * @get_temp: a pointer to a function that reads the sensor temperature. + * + * Optional: + * @get_trend: a pointer to a function that reads the sensor temperature trend. + * @set_emul_temp: a pointer to a function that sets sensor emulated + * temperature. + */ +struct thermal_zone_of_device_ops { + int (*get_temp)(void *, long *); + int (*get_trend)(void *, long *); + int (*set_emul_temp)(void *, unsigned long); +}; + +/** + * struct thermal_trip - representation of a point in temperature domain + * @np: pointer to struct device_node that this trip point was created from + * @temperature: temperature value in miliCelsius + * @hysteresis: relative hysteresis in miliCelsius + * @type: trip point type + */ + +struct thermal_trip { + struct device_node *np; + unsigned long int temperature; + unsigned long int hysteresis; + enum thermal_trip_type type; +}; + /* Function declarations */ #ifdef CONFIG_THERMAL_OF struct thermal_zone_device * -thermal_zone_of_sensor_register(struct device *dev, int id, - void *data, int (*get_temp)(void *, long *), - int (*get_trend)(void *, long *)); +thermal_zone_of_sensor_register(struct device *dev, int id, void *data, + const struct thermal_zone_of_device_ops *ops); void thermal_zone_of_sensor_unregister(struct device *dev, struct thermal_zone_device *tz); #else static inline struct thermal_zone_device * -thermal_zone_of_sensor_register(struct device *dev, int id, - void *data, int (*get_temp)(void *, long *), - int (*get_trend)(void *, long *)) +thermal_zone_of_sensor_register(struct device *dev, int id, void *data, + const struct thermal_zone_of_device_ops *ops) { return NULL; } @@ -320,14 +381,27 @@ void thermal_zone_of_sensor_unregister(struct device *dev, } #endif + +#if IS_ENABLED(CONFIG_THERMAL) +static inline bool cdev_is_power_actor(struct thermal_cooling_device *cdev) +{ + return cdev->ops->get_requested_power && cdev->ops->state2power && + cdev->ops->power2state; +} + +int power_actor_get_max_power(struct thermal_cooling_device *, + struct thermal_zone_device *tz, u32 *max_power); +int power_actor_set_power(struct thermal_cooling_device *, + struct thermal_instance *, u32); struct thermal_zone_device *thermal_zone_device_register(const char *, int, int, void *, struct thermal_zone_device_ops *, - const struct thermal_zone_params *, int, int); + struct thermal_zone_params *, int, int); void thermal_zone_device_unregister(struct thermal_zone_device *); int thermal_zone_bind_cooling_device(struct thermal_zone_device *, int, struct thermal_cooling_device *, - unsigned long, unsigned long); + unsigned long, unsigned long, + unsigned int); int thermal_zone_unbind_cooling_device(struct thermal_zone_device *, int, struct thermal_cooling_device *); void thermal_zone_device_update(struct thermal_zone_device *); @@ -346,8 +420,66 @@ struct thermal_instance *get_thermal_instance(struct thermal_zone_device *, struct thermal_cooling_device *, int); void thermal_cdev_update(struct thermal_cooling_device *); void thermal_notify_framework(struct thermal_zone_device *, int); - -#ifdef CONFIG_NET +#else +static inline bool cdev_is_power_actor(struct thermal_cooling_device *cdev) +{ return false; } +static inline int power_actor_get_max_power(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, u32 *max_power) +{ return 0; } +static inline int power_actor_set_power(struct thermal_cooling_device *cdev, + struct thermal_instance *tz, u32 power) +{ return 0; } +static inline struct thermal_zone_device *thermal_zone_device_register( + const char *type, int trips, int mask, void *devdata, + struct thermal_zone_device_ops *ops, + const struct thermal_zone_params *tzp, + int passive_delay, int polling_delay) +{ return ERR_PTR(-ENODEV); } +static inline void thermal_zone_device_unregister( + struct thermal_zone_device *tz) +{ } +static inline int thermal_zone_bind_cooling_device( + struct thermal_zone_device *tz, int trip, + struct thermal_cooling_device *cdev, + unsigned long upper, unsigned long lower) +{ return -ENODEV; } +static inline int thermal_zone_unbind_cooling_device( + struct thermal_zone_device *tz, int trip, + struct thermal_cooling_device *cdev) +{ return -ENODEV; } +static inline void thermal_zone_device_update(struct thermal_zone_device *tz) +{ } +static inline struct thermal_cooling_device * +thermal_cooling_device_register(char *type, void *devdata, + const struct thermal_cooling_device_ops *ops) +{ return ERR_PTR(-ENODEV); } +static inline struct thermal_cooling_device * +thermal_of_cooling_device_register(struct device_node *np, + char *type, void *devdata, const struct thermal_cooling_device_ops *ops) +{ return ERR_PTR(-ENODEV); } +static inline void thermal_cooling_device_unregister( + struct thermal_cooling_device *cdev) +{ } +static inline struct thermal_zone_device *thermal_zone_get_zone_by_name( + const char *name) +{ return ERR_PTR(-ENODEV); } +static inline int thermal_zone_get_temp( + struct thermal_zone_device *tz, unsigned long *temp) +{ return -ENODEV; } +static inline int get_tz_trend(struct thermal_zone_device *tz, int trip) +{ return -ENODEV; } +static inline struct thermal_instance * +get_thermal_instance(struct thermal_zone_device *tz, + struct thermal_cooling_device *cdev, int trip) +{ return ERR_PTR(-ENODEV); } +static inline void thermal_cdev_update(struct thermal_cooling_device *cdev) +{ } +static inline void thermal_notify_framework(struct thermal_zone_device *tz, + int trip) +{ } +#endif /* CONFIG_THERMAL */ + +#if defined(CONFIG_NET) && IS_ENABLED(CONFIG_THERMAL) extern int thermal_generate_netlink_event(struct thermal_zone_device *tz, enum events event); #else diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index b87696fdf06a..0ec598381f97 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -16,6 +16,8 @@ struct vm_area_struct; /* vma defining user mapping in mm_types.h */ #define VM_USERMAP 0x00000008 /* suitable for remap_vmalloc_range */ #define VM_VPAGES 0x00000010 /* buffer for pages was vmalloc'ed */ #define VM_UNINITIALIZED 0x00000020 /* vm_struct is not fully initialized */ +#define VM_NO_GUARD 0x00000040 /* don't add guard page */ +#define VM_KASAN 0x00000080 /* has allocated kasan shadow memory */ /* bits [20..32] reserved for arch specific ioremap internals */ /* @@ -75,7 +77,9 @@ extern void *vmalloc_32_user(unsigned long size); extern void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot); extern void *__vmalloc_node_range(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, - pgprot_t prot, int node, const void *caller); + pgprot_t prot, unsigned long vm_flags, int node, + const void *caller); + extern void vfree(const void *addr); extern void *vmap(struct page **pages, unsigned int count, @@ -96,8 +100,12 @@ void vmalloc_sync_all(void); static inline size_t get_vm_area_size(const struct vm_struct *area) { - /* return actual size without guard page */ - return area->size - PAGE_SIZE; + if (!(area->flags & VM_NO_GUARD)) + /* return actual size without guard page */ + return area->size - PAGE_SIZE; + else + return area->size; + } extern struct vm_struct *get_vm_area(unsigned long size, unsigned long flags); diff --git a/include/trace/events/thermal.h b/include/trace/events/thermal.h index 0f4f95d63c03..8b1f80682b80 100644 --- a/include/trace/events/thermal.h +++ b/include/trace/events/thermal.h @@ -77,6 +77,64 @@ TRACE_EVENT(thermal_zone_trip, __entry->trip_type) ); +TRACE_EVENT(thermal_power_cpu_get_power, + TP_PROTO(const struct cpumask *cpus, unsigned long freq, u32 *load, + size_t load_len, u32 dynamic_power, u32 static_power), + + TP_ARGS(cpus, freq, load, load_len, dynamic_power, static_power), + + TP_STRUCT__entry( + __bitmask(cpumask, num_possible_cpus()) + __field(unsigned long, freq ) + __dynamic_array(u32, load, load_len) + __field(size_t, load_len ) + __field(u32, dynamic_power ) + __field(u32, static_power ) + ), + + TP_fast_assign( + __assign_bitmask(cpumask, cpumask_bits(cpus), + num_possible_cpus()); + __entry->freq = freq; + memcpy(__get_dynamic_array(load), load, + load_len * sizeof(*load)); + __entry->load_len = load_len; + __entry->dynamic_power = dynamic_power; + __entry->static_power = static_power; + ), + + TP_printk("cpus=%s freq=%lu load={%s} dynamic_power=%d static_power=%d", + __get_bitmask(cpumask), __entry->freq, + __print_array(__get_dynamic_array(load), __entry->load_len, 4), + __entry->dynamic_power, __entry->static_power) +); + +TRACE_EVENT(thermal_power_cpu_limit, + TP_PROTO(const struct cpumask *cpus, unsigned int freq, + unsigned long cdev_state, u32 power), + + TP_ARGS(cpus, freq, cdev_state, power), + + TP_STRUCT__entry( + __bitmask(cpumask, num_possible_cpus()) + __field(unsigned int, freq ) + __field(unsigned long, cdev_state) + __field(u32, power ) + ), + + TP_fast_assign( + __assign_bitmask(cpumask, cpumask_bits(cpus), + num_possible_cpus()); + __entry->freq = freq; + __entry->cdev_state = cdev_state; + __entry->power = power; + ), + + TP_printk("cpus=%s freq=%u cdev_state=%lu power=%u", + __get_bitmask(cpumask), __entry->freq, __entry->cdev_state, + __entry->power) +); + #endif /* _TRACE_THERMAL_H */ /* This part must be outside protection */ diff --git a/include/trace/events/thermal_power_allocator.h b/include/trace/events/thermal_power_allocator.h new file mode 100644 index 000000000000..12e1321c4e0c --- /dev/null +++ b/include/trace/events/thermal_power_allocator.h @@ -0,0 +1,87 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM thermal_power_allocator + +#if !defined(_TRACE_THERMAL_POWER_ALLOCATOR_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_THERMAL_POWER_ALLOCATOR_H + +#include <linux/tracepoint.h> + +TRACE_EVENT(thermal_power_allocator, + TP_PROTO(struct thermal_zone_device *tz, u32 *req_power, + u32 total_req_power, u32 *granted_power, + u32 total_granted_power, size_t num_actors, + u32 power_range, u32 max_allocatable_power, + unsigned long current_temp, s32 delta_temp), + TP_ARGS(tz, req_power, total_req_power, granted_power, + total_granted_power, num_actors, power_range, + max_allocatable_power, current_temp, delta_temp), + TP_STRUCT__entry( + __field(int, tz_id ) + __dynamic_array(u32, req_power, num_actors ) + __field(u32, total_req_power ) + __dynamic_array(u32, granted_power, num_actors) + __field(u32, total_granted_power ) + __field(size_t, num_actors ) + __field(u32, power_range ) + __field(u32, max_allocatable_power ) + __field(unsigned long, current_temp ) + __field(s32, delta_temp ) + ), + TP_fast_assign( + __entry->tz_id = tz->id; + memcpy(__get_dynamic_array(req_power), req_power, + num_actors * sizeof(*req_power)); + __entry->total_req_power = total_req_power; + memcpy(__get_dynamic_array(granted_power), granted_power, + num_actors * sizeof(*granted_power)); + __entry->total_granted_power = total_granted_power; + __entry->num_actors = num_actors; + __entry->power_range = power_range; + __entry->max_allocatable_power = max_allocatable_power; + __entry->current_temp = current_temp; + __entry->delta_temp = delta_temp; + ), + + TP_printk("thermal_zone_id=%d req_power={%s} total_req_power=%u granted_power={%s} total_granted_power=%u power_range=%u max_allocatable_power=%u current_temperature=%lu delta_temperature=%d", + __entry->tz_id, + __print_array(__get_dynamic_array(req_power), + __entry->num_actors, 4), + __entry->total_req_power, + __print_array(__get_dynamic_array(granted_power), + __entry->num_actors, 4), + __entry->total_granted_power, __entry->power_range, + __entry->max_allocatable_power, __entry->current_temp, + __entry->delta_temp) +); + +TRACE_EVENT(thermal_power_allocator_pid, + TP_PROTO(struct thermal_zone_device *tz, s32 err, s32 err_integral, + s64 p, s64 i, s64 d, s32 output), + TP_ARGS(tz, err, err_integral, p, i, d, output), + TP_STRUCT__entry( + __field(int, tz_id ) + __field(s32, err ) + __field(s32, err_integral) + __field(s64, p ) + __field(s64, i ) + __field(s64, d ) + __field(s32, output ) + ), + TP_fast_assign( + __entry->tz_id = tz->id; + __entry->err = err; + __entry->err_integral = err_integral; + __entry->p = p; + __entry->i = i; + __entry->d = d; + __entry->output = output; + ), + + TP_printk("thermal_zone_id=%d err=%d err_integral=%d p=%lld i=%lld d=%lld output=%d", + __entry->tz_id, __entry->err, __entry->err_integral, + __entry->p, __entry->i, __entry->d, __entry->output) +); +#endif /* _TRACE_THERMAL_POWER_ALLOCATOR_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h index bb1f5d82ad49..2de738e1e561 100644 --- a/include/trace/ftrace.h +++ b/include/trace/ftrace.h @@ -263,6 +263,14 @@ #undef __print_hex #define __print_hex(buf, buf_len) ftrace_print_hex_seq(p, buf, buf_len) +#undef __print_array +#define __print_array(array, count, el_size) \ + ({ \ + BUILD_BUG_ON(el_size != 1 && el_size != 2 && \ + el_size != 4 && el_size != 8); \ + ftrace_print_array_seq(p, array, count, el_size); \ + }) + #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ static notrace enum print_line_t \ @@ -676,6 +684,7 @@ static inline void ftrace_test_probe_##call(void) \ #undef __get_dynamic_array_len #undef __get_str #undef __get_bitmask +#undef __print_array #undef TP_printk #define TP_printk(fmt, args...) "\"" fmt "\", " __stringify(args) diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild index 8523f9bb72f2..fe4739531168 100644 --- a/include/uapi/linux/Kbuild +++ b/include/uapi/linux/Kbuild @@ -383,6 +383,7 @@ header-y += tcp.h header-y += tcp_metrics.h header-y += telephony.h header-y += termios.h +header-y += thermal.h header-y += time.h header-y += times.h header-y += timex.h diff --git a/include/uapi/linux/thermal.h b/include/uapi/linux/thermal.h new file mode 100644 index 000000000000..ac5535855982 --- /dev/null +++ b/include/uapi/linux/thermal.h @@ -0,0 +1,35 @@ +#ifndef _UAPI_LINUX_THERMAL_H +#define _UAPI_LINUX_THERMAL_H + +#define THERMAL_NAME_LENGTH 20 + +/* Adding event notification support elements */ +#define THERMAL_GENL_FAMILY_NAME "thermal_event" +#define THERMAL_GENL_VERSION 0x01 +#define THERMAL_GENL_MCAST_GROUP_NAME "thermal_mc_grp" + +/* Events supported by Thermal Netlink */ +enum events { + THERMAL_AUX0, + THERMAL_AUX1, + THERMAL_CRITICAL, + THERMAL_DEV_FAULT, +}; + +/* attributes of thermal_genl_family */ +enum { + THERMAL_GENL_ATTR_UNSPEC, + THERMAL_GENL_ATTR_EVENT, + __THERMAL_GENL_ATTR_MAX, +}; +#define THERMAL_GENL_ATTR_MAX (__THERMAL_GENL_ATTR_MAX - 1) + +/* commands supported by the thermal_genl_family */ +enum { + THERMAL_GENL_CMD_UNSPEC, + THERMAL_GENL_CMD_EVENT, + __THERMAL_GENL_CMD_MAX, +}; +#define THERMAL_GENL_CMD_MAX (__THERMAL_GENL_CMD_MAX - 1) + +#endif /* _UAPI_LINUX_THERMAL_H */ diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig index 225086b2652e..9a76e3beda54 100644 --- a/kernel/irq/Kconfig +++ b/kernel/irq/Kconfig @@ -55,6 +55,21 @@ config GENERIC_IRQ_CHIP config IRQ_DOMAIN bool +# Support for hierarchical irq domains +config IRQ_DOMAIN_HIERARCHY + bool + select IRQ_DOMAIN + +# Generic MSI interrupt support +config GENERIC_MSI_IRQ + bool + +# Generic MSI hierarchical interrupt domain support +config GENERIC_MSI_IRQ_DOMAIN + bool + select IRQ_DOMAIN_HIERARCHY + select GENERIC_MSI_IRQ + config HANDLE_DOMAIN_IRQ bool diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index fff17381f0af..d12123526e2b 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -6,3 +6,4 @@ obj-$(CONFIG_IRQ_DOMAIN) += irqdomain.o obj-$(CONFIG_PROC_FS) += proc.o obj-$(CONFIG_GENERIC_PENDING_IRQ) += migration.o obj-$(CONFIG_PM_SLEEP) += pm.o +obj-$(CONFIG_GENERIC_MSI_IRQ) += msi.o diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index e5202f00cabc..6f1c7a566b95 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -15,6 +15,7 @@ #include <linux/module.h> #include <linux/interrupt.h> #include <linux/kernel_stat.h> +#include <linux/irqdomain.h> #include <trace/events/irq.h> @@ -178,6 +179,7 @@ int irq_startup(struct irq_desc *desc, bool resend) irq_state_clr_disabled(desc); desc->depth = 0; + irq_domain_activate_irq(&desc->irq_data); if (desc->irq_data.chip->irq_startup) { ret = desc->irq_data.chip->irq_startup(&desc->irq_data); irq_state_clr_masked(desc); @@ -199,6 +201,7 @@ void irq_shutdown(struct irq_desc *desc) desc->irq_data.chip->irq_disable(&desc->irq_data); else desc->irq_data.chip->irq_mask(&desc->irq_data); + irq_domain_deactivate_irq(&desc->irq_data); irq_state_set_masked(desc); } @@ -728,7 +731,30 @@ __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, if (!handle) { handle = handle_bad_irq; } else { - if (WARN_ON(desc->irq_data.chip == &no_irq_chip)) + struct irq_data *irq_data = &desc->irq_data; +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY + /* + * With hierarchical domains we might run into a + * situation where the outermost chip is not yet set + * up, but the inner chips are there. Instead of + * bailing we install the handler, but obviously we + * cannot enable/startup the interrupt at this point. + */ + while (irq_data) { + if (irq_data->chip != &no_irq_chip) + break; + /* + * Bail out if the outer chip is not set up + * and the interrrupt supposed to be started + * right away. + */ + if (WARN_ON(is_chained)) + goto out; + /* Try the parent */ + irq_data = irq_data->parent_data; + } +#endif + if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip)) goto out; } @@ -847,3 +873,105 @@ void irq_cpu_offline(void) raw_spin_unlock_irqrestore(&desc->lock, flags); } } + +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY +/** + * irq_chip_ack_parent - Acknowledge the parent interrupt + * @data: Pointer to interrupt specific data + */ +void irq_chip_ack_parent(struct irq_data *data) +{ + data = data->parent_data; + data->chip->irq_ack(data); +} + +/** + * irq_chip_mask_parent - Mask the parent interrupt + * @data: Pointer to interrupt specific data + */ +void irq_chip_mask_parent(struct irq_data *data) +{ + data = data->parent_data; + data->chip->irq_mask(data); +} + +/** + * irq_chip_unmask_parent - Unmask the parent interrupt + * @data: Pointer to interrupt specific data + */ +void irq_chip_unmask_parent(struct irq_data *data) +{ + data = data->parent_data; + data->chip->irq_unmask(data); +} + +/** + * irq_chip_eoi_parent - Invoke EOI on the parent interrupt + * @data: Pointer to interrupt specific data + */ +void irq_chip_eoi_parent(struct irq_data *data) +{ + data = data->parent_data; + data->chip->irq_eoi(data); +} + +/** + * irq_chip_set_affinity_parent - Set affinity on the parent interrupt + * @data: Pointer to interrupt specific data + * @dest: The affinity mask to set + * @force: Flag to enforce setting (disable online checks) + * + * Conditinal, as the underlying parent chip might not implement it. + */ +int irq_chip_set_affinity_parent(struct irq_data *data, + const struct cpumask *dest, bool force) +{ + data = data->parent_data; + if (data->chip->irq_set_affinity) + return data->chip->irq_set_affinity(data, dest, force); + + return -ENOSYS; +} + +/** + * irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware + * @data: Pointer to interrupt specific data + * + * Iterate through the domain hierarchy of the interrupt and check + * whether a hw retrigger function exists. If yes, invoke it. + */ +int irq_chip_retrigger_hierarchy(struct irq_data *data) +{ + for (data = data->parent_data; data; data = data->parent_data) + if (data->chip && data->chip->irq_retrigger) + return data->chip->irq_retrigger(data); + + return -ENOSYS; +} +#endif + +/** + * irq_chip_compose_msi_msg - Componse msi message for a irq chip + * @data: Pointer to interrupt specific data + * @msg: Pointer to the MSI message + * + * For hierarchical domains we find the first chip in the hierarchy + * which implements the irq_compose_msi_msg callback. For non + * hierarchical we use the top level chip. + */ +int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct irq_data *pos = NULL; + +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY + for (; data; data = data->parent_data) +#endif + if (data->chip && data->chip->irq_compose_msi_msg) + pos = data; + if (!pos) + return -ENOSYS; + + pos->chip->irq_compose_msi_msg(pos, msg); + + return 0; +} diff --git a/kernel/irq/generic-chip.c b/kernel/irq/generic-chip.c index db458c68e392..61024e8abdef 100644 --- a/kernel/irq/generic-chip.c +++ b/kernel/irq/generic-chip.c @@ -191,6 +191,16 @@ int irq_gc_set_wake(struct irq_data *d, unsigned int on) return 0; } +static u32 irq_readl_be(void __iomem *addr) +{ + return ioread32be(addr); +} + +static void irq_writel_be(u32 val, void __iomem *addr) +{ + iowrite32be(val, addr); +} + static void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name, int num_ct, unsigned int irq_base, @@ -300,7 +310,13 @@ int irq_alloc_domain_generic_chips(struct irq_domain *d, int irqs_per_chip, dgc->gc[i] = gc = tmp; irq_init_generic_chip(gc, name, num_ct, i * irqs_per_chip, NULL, handler); + gc->domain = d; + if (gcflags & IRQ_GC_BE_IO) { + gc->reg_readl = &irq_readl_be; + gc->reg_writel = &irq_writel_be; + } + raw_spin_lock_irqsave(&gc_lock, flags); list_add_tail(&gc->list, &gc_list); raw_spin_unlock_irqrestore(&gc_lock, flags); diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 6534ff6ce02e..021f823794e7 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -23,6 +23,10 @@ static DEFINE_MUTEX(irq_domain_mutex); static DEFINE_MUTEX(revmap_trees_mutex); static struct irq_domain *irq_default_domain; +static int irq_domain_alloc_descs(int virq, unsigned int nr_irqs, + irq_hw_number_t hwirq, int node); +static void irq_domain_check_hierarchy(struct irq_domain *domain); + /** * __irq_domain_add() - Allocate a new irq_domain data structure * @of_node: optional device-tree node of the interrupt controller @@ -30,7 +34,7 @@ static struct irq_domain *irq_default_domain; * @hwirq_max: Maximum number of interrupts supported by controller * @direct_max: Maximum value of direct maps; Use ~0 for no limit; 0 for no * direct mapping - * @ops: map/unmap domain callbacks + * @ops: domain callbacks * @host_data: Controller private data pointer * * Allocates and initialize and irq_domain structure. @@ -56,6 +60,7 @@ struct irq_domain *__irq_domain_add(struct device_node *of_node, int size, domain->hwirq_max = hwirq_max; domain->revmap_size = size; domain->revmap_direct_max_irq = direct_max; + irq_domain_check_hierarchy(domain); mutex_lock(&irq_domain_mutex); list_add(&domain->link, &irq_domain_list); @@ -109,7 +114,7 @@ EXPORT_SYMBOL_GPL(irq_domain_remove); * @first_irq: first number of irq block assigned to the domain, * pass zero to assign irqs on-the-fly. If first_irq is non-zero, then * pre-map all of the irqs in the domain to virqs starting at first_irq. - * @ops: map/unmap domain callbacks + * @ops: domain callbacks * @host_data: Controller private data pointer * * Allocates an irq_domain, and optionally if first_irq is positive then also @@ -174,20 +179,20 @@ struct irq_domain *irq_domain_add_legacy(struct device_node *of_node, domain = __irq_domain_add(of_node, first_hwirq + size, first_hwirq + size, 0, ops, host_data); - if (!domain) - return NULL; - - irq_domain_associate_many(domain, first_irq, first_hwirq, size); + if (domain) + irq_domain_associate_many(domain, first_irq, first_hwirq, size); return domain; } EXPORT_SYMBOL_GPL(irq_domain_add_legacy); /** - * irq_find_host() - Locates a domain for a given device node + * irq_find_matching_host() - Locates a domain for a given device node * @node: device-tree node of the interrupt controller + * @bus_token: domain-specific data */ -struct irq_domain *irq_find_host(struct device_node *node) +struct irq_domain *irq_find_matching_host(struct device_node *node, + enum irq_domain_bus_token bus_token) { struct irq_domain *h, *found = NULL; int rc; @@ -196,13 +201,19 @@ struct irq_domain *irq_find_host(struct device_node *node) * it might potentially be set to match all interrupts in * the absence of a device node. This isn't a problem so far * yet though... + * + * bus_token == DOMAIN_BUS_ANY matches any domain, any other + * values must generate an exact match for the domain to be + * selected. */ mutex_lock(&irq_domain_mutex); list_for_each_entry(h, &irq_domain_list, link) { if (h->ops->match) - rc = h->ops->match(h, node); + rc = h->ops->match(h, node, bus_token); else - rc = (h->of_node != NULL) && (h->of_node == node); + rc = ((h->of_node != NULL) && (h->of_node == node) && + ((bus_token == DOMAIN_BUS_ANY) || + (h->bus_token == bus_token))); if (rc) { found = h; @@ -212,7 +223,7 @@ struct irq_domain *irq_find_host(struct device_node *node) mutex_unlock(&irq_domain_mutex); return found; } -EXPORT_SYMBOL_GPL(irq_find_host); +EXPORT_SYMBOL_GPL(irq_find_matching_host); /** * irq_set_default_host() - Set a "default" irq domain @@ -388,7 +399,6 @@ EXPORT_SYMBOL_GPL(irq_create_direct_mapping); unsigned int irq_create_mapping(struct irq_domain *domain, irq_hw_number_t hwirq) { - unsigned int hint; int virq; pr_debug("irq_create_mapping(0x%p, 0x%lx)\n", domain, hwirq); @@ -410,12 +420,8 @@ unsigned int irq_create_mapping(struct irq_domain *domain, } /* Allocate a virtual interrupt number */ - hint = hwirq % nr_irqs; - if (hint == 0) - hint++; - virq = irq_alloc_desc_from(hint, of_node_to_nid(domain->of_node)); - if (virq <= 0) - virq = irq_alloc_desc_from(1, of_node_to_nid(domain->of_node)); + virq = irq_domain_alloc_descs(-1, 1, hwirq, + of_node_to_nid(domain->of_node)); if (virq <= 0) { pr_debug("-> virq allocation failed\n"); return 0; @@ -471,7 +477,7 @@ unsigned int irq_create_of_mapping(struct of_phandle_args *irq_data) struct irq_domain *domain; irq_hw_number_t hwirq; unsigned int type = IRQ_TYPE_NONE; - unsigned int virq; + int virq; domain = irq_data->np ? irq_find_host(irq_data->np) : irq_default_domain; if (!domain) { @@ -489,10 +495,24 @@ unsigned int irq_create_of_mapping(struct of_phandle_args *irq_data) return 0; } - /* Create mapping */ - virq = irq_create_mapping(domain, hwirq); - if (!virq) - return virq; + if (irq_domain_is_hierarchy(domain)) { + /* + * If we've already configured this interrupt, + * don't do it again, or hell will break loose. + */ + virq = irq_find_mapping(domain, hwirq); + if (virq) + return virq; + + virq = irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, irq_data); + if (virq <= 0) + return 0; + } else { + /* Create mapping */ + virq = irq_create_mapping(domain, hwirq); + if (!virq) + return virq; + } /* Set type if specified and different than the current one */ if (type != IRQ_TYPE_NONE && @@ -540,8 +560,8 @@ unsigned int irq_find_mapping(struct irq_domain *domain, return 0; if (hwirq < domain->revmap_direct_max_irq) { - data = irq_get_irq_data(hwirq); - if (data && (data->domain == domain) && (data->hwirq == hwirq)) + data = irq_domain_get_irq_data(domain, hwirq); + if (data && data->hwirq == hwirq) return hwirq; } @@ -709,3 +729,518 @@ const struct irq_domain_ops irq_domain_simple_ops = { .xlate = irq_domain_xlate_onetwocell, }; EXPORT_SYMBOL_GPL(irq_domain_simple_ops); + +static int irq_domain_alloc_descs(int virq, unsigned int cnt, + irq_hw_number_t hwirq, int node) +{ + unsigned int hint; + + if (virq >= 0) { + virq = irq_alloc_descs(virq, virq, cnt, node); + } else { + hint = hwirq % nr_irqs; + if (hint == 0) + hint++; + virq = irq_alloc_descs_from(hint, cnt, node); + if (virq <= 0 && hint > 1) + virq = irq_alloc_descs_from(1, cnt, node); + } + + return virq; +} + +#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY +/** + * irq_domain_add_hierarchy - Add a irqdomain into the hierarchy + * @parent: Parent irq domain to associate with the new domain + * @flags: Irq domain flags associated to the domain + * @size: Size of the domain. See below + * @node: Optional device-tree node of the interrupt controller + * @ops: Pointer to the interrupt domain callbacks + * @host_data: Controller private data pointer + * + * If @size is 0 a tree domain is created, otherwise a linear domain. + * + * If successful the parent is associated to the new domain and the + * domain flags are set. + * Returns pointer to IRQ domain, or NULL on failure. + */ +struct irq_domain *irq_domain_add_hierarchy(struct irq_domain *parent, + unsigned int flags, + unsigned int size, + struct device_node *node, + const struct irq_domain_ops *ops, + void *host_data) +{ + struct irq_domain *domain; + + if (size) + domain = irq_domain_add_linear(node, size, ops, host_data); + else + domain = irq_domain_add_tree(node, ops, host_data); + if (domain) { + domain->parent = parent; + domain->flags |= flags; + } + + return domain; +} + +static void irq_domain_insert_irq(int virq) +{ + struct irq_data *data; + + for (data = irq_get_irq_data(virq); data; data = data->parent_data) { + struct irq_domain *domain = data->domain; + irq_hw_number_t hwirq = data->hwirq; + + if (hwirq < domain->revmap_size) { + domain->linear_revmap[hwirq] = virq; + } else { + mutex_lock(&revmap_trees_mutex); + radix_tree_insert(&domain->revmap_tree, hwirq, data); + mutex_unlock(&revmap_trees_mutex); + } + + /* If not already assigned, give the domain the chip's name */ + if (!domain->name && data->chip) + domain->name = data->chip->name; + } + + irq_clear_status_flags(virq, IRQ_NOREQUEST); +} + +static void irq_domain_remove_irq(int virq) +{ + struct irq_data *data; + + irq_set_status_flags(virq, IRQ_NOREQUEST); + irq_set_chip_and_handler(virq, NULL, NULL); + synchronize_irq(virq); + smp_mb(); + + for (data = irq_get_irq_data(virq); data; data = data->parent_data) { + struct irq_domain *domain = data->domain; + irq_hw_number_t hwirq = data->hwirq; + + if (hwirq < domain->revmap_size) { + domain->linear_revmap[hwirq] = 0; + } else { + mutex_lock(&revmap_trees_mutex); + radix_tree_delete(&domain->revmap_tree, hwirq); + mutex_unlock(&revmap_trees_mutex); + } + } +} + +static struct irq_data *irq_domain_insert_irq_data(struct irq_domain *domain, + struct irq_data *child) +{ + struct irq_data *irq_data; + + irq_data = kzalloc_node(sizeof(*irq_data), GFP_KERNEL, child->node); + if (irq_data) { + child->parent_data = irq_data; + irq_data->irq = child->irq; + irq_data->node = child->node; + irq_data->domain = domain; + } + + return irq_data; +} + +static void irq_domain_free_irq_data(unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *irq_data, *tmp; + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_get_irq_data(virq + i); + tmp = irq_data->parent_data; + irq_data->parent_data = NULL; + irq_data->domain = NULL; + + while (tmp) { + irq_data = tmp; + tmp = tmp->parent_data; + kfree(irq_data); + } + } +} + +static int irq_domain_alloc_irq_data(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *irq_data; + struct irq_domain *parent; + int i; + + /* The outermost irq_data is embedded in struct irq_desc */ + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_get_irq_data(virq + i); + irq_data->domain = domain; + + for (parent = domain->parent; parent; parent = parent->parent) { + irq_data = irq_domain_insert_irq_data(parent, irq_data); + if (!irq_data) { + irq_domain_free_irq_data(virq, i + 1); + return -ENOMEM; + } + } + } + + return 0; +} + +/** + * irq_domain_get_irq_data - Get irq_data associated with @virq and @domain + * @domain: domain to match + * @virq: IRQ number to get irq_data + */ +struct irq_data *irq_domain_get_irq_data(struct irq_domain *domain, + unsigned int virq) +{ + struct irq_data *irq_data; + + for (irq_data = irq_get_irq_data(virq); irq_data; + irq_data = irq_data->parent_data) + if (irq_data->domain == domain) + return irq_data; + + return NULL; +} + +/** + * irq_domain_set_hwirq_and_chip - Set hwirq and irqchip of @virq at @domain + * @domain: Interrupt domain to match + * @virq: IRQ number + * @hwirq: The hwirq number + * @chip: The associated interrupt chip + * @chip_data: The associated chip data + */ +int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, unsigned int virq, + irq_hw_number_t hwirq, struct irq_chip *chip, + void *chip_data) +{ + struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq); + + if (!irq_data) + return -ENOENT; + + irq_data->hwirq = hwirq; + irq_data->chip = chip ? chip : &no_irq_chip; + irq_data->chip_data = chip_data; + + return 0; +} + +/** + * irq_domain_set_info - Set the complete data for a @virq in @domain + * @domain: Interrupt domain to match + * @virq: IRQ number + * @hwirq: The hardware interrupt number + * @chip: The associated interrupt chip + * @chip_data: The associated interrupt chip data + * @handler: The interrupt flow handler + * @handler_data: The interrupt flow handler data + * @handler_name: The interrupt handler name + */ +void irq_domain_set_info(struct irq_domain *domain, unsigned int virq, + irq_hw_number_t hwirq, struct irq_chip *chip, + void *chip_data, irq_flow_handler_t handler, + void *handler_data, const char *handler_name) +{ + irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip, chip_data); + __irq_set_handler(virq, handler, 0, handler_name); + irq_set_handler_data(virq, handler_data); +} + +/** + * irq_domain_reset_irq_data - Clear hwirq, chip and chip_data in @irq_data + * @irq_data: The pointer to irq_data + */ +void irq_domain_reset_irq_data(struct irq_data *irq_data) +{ + irq_data->hwirq = 0; + irq_data->chip = &no_irq_chip; + irq_data->chip_data = NULL; +} + +/** + * irq_domain_free_irqs_common - Clear irq_data and free the parent + * @domain: Interrupt domain to match + * @virq: IRQ number to start with + * @nr_irqs: The number of irqs to free + */ +void irq_domain_free_irqs_common(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *irq_data; + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + if (irq_data) + irq_domain_reset_irq_data(irq_data); + } + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +/** + * irq_domain_free_irqs_top - Clear handler and handler data, clear irqdata and free parent + * @domain: Interrupt domain to match + * @virq: IRQ number to start with + * @nr_irqs: The number of irqs to free + */ +void irq_domain_free_irqs_top(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_handler_data(virq + i, NULL); + irq_set_handler(virq + i, NULL); + } + irq_domain_free_irqs_common(domain, virq, nr_irqs); +} + +static bool irq_domain_is_auto_recursive(struct irq_domain *domain) +{ + return domain->flags & IRQ_DOMAIN_FLAG_AUTO_RECURSIVE; +} + +static void irq_domain_free_irqs_recursive(struct irq_domain *domain, + unsigned int irq_base, + unsigned int nr_irqs) +{ + domain->ops->free(domain, irq_base, nr_irqs); + if (irq_domain_is_auto_recursive(domain)) { + BUG_ON(!domain->parent); + irq_domain_free_irqs_recursive(domain->parent, irq_base, + nr_irqs); + } +} + +static int irq_domain_alloc_irqs_recursive(struct irq_domain *domain, + unsigned int irq_base, + unsigned int nr_irqs, void *arg) +{ + int ret = 0; + struct irq_domain *parent = domain->parent; + bool recursive = irq_domain_is_auto_recursive(domain); + + BUG_ON(recursive && !parent); + if (recursive) + ret = irq_domain_alloc_irqs_recursive(parent, irq_base, + nr_irqs, arg); + if (ret >= 0) + ret = domain->ops->alloc(domain, irq_base, nr_irqs, arg); + if (ret < 0 && recursive) + irq_domain_free_irqs_recursive(parent, irq_base, nr_irqs); + + return ret; +} + +/** + * __irq_domain_alloc_irqs - Allocate IRQs from domain + * @domain: domain to allocate from + * @irq_base: allocate specified IRQ nubmer if irq_base >= 0 + * @nr_irqs: number of IRQs to allocate + * @node: NUMA node id for memory allocation + * @arg: domain specific argument + * @realloc: IRQ descriptors have already been allocated if true + * + * Allocate IRQ numbers and initialized all data structures to support + * hierarchy IRQ domains. + * Parameter @realloc is mainly to support legacy IRQs. + * Returns error code or allocated IRQ number + * + * The whole process to setup an IRQ has been split into two steps. + * The first step, __irq_domain_alloc_irqs(), is to allocate IRQ + * descriptor and required hardware resources. The second step, + * irq_domain_activate_irq(), is to program hardwares with preallocated + * resources. In this way, it's easier to rollback when failing to + * allocate resources. + */ +int __irq_domain_alloc_irqs(struct irq_domain *domain, int irq_base, + unsigned int nr_irqs, int node, void *arg, + bool realloc) +{ + int i, ret, virq; + + if (domain == NULL) { + domain = irq_default_domain; + if (WARN(!domain, "domain is NULL; cannot allocate IRQ\n")) + return -EINVAL; + } + + if (!domain->ops->alloc) { + pr_debug("domain->ops->alloc() is NULL\n"); + return -ENOSYS; + } + + if (realloc && irq_base >= 0) { + virq = irq_base; + } else { + virq = irq_domain_alloc_descs(irq_base, nr_irqs, 0, node); + if (virq < 0) { + pr_debug("cannot allocate IRQ(base %d, count %d)\n", + irq_base, nr_irqs); + return virq; + } + } + + if (irq_domain_alloc_irq_data(domain, virq, nr_irqs)) { + pr_debug("cannot allocate memory for IRQ%d\n", virq); + ret = -ENOMEM; + goto out_free_desc; + } + + mutex_lock(&irq_domain_mutex); + ret = irq_domain_alloc_irqs_recursive(domain, virq, nr_irqs, arg); + if (ret < 0) { + mutex_unlock(&irq_domain_mutex); + goto out_free_irq_data; + } + for (i = 0; i < nr_irqs; i++) + irq_domain_insert_irq(virq + i); + mutex_unlock(&irq_domain_mutex); + + return virq; + +out_free_irq_data: + irq_domain_free_irq_data(virq, nr_irqs); +out_free_desc: + irq_free_descs(virq, nr_irqs); + return ret; +} + +/** + * irq_domain_free_irqs - Free IRQ number and associated data structures + * @virq: base IRQ number + * @nr_irqs: number of IRQs to free + */ +void irq_domain_free_irqs(unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *data = irq_get_irq_data(virq); + int i; + + if (WARN(!data || !data->domain || !data->domain->ops->free, + "NULL pointer, cannot free irq\n")) + return; + + mutex_lock(&irq_domain_mutex); + for (i = 0; i < nr_irqs; i++) + irq_domain_remove_irq(virq + i); + irq_domain_free_irqs_recursive(data->domain, virq, nr_irqs); + mutex_unlock(&irq_domain_mutex); + + irq_domain_free_irq_data(virq, nr_irqs); + irq_free_descs(virq, nr_irqs); +} + +/** + * irq_domain_alloc_irqs_parent - Allocate interrupts from parent domain + * @irq_base: Base IRQ number + * @nr_irqs: Number of IRQs to allocate + * @arg: Allocation data (arch/domain specific) + * + * Check whether the domain has been setup recursive. If not allocate + * through the parent domain. + */ +int irq_domain_alloc_irqs_parent(struct irq_domain *domain, + unsigned int irq_base, unsigned int nr_irqs, + void *arg) +{ + /* irq_domain_alloc_irqs_recursive() has called parent's alloc() */ + if (irq_domain_is_auto_recursive(domain)) + return 0; + + domain = domain->parent; + if (domain) + return irq_domain_alloc_irqs_recursive(domain, irq_base, + nr_irqs, arg); + return -ENOSYS; +} + +/** + * irq_domain_free_irqs_parent - Free interrupts from parent domain + * @irq_base: Base IRQ number + * @nr_irqs: Number of IRQs to free + * + * Check whether the domain has been setup recursive. If not free + * through the parent domain. + */ +void irq_domain_free_irqs_parent(struct irq_domain *domain, + unsigned int irq_base, unsigned int nr_irqs) +{ + /* irq_domain_free_irqs_recursive() will call parent's free */ + if (!irq_domain_is_auto_recursive(domain) && domain->parent) + irq_domain_free_irqs_recursive(domain->parent, irq_base, + nr_irqs); +} + +/** + * irq_domain_activate_irq - Call domain_ops->activate recursively to activate + * interrupt + * @irq_data: outermost irq_data associated with interrupt + * + * This is the second step to call domain_ops->activate to program interrupt + * controllers, so the interrupt could actually get delivered. + */ +void irq_domain_activate_irq(struct irq_data *irq_data) +{ + if (irq_data && irq_data->domain) { + struct irq_domain *domain = irq_data->domain; + + if (irq_data->parent_data) + irq_domain_activate_irq(irq_data->parent_data); + if (domain->ops->activate) + domain->ops->activate(domain, irq_data); + } +} + +/** + * irq_domain_deactivate_irq - Call domain_ops->deactivate recursively to + * deactivate interrupt + * @irq_data: outermost irq_data associated with interrupt + * + * It calls domain_ops->deactivate to program interrupt controllers to disable + * interrupt delivery. + */ +void irq_domain_deactivate_irq(struct irq_data *irq_data) +{ + if (irq_data && irq_data->domain) { + struct irq_domain *domain = irq_data->domain; + + if (domain->ops->deactivate) + domain->ops->deactivate(domain, irq_data); + if (irq_data->parent_data) + irq_domain_deactivate_irq(irq_data->parent_data); + } +} + +static void irq_domain_check_hierarchy(struct irq_domain *domain) +{ + /* Hierarchy irq_domains must implement callback alloc() */ + if (domain->ops->alloc) + domain->flags |= IRQ_DOMAIN_FLAG_HIERARCHY; +} +#else /* CONFIG_IRQ_DOMAIN_HIERARCHY */ +/** + * irq_domain_get_irq_data - Get irq_data associated with @virq and @domain + * @domain: domain to match + * @virq: IRQ number to get irq_data + */ +struct irq_data *irq_domain_get_irq_data(struct irq_domain *domain, + unsigned int virq) +{ + struct irq_data *irq_data = irq_get_irq_data(virq); + + return (irq_data && irq_data->domain == domain) ? irq_data : NULL; +} + +static void irq_domain_check_hierarchy(struct irq_domain *domain) +{ +} +#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */ diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 5f23a04789db..73083e41f6b7 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -183,6 +183,7 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, ret = chip->irq_set_affinity(data, mask, force); switch (ret) { case IRQ_SET_MASK_OK: + case IRQ_SET_MASK_OK_DONE: cpumask_copy(data->affinity, mask); case IRQ_SET_MASK_OK_NOCOPY: irq_set_thread_affinity(desc); @@ -600,6 +601,7 @@ int __irq_set_trigger(struct irq_desc *desc, unsigned int irq, switch (ret) { case IRQ_SET_MASK_OK: + case IRQ_SET_MASK_OK_DONE: irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK); irqd_set(&desc->irq_data, flags); diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c new file mode 100644 index 000000000000..3e18163f336f --- /dev/null +++ b/kernel/irq/msi.c @@ -0,0 +1,330 @@ +/* + * linux/kernel/irq/msi.c + * + * Copyright (C) 2014 Intel Corp. + * Author: Jiang Liu <jiang.liu@linux.intel.com> + * + * This file is licensed under GPLv2. + * + * This file contains common code to support Message Signalled Interrupt for + * PCI compatible and non PCI compatible devices. + */ +#include <linux/types.h> +#include <linux/device.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/msi.h> + +/* Temparory solution for building, will be removed later */ +#include <linux/pci.h> + +void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) +{ + *msg = entry->msg; +} + +void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) +{ + struct msi_desc *entry = irq_get_msi_desc(irq); + + __get_cached_msi_msg(entry, msg); +} +EXPORT_SYMBOL_GPL(get_cached_msi_msg); + +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN +static inline void irq_chip_write_msi_msg(struct irq_data *data, + struct msi_msg *msg) +{ + data->chip->irq_write_msi_msg(data, msg); +} + +/** + * msi_domain_set_affinity - Generic affinity setter function for MSI domains + * @irq_data: The irq data associated to the interrupt + * @mask: The affinity mask to set + * @force: Flag to enforce setting (disable online checks) + * + * Intended to be used by MSI interrupt controllers which are + * implemented with hierarchical domains. + */ +int msi_domain_set_affinity(struct irq_data *irq_data, + const struct cpumask *mask, bool force) +{ + struct irq_data *parent = irq_data->parent_data; + struct msi_msg msg; + int ret; + + ret = parent->chip->irq_set_affinity(parent, mask, force); + if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) { + BUG_ON(irq_chip_compose_msi_msg(irq_data, &msg)); + irq_chip_write_msi_msg(irq_data, &msg); + } + + return ret; +} + +static void msi_domain_activate(struct irq_domain *domain, + struct irq_data *irq_data) +{ + struct msi_msg msg; + + BUG_ON(irq_chip_compose_msi_msg(irq_data, &msg)); + irq_chip_write_msi_msg(irq_data, &msg); +} + +static void msi_domain_deactivate(struct irq_domain *domain, + struct irq_data *irq_data) +{ + struct msi_msg msg; + + memset(&msg, 0, sizeof(msg)); + irq_chip_write_msi_msg(irq_data, &msg); +} + +static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + struct msi_domain_info *info = domain->host_data; + struct msi_domain_ops *ops = info->ops; + irq_hw_number_t hwirq = ops->get_hwirq(info, arg); + int i, ret; + + if (irq_find_mapping(domain, hwirq) > 0) + return -EEXIST; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); + if (ret < 0) + return ret; + + for (i = 0; i < nr_irqs; i++) { + ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg); + if (ret < 0) { + if (ops->msi_free) { + for (i--; i > 0; i--) + ops->msi_free(domain, info, virq + i); + } + irq_domain_free_irqs_top(domain, virq, nr_irqs); + return ret; + } + } + + return 0; +} + +static void msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct msi_domain_info *info = domain->host_data; + int i; + + if (info->ops->msi_free) { + for (i = 0; i < nr_irqs; i++) + info->ops->msi_free(domain, info, virq + i); + } + irq_domain_free_irqs_top(domain, virq, nr_irqs); +} + +static struct irq_domain_ops msi_domain_ops = { + .alloc = msi_domain_alloc, + .free = msi_domain_free, + .activate = msi_domain_activate, + .deactivate = msi_domain_deactivate, +}; + +#ifdef GENERIC_MSI_DOMAIN_OPS +static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info, + msi_alloc_info_t *arg) +{ + return arg->hwirq; +} + +static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *arg) +{ + memset(arg, 0, sizeof(*arg)); + return 0; +} + +static void msi_domain_ops_set_desc(msi_alloc_info_t *arg, + struct msi_desc *desc) +{ + arg->desc = desc; +} +#else +#define msi_domain_ops_get_hwirq NULL +#define msi_domain_ops_prepare NULL +#define msi_domain_ops_set_desc NULL +#endif /* !GENERIC_MSI_DOMAIN_OPS */ + +static int msi_domain_ops_init(struct irq_domain *domain, + struct msi_domain_info *info, + unsigned int virq, irq_hw_number_t hwirq, + msi_alloc_info_t *arg) +{ + irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip, + info->chip_data); + if (info->handler && info->handler_name) { + __irq_set_handler(virq, info->handler, 0, info->handler_name); + if (info->handler_data) + irq_set_handler_data(virq, info->handler_data); + } + return 0; +} + +static int msi_domain_ops_check(struct irq_domain *domain, + struct msi_domain_info *info, + struct device *dev) +{ + return 0; +} + +static struct msi_domain_ops msi_domain_ops_default = { + .get_hwirq = msi_domain_ops_get_hwirq, + .msi_init = msi_domain_ops_init, + .msi_check = msi_domain_ops_check, + .msi_prepare = msi_domain_ops_prepare, + .set_desc = msi_domain_ops_set_desc, +}; + +static void msi_domain_update_dom_ops(struct msi_domain_info *info) +{ + struct msi_domain_ops *ops = info->ops; + + if (ops == NULL) { + info->ops = &msi_domain_ops_default; + return; + } + + if (ops->get_hwirq == NULL) + ops->get_hwirq = msi_domain_ops_default.get_hwirq; + if (ops->msi_init == NULL) + ops->msi_init = msi_domain_ops_default.msi_init; + if (ops->msi_check == NULL) + ops->msi_check = msi_domain_ops_default.msi_check; + if (ops->msi_prepare == NULL) + ops->msi_prepare = msi_domain_ops_default.msi_prepare; + if (ops->set_desc == NULL) + ops->set_desc = msi_domain_ops_default.set_desc; +} + +static void msi_domain_update_chip_ops(struct msi_domain_info *info) +{ + struct irq_chip *chip = info->chip; + + BUG_ON(!chip); + if (!chip->irq_mask) + chip->irq_mask = pci_msi_mask_irq; + if (!chip->irq_unmask) + chip->irq_unmask = pci_msi_unmask_irq; + if (!chip->irq_set_affinity) + chip->irq_set_affinity = msi_domain_set_affinity; +} + +/** + * msi_create_irq_domain - Create a MSI interrupt domain + * @of_node: Optional device-tree node of the interrupt controller + * @info: MSI domain info + * @parent: Parent irq domain + */ +struct irq_domain *msi_create_irq_domain(struct device_node *node, + struct msi_domain_info *info, + struct irq_domain *parent) +{ + if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) + msi_domain_update_dom_ops(info); + if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) + msi_domain_update_chip_ops(info); + + return irq_domain_add_hierarchy(parent, 0, 0, node, &msi_domain_ops, + info); +} + +/** + * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain + * @domain: The domain to allocate from + * @dev: Pointer to device struct of the device for which the interrupts + * are allocated + * @nvec: The number of interrupts to allocate + * + * Returns 0 on success or an error code. + */ +int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, + int nvec) +{ + struct msi_domain_info *info = domain->host_data; + struct msi_domain_ops *ops = info->ops; + msi_alloc_info_t arg; + struct msi_desc *desc; + int i, ret, virq = -1; + + ret = ops->msi_check(domain, info, dev); + if (ret == 0) + ret = ops->msi_prepare(domain, dev, nvec, &arg); + if (ret) + return ret; + + for_each_msi_entry(desc, dev) { + ops->set_desc(&arg, desc); + if (info->flags & MSI_FLAG_IDENTITY_MAP) + virq = (int)ops->get_hwirq(info, &arg); + else + virq = -1; + + virq = __irq_domain_alloc_irqs(domain, virq, desc->nvec_used, + dev_to_node(dev), &arg, false); + if (virq < 0) { + ret = -ENOSPC; + if (ops->handle_error) + ret = ops->handle_error(domain, desc, ret); + if (ops->msi_finish) + ops->msi_finish(&arg, ret); + return ret; + } + + for (i = 0; i < desc->nvec_used; i++) + irq_set_msi_desc_off(virq, i, desc); + } + + if (ops->msi_finish) + ops->msi_finish(&arg, 0); + + for_each_msi_entry(desc, dev) { + if (desc->nvec_used == 1) + dev_dbg(dev, "irq %d for MSI\n", virq); + else + dev_dbg(dev, "irq [%d-%d] for MSI\n", + virq, virq + desc->nvec_used - 1); + } + + return 0; +} + +/** + * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated tp @dev + * @domain: The domain to managing the interrupts + * @dev: Pointer to device struct of the device for which the interrupts + * are free + */ +void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) +{ + struct msi_desc *desc; + + for_each_msi_entry(desc, dev) { + irq_domain_free_irqs(desc->irq, desc->nvec_used); + desc->irq = 0; + } +} + +/** + * msi_get_domain_info - Get the MSI interrupt domain info for @domain + * @domain: The interrupt domain to retrieve data from + * + * Returns the pointer to the msi_domain_info stored in + * @domain->host_data. + */ +struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain) +{ + return (struct msi_domain_info *)domain->host_data; +} + +#endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */ diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index c6977d5a9b12..0ae06473dd8a 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c @@ -186,6 +186,50 @@ ftrace_print_hex_seq(struct trace_seq *p, const unsigned char *buf, int buf_len) } EXPORT_SYMBOL(ftrace_print_hex_seq); +const char * +ftrace_print_array_seq(struct trace_seq *p, const void *buf, int buf_len, + size_t el_size) +{ + const char *ret = trace_seq_buffer_ptr(p); + const char *prefix = ""; + void *ptr = (void *)buf; + + trace_seq_putc(p, '{'); + + while (ptr < buf + buf_len) { + switch (el_size) { + case 1: + trace_seq_printf(p, "%s0x%x", prefix, + *(u8 *)ptr); + break; + case 2: + trace_seq_printf(p, "%s0x%x", prefix, + *(u16 *)ptr); + break; + case 4: + trace_seq_printf(p, "%s0x%x", prefix, + *(u32 *)ptr); + break; + case 8: + trace_seq_printf(p, "%s0x%llx", prefix, + *(u64 *)ptr); + break; + default: + trace_seq_printf(p, "BAD SIZE:%zu 0x%x", el_size, + *(u8 *)ptr); + el_size = 1; + } + prefix = ","; + ptr += el_size; + } + + trace_seq_putc(p, '}'); + trace_seq_putc(p, 0); + + return ret; +} +EXPORT_SYMBOL(ftrace_print_array_seq); + int ftrace_raw_output_prep(struct trace_iterator *iter, struct trace_event *trace_event) { diff --git a/known-bugs b/known-bugs new file mode 100644 index 000000000000..9b8758452ffe --- /dev/null +++ b/known-bugs @@ -0,0 +1,7 @@ +1, Kernel build error when ARCH=arm with multi_v7_defconfig+CONFIG_SMP=n + arch/arm/kernel/psci.c:287:115: error: redefinition of 'psci_init' + This bug start from commit dbcfee724255ae1 ARM: 8457/1: psci-smp is built only for SMP +2, Kernel build error when ARCH=x86 with allmodconfig+CONFIG_OF=n or defconfig+CONFIG_KASAN=y + include/linux/kasan.h:21:14: error: conflicting types for 'kasan_zero_pud' + include/linux/kasan.h:20:14: error: conflicting types for 'kasan_zero_pmd' + This bug is due to KASAN feature backporting diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index aab836755e21..1c7c03c6101a 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -635,6 +635,8 @@ config DEBUG_STACKOVERFLOW source "lib/Kconfig.kmemcheck" +source "lib/Kconfig.kasan" + endmenu # "Memory Debugging" config DEBUG_SHIRQ diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan new file mode 100644 index 000000000000..53362c1e95a4 --- /dev/null +++ b/lib/Kconfig.kasan @@ -0,0 +1,57 @@ +config HAVE_ARCH_KASAN + bool + +if HAVE_ARCH_KASAN + +config KASAN + bool "KASan: runtime memory debugger" + depends on SLUB_DEBUG + select CONSTRUCTORS + help + Enables kernel address sanitizer - runtime memory debugger, + designed to find out-of-bounds accesses and use-after-free bugs. + This is strictly a debugging feature and it requires a gcc version + of 4.9.2 or later. Detection of out of bounds accesses to stack or + global variables requires gcc 5.0 or later. + This feature consumes about 1/8 of available memory and brings about + ~x3 performance slowdown. + For better error detection enable CONFIG_STACKTRACE. + +config KASAN_SHADOW_OFFSET + hex + default 0xdffffc0000000000 if X86_64 + +choice + prompt "Instrumentation type" + depends on KASAN + default KASAN_OUTLINE + +config KASAN_OUTLINE + bool "Outline instrumentation" + help + Before every memory access compiler insert function call + __asan_load*/__asan_store*. These functions performs check + of shadow memory. This is slower than inline instrumentation, + however it doesn't bloat size of kernel's .text section so + much as inline does. + +config KASAN_INLINE + bool "Inline instrumentation" + help + Compiler directly inserts code checking shadow memory before + memory accesses. This is faster than outline (in some workloads + it gives about x2 boost over outline instrumentation), but + make kernel's .text size much bigger. + This requires a gcc version of 5.0 or later. + +endchoice + +config TEST_KASAN + tristate "Module for testing kasan for bug detection" + depends on m && KASAN + help + This is a test module doing various nasty things like + out of bounds accesses, use after free. It is useful for testing + kernel debugging features like kernel address sanitizer. + +endif diff --git a/lib/Makefile b/lib/Makefile index 0211d2bd5e17..de5c0b494761 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -30,11 +30,12 @@ obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \ obj-y += string_helpers.o obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o obj-y += kstrtox.o +obj-$(CONFIG_TEST_BPF) += test_bpf.o +obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o +obj-$(CONFIG_TEST_KASAN) += test_kasan.o obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o obj-$(CONFIG_TEST_LKM) += test_module.o obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o -obj-$(CONFIG_TEST_BPF) += test_bpf.o -obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o ifeq ($(CONFIG_DEBUG_KOBJECT),y) CFLAGS_kobject.o += -DDEBUG @@ -142,7 +143,7 @@ obj-$(CONFIG_GLOB) += glob.o obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_SIGNATURE) += digsig.o -obj-$(CONFIG_CLZ_TAB) += clz_tab.o +lib-$(CONFIG_CLZ_TAB) += clz_tab.o obj-$(CONFIG_DDR) += jedec_ddr_data.o diff --git a/lib/test_kasan.c b/lib/test_kasan.c new file mode 100644 index 000000000000..dc340b321624 --- /dev/null +++ b/lib/test_kasan.c @@ -0,0 +1,346 @@ +/* + * + * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * Author: Andrey Ryabinin <a.ryabinin@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define pr_fmt(fmt) "kasan test: %s " fmt, __func__ + +#include <linux/kernel.h> +#include <linux/printk.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/module.h> + +static noinline void __init kmalloc_oob_right(void) +{ + char *ptr; + size_t size = 123; + + pr_info("out-of-bounds to right\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + ptr[size] = 'x'; + kfree(ptr); +} + +static noinline void __init kmalloc_oob_left(void) +{ + char *ptr; + size_t size = 15; + + pr_info("out-of-bounds to left\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + *ptr = *(ptr - 1); + kfree(ptr); +} + +static noinline void __init kmalloc_node_oob_right(void) +{ + char *ptr; + size_t size = 4096; + + pr_info("kmalloc_node(): out-of-bounds to right\n"); + ptr = kmalloc_node(size, GFP_KERNEL, 0); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + ptr[size] = 0; + kfree(ptr); +} + +static noinline void __init kmalloc_large_oob_rigth(void) +{ + char *ptr; + size_t size = KMALLOC_MAX_CACHE_SIZE + 10; + + pr_info("kmalloc large allocation: out-of-bounds to right\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + ptr[size] = 0; + kfree(ptr); +} + +static noinline void __init kmalloc_oob_krealloc_more(void) +{ + char *ptr1, *ptr2; + size_t size1 = 17; + size_t size2 = 19; + + pr_info("out-of-bounds after krealloc more\n"); + ptr1 = kmalloc(size1, GFP_KERNEL); + ptr2 = krealloc(ptr1, size2, GFP_KERNEL); + if (!ptr1 || !ptr2) { + pr_err("Allocation failed\n"); + kfree(ptr1); + return; + } + + ptr2[size2] = 'x'; + kfree(ptr2); +} + +static noinline void __init kmalloc_oob_krealloc_less(void) +{ + char *ptr1, *ptr2; + size_t size1 = 17; + size_t size2 = 15; + + pr_info("out-of-bounds after krealloc less\n"); + ptr1 = kmalloc(size1, GFP_KERNEL); + ptr2 = krealloc(ptr1, size2, GFP_KERNEL); + if (!ptr1 || !ptr2) { + pr_err("Allocation failed\n"); + kfree(ptr1); + return; + } + ptr2[size1] = 'x'; + kfree(ptr2); +} + +static noinline void __init kmalloc_oob_16(void) +{ + struct { + u64 words[2]; + } *ptr1, *ptr2; + + pr_info("kmalloc out-of-bounds for 16-bytes access\n"); + ptr1 = kmalloc(sizeof(*ptr1) - 3, GFP_KERNEL); + ptr2 = kmalloc(sizeof(*ptr2), GFP_KERNEL); + if (!ptr1 || !ptr2) { + pr_err("Allocation failed\n"); + kfree(ptr1); + kfree(ptr2); + return; + } + *ptr1 = *ptr2; + kfree(ptr1); + kfree(ptr2); +} + +static noinline void __init kmalloc_oob_memset_2(void) +{ + char *ptr; + size_t size = 8; + + pr_info("out-of-bounds in memset2\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + memset(ptr+7, 0, 2); + kfree(ptr); +} + +static noinline void __init kmalloc_oob_memset_4(void) +{ + char *ptr; + size_t size = 8; + + pr_info("out-of-bounds in memset4\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + memset(ptr+5, 0, 4); + kfree(ptr); +} + + +static noinline void __init kmalloc_oob_memset_8(void) +{ + char *ptr; + size_t size = 8; + + pr_info("out-of-bounds in memset8\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + memset(ptr+1, 0, 8); + kfree(ptr); +} + +static noinline void __init kmalloc_oob_memset_16(void) +{ + char *ptr; + size_t size = 16; + + pr_info("out-of-bounds in memset16\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + memset(ptr+1, 0, 16); + kfree(ptr); +} + +static noinline void __init kmalloc_oob_in_memset(void) +{ + char *ptr; + size_t size = 666; + + pr_info("out-of-bounds in memset\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + memset(ptr, 0, size+5); + kfree(ptr); +} + +static noinline void __init kmalloc_uaf(void) +{ + char *ptr; + size_t size = 10; + + pr_info("use-after-free\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + kfree(ptr); + *(ptr + 8) = 'x'; +} + +static noinline void __init kmalloc_uaf_memset(void) +{ + char *ptr; + size_t size = 33; + + pr_info("use-after-free in memset\n"); + ptr = kmalloc(size, GFP_KERNEL); + if (!ptr) { + pr_err("Allocation failed\n"); + return; + } + + kfree(ptr); + memset(ptr, 0, size); +} + +static noinline void __init kmalloc_uaf2(void) +{ + char *ptr1, *ptr2; + size_t size = 43; + + pr_info("use-after-free after another kmalloc\n"); + ptr1 = kmalloc(size, GFP_KERNEL); + if (!ptr1) { + pr_err("Allocation failed\n"); + return; + } + + kfree(ptr1); + ptr2 = kmalloc(size, GFP_KERNEL); + if (!ptr2) { + pr_err("Allocation failed\n"); + return; + } + + ptr1[40] = 'x'; + kfree(ptr2); +} + +static noinline void __init kmem_cache_oob(void) +{ + char *p; + size_t size = 200; + struct kmem_cache *cache = kmem_cache_create("test_cache", + size, 0, + 0, NULL); + if (!cache) { + pr_err("Cache allocation failed\n"); + return; + } + pr_info("out-of-bounds in kmem_cache_alloc\n"); + p = kmem_cache_alloc(cache, GFP_KERNEL); + if (!p) { + pr_err("Allocation failed\n"); + kmem_cache_destroy(cache); + return; + } + + *p = p[size]; + kmem_cache_free(cache, p); + kmem_cache_destroy(cache); +} + +static char global_array[10]; + +static noinline void __init kasan_global_oob(void) +{ + volatile int i = 3; + char *p = &global_array[ARRAY_SIZE(global_array) + i]; + + pr_info("out-of-bounds global variable\n"); + *(volatile char *)p; +} + +static noinline void __init kasan_stack_oob(void) +{ + char stack_array[10]; + volatile int i = 0; + char *p = &stack_array[ARRAY_SIZE(stack_array) + i]; + + pr_info("out-of-bounds on stack\n"); + *(volatile char *)p; +} + +static int __init kmalloc_tests_init(void) +{ + kmalloc_oob_right(); + kmalloc_oob_left(); + kmalloc_node_oob_right(); + kmalloc_large_oob_rigth(); + kmalloc_oob_krealloc_more(); + kmalloc_oob_krealloc_less(); + kmalloc_oob_16(); + kmalloc_oob_in_memset(); + kmalloc_oob_memset_2(); + kmalloc_oob_memset_4(); + kmalloc_oob_memset_8(); + kmalloc_oob_memset_16(); + kmalloc_uaf(); + kmalloc_uaf_memset(); + kmalloc_uaf2(); + kmem_cache_oob(); + kasan_stack_oob(); + kasan_global_oob(); + return -EAGAIN; +} + +module_init(kmalloc_tests_init); +MODULE_LICENSE("GPL"); diff --git a/linaro/configs/arndale.conf b/linaro/configs/arndale.conf new file mode 100644 index 000000000000..109052f2f6ec --- /dev/null +++ b/linaro/configs/arndale.conf @@ -0,0 +1,66 @@ +CONFIG_KALLSYMS_ALL=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_BSD_DISKLABEL=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_ARCH_EXYNOS=y +CONFIG_S3C_LOWLEVEL_UART_PORT=2 +CONFIG_ARCH_EXYNOS5=y +# CONFIG_EXYNOS_ATAGS is not set +CONFIG_MACH_EXYNOS4_DT=y +CONFIG_VMSPLIT_2G=y +CONFIG_NR_CPUS=2 +CONFIG_HIGHMEM=y +# CONFIG_COMPACTION is not set +CONFIG_ARM_APPENDED_DTB=y +CONFIG_ARM_ATAG_DTB_COMPAT=y +CONFIG_CMDLINE="root=/dev/ram0 rw ramdisk=8192 initrd=0x41000000,8M console=ttySAC1,115200 init= mem=256M" +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_VFP=y +CONFIG_NEON=y +CONFIG_PM_RUNTIME=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_SG=y +CONFIG_ATA=y +CONFIG_SATA_AHCI_PLATFORM=y +CONFIG_SATA_EXYNOS=y +CONFIG_AX88796=y +CONFIG_AX88796_93CX6=y +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_GPIO=y +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_HW_RANDOM=y +CONFIG_I2C=y +CONFIG_I2C_S3C2410=y +CONFIG_THERMAL=y +CONFIG_CPU_THERMAL=y +CONFIG_EXYNOS_THERMAL=y +CONFIG_MFD_SEC_CORE=y +CONFIG_REGULATOR=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_S5M8767=y +CONFIG_DRM=y +CONFIG_DRM_LOAD_EDID_FIRMWARE=y +CONFIG_DRM_EXYNOS=y +CONFIG_DRM_EXYNOS_DMABUF=y +CONFIG_DRM_EXYNOS_HDMI=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_LOGO=y +CONFIG_MMC=y +CONFIG_MMC_UNSAFE_RESUME=y +CONFIG_MMC_DW=y +CONFIG_MMC_DW_IDMAC=y +CONFIG_MMC_DW_EXYNOS=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_S3C=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DETECT_HUNG_TASK=y +CONFIG_DEBUG_RT_MUTEXES=y +CONFIG_DEBUG_SPINLOCK=y +CONFIG_DEBUG_INFO=y +CONFIG_RCU_CPU_STALL_TIMEOUT=60 +CONFIG_DEBUG_USER=y +CONFIG_TUN=y diff --git a/linaro/configs/big-LITTLE-IKS.conf b/linaro/configs/big-LITTLE-IKS.conf new file mode 100644 index 000000000000..b067fde86eaa --- /dev/null +++ b/linaro/configs/big-LITTLE-IKS.conf @@ -0,0 +1,5 @@ +CONFIG_BIG_LITTLE=y +CONFIG_BL_SWITCHER=y +CONFIG_ARM_DT_BL_CPUFREQ=y +CONFIG_ARM_VEXPRESS_BL_CPUFREQ=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y diff --git a/linaro/configs/bigendian.conf b/linaro/configs/bigendian.conf new file mode 100644 index 000000000000..6a1020299e85 --- /dev/null +++ b/linaro/configs/bigendian.conf @@ -0,0 +1,4 @@ +CONFIG_CPU_BIG_ENDIAN=y +CONFIG_CPU_ENDIAN_BE8=y +# CONFIG_VIRTUALIZATION is not set +# CONFIG_MMC_DW_IDMAC is not set diff --git a/linaro/configs/distribution.conf b/linaro/configs/distribution.conf new file mode 100644 index 000000000000..729b9b8979e4 --- /dev/null +++ b/linaro/configs/distribution.conf @@ -0,0 +1,49 @@ +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_CGROUPS=y +# CONFIG_COMPAT_BRK is not set +CONFIG_DEFAULT_MMAP_MIN_ADDR=32768 +CONFIG_SECCOMP=y +CONFIG_CC_STACKPROTECTOR=y +CONFIG_SYN_COOKIES=y +CONFIG_IPV6=y +CONFIG_NETLABEL=y +CONFIG_BRIDGE_NETFILTER=y +CONFIG_NF_CONNTRACK=m +CONFIG_NETFILTER_XT_CONNMARK=m +CONFIG_NETFILTER_XT_MARK=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NF_CONNTRACK_IPV4=m +CONFIG_NF_NAT_IPV4=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_MANGLE=m +CONFIG_NF_CONNTRACK_IPV6=m +CONFIG_NF_NAT_IPV6=m +CONFIG_IP6_NF_IPTABLES=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE=m +CONFIG_TUN=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=65536 +CONFIG_INPUT_MISC=y +CONFIG_INPUT_UINPUT=y +# CONFIG_DEVKMEM is not set +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_AUTOFS4_FS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_STRICT_DEVMEM=y +CONFIG_SECURITY=y +CONFIG_LSM_MMAP_MIN_ADDR=0 +CONFIG_SECURITY_SELINUX=y +CONFIG_SECURITY_SMACK=y +CONFIG_SECURITY_APPARMOR=y +CONFIG_DEFAULT_SECURITY_APPARMOR=y +CONFIG_HUGETLBFS=y +CONFIG_HUGETLB_PAGE=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS=y diff --git a/linaro/configs/kvm-guest.conf b/linaro/configs/kvm-guest.conf new file mode 100644 index 000000000000..00e84a3ba1ec --- /dev/null +++ b/linaro/configs/kvm-guest.conf @@ -0,0 +1,11 @@ +CONFIG_BALLOON_COMPACTION=y +CONFIG_VIRTIO_BLK=y +CONFIG_VIRTIO_NET=y +CONFIG_HVC_DRIVER=y +CONFIG_VIRTIO_CONSOLE=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y +CONFIG_VIRTUALIZATION=y +# CONFIG_THUMB2_KERNEL is not set diff --git a/linaro/configs/kvm-host.conf b/linaro/configs/kvm-host.conf new file mode 100644 index 000000000000..21a40e031372 --- /dev/null +++ b/linaro/configs/kvm-host.conf @@ -0,0 +1,11 @@ +CONFIG_VIRTUALIZATION=y +CONFIG_ARM_LPAE=y +CONFIG_ARM_VIRT_EXT=y +CONFIG_HAVE_KVM_IRQCHIP=y +CONFIG_KVM_ARM_HOST=y +CONFIG_KVM_ARM_MAX_VCPUS=4 +CONFIG_KVM_ARM_TIMER=y +CONFIG_KVM_ARM_VGIC=y +CONFIG_KVM_MMIO=y +CONFIG_KVM=y +CONFIG_BLK_DEV_NBD=m diff --git a/linaro/configs/linaro-base.conf b/linaro/configs/linaro-base.conf new file mode 100644 index 000000000000..e9a76314a2a8 --- /dev/null +++ b/linaro/configs/linaro-base.conf @@ -0,0 +1,128 @@ +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=16 +CONFIG_BLK_DEV_INITRD=y +CONFIG_EMBEDDED=y +CONFIG_HOTPLUG=y +CONFIG_PERF_EVENTS=y +CONFIG_SLAB=y +CONFIG_PROFILING=y +CONFIG_OPROFILE=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_SMP=y +CONFIG_SCHED_MC=y +CONFIG_SCHED_SMT=y +CONFIG_THUMB2_KERNEL=y +CONFIG_AEABI=y +# CONFIG_OABI_COMPAT is not set +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y +CONFIG_CPU_IDLE=y +CONFIG_BINFMT_MISC=y +CONFIG_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +# CONFIG_INET_LRO is not set +CONFIG_NETFILTER=y +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +CONFIG_CONNECTOR=y +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_OOPS=y +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_NAND=y +CONFIG_NETDEVICES=y +CONFIG_EXT2_FS=y +CONFIG_EXT3_FS=y +CONFIG_EXT4_FS=y +CONFIG_BTRFS_FS=y +CONFIG_QUOTA=y +CONFIG_QFMT_V2=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_TMPFS=y +CONFIG_ECRYPT_FS=y +CONFIG_JFFS2_FS=y +CONFIG_JFFS2_SUMMARY=y +CONFIG_JFFS2_FS_XATTR=y +CONFIG_JFFS2_COMPRESSION_OPTIONS=y +CONFIG_JFFS2_LZO=y +CONFIG_JFFS2_RUBIN=y +CONFIG_CRAMFS=y +CONFIG_NETWORK_FILESYSTEMS=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_ROOT_NFS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_PRINTK_TIME=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_FS=y +CONFIG_SCHEDSTATS=y +CONFIG_TIMER_STATS=y +CONFIG_KEYS=y +CONFIG_CRYPTO_MICHAEL_MIC=y +CONFIG_CRC_CCITT=y +CONFIG_CRC_T10DIF=y +CONFIG_CRC_ITU_T=y +CONFIG_CRC7=y +CONFIG_HW_PERF_EVENTS=y +CONFIG_FUNCTION_TRACER=y +CONFIG_ENABLE_DEFAULT_TRACERS=y +CONFIG_PROC_DEVICETREE=y +CONFIG_JUMP_LABEL=y +CONFIG_STRICT_DEVMEM=y +CONFIG_KGDB=y +CONFIG_KGDB_TESTS=y +CONFIG_OF_IDLE_STATES=y +CONFIG_FTRACE=y +CONFIG_FUNCTION_TRACER=y +CONFIG_FTRACE_SYSCALLS=y +CONFIG_STACK_TRACER=y +CONFIG_FUNCTION_PROFILER=y +CONFIG_MAILBOX=y +CONFIG_AUDIT=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y +CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_IP_NF_SECURITY=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_LSM_MMAP_MIN_ADDR=4096 +CONFIG_SECURITY_SELINUX=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_DEBUG_INFO=y +CONFIG_FANOTIFY=y +CONFIG_RCU_TORTURE_TEST=m +CONFIG_RCU_TORTURE_TEST_RUNNABLE=n +CONFIG_CORESIGHT=y +CONFIG_CORESIGHT_LINKS_AND_SINKS=y +CONFIG_CORESIGHT_SOURCE_ETM=y + +# enable ZRAM features +CONFIG_ZPOOL=y +CONFIG_ZSMALLOC=y +CONFIG_ZRAM=y +CONFIG_ZRAM_LZ4_COMPRESS=y diff --git a/linaro/configs/omap4.conf b/linaro/configs/omap4.conf new file mode 100644 index 000000000000..d0a2b808ff12 --- /dev/null +++ b/linaro/configs/omap4.conf @@ -0,0 +1,196 @@ +CONFIG_EXPERT=y +CONFIG_KPROBES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_MODULE_SRCVERSION_ALL=y +# CONFIG_BLK_DEV_BSG is not set +CONFIG_PARTITION_ADVANCED=y +CONFIG_GPIO_PCA953X=y +CONFIG_OMAP_RESET_CLOCKS=y +CONFIG_OMAP_MUX_DEBUG=y +CONFIG_ARCH_OMAP3=y +CONFIG_ARCH_OMAP4=y +CONFIG_ARCH_OMAP2PLUS=y +CONFIG_SOC_OMAP5=y +# CONFIG_ARCH_OMAP2 is not set +CONFIG_ARCH_VEXPRESS_CA9X4=y +CONFIG_ARM_THUMBEE=y +CONFIG_ARM_ERRATA_411920=y +CONFIG_NR_CPUS=2 +CONFIG_ZBOOT_ROM_TEXT=0x0 +CONFIG_ZBOOT_ROM_BSS=0x0 +CONFIG_CMDLINE="root=/dev/mmcblk0p2 rootwait console=ttyO2,115200" +CONFIG_KEXEC=y +CONFIG_PM_DEBUG=y +CONFIG_CAN=m +CONFIG_CAN_C_CAN=m +CONFIG_CAN_C_CAN_PLATFORM=m +CONFIG_BT=m +CONFIG_BT_HCIUART=m +CONFIG_BT_HCIUART_H4=y +CONFIG_BT_HCIUART_BCSP=y +CONFIG_BT_HCIUART_LL=y +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBPA10X=m +CONFIG_CFG80211=m +CONFIG_MAC80211=m +CONFIG_MAC80211_RC_PID=y +CONFIG_MAC80211_RC_DEFAULT_PID=y +CONFIG_CMA=y +CONFIG_MTD_NAND_OMAP2=y +CONFIG_MTD_ONENAND=y +CONFIG_MTD_ONENAND_VERIFY_WRITE=y +CONFIG_MTD_ONENAND_OMAP2=y +CONFIG_MTD_UBI=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_SENSORS_TSL2550=m +CONFIG_SENSORS_LIS3_I2C=m +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_SCSI_MULTI_LUN=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_KS8851=y +CONFIG_KS8851_MLL=y +CONFIG_SMC91X=y +CONFIG_SMSC911X=y +CONFIG_TI_CPSW=y +CONFIG_SMSC_PHY=y +CONFIG_USB_USBNET=y +CONFIG_USB_NET_SMSC95XX=y +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_EPSON2888=y +CONFIG_USB_KC2190=y +CONFIG_LIBERTAS=m +CONFIG_LIBERTAS_USB=m +CONFIG_LIBERTAS_SDIO=m +CONFIG_LIBERTAS_DEBUG=y +CONFIG_INPUT_JOYDEV=y +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_GPIO=y +CONFIG_KEYBOARD_MATRIX=m +CONFIG_KEYBOARD_TWL4030=y +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_ADS7846=y +CONFIG_INPUT_TWL4030_PWRBUTTON=y +CONFIG_VT_HW_CONSOLE_BINDING=y +# CONFIG_LEGACY_PTYS is not set +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_NR_UARTS=32 +CONFIG_SERIAL_8250_EXTENDED=y +CONFIG_SERIAL_8250_MANY_PORTS=y +CONFIG_SERIAL_8250_SHARE_IRQ=y +CONFIG_SERIAL_8250_DETECT_IRQ=y +CONFIG_SERIAL_8250_RSA=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_SERIAL_OMAP=y +CONFIG_SERIAL_OMAP_CONSOLE=y +CONFIG_HW_RANDOM=y +CONFIG_I2C_CHARDEV=y +CONFIG_SPI=y +CONFIG_SPI_OMAP24XX=y +CONFIG_PINCTRL_SINGLE=y +CONFIG_DEBUG_GPIO=y +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_TWL4030=y +CONFIG_W1=y +CONFIG_SENSORS_LM75=m +CONFIG_WATCHDOG=y +CONFIG_OMAP_WATCHDOG=y +CONFIG_TWL4030_WATCHDOG=y +CONFIG_MFD_TPS65217=y +CONFIG_MFD_TPS65910=y +CONFIG_TWL6040_CORE=y +CONFIG_REGULATOR_TPS65023=y +CONFIG_REGULATOR_TPS6507X=y +CONFIG_REGULATOR_TPS65217=y +CONFIG_REGULATOR_TPS65910=y +CONFIG_REGULATOR_TWL4030=y +CONFIG_FB=y +CONFIG_FIRMWARE_EDID=y +CONFIG_FB_MODE_HELPERS=y +CONFIG_FB_TILEBLITTING=y +CONFIG_OMAP2_DSS=m +CONFIG_OMAP2_DSS_RFBI=y +CONFIG_OMAP2_DSS_SDI=y +CONFIG_OMAP2_DSS_DSI=y +CONFIG_FB_OMAP2=m +CONFIG_PANEL_GENERIC_DPI=m +CONFIG_PANEL_TFP410=m +CONFIG_PANEL_SHARP_LS037V7DW01=m +CONFIG_PANEL_NEC_NL8048HL11_01B=m +CONFIG_PANEL_TAAL=m +CONFIG_PANEL_TPO_TD043MTEA1=m +CONFIG_PANEL_ACX565AKM=m +CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LCD_CLASS_DEVICE=y +CONFIG_LCD_PLATFORM=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_FONTS=y +CONFIG_FONT_8x8=y +CONFIG_FONT_8x16=y +CONFIG_LOGO=y +CONFIG_SOUND=m +CONFIG_SND=m +CONFIG_SND_VERBOSE_PRINTK=y +CONFIG_SND_DEBUG=y +CONFIG_SND_USB_AUDIO=m +CONFIG_SND_SOC=m +CONFIG_SND_OMAP_SOC=m +CONFIG_SND_OMAP_SOC_OMAP_TWL4030=m +CONFIG_SND_OMAP_SOC_OMAP_ABE_TWL6040=m +CONFIG_SND_OMAP_SOC_OMAP3_PANDORA=m +CONFIG_USB=y +CONFIG_USB_DEBUG=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_MON=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_WDM=y +CONFIG_USB_STORAGE=y +CONFIG_USB_TEST=y +CONFIG_USB_PHY=y +CONFIG_NOP_USB_XCEIV=y +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_DEBUG=y +CONFIG_USB_GADGET_DEBUG_FILES=y +CONFIG_USB_GADGET_DEBUG_FS=y +CONFIG_USB_ZERO=m +CONFIG_MMC=y +CONFIG_MMC_UNSAFE_RESUME=y +CONFIG_SDIO_UART=y +CONFIG_MMC_ARMMMCI=y +CONFIG_MMC_OMAP=y +CONFIG_MMC_OMAP_HS=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_TRIGGERS=y +CONFIG_LEDS_TRIGGER_TIMER=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=y +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_GPIO=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_TWL92330=y +CONFIG_RTC_DRV_TWL4030=y +CONFIG_RTC_DRV_OMAP=y +CONFIG_DMADEVICES=y +CONFIG_DMA_OMAP=y +# CONFIG_EXT3_FS_XATTR is not set +CONFIG_UBIFS_FS=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_ROOT_NFS=y +# CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO=y +# CONFIG_CRYPTO_ANSI_CPRNG is not set +CONFIG_LIBCRC32C=y +# CONFIG_CPU_FREQ is not set diff --git a/linaro/configs/preempt-rt.conf b/linaro/configs/preempt-rt.conf new file mode 100644 index 000000000000..7c6594f3a016 --- /dev/null +++ b/linaro/configs/preempt-rt.conf @@ -0,0 +1,3 @@ +CONFIG_PREEMPT=y +CONFIG_PREEMPT_RT_FULL=y +# CONFIG_CPU_FREQ is not set diff --git a/linaro/configs/vexpress64.conf b/linaro/configs/vexpress64.conf new file mode 100644 index 000000000000..a3bb8d432ccc --- /dev/null +++ b/linaro/configs/vexpress64.conf @@ -0,0 +1,58 @@ +CONFIG_ARCH_VEXPRESS=y +CONFIG_SMP=y +CONFIG_NR_CPUS=8 +CONFIG_CMDLINE="console=ttyAMA0" +CONFIG_COMPAT=y +CONFIG_SMC91X=y +CONFIG_INPUT_EVDEV=y +CONFIG_SERIO_AMBAKMI=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +# CONFIG_SERIO_I8042 is not set +CONFIG_FB=y +CONFIG_FB_ARMCLCD=y +CONFIG_FRAMEBUFFER_CONSOLE=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_LOGO=y +# CONFIG_LOGO_LINUX_MONO is not set +# CONFIG_LOGO_LINUX_VGA16 is not set +CONFIG_MMC=y +CONFIG_MMC_ARMMMCI=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_PL031=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_ROOT_NFS=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_VIRTIO_MMIO=y +CONFIG_REGULATOR=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_CMA=y +CONFIG_DMA_CMA=y +CONFIG_COMMON_CLK_SCPI=y +CONFIG_SMSC911X=y +CONFIG_I2C=y +CONFIG_ARM_MHU_MBOX=y +CONFIG_ARM_SCPI_PROTOCOL=y +CONFIG_USB_HIDDEV=y +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_USB_STORAGE=y +CONFIG_USB=y +CONFIG_USB_ULPI=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_HCD_SYNOPSYS=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_PHY=y +CONFIG_USB_ISP1301=y +CONFIG_PM_OPP=y +CONFIG_GENERIC_CPUFREQ_CPU0=y +CONFIG_ARM_BIG_LITTLE_CPUFREQ=y +CONFIG_ARM_DT_BL_CPUFREQ=y +CONFIG_ARM64_CPUIDLE=y +CONFIG_ARM64_CRYPTO=y +CONFIG_SATA_SIL24=y +CONFIG_SKY2=y diff --git a/mm/Makefile b/mm/Makefile index 8405eb0023a9..3614adbf6d32 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -2,6 +2,9 @@ # Makefile for the linux memory manager. # +KASAN_SANITIZE_slab_common.o := n +KASAN_SANITIZE_slub.o := n + mmu-y := nommu.o mmu-$(CONFIG_MMU) := fremap.o gup.o highmem.o memory.o mincore.o \ mlock.o mmap.o mprotect.o mremap.o msync.o rmap.o \ @@ -49,6 +52,7 @@ obj-$(CONFIG_PAGE_POISONING) += debug-pagealloc.o obj-$(CONFIG_SLAB) += slab.o obj-$(CONFIG_SLUB) += slub.o obj-$(CONFIG_KMEMCHECK) += kmemcheck.o +obj-$(CONFIG_KASAN) += kasan/ obj-$(CONFIG_FAILSLAB) += failslab.o obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o obj-$(CONFIG_FS_XIP) += filemap_xip.o diff --git a/mm/compaction.c b/mm/compaction.c index 9f8b1dd5303c..be7729b706af 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -16,6 +16,7 @@ #include <linux/sysfs.h> #include <linux/balloon_compaction.h> #include <linux/page-isolation.h> +#include <linux/kasan.h> #include "internal.h" #ifdef CONFIG_COMPACTION @@ -59,6 +60,7 @@ static void map_pages(struct list_head *list) list_for_each_entry(page, list, lru) { arch_alloc_page(page, 0); kernel_map_pages(page, 1, 1); + kasan_alloc_pages(page, 0); } } diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c index 5edcf1b37fa6..f283350049de 100644 --- a/mm/early_ioremap.c +++ b/mm/early_ioremap.c @@ -15,6 +15,7 @@ #include <linux/mm.h> #include <linux/vmalloc.h> #include <asm/fixmap.h> +#include <asm/early_ioremap.h> #ifdef CONFIG_MMU static int early_ioremap_debug __initdata; @@ -217,6 +218,28 @@ early_memremap(resource_size_t phys_addr, unsigned long size) return (__force void *)__early_ioremap(phys_addr, size, FIXMAP_PAGE_NORMAL); } + +#define MAX_MAP_CHUNK (NR_FIX_BTMAPS << PAGE_SHIFT) + +void __init copy_from_early_mem(void *dest, phys_addr_t src, unsigned long size) +{ + unsigned long slop, clen; + char *p; + + while (size) { + slop = src & ~PAGE_MASK; + clen = size; + if (clen > MAX_MAP_CHUNK - slop) + clen = MAX_MAP_CHUNK - slop; + p = early_memremap(src & PAGE_MASK, clen + slop); + memcpy(dest, p + slop, clen); + early_memunmap(p, clen + slop); + dest += clen; + src += clen; + size -= clen; + } +} + #else /* CONFIG_MMU */ void __init __iomem * diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile new file mode 100644 index 000000000000..64710148941e --- /dev/null +++ b/mm/kasan/Makefile @@ -0,0 +1,8 @@ +KASAN_SANITIZE := n + +CFLAGS_REMOVE_kasan.o = -pg +# Function splitter causes unnecessary splits in __asan_load1/__asan_store1 +# see: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63533 +CFLAGS_kasan.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) + +obj-y := kasan.o report.o kasan_init.o diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c new file mode 100644 index 000000000000..8f6ff54e7440 --- /dev/null +++ b/mm/kasan/kasan.c @@ -0,0 +1,572 @@ +/* + * This file contains shadow memory manipulation code. + * + * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * Author: Andrey Ryabinin <a.ryabinin@samsung.com> + * + * Some code borrowed from https://github.com/xairy/kasan-prototype by + * Andrey Konovalov <adech.fo@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define DISABLE_BRANCH_PROFILING + +#include <linux/export.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/kmemleak.h> +#include <linux/linkage.h> +#include <linux/memblock.h> +#include <linux/memory.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/stacktrace.h> +#include <linux/string.h> +#include <linux/types.h> +#include <linux/vmalloc.h> +#include <linux/kasan.h> + +#include "kasan.h" +#include "../slab.h" + +/* + * Poisons the shadow memory for 'size' bytes starting from 'addr'. + * Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE. + */ +static void kasan_poison_shadow(const void *address, size_t size, u8 value) +{ + void *shadow_start, *shadow_end; + + shadow_start = kasan_mem_to_shadow(address); + shadow_end = kasan_mem_to_shadow(address + size); + + memset(shadow_start, value, shadow_end - shadow_start); +} + +void kasan_unpoison_shadow(const void *address, size_t size) +{ + kasan_poison_shadow(address, size, 0); + + if (size & KASAN_SHADOW_MASK) { + u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size); + *shadow = size & KASAN_SHADOW_MASK; + } +} + +static void __kasan_unpoison_stack(struct task_struct *task, void *sp) +{ + void *base = task_stack_page(task); + size_t size = sp - base; + + kasan_unpoison_shadow(base, size); +} + +/* Unpoison the entire stack for a task. */ +void kasan_unpoison_task_stack(struct task_struct *task) +{ + __kasan_unpoison_stack(task, task_stack_page(task) + THREAD_SIZE); +} + +/* Unpoison the stack for the current task beyond a watermark sp value. */ +asmlinkage void kasan_unpoison_remaining_stack(void *sp) +{ + __kasan_unpoison_stack(current, sp); +} + +/* + * All functions below always inlined so compiler could + * perform better optimizations in each of __asan_loadX/__assn_storeX + * depending on memory access size X. + */ + +static __always_inline bool memory_is_poisoned_1(unsigned long addr) +{ + s8 shadow_value = *(s8 *)kasan_mem_to_shadow((void *)addr); + + if (unlikely(shadow_value)) { + s8 last_accessible_byte = addr & KASAN_SHADOW_MASK; + return unlikely(last_accessible_byte >= shadow_value); + } + + return false; +} + +static __always_inline bool memory_is_poisoned_2(unsigned long addr) +{ + u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); + + if (unlikely(*shadow_addr)) { + if (memory_is_poisoned_1(addr + 1)) + return true; + + /* + * If single shadow byte covers 2-byte access, we don't + * need to do anything more. Otherwise, test the first + * shadow byte. + */ + if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0)) + return false; + + return unlikely(*(u8 *)shadow_addr); + } + + return false; +} + +static __always_inline bool memory_is_poisoned_4(unsigned long addr) +{ + u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); + + if (unlikely(*shadow_addr)) { + if (memory_is_poisoned_1(addr + 3)) + return true; + + /* + * If single shadow byte covers 4-byte access, we don't + * need to do anything more. Otherwise, test the first + * shadow byte. + */ + if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3)) + return false; + + return unlikely(*(u8 *)shadow_addr); + } + + return false; +} + +static __always_inline bool memory_is_poisoned_8(unsigned long addr) +{ + u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); + + if (unlikely(*shadow_addr)) { + if (memory_is_poisoned_1(addr + 7)) + return true; + + /* + * If single shadow byte covers 8-byte access, we don't + * need to do anything more. Otherwise, test the first + * shadow byte. + */ + if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE))) + return false; + + return unlikely(*(u8 *)shadow_addr); + } + + return false; +} + +static __always_inline bool memory_is_poisoned_16(unsigned long addr) +{ + u32 *shadow_addr = (u32 *)kasan_mem_to_shadow((void *)addr); + + if (unlikely(*shadow_addr)) { + u16 shadow_first_bytes = *(u16 *)shadow_addr; + + if (unlikely(shadow_first_bytes)) + return true; + + /* + * If two shadow bytes covers 16-byte access, we don't + * need to do anything more. Otherwise, test the last + * shadow byte. + */ + if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE))) + return false; + + return memory_is_poisoned_1(addr + 15); + } + + return false; +} + +static __always_inline unsigned long bytes_is_zero(const u8 *start, + size_t size) +{ + while (size) { + if (unlikely(*start)) + return (unsigned long)start; + start++; + size--; + } + + return 0; +} + +static __always_inline unsigned long memory_is_zero(const void *start, + const void *end) +{ + unsigned int words; + unsigned long ret; + unsigned int prefix = (unsigned long)start % 8; + + if (end - start <= 16) + return bytes_is_zero(start, end - start); + + if (prefix) { + prefix = 8 - prefix; + ret = bytes_is_zero(start, prefix); + if (unlikely(ret)) + return ret; + start += prefix; + } + + words = (end - start) / 8; + while (words) { + if (unlikely(*(u64 *)start)) + return bytes_is_zero(start, 8); + start += 8; + words--; + } + + return bytes_is_zero(start, (end - start) % 8); +} + +static __always_inline bool memory_is_poisoned_n(unsigned long addr, + size_t size) +{ + unsigned long ret; + + ret = memory_is_zero(kasan_mem_to_shadow((void *)addr), + kasan_mem_to_shadow((void *)addr + size - 1) + 1); + + if (unlikely(ret)) { + unsigned long last_byte = addr + size - 1; + s8 *last_shadow = (s8 *)kasan_mem_to_shadow((void *)last_byte); + + if (unlikely(ret != (unsigned long)last_shadow || + ((long)(last_byte & KASAN_SHADOW_MASK) >= *last_shadow))) + return true; + } + return false; +} + +static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size) +{ + if (__builtin_constant_p(size)) { + switch (size) { + case 1: + return memory_is_poisoned_1(addr); + case 2: + return memory_is_poisoned_2(addr); + case 4: + return memory_is_poisoned_4(addr); + case 8: + return memory_is_poisoned_8(addr); + case 16: + return memory_is_poisoned_16(addr); + default: + BUILD_BUG(); + } + } + + return memory_is_poisoned_n(addr, size); +} + + +static __always_inline void check_memory_region(unsigned long addr, + size_t size, bool write) +{ + if (unlikely(size == 0)) + return; + + if (unlikely((void *)addr < + kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) { + kasan_report(addr, size, write, _RET_IP_); + return; + } + + if (likely(!memory_is_poisoned(addr, size))) + return; + + kasan_report(addr, size, write, _RET_IP_); +} + +void __asan_loadN(unsigned long addr, size_t size); +void __asan_storeN(unsigned long addr, size_t size); + +#undef memset +void *memset(void *addr, int c, size_t len) +{ + __asan_storeN((unsigned long)addr, len); + + return __memset(addr, c, len); +} + +#undef memmove +void *memmove(void *dest, const void *src, size_t len) +{ + __asan_loadN((unsigned long)src, len); + __asan_storeN((unsigned long)dest, len); + + return __memmove(dest, src, len); +} + +#undef memcpy +void *memcpy(void *dest, const void *src, size_t len) +{ + __asan_loadN((unsigned long)src, len); + __asan_storeN((unsigned long)dest, len); + + return __memcpy(dest, src, len); +} + +void kasan_alloc_pages(struct page *page, unsigned int order) +{ + if (likely(!PageHighMem(page))) + kasan_unpoison_shadow(page_address(page), PAGE_SIZE << order); +} + +void kasan_free_pages(struct page *page, unsigned int order) +{ + if (likely(!PageHighMem(page))) + kasan_poison_shadow(page_address(page), + PAGE_SIZE << order, + KASAN_FREE_PAGE); +} + +void kasan_poison_slab(struct page *page) +{ + kasan_poison_shadow(page_address(page), + PAGE_SIZE << compound_order(page), + KASAN_KMALLOC_REDZONE); +} + +void kasan_unpoison_object_data(struct kmem_cache *cache, void *object) +{ + kasan_unpoison_shadow(object, cache->object_size); +} + +void kasan_poison_object_data(struct kmem_cache *cache, void *object) +{ + kasan_poison_shadow(object, + round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE), + KASAN_KMALLOC_REDZONE); +} + +void kasan_slab_alloc(struct kmem_cache *cache, void *object) +{ + kasan_kmalloc(cache, object, cache->object_size); +} + +void kasan_slab_free(struct kmem_cache *cache, void *object) +{ + unsigned long size = cache->object_size; + unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE); + + /* RCU slabs could be legally used after free within the RCU period */ + if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU)) + return; + + kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE); +} + +void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size) +{ + unsigned long redzone_start; + unsigned long redzone_end; + + if (unlikely(object == NULL)) + return; + + redzone_start = round_up((unsigned long)(object + size), + KASAN_SHADOW_SCALE_SIZE); + redzone_end = round_up((unsigned long)object + cache->object_size, + KASAN_SHADOW_SCALE_SIZE); + + kasan_unpoison_shadow(object, size); + kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start, + KASAN_KMALLOC_REDZONE); +} +EXPORT_SYMBOL(kasan_kmalloc); + +void kasan_kmalloc_large(const void *ptr, size_t size) +{ + struct page *page; + unsigned long redzone_start; + unsigned long redzone_end; + + if (unlikely(ptr == NULL)) + return; + + page = virt_to_page(ptr); + redzone_start = round_up((unsigned long)(ptr + size), + KASAN_SHADOW_SCALE_SIZE); + redzone_end = (unsigned long)ptr + (PAGE_SIZE << compound_order(page)); + + kasan_unpoison_shadow(ptr, size); + kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start, + KASAN_PAGE_REDZONE); +} + +void kasan_krealloc(const void *object, size_t size) +{ + struct page *page; + + if (unlikely(object == ZERO_SIZE_PTR)) + return; + + page = virt_to_head_page(object); + + if (unlikely(!PageSlab(page))) + kasan_kmalloc_large(object, size); + else + kasan_kmalloc(page->slab_cache, object, size); +} + +void kasan_kfree(void *ptr) +{ + struct page *page; + + page = virt_to_head_page(ptr); + + if (unlikely(!PageSlab(page))) + kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page), + KASAN_FREE_PAGE); + else + kasan_slab_free(page->slab_cache, ptr); +} + +void kasan_kfree_large(const void *ptr) +{ + struct page *page = virt_to_page(ptr); + + kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page), + KASAN_FREE_PAGE); +} + +int kasan_module_alloc(void *addr, size_t size) +{ + void *ret; + size_t shadow_size; + unsigned long shadow_start; + + shadow_start = (unsigned long)kasan_mem_to_shadow(addr); + shadow_size = round_up(size >> KASAN_SHADOW_SCALE_SHIFT, + PAGE_SIZE); + + if (WARN_ON(!PAGE_ALIGNED(shadow_start))) + return -EINVAL; + + ret = __vmalloc_node_range(shadow_size, 1, shadow_start, + shadow_start + shadow_size, + GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, + PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE, + __builtin_return_address(0)); + + if (ret) { + find_vm_area(addr)->flags |= VM_KASAN; + kmemleak_ignore(ret); + return 0; + } + + return -ENOMEM; +} + +void kasan_free_shadow(const struct vm_struct *vm) +{ + if (vm->flags & VM_KASAN) + vfree(kasan_mem_to_shadow(vm->addr)); +} + +static void register_global(struct kasan_global *global) +{ + size_t aligned_size = round_up(global->size, KASAN_SHADOW_SCALE_SIZE); + + kasan_unpoison_shadow(global->beg, global->size); + + kasan_poison_shadow(global->beg + aligned_size, + global->size_with_redzone - aligned_size, + KASAN_GLOBAL_REDZONE); +} + +void __asan_register_globals(struct kasan_global *globals, size_t size) +{ + int i; + + for (i = 0; i < size; i++) + register_global(&globals[i]); +} +EXPORT_SYMBOL(__asan_register_globals); + +void __asan_unregister_globals(struct kasan_global *globals, size_t size) +{ +} +EXPORT_SYMBOL(__asan_unregister_globals); + +#define DEFINE_ASAN_LOAD_STORE(size) \ + void __asan_load##size(unsigned long addr) \ + { \ + check_memory_region(addr, size, false); \ + } \ + EXPORT_SYMBOL(__asan_load##size); \ + __alias(__asan_load##size) \ + void __asan_load##size##_noabort(unsigned long); \ + EXPORT_SYMBOL(__asan_load##size##_noabort); \ + void __asan_store##size(unsigned long addr) \ + { \ + check_memory_region(addr, size, true); \ + } \ + EXPORT_SYMBOL(__asan_store##size); \ + __alias(__asan_store##size) \ + void __asan_store##size##_noabort(unsigned long); \ + EXPORT_SYMBOL(__asan_store##size##_noabort) + +DEFINE_ASAN_LOAD_STORE(1); +DEFINE_ASAN_LOAD_STORE(2); +DEFINE_ASAN_LOAD_STORE(4); +DEFINE_ASAN_LOAD_STORE(8); +DEFINE_ASAN_LOAD_STORE(16); + +void __asan_loadN(unsigned long addr, size_t size) +{ + check_memory_region(addr, size, false); +} +EXPORT_SYMBOL(__asan_loadN); + +__alias(__asan_loadN) +void __asan_loadN_noabort(unsigned long, size_t); +EXPORT_SYMBOL(__asan_loadN_noabort); + +void __asan_storeN(unsigned long addr, size_t size) +{ + check_memory_region(addr, size, true); +} +EXPORT_SYMBOL(__asan_storeN); + +__alias(__asan_storeN) +void __asan_storeN_noabort(unsigned long, size_t); +EXPORT_SYMBOL(__asan_storeN_noabort); + +/* to shut up compiler complaints */ +void __asan_handle_no_return(void) {} +EXPORT_SYMBOL(__asan_handle_no_return); + +#ifdef CONFIG_MEMORY_HOTPLUG +static int kasan_mem_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + return (action == MEM_GOING_ONLINE) ? NOTIFY_BAD : NOTIFY_OK; +} + +static int __init kasan_memhotplug_init(void) +{ + pr_err("WARNING: KASAN doesn't support memory hot-add\n"); + pr_err("Memory hot-add will be disabled\n"); + + hotplug_memory_notifier(kasan_mem_notifier, 0); + + return 0; +} + +module_init(kasan_memhotplug_init); +#endif diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h new file mode 100644 index 000000000000..4f6c62e5c21e --- /dev/null +++ b/mm/kasan/kasan.h @@ -0,0 +1,71 @@ +#ifndef __MM_KASAN_KASAN_H +#define __MM_KASAN_KASAN_H + +#include <linux/kasan.h> + +#define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) +#define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1) + +#define KASAN_FREE_PAGE 0xFF /* page was freed */ +#define KASAN_PAGE_REDZONE 0xFE /* redzone for kmalloc_large allocations */ +#define KASAN_KMALLOC_REDZONE 0xFC /* redzone inside slub object */ +#define KASAN_KMALLOC_FREE 0xFB /* object was freed (kmem_cache_free/kfree) */ +#define KASAN_GLOBAL_REDZONE 0xFA /* redzone for global variable */ + +/* + * Stack redzone shadow values + * (Those are compiler's ABI, don't change them) + */ +#define KASAN_STACK_LEFT 0xF1 +#define KASAN_STACK_MID 0xF2 +#define KASAN_STACK_RIGHT 0xF3 +#define KASAN_STACK_PARTIAL 0xF4 + +/* Don't break randconfig/all*config builds */ +#ifndef KASAN_ABI_VERSION +#define KASAN_ABI_VERSION 1 +#endif + +struct kasan_access_info { + const void *access_addr; + const void *first_bad_addr; + size_t access_size; + bool is_write; + unsigned long ip; +}; + +/* The layout of struct dictated by compiler */ +struct kasan_source_location { + const char *filename; + int line_no; + int column_no; +}; + +/* The layout of struct dictated by compiler */ +struct kasan_global { + const void *beg; /* Address of the beginning of the global variable. */ + size_t size; /* Size of the global variable. */ + size_t size_with_redzone; /* Size of the variable + size of the red zone. 32 bytes aligned */ + const void *name; + const void *module_name; /* Name of the module where the global variable is declared. */ + unsigned long has_dynamic_init; /* This needed for C++ */ +#if KASAN_ABI_VERSION >= 4 + struct kasan_source_location *location; +#endif +}; + +static inline const void *kasan_shadow_to_mem(const void *shadow_addr) +{ + return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET) + << KASAN_SHADOW_SCALE_SHIFT); +} + +static inline bool kasan_report_enabled(void) +{ + return !current->kasan_depth; +} + +void kasan_report(unsigned long addr, size_t size, + bool is_write, unsigned long ip); + +#endif diff --git a/mm/kasan/kasan_init.c b/mm/kasan/kasan_init.c new file mode 100644 index 000000000000..f41ac9a551cc --- /dev/null +++ b/mm/kasan/kasan_init.c @@ -0,0 +1,152 @@ +/* + * This file contains some kasan initialization code. + * + * Copyright (c) 2015 Samsung Electronics Co., Ltd. + * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include <linux/bootmem.h> +#include <linux/init.h> +#include <linux/kasan.h> +#include <linux/kernel.h> +#include <linux/memblock.h> +#include <linux/pfn.h> + +#include <asm/page.h> +#include <asm/pgalloc.h> + +/* + * This page serves two purposes: + * - It used as early shadow memory. The entire shadow region populated + * with this page, before we will be able to setup normal shadow memory. + * - Latter it reused it as zero shadow to cover large ranges of memory + * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...). + */ +unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss; + +#if CONFIG_ARM64_PGTABLE_LEVELS > 3 +pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss; +#endif +#if CONFIG_ARM64_PGTABLE_LEVELS > 2 +pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss; +#endif +pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss; + +static __init void *early_alloc(size_t size, int node) +{ + return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), + BOOTMEM_ALLOC_ACCESSIBLE, node); +} + +static void __init zero_pte_populate(pmd_t *pmd, unsigned long addr, + unsigned long end) +{ + pte_t *pte = pte_offset_kernel(pmd, addr); + pte_t zero_pte; + + zero_pte = pfn_pte(PFN_DOWN(__pa(kasan_zero_page)), PAGE_KERNEL); + zero_pte = pte_wrprotect(zero_pte); + + while (addr + PAGE_SIZE <= end) { + set_pte_at(&init_mm, addr, pte, zero_pte); + addr += PAGE_SIZE; + pte = pte_offset_kernel(pmd, addr); + } +} + +static void __init zero_pmd_populate(pud_t *pud, unsigned long addr, + unsigned long end) +{ + pmd_t *pmd = pmd_offset(pud, addr); + unsigned long next; + + do { + next = pmd_addr_end(addr, end); + + if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); + continue; + } + + if (pmd_none(*pmd)) { + pmd_populate_kernel(&init_mm, pmd, + early_alloc(PAGE_SIZE, NUMA_NO_NODE)); + } + zero_pte_populate(pmd, addr, next); + } while (pmd++, addr = next, addr != end); +} + +static void __init zero_pud_populate(pgd_t *pgd, unsigned long addr, + unsigned long end) +{ + pud_t *pud = pud_offset(pgd, addr); + unsigned long next; + + do { + next = pud_addr_end(addr, end); + if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { + pmd_t *pmd; + + pud_populate(&init_mm, pud, kasan_zero_pmd); + pmd = pmd_offset(pud, addr); + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); + continue; + } + + if (pud_none(*pud)) { + pud_populate(&init_mm, pud, + early_alloc(PAGE_SIZE, NUMA_NO_NODE)); + } + zero_pmd_populate(pud, addr, next); + } while (pud++, addr = next, addr != end); +} + +/** + * kasan_populate_zero_shadow - populate shadow memory region with + * kasan_zero_page + * @shadow_start - start of the memory range to populate + * @shadow_end - end of the memory range to populate + */ +void __init kasan_populate_zero_shadow(const void *shadow_start, + const void *shadow_end) +{ + unsigned long addr = (unsigned long)shadow_start; + unsigned long end = (unsigned long)shadow_end; + pgd_t *pgd = pgd_offset_k(addr); + unsigned long next; + + do { + next = pgd_addr_end(addr, end); + + if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { + pud_t *pud; + pmd_t *pmd; + + /* + * kasan_zero_pud should be populated with pmds + * at this moment. + * [pud,pmd]_populate*() below needed only for + * 3,2 - level page tables where we don't have + * puds,pmds, so pgd_populate(), pud_populate() + * is noops. + */ + pgd_populate(&init_mm, pgd, kasan_zero_pud); + pud = pud_offset(pgd, addr); + pud_populate(&init_mm, pud, kasan_zero_pmd); + pmd = pmd_offset(pud, addr); + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); + continue; + } + + if (pgd_none(*pgd)) { + pgd_populate(&init_mm, pgd, + early_alloc(PAGE_SIZE, NUMA_NO_NODE)); + } + zero_pud_populate(pgd, addr, next); + } while (pgd++, addr = next, addr != end); +} diff --git a/mm/kasan/report.c b/mm/kasan/report.c new file mode 100644 index 000000000000..e0b3e94ecdb5 --- /dev/null +++ b/mm/kasan/report.c @@ -0,0 +1,298 @@ +/* + * This file contains error reporting code. + * + * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * Author: Andrey Ryabinin <a.ryabinin@samsung.com> + * + * Some code borrowed from https://github.com/xairy/kasan-prototype by + * Andrey Konovalov <adech.fo@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/printk.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/stacktrace.h> +#include <linux/string.h> +#include <linux/types.h> +#include <linux/kasan.h> +#include <linux/module.h> + +#include <asm/sections.h> + +#include "kasan.h" +#include "../slab.h" + +/* Shadow layout customization. */ +#define SHADOW_BYTES_PER_BLOCK 1 +#define SHADOW_BLOCKS_PER_ROW 16 +#define SHADOW_BYTES_PER_ROW (SHADOW_BLOCKS_PER_ROW * SHADOW_BYTES_PER_BLOCK) +#define SHADOW_ROWS_AROUND_ADDR 2 + +static const void *find_first_bad_addr(const void *addr, size_t size) +{ + u8 shadow_val = *(u8 *)kasan_mem_to_shadow(addr); + const void *first_bad_addr = addr; + + while (!shadow_val && first_bad_addr < addr + size) { + first_bad_addr += KASAN_SHADOW_SCALE_SIZE; + shadow_val = *(u8 *)kasan_mem_to_shadow(first_bad_addr); + } + return first_bad_addr; +} + +static void print_error_description(struct kasan_access_info *info) +{ + const char *bug_type = "unknown-crash"; + u8 *shadow_addr; + + info->first_bad_addr = find_first_bad_addr(info->access_addr, + info->access_size); + + shadow_addr = (u8 *)kasan_mem_to_shadow(info->first_bad_addr); + + /* + * If shadow byte value is in [0, KASAN_SHADOW_SCALE_SIZE) we can look + * at the next shadow byte to determine the type of the bad access. + */ + if (*shadow_addr > 0 && *shadow_addr <= KASAN_SHADOW_SCALE_SIZE - 1) + shadow_addr++; + + switch (*shadow_addr) { + case 0 ... KASAN_SHADOW_SCALE_SIZE - 1: + /* + * In theory it's still possible to see these shadow values + * due to a data race in the kernel code. + */ + bug_type = "out-of-bounds"; + break; + case KASAN_PAGE_REDZONE: + case KASAN_KMALLOC_REDZONE: + bug_type = "slab-out-of-bounds"; + break; + case KASAN_GLOBAL_REDZONE: + bug_type = "global-out-of-bounds"; + break; + case KASAN_STACK_LEFT: + case KASAN_STACK_MID: + case KASAN_STACK_RIGHT: + case KASAN_STACK_PARTIAL: + bug_type = "stack-out-of-bounds"; + break; + case KASAN_FREE_PAGE: + case KASAN_KMALLOC_FREE: + bug_type = "use-after-free"; + break; + } + + pr_err("BUG: KASAN: %s in %pS at addr %p\n", + bug_type, (void *)info->ip, + info->access_addr); + pr_err("%s of size %zu by task %s/%d\n", + info->is_write ? "Write" : "Read", + info->access_size, current->comm, task_pid_nr(current)); +} + +static inline bool kernel_or_module_addr(const void *addr) +{ + if (addr >= (void *)_stext && addr < (void *)_end) + return true; + if (is_module_address((unsigned long)addr)) + return true; + return false; +} + +static inline bool init_task_stack_addr(const void *addr) +{ + return addr >= (void *)&init_thread_union.stack && + (addr <= (void *)&init_thread_union.stack + + sizeof(init_thread_union.stack)); +} + +static void print_address_description(struct kasan_access_info *info) +{ + const void *addr = info->access_addr; + + if ((addr >= (void *)PAGE_OFFSET) && + (addr < high_memory)) { + struct page *page = virt_to_head_page(addr); + + if (PageSlab(page)) { + void *object; + struct kmem_cache *cache = page->slab_cache; + void *last_object; + + object = virt_to_obj(cache, page_address(page), addr); + last_object = page_address(page) + + page->objects * cache->size; + + if (unlikely(object > last_object)) + object = last_object; /* we hit into padding */ + + object_err(cache, page, object, + "kasan: bad access detected"); + return; + } + dump_page(page, "kasan: bad access detected"); + } + + if (kernel_or_module_addr(addr)) { + if (!init_task_stack_addr(addr)) + pr_err("Address belongs to variable %pS\n", addr); + } + + dump_stack(); +} + +static bool row_is_guilty(const void *row, const void *guilty) +{ + return (row <= guilty) && (guilty < row + SHADOW_BYTES_PER_ROW); +} + +static int shadow_pointer_offset(const void *row, const void *shadow) +{ + /* The length of ">ff00ff00ff00ff00: " is + * 3 + (BITS_PER_LONG/8)*2 chars. + */ + return 3 + (BITS_PER_LONG/8)*2 + (shadow - row)*2 + + (shadow - row) / SHADOW_BYTES_PER_BLOCK + 1; +} + +static void print_shadow_for_address(const void *addr) +{ + int i; + const void *shadow = kasan_mem_to_shadow(addr); + const void *shadow_row; + + shadow_row = (void *)round_down((unsigned long)shadow, + SHADOW_BYTES_PER_ROW) + - SHADOW_ROWS_AROUND_ADDR * SHADOW_BYTES_PER_ROW; + + pr_err("Memory state around the buggy address:\n"); + + for (i = -SHADOW_ROWS_AROUND_ADDR; i <= SHADOW_ROWS_AROUND_ADDR; i++) { + const void *kaddr = kasan_shadow_to_mem(shadow_row); + char buffer[4 + (BITS_PER_LONG/8)*2]; + char shadow_buf[SHADOW_BYTES_PER_ROW]; + + snprintf(buffer, sizeof(buffer), + (i == 0) ? ">%p: " : " %p: ", kaddr); + /* + * We should not pass a shadow pointer to generic + * function, because generic functions may try to + * access kasan mapping for the passed address. + */ + memcpy(shadow_buf, shadow_row, SHADOW_BYTES_PER_ROW); + print_hex_dump(KERN_ERR, buffer, + DUMP_PREFIX_NONE, SHADOW_BYTES_PER_ROW, 1, + shadow_buf, SHADOW_BYTES_PER_ROW, 0); + + if (row_is_guilty(shadow_row, shadow)) + pr_err("%*c\n", + shadow_pointer_offset(shadow_row, shadow), + '^'); + + shadow_row += SHADOW_BYTES_PER_ROW; + } +} + +static DEFINE_SPINLOCK(report_lock); + +static void kasan_report_error(struct kasan_access_info *info) +{ + unsigned long flags; + const char *bug_type; + + /* + * Make sure we don't end up in loop. + */ + kasan_disable_current(); + spin_lock_irqsave(&report_lock, flags); + pr_err("=================================" + "=================================\n"); + if (info->access_addr < + kasan_shadow_to_mem((void *)KASAN_SHADOW_START)) { + if ((unsigned long)info->access_addr < PAGE_SIZE) + bug_type = "null-ptr-deref"; + else if ((unsigned long)info->access_addr < TASK_SIZE) + bug_type = "user-memory-access"; + else + bug_type = "wild-memory-access"; + pr_err("BUG: KASAN: %s on address %p\n", + bug_type, info->access_addr); + pr_err("%s of size %zu by task %s/%d\n", + info->is_write ? "Write" : "Read", + info->access_size, current->comm, + task_pid_nr(current)); + dump_stack(); + } else { + print_error_description(info); + print_address_description(info); + print_shadow_for_address(info->first_bad_addr); + } + pr_err("=================================" + "=================================\n"); + add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); + spin_unlock_irqrestore(&report_lock, flags); + kasan_enable_current(); +} + +void kasan_report(unsigned long addr, size_t size, + bool is_write, unsigned long ip) +{ + struct kasan_access_info info; + + if (likely(!kasan_report_enabled())) + return; + + info.access_addr = (void *)addr; + info.access_size = size; + info.is_write = is_write; + info.ip = ip; + + kasan_report_error(&info); +} + + +#define DEFINE_ASAN_REPORT_LOAD(size) \ +void __asan_report_load##size##_noabort(unsigned long addr) \ +{ \ + kasan_report(addr, size, false, _RET_IP_); \ +} \ +EXPORT_SYMBOL(__asan_report_load##size##_noabort) + +#define DEFINE_ASAN_REPORT_STORE(size) \ +void __asan_report_store##size##_noabort(unsigned long addr) \ +{ \ + kasan_report(addr, size, true, _RET_IP_); \ +} \ +EXPORT_SYMBOL(__asan_report_store##size##_noabort) + +DEFINE_ASAN_REPORT_LOAD(1); +DEFINE_ASAN_REPORT_LOAD(2); +DEFINE_ASAN_REPORT_LOAD(4); +DEFINE_ASAN_REPORT_LOAD(8); +DEFINE_ASAN_REPORT_LOAD(16); +DEFINE_ASAN_REPORT_STORE(1); +DEFINE_ASAN_REPORT_STORE(2); +DEFINE_ASAN_REPORT_STORE(4); +DEFINE_ASAN_REPORT_STORE(8); +DEFINE_ASAN_REPORT_STORE(16); + +void __asan_report_load_n_noabort(unsigned long addr, size_t size) +{ + kasan_report(addr, size, false, _RET_IP_); +} +EXPORT_SYMBOL(__asan_report_load_n_noabort); + +void __asan_report_store_n_noabort(unsigned long addr, size_t size) +{ + kasan_report(addr, size, true, _RET_IP_); +} +EXPORT_SYMBOL(__asan_report_store_n_noabort); diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 1f14ef68a5c8..c607bc059664 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -98,6 +98,7 @@ #include <asm/processor.h> #include <linux/atomic.h> +#include <linux/kasan.h> #include <linux/kmemcheck.h> #include <linux/kmemleak.h> #include <linux/memory_hotplug.h> @@ -1116,7 +1117,10 @@ static bool update_checksum(struct kmemleak_object *object) if (!kmemcheck_is_obj_initialized(object->pointer, object->size)) return false; + kasan_disable_current(); object->checksum = crc32(0, (void *)object->pointer, object->size); + kasan_enable_current(); + return object->checksum != old_csum; } @@ -1167,7 +1171,9 @@ static void scan_block(void *_start, void *_end, BYTES_PER_POINTER)) continue; + kasan_disable_current(); pointer = *ptr; + kasan_enable_current(); object = find_and_get_object(pointer, 1); if (!object) diff --git a/mm/mempool.c b/mm/mempool.c index e209c98c7203..d4c4e4c1bc56 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -6,26 +6,139 @@ * extreme VM load. * * started by Ingo Molnar, Copyright (C) 2001 + * debugging by David Rientjes, Copyright (C) 2015 */ #include <linux/mm.h> #include <linux/slab.h> +#include <linux/highmem.h> +#include <linux/kasan.h> #include <linux/kmemleak.h> #include <linux/export.h> #include <linux/mempool.h> #include <linux/blkdev.h> #include <linux/writeback.h> +#include "slab.h" + +#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON) +static void poison_error(mempool_t *pool, void *element, size_t size, + size_t byte) +{ + const int nr = pool->curr_nr; + const int start = max_t(int, byte - (BITS_PER_LONG / 8), 0); + const int end = min_t(int, byte + (BITS_PER_LONG / 8), size); + int i; + + pr_err("BUG: mempool element poison mismatch\n"); + pr_err("Mempool %p size %zu\n", pool, size); + pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : ""); + for (i = start; i < end; i++) + pr_cont("%x ", *(u8 *)(element + i)); + pr_cont("%s\n", end < size ? "..." : ""); + dump_stack(); +} + +static void __check_element(mempool_t *pool, void *element, size_t size) +{ + u8 *obj = element; + size_t i; + + for (i = 0; i < size; i++) { + u8 exp = (i < size - 1) ? POISON_FREE : POISON_END; + + if (obj[i] != exp) { + poison_error(pool, element, size, i); + return; + } + } + memset(obj, POISON_INUSE, size); +} + +static void check_element(mempool_t *pool, void *element) +{ + /* Mempools backed by slab allocator */ + if (pool->free == mempool_free_slab || pool->free == mempool_kfree) + __check_element(pool, element, ksize(element)); + + /* Mempools backed by page allocator */ + if (pool->free == mempool_free_pages) { + int order = (int)(long)pool->pool_data; + void *addr = kmap_atomic((struct page *)element); + + __check_element(pool, addr, 1UL << (PAGE_SHIFT + order)); + kunmap_atomic(addr); + } +} + +static void __poison_element(void *element, size_t size) +{ + u8 *obj = element; + + memset(obj, POISON_FREE, size - 1); + obj[size - 1] = POISON_END; +} + +static void poison_element(mempool_t *pool, void *element) +{ + /* Mempools backed by slab allocator */ + if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) + __poison_element(element, ksize(element)); + + /* Mempools backed by page allocator */ + if (pool->alloc == mempool_alloc_pages) { + int order = (int)(long)pool->pool_data; + void *addr = kmap_atomic((struct page *)element); + + __poison_element(addr, 1UL << (PAGE_SHIFT + order)); + kunmap_atomic(addr); + } +} +#else /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */ +static inline void check_element(mempool_t *pool, void *element) +{ +} +static inline void poison_element(mempool_t *pool, void *element) +{ +} +#endif /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */ + +static void kasan_poison_element(mempool_t *pool, void *element) +{ + if (pool->alloc == mempool_alloc_slab) + kasan_slab_free(pool->pool_data, element); + if (pool->alloc == mempool_kmalloc) + kasan_kfree(element); + if (pool->alloc == mempool_alloc_pages) + kasan_free_pages(element, (unsigned long)pool->pool_data); +} + +static void kasan_unpoison_element(mempool_t *pool, void *element) +{ + if (pool->alloc == mempool_alloc_slab) + kasan_slab_alloc(pool->pool_data, element); + if (pool->alloc == mempool_kmalloc) + kasan_krealloc(element, (size_t)pool->pool_data); + if (pool->alloc == mempool_alloc_pages) + kasan_alloc_pages(element, (unsigned long)pool->pool_data); +} static void add_element(mempool_t *pool, void *element) { BUG_ON(pool->curr_nr >= pool->min_nr); + poison_element(pool, element); + kasan_poison_element(pool, element); pool->elements[pool->curr_nr++] = element; } static void *remove_element(mempool_t *pool) { - BUG_ON(pool->curr_nr <= 0); - return pool->elements[--pool->curr_nr]; + void *element = pool->elements[--pool->curr_nr]; + + BUG_ON(pool->curr_nr < 0); + kasan_unpoison_element(pool, element); + check_element(pool, element); + kasan_unpoison_element(pool, element); + return element; } /** @@ -38,6 +151,9 @@ static void *remove_element(mempool_t *pool) */ void mempool_destroy(mempool_t *pool) { + if (unlikely(!pool)) + return; + while (pool->curr_nr) { void *element = remove_element(pool); pool->free(element, pool->pool_data); @@ -113,23 +229,24 @@ EXPORT_SYMBOL(mempool_create_node); * mempool_create(). * @new_min_nr: the new minimum number of elements guaranteed to be * allocated for this pool. - * @gfp_mask: the usual allocation bitmask. * * This function shrinks/grows the pool. In the case of growing, * it cannot be guaranteed that the pool will be grown to the new * size immediately, but new mempool_free() calls will refill it. + * This function may sleep. * * Note, the caller must guarantee that no mempool_destroy is called * while this function is running. mempool_alloc() & mempool_free() * might be called (eg. from IRQ contexts) while this function executes. */ -int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) +int mempool_resize(mempool_t *pool, int new_min_nr) { void *element; void **new_elements; unsigned long flags; BUG_ON(new_min_nr <= 0); + might_sleep(); spin_lock_irqsave(&pool->lock, flags); if (new_min_nr <= pool->min_nr) { @@ -145,7 +262,8 @@ int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) spin_unlock_irqrestore(&pool->lock, flags); /* Grow the pool */ - new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask); + new_elements = kmalloc_array(new_min_nr, sizeof(*new_elements), + GFP_KERNEL); if (!new_elements) return -ENOMEM; @@ -164,7 +282,7 @@ int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) while (pool->curr_nr < pool->min_nr) { spin_unlock_irqrestore(&pool->lock, flags); - element = pool->alloc(gfp_mask, pool->pool_data); + element = pool->alloc(GFP_KERNEL, pool->pool_data); if (!element) goto out; spin_lock_irqsave(&pool->lock, flags); @@ -332,6 +450,7 @@ EXPORT_SYMBOL(mempool_free); void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) { struct kmem_cache *mem = pool_data; + VM_BUG_ON(mem->ctor); return kmem_cache_alloc(mem, gfp_mask); } EXPORT_SYMBOL(mempool_alloc_slab); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index fcd8a8ce5cc0..bf165579f1ee 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -25,6 +25,7 @@ #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/kmemcheck.h> +#include <linux/kasan.h> #include <linux/module.h> #include <linux/suspend.h> #include <linux/pagevec.h> @@ -764,6 +765,7 @@ static bool free_pages_prepare(struct page *page, unsigned int order) trace_mm_page_free(page, order); kmemcheck_free_shadow(page, order); + kasan_free_pages(page, order); if (PageAnon(page)) page->mapping = NULL; @@ -944,6 +946,7 @@ static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags) arch_alloc_page(page, order); kernel_map_pages(page, 1 << order, 1); + kasan_alloc_pages(page, order); if (gfp_flags & __GFP_ZERO) prep_zero_page(page, order, gfp_flags); diff --git a/mm/slab_common.c b/mm/slab_common.c index dcdab81bd240..9a18f19360d3 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -789,6 +789,7 @@ void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) page = alloc_kmem_pages(flags, order); ret = page ? page_address(page) : NULL; kmemleak_alloc(ret, size, 1, flags); + kasan_kmalloc_large(ret, size); return ret; } EXPORT_SYMBOL(kmalloc_order); @@ -963,8 +964,10 @@ static __always_inline void *__do_krealloc(const void *p, size_t new_size, if (p) ks = ksize(p); - if (ks >= new_size) + if (ks >= new_size) { + kasan_krealloc((void *)p, new_size); return (void *)p; + } ret = kmalloc_track_caller(new_size, flags); if (ret && p) diff --git a/mm/slub.c b/mm/slub.c index 332ce18463cb..1256a8f603a2 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -20,6 +20,7 @@ #include <linux/proc_fs.h> #include <linux/notifier.h> #include <linux/seq_file.h> +#include <linux/kasan.h> #include <linux/kmemcheck.h> #include <linux/cpu.h> #include <linux/cpuset.h> @@ -458,8 +459,10 @@ static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) /* * Debug settings: */ -#ifdef CONFIG_SLUB_DEBUG_ON +#if defined(CONFIG_SLUB_DEBUG_ON) static int slub_debug = DEBUG_DEFAULT_FLAGS; +#elif defined(CONFIG_KASAN) +static int slub_debug = SLAB_STORE_USER; #else static int slub_debug; #endif @@ -468,12 +471,30 @@ static char *slub_debug_slabs; static int disable_higher_order_debug; /* + * slub is about to manipulate internal object metadata. This memory lies + * outside the range of the allocated object, so accessing it would normally + * be reported by kasan as a bounds error. metadata_access_enable() is used + * to tell kasan that these accesses are OK. + */ +static inline void metadata_access_enable(void) +{ + kasan_disable_current(); +} + +static inline void metadata_access_disable(void) +{ + kasan_enable_current(); +} + +/* * Object debugging */ static void print_section(char *text, u8 *addr, unsigned int length) { + metadata_access_enable(); print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr, length, 1); + metadata_access_disable(); } static struct track *get_track(struct kmem_cache *s, void *object, @@ -503,7 +524,9 @@ static void set_track(struct kmem_cache *s, void *object, trace.max_entries = TRACK_ADDRS_COUNT; trace.entries = p->addrs; trace.skip = 3; + metadata_access_enable(); save_stack_trace(&trace); + metadata_access_disable(); /* See rant in lockdep.c */ if (trace.nr_entries != 0 && @@ -629,7 +652,7 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) dump_stack(); } -static void object_err(struct kmem_cache *s, struct page *page, +void object_err(struct kmem_cache *s, struct page *page, u8 *object, char *reason) { slab_bug(s, "%s", reason); @@ -677,7 +700,9 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, u8 *fault; u8 *end; + metadata_access_enable(); fault = memchr_inv(start, value, bytes); + metadata_access_disable(); if (!fault) return 1; @@ -770,7 +795,9 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page) if (!remainder) return 1; + metadata_access_enable(); fault = memchr_inv(end - remainder, POISON_INUSE, remainder); + metadata_access_disable(); if (!fault) return 1; while (end > fault && end[-1] == POISON_INUSE) @@ -1226,11 +1253,13 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node, static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) { kmemleak_alloc(ptr, size, 1, flags); + kasan_kmalloc_large(ptr, size); } static inline void kfree_hook(const void *x) { kmemleak_free(x); + kasan_kfree_large(x); } static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) @@ -1248,6 +1277,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, flags &= gfp_allowed_mask; kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); + kasan_slab_alloc(s, object); } static inline void slab_free_hook(struct kmem_cache *s, void *x) @@ -1271,6 +1301,8 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x) #endif if (!(s->flags & SLAB_DEBUG_OBJECTS)) debug_check_no_obj_freed(x, s->object_size); + + kasan_slab_free(s, x); } /* @@ -1365,8 +1397,11 @@ static void setup_object(struct kmem_cache *s, struct page *page, void *object) { setup_object_debug(s, page, object); - if (unlikely(s->ctor)) + if (unlikely(s->ctor)) { + kasan_unpoison_object_data(s, object); s->ctor(object); + kasan_poison_object_data(s, object); + } } static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) @@ -1396,6 +1431,8 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) if (unlikely(s->flags & SLAB_POISON)) memset(start, POISON_INUSE, PAGE_SIZE << order); + kasan_poison_slab(page); + for_each_object_idx(p, idx, s, start, page->objects) { setup_object(s, page, p); if (likely(idx < page->objects)) @@ -2472,6 +2509,7 @@ void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) { void *ret = slab_alloc(s, gfpflags, _RET_IP_); trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); + kasan_kmalloc(s, ret, size); return ret; } EXPORT_SYMBOL(kmem_cache_alloc_trace); @@ -2498,6 +2536,8 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s, trace_kmalloc_node(_RET_IP_, ret, size, s->size, gfpflags, node); + + kasan_kmalloc(s, ret, size); return ret; } EXPORT_SYMBOL(kmem_cache_alloc_node_trace); @@ -2881,6 +2921,7 @@ static void early_kmem_cache_node_alloc(int node) init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); init_tracking(kmem_cache_node, n); #endif + kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node)); init_kmem_cache_node(n); inc_slabs_node(kmem_cache_node, node, page->objects); @@ -3253,6 +3294,8 @@ void *__kmalloc(size_t size, gfp_t flags) trace_kmalloc(_RET_IP_, ret, size, s->size, flags); + kasan_kmalloc(s, ret, size); + return ret; } EXPORT_SYMBOL(__kmalloc); @@ -3296,12 +3339,14 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); + kasan_kmalloc(s, ret, size); + return ret; } EXPORT_SYMBOL(__kmalloc_node); #endif -size_t ksize(const void *object) +static size_t __ksize(const void *object) { struct page *page; @@ -3317,6 +3362,15 @@ size_t ksize(const void *object) return slab_ksize(page->slab_cache); } + +size_t ksize(const void *object) +{ + size_t size = __ksize(object); + /* We assume that ksize callers could use whole allocated area, + so we need unpoison this area. */ + kasan_krealloc(object, size); + return size; +} EXPORT_SYMBOL(ksize); void kfree(const void *x) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 90520af7f186..3f814de8eb8d 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1325,10 +1325,8 @@ static struct vm_struct *__get_vm_area_node(unsigned long size, if (unlikely(!area)) return NULL; - /* - * We always allocate a guard page. - */ - size += PAGE_SIZE; + if (!(flags & VM_NO_GUARD)) + size += PAGE_SIZE; va = alloc_vmap_area(size, align, start, end, node, gfp_mask); if (IS_ERR(va)) { @@ -1421,8 +1419,8 @@ struct vm_struct *remove_vm_area(const void *addr) spin_unlock(&vmap_area_lock); vmap_debug_free_range(va->va_start, va->va_end); + kasan_free_shadow(vm); free_unmap_vmap_area(va); - vm->size -= PAGE_SIZE; return vm; } @@ -1447,8 +1445,8 @@ static void __vunmap(const void *addr, int deallocate_pages) return; } - debug_check_no_locks_freed(addr, area->size); - debug_check_no_obj_freed(addr, area->size); + debug_check_no_locks_freed(addr, get_vm_area_size(area)); + debug_check_no_obj_freed(addr, get_vm_area_size(area)); if (deallocate_pages) { int i; @@ -1622,6 +1620,7 @@ fail: * @end: vm area range end * @gfp_mask: flags for the page level allocator * @prot: protection mask for the allocated pages + * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) * @node: node to use for allocation or NUMA_NO_NODE * @caller: caller's return address * @@ -1631,7 +1630,8 @@ fail: */ void *__vmalloc_node_range(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, - pgprot_t prot, int node, const void *caller) + pgprot_t prot, unsigned long vm_flags, int node, + const void *caller) { struct vm_struct *area; void *addr; @@ -1641,8 +1641,8 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, if (!size || (size >> PAGE_SHIFT) > totalram_pages) goto fail; - area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED, - start, end, node, gfp_mask, caller); + area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED | + vm_flags, start, end, node, gfp_mask, caller); if (!area) goto fail; @@ -1691,7 +1691,7 @@ static void *__vmalloc_node(unsigned long size, unsigned long align, int node, const void *caller) { return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, - gfp_mask, prot, node, caller); + gfp_mask, prot, 0, node, caller); } void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 65e7b08bb2cc..5374b1bdf02f 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -179,6 +179,12 @@ build := -f $(srctree)/scripts/Makefile.build obj # $(Q)$(MAKE) $(modbuiltin)=dir modbuiltin := -f $(srctree)/scripts/Makefile.modbuiltin obj +### +# Shorthand for $(Q)$(MAKE) -f scripts/Makefile.dtbinst obj= +# Usage: +# $(Q)$(MAKE) $(dtbinst)=dir +dtbinst := -f $(if $(KBUILD_SRC),$(srctree)/)scripts/Makefile.dtbinst obj + # Prefix -I with $(srctree) if it is not an absolute path. # skip if -I has no parameter addtree = $(if $(patsubst -I%,%,$(1)), \ diff --git a/scripts/Makefile.dtbinst b/scripts/Makefile.dtbinst new file mode 100644 index 000000000000..909ed7a2ac61 --- /dev/null +++ b/scripts/Makefile.dtbinst @@ -0,0 +1,51 @@ +# ========================================================================== +# Installing dtb files +# +# Installs all dtb files listed in $(dtb-y) either in the +# INSTALL_DTBS_PATH directory or the default location: +# +# $INSTALL_PATH/dtbs/$KERNELRELEASE +# +# Traverse through subdirectories listed in $(dts-dirs). +# ========================================================================== + +src := $(obj) + +PHONY := __dtbs_install +__dtbs_install: + +export dtbinst-root ?= $(obj) + +include include/config/auto.conf +include scripts/Kbuild.include +include $(srctree)/$(obj)/Makefile + +PHONY += __dtbs_install_prep +__dtbs_install_prep: +ifeq ("$(dtbinst-root)", "$(obj)") + $(Q)if [ -d $(INSTALL_DTBS_PATH).old ]; then rm -rf $(INSTALL_DTBS_PATH).old; fi + $(Q)if [ -d $(INSTALL_DTBS_PATH) ]; then mv $(INSTALL_DTBS_PATH) $(INSTALL_DTBS_PATH).old; fi + $(Q)mkdir -p $(INSTALL_DTBS_PATH) +endif + +dtbinst-files := $(dtb-y) +dtbinst-dirs := $(dts-dirs) + +# Helper targets for Installing DTBs into the boot directory +quiet_cmd_dtb_install = INSTALL $< + cmd_dtb_install = mkdir -p $(2); cp $< $(2) + +install-dir = $(patsubst $(dtbinst-root)%,$(INSTALL_DTBS_PATH)%,$(obj)) + +$(dtbinst-files) $(dtbinst-dirs): | __dtbs_install_prep + +$(dtbinst-files): %.dtb: $(obj)/%.dtb + $(call cmd,dtb_install,$(install-dir)) + +$(dtbinst-dirs): + $(Q)$(MAKE) $(dtbinst)=$(obj)/$@ + +PHONY += $(dtbinst-files) $(dtbinst-dirs) +__dtbs_install: $(dtbinst-files) $(dtbinst-dirs) + +.PHONY: $(PHONY) diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan new file mode 100644 index 000000000000..37323b0df374 --- /dev/null +++ b/scripts/Makefile.kasan @@ -0,0 +1,31 @@ +ifdef CONFIG_KASAN +ifdef CONFIG_KASAN_INLINE + call_threshold := 10000 +else + call_threshold := 0 +endif + +KASAN_SHADOW_OFFSET ?= $(CONFIG_KASAN_SHADOW_OFFSET) + +CFLAGS_KASAN_MINIMAL := -fsanitize=kernel-address + +CFLAGS_KASAN := $(call cc-option, -fsanitize=kernel-address \ + -fasan-shadow-offset=$(KASAN_SHADOW_OFFSET) \ + --param asan-stack=1 --param asan-globals=1 \ + --param asan-instrumentation-with-call-threshold=$(call_threshold)) + +ifeq ($(call cc-option, $(CFLAGS_KASAN_MINIMAL) -Werror),) + ifneq ($(CONFIG_COMPILE_TEST),y) + $(warning Cannot use CONFIG_KASAN: \ + -fsanitize=kernel-address is not supported by compiler) + endif +else + ifeq ($(CFLAGS_KASAN),) + ifneq ($(CONFIG_COMPILE_TEST),y) + $(warning CONFIG_KASAN: compiler does not support all options.\ + Trying minimal configuration) + endif + CFLAGS_KASAN := $(CFLAGS_KASAN_MINIMAL) + endif +endif +endif diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 54be19a0fa51..044eb4f89a91 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -119,6 +119,16 @@ _c_flags += $(if $(patsubst n%,, \ $(CFLAGS_GCOV)) endif +# +# Enable address sanitizer flags for kernel except some files or directories +# we don't want to check (depends on variables KASAN_SANITIZE_obj.o, KASAN_SANITIZE) +# +ifeq ($(CONFIG_KASAN),y) +_c_flags += $(if $(patsubst n%,, \ + $(KASAN_SANITIZE_$(basetarget).o)$(KASAN_SANITIZE)y), \ + $(CFLAGS_KASAN)) +endif + # If building the kernel in a separate objtree expand all occurrences # of -Idir to -I$(srctree)/dir except for absolute paths (starting with '/'). @@ -283,18 +293,6 @@ $(obj)/%.dtb: $(src)/%.dts FORCE dtc-tmp = $(subst $(comma),_,$(dot-target).dts.tmp) -# Helper targets for Installing DTBs into the boot directory -quiet_cmd_dtb_install = INSTALL $< - cmd_dtb_install = cp $< $(2) - -_dtbinst_pre_: - $(Q)if [ -d $(INSTALL_DTBS_PATH).old ]; then rm -rf $(INSTALL_DTBS_PATH).old; fi - $(Q)if [ -d $(INSTALL_DTBS_PATH) ]; then mv $(INSTALL_DTBS_PATH) $(INSTALL_DTBS_PATH).old; fi - $(Q)mkdir -p $(INSTALL_DTBS_PATH) - -%.dtb_dtbinst_: $(obj)/%.dtb _dtbinst_pre_ - $(call cmd,dtb_install,$(INSTALL_DTBS_PATH)) - # Bzip2 # --------------------------------------------------------------------------- diff --git a/scripts/module-common.lds b/scripts/module-common.lds index 0865b3e752be..c25bbd1c5f2e 100644 --- a/scripts/module-common.lds +++ b/scripts/module-common.lds @@ -16,4 +16,7 @@ SECTIONS { __kcrctab_unused : { *(SORT(___kcrctab_unused+*)) } __kcrctab_unused_gpl : { *(SORT(___kcrctab_unused_gpl+*)) } __kcrctab_gpl_future : { *(SORT(___kcrctab_gpl_future+*)) } + + . = ALIGN(8); + .init_array 0 : { *(SORT(.init_array.*)) *(.init_array) } } |