blob: ae3864d95e6c8501ac6dbe4c8991ea69467729dc [file] [log] [blame]
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001/*
2 * CCI cache coherent interconnect driver
3 *
4 * Copyright (C) 2013 ARM Ltd.
5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
12 * kind, whether express or implied; without even the implied warranty
13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 */
16
17#include <linux/arm-cci.h>
18#include <linux/io.h>
Mark Rutlandc6f85cb2014-06-30 12:20:21 +010019#include <linux/interrupt.h>
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010020#include <linux/module.h>
21#include <linux/of_address.h>
Punit Agrawalb91c8f22013-08-22 14:41:51 +010022#include <linux/of_irq.h>
23#include <linux/of_platform.h>
Mark Rutlandc6f85cb2014-06-30 12:20:21 +010024#include <linux/perf_event.h>
Punit Agrawalb91c8f22013-08-22 14:41:51 +010025#include <linux/platform_device.h>
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010026#include <linux/slab.h>
Punit Agrawalb91c8f22013-08-22 14:41:51 +010027#include <linux/spinlock.h>
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010028
29#include <asm/cacheflush.h>
30#include <asm/smp_plat.h>
31
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +000032static void __iomem *cci_ctrl_base;
33static unsigned long cci_ctrl_phys;
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010034
35struct cci_nb_ports {
36 unsigned int nb_ace;
37 unsigned int nb_ace_lite;
38};
39
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +000040static const struct cci_nb_ports cci400_ports = {
41 .nb_ace = 2,
42 .nb_ace_lite = 3
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010043};
44
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +000045static const struct of_device_id arm_cci_matches[] = {
46 {.compatible = "arm,cci-400", .data = &cci400_ports },
47 {},
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +010048};
49
Punit Agrawalb91c8f22013-08-22 14:41:51 +010050#ifdef CONFIG_HW_PERF_EVENTS
51
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +000052#define DRIVER_NAME "CCI-400"
53#define DRIVER_NAME_PMU DRIVER_NAME " PMU"
54
Punit Agrawalb91c8f22013-08-22 14:41:51 +010055#define CCI_PMCR 0x0100
56#define CCI_PID2 0x0fe8
57
58#define CCI_PMCR_CEN 0x00000001
59#define CCI_PMCR_NCNT_MASK 0x0000f800
60#define CCI_PMCR_NCNT_SHIFT 11
61
62#define CCI_PID2_REV_MASK 0xf0
63#define CCI_PID2_REV_SHIFT 4
64
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +000065#define CCI_PMU_EVT_SEL 0x000
66#define CCI_PMU_CNTR 0x004
67#define CCI_PMU_CNTR_CTRL 0x008
68#define CCI_PMU_OVRFLW 0x00c
69
70#define CCI_PMU_OVRFLW_FLAG 1
71
72#define CCI_PMU_CNTR_BASE(idx) ((idx) * SZ_4K)
73
74#define CCI_PMU_CNTR_MASK ((1ULL << 32) -1)
75
76#define CCI_PMU_EVENT_MASK 0xff
77#define CCI_PMU_EVENT_SOURCE(event) ((event >> 5) & 0x7)
78#define CCI_PMU_EVENT_CODE(event) (event & 0x1f)
79
80#define CCI_PMU_MAX_HW_EVENTS 5 /* CCI PMU has 4 counters + 1 cycle counter */
81
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +000082/* Types of interfaces that can generate events */
83enum {
84 CCI_IF_SLAVE,
85 CCI_IF_MASTER,
86 CCI_IF_MAX,
87};
88
89struct event_range {
90 u32 min;
91 u32 max;
92};
93
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +000094struct cci_pmu_hw_events {
95 struct perf_event *events[CCI_PMU_MAX_HW_EVENTS];
96 unsigned long used_mask[BITS_TO_LONGS(CCI_PMU_MAX_HW_EVENTS)];
97 raw_spinlock_t pmu_lock;
98};
99
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000100struct cci_pmu_model {
101 char *name;
102 struct event_range event_ranges[CCI_IF_MAX];
103};
104
105static struct cci_pmu_model cci_pmu_models[];
106
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +0000107struct cci_pmu {
108 void __iomem *base;
109 struct pmu pmu;
110 int nr_irqs;
111 int irqs[CCI_PMU_MAX_HW_EVENTS];
112 unsigned long active_irqs;
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000113 const struct cci_pmu_model *model;
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +0000114 struct cci_pmu_hw_events hw_events;
115 struct platform_device *plat_device;
116 int num_events;
117 atomic_t active_events;
118 struct mutex reserve_mutex;
119 cpumask_t cpus;
120};
121static struct cci_pmu *pmu;
122
123#define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu))
124
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100125/* Port ids */
126#define CCI_PORT_S0 0
127#define CCI_PORT_S1 1
128#define CCI_PORT_S2 2
129#define CCI_PORT_S3 3
130#define CCI_PORT_S4 4
131#define CCI_PORT_M0 5
132#define CCI_PORT_M1 6
133#define CCI_PORT_M2 7
134
135#define CCI_REV_R0 0
136#define CCI_REV_R1 1
Punit Agrawal6fb0c4a2014-02-19 12:17:02 +0000137#define CCI_REV_R1_PX 5
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100138
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100139/*
140 * Instead of an event id to monitor CCI cycles, a dedicated counter is
141 * provided. Use 0xff to represent CCI cycles and hope that no future revisions
142 * make use of this event in hardware.
143 */
144enum cci400_perf_events {
145 CCI_PMU_CYCLES = 0xff
146};
147
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100148#define CCI_PMU_CYCLE_CNTR_IDX 0
149#define CCI_PMU_CNTR0_IDX 1
150#define CCI_PMU_CNTR_LAST(cci_pmu) (CCI_PMU_CYCLE_CNTR_IDX + cci_pmu->num_events - 1)
151
152/*
153 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8
154 * ports and bits 4:0 are event codes. There are different event codes
155 * associated with each port type.
156 *
157 * Additionally, the range of events associated with the port types changed
158 * between Rev0 and Rev1.
159 *
160 * The constants below define the range of valid codes for each port type for
161 * the different revisions and are used to validate the event to be monitored.
162 */
163
164#define CCI_REV_R0_SLAVE_PORT_MIN_EV 0x00
165#define CCI_REV_R0_SLAVE_PORT_MAX_EV 0x13
166#define CCI_REV_R0_MASTER_PORT_MIN_EV 0x14
167#define CCI_REV_R0_MASTER_PORT_MAX_EV 0x1a
168
169#define CCI_REV_R1_SLAVE_PORT_MIN_EV 0x00
170#define CCI_REV_R1_SLAVE_PORT_MAX_EV 0x14
171#define CCI_REV_R1_MASTER_PORT_MIN_EV 0x00
172#define CCI_REV_R1_MASTER_PORT_MAX_EV 0x11
173
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100174static int pmu_validate_hw_event(u8 hw_event)
175{
176 u8 ev_source = CCI_PMU_EVENT_SOURCE(hw_event);
177 u8 ev_code = CCI_PMU_EVENT_CODE(hw_event);
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000178 int if_type;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100179
180 switch (ev_source) {
181 case CCI_PORT_S0:
182 case CCI_PORT_S1:
183 case CCI_PORT_S2:
184 case CCI_PORT_S3:
185 case CCI_PORT_S4:
186 /* Slave Interface */
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000187 if_type = CCI_IF_SLAVE;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100188 break;
189 case CCI_PORT_M0:
190 case CCI_PORT_M1:
191 case CCI_PORT_M2:
192 /* Master Interface */
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000193 if_type = CCI_IF_MASTER;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100194 break;
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000195 default:
196 return -ENOENT;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100197 }
198
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000199 if (ev_code >= pmu->model->event_ranges[if_type].min &&
200 ev_code <= pmu->model->event_ranges[if_type].max)
201 return hw_event;
202
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100203 return -ENOENT;
204}
205
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +0000206static int probe_cci_revision(void)
207{
208 int rev;
209 rev = readl_relaxed(cci_ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK;
210 rev >>= CCI_PID2_REV_SHIFT;
211
212 if (rev < CCI_REV_R1_PX)
213 return CCI_REV_R0;
214 else
215 return CCI_REV_R1;
216}
217
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000218static const struct cci_pmu_model *probe_cci_model(struct platform_device *pdev)
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +0000219{
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000220 return &cci_pmu_models[probe_cci_revision()];
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +0000221}
222
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100223static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100224{
225 return CCI_PMU_CYCLE_CNTR_IDX <= idx &&
226 idx <= CCI_PMU_CNTR_LAST(cci_pmu);
227}
228
229static u32 pmu_read_register(int idx, unsigned int offset)
230{
231 return readl_relaxed(pmu->base + CCI_PMU_CNTR_BASE(idx) + offset);
232}
233
234static void pmu_write_register(u32 value, int idx, unsigned int offset)
235{
236 return writel_relaxed(value, pmu->base + CCI_PMU_CNTR_BASE(idx) + offset);
237}
238
239static void pmu_disable_counter(int idx)
240{
241 pmu_write_register(0, idx, CCI_PMU_CNTR_CTRL);
242}
243
244static void pmu_enable_counter(int idx)
245{
246 pmu_write_register(1, idx, CCI_PMU_CNTR_CTRL);
247}
248
249static void pmu_set_event(int idx, unsigned long event)
250{
251 event &= CCI_PMU_EVENT_MASK;
252 pmu_write_register(event, idx, CCI_PMU_EVT_SEL);
253}
254
255static u32 pmu_get_max_counters(void)
256{
257 u32 n_cnts = (readl_relaxed(cci_ctrl_base + CCI_PMCR) &
258 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT;
259
260 /* add 1 for cycle counter */
261 return n_cnts + 1;
262}
263
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100264static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100265{
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100266 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100267 struct hw_perf_event *hw_event = &event->hw;
268 unsigned long cci_event = hw_event->config_base & CCI_PMU_EVENT_MASK;
269 int idx;
270
271 if (cci_event == CCI_PMU_CYCLES) {
272 if (test_and_set_bit(CCI_PMU_CYCLE_CNTR_IDX, hw->used_mask))
273 return -EAGAIN;
274
275 return CCI_PMU_CYCLE_CNTR_IDX;
276 }
277
278 for (idx = CCI_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx)
279 if (!test_and_set_bit(idx, hw->used_mask))
280 return idx;
281
282 /* No counters available */
283 return -EAGAIN;
284}
285
286static int pmu_map_event(struct perf_event *event)
287{
288 int mapping;
289 u8 config = event->attr.config & CCI_PMU_EVENT_MASK;
290
291 if (event->attr.type < PERF_TYPE_MAX)
292 return -ENOENT;
293
294 if (config == CCI_PMU_CYCLES)
295 mapping = config;
296 else
297 mapping = pmu_validate_hw_event(config);
298
299 return mapping;
300}
301
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100302static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100303{
304 int i;
305 struct platform_device *pmu_device = cci_pmu->plat_device;
306
307 if (unlikely(!pmu_device))
308 return -ENODEV;
309
310 if (pmu->nr_irqs < 1) {
311 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n");
312 return -ENODEV;
313 }
314
315 /*
316 * Register all available CCI PMU interrupts. In the interrupt handler
317 * we iterate over the counters checking for interrupt source (the
318 * overflowing counter) and clear it.
319 *
320 * This should allow handling of non-unique interrupt for the counters.
321 */
322 for (i = 0; i < pmu->nr_irqs; i++) {
323 int err = request_irq(pmu->irqs[i], handler, IRQF_SHARED,
324 "arm-cci-pmu", cci_pmu);
325 if (err) {
326 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n",
327 pmu->irqs[i]);
328 return err;
329 }
330
331 set_bit(i, &pmu->active_irqs);
332 }
333
334 return 0;
335}
336
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100337static void pmu_free_irq(struct cci_pmu *cci_pmu)
338{
339 int i;
340
341 for (i = 0; i < pmu->nr_irqs; i++) {
342 if (!test_and_clear_bit(i, &pmu->active_irqs))
343 continue;
344
345 free_irq(pmu->irqs[i], cci_pmu);
346 }
347}
348
349static u32 pmu_read_counter(struct perf_event *event)
350{
351 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
352 struct hw_perf_event *hw_counter = &event->hw;
353 int idx = hw_counter->idx;
354 u32 value;
355
356 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
357 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
358 return 0;
359 }
360 value = pmu_read_register(idx, CCI_PMU_CNTR);
361
362 return value;
363}
364
365static void pmu_write_counter(struct perf_event *event, u32 value)
366{
367 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
368 struct hw_perf_event *hw_counter = &event->hw;
369 int idx = hw_counter->idx;
370
371 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx)))
372 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
373 else
374 pmu_write_register(value, idx, CCI_PMU_CNTR);
375}
376
377static u64 pmu_event_update(struct perf_event *event)
378{
379 struct hw_perf_event *hwc = &event->hw;
380 u64 delta, prev_raw_count, new_raw_count;
381
382 do {
383 prev_raw_count = local64_read(&hwc->prev_count);
384 new_raw_count = pmu_read_counter(event);
385 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
386 new_raw_count) != prev_raw_count);
387
388 delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK;
389
390 local64_add(delta, &event->count);
391
392 return new_raw_count;
393}
394
395static void pmu_read(struct perf_event *event)
396{
397 pmu_event_update(event);
398}
399
400void pmu_event_set_period(struct perf_event *event)
401{
402 struct hw_perf_event *hwc = &event->hw;
403 /*
404 * The CCI PMU counters have a period of 2^32. To account for the
405 * possiblity of extreme interrupt latency we program for a period of
406 * half that. Hopefully we can handle the interrupt before another 2^31
407 * events occur and the counter overtakes its previous value.
408 */
409 u64 val = 1ULL << 31;
410 local64_set(&hwc->prev_count, val);
411 pmu_write_counter(event, val);
412}
413
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100414static irqreturn_t pmu_handle_irq(int irq_num, void *dev)
415{
416 unsigned long flags;
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100417 struct cci_pmu *cci_pmu = dev;
418 struct cci_pmu_hw_events *events = &pmu->hw_events;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100419 int idx, handled = IRQ_NONE;
420
421 raw_spin_lock_irqsave(&events->pmu_lock, flags);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100422 /*
423 * Iterate over counters and update the corresponding perf events.
424 * This should work regardless of whether we have per-counter overflow
425 * interrupt or a combined overflow interrupt.
426 */
427 for (idx = CCI_PMU_CYCLE_CNTR_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) {
428 struct perf_event *event = events->events[idx];
429 struct hw_perf_event *hw_counter;
430
431 if (!event)
432 continue;
433
434 hw_counter = &event->hw;
435
436 /* Did this counter overflow? */
Himangi Saraogifc5130d2014-07-30 11:37:35 +0100437 if (!(pmu_read_register(idx, CCI_PMU_OVRFLW) &
438 CCI_PMU_OVRFLW_FLAG))
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100439 continue;
440
441 pmu_write_register(CCI_PMU_OVRFLW_FLAG, idx, CCI_PMU_OVRFLW);
442
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100443 pmu_event_update(event);
444 pmu_event_set_period(event);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100445 handled = IRQ_HANDLED;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100446 }
447 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
448
449 return IRQ_RETVAL(handled);
450}
451
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100452static int cci_pmu_get_hw(struct cci_pmu *cci_pmu)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100453{
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100454 int ret = pmu_request_irq(cci_pmu, pmu_handle_irq);
455 if (ret) {
456 pmu_free_irq(cci_pmu);
457 return ret;
458 }
459 return 0;
460}
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100461
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100462static void cci_pmu_put_hw(struct cci_pmu *cci_pmu)
463{
464 pmu_free_irq(cci_pmu);
465}
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100466
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100467static void hw_perf_event_destroy(struct perf_event *event)
468{
469 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
470 atomic_t *active_events = &cci_pmu->active_events;
471 struct mutex *reserve_mutex = &cci_pmu->reserve_mutex;
472
473 if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) {
474 cci_pmu_put_hw(cci_pmu);
475 mutex_unlock(reserve_mutex);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100476 }
477}
478
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100479static void cci_pmu_enable(struct pmu *pmu)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100480{
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100481 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
482 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
483 int enabled = bitmap_weight(hw_events->used_mask, cci_pmu->num_events);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100484 unsigned long flags;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100485 u32 val;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100486
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100487 if (!enabled)
488 return;
489
490 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100491
492 /* Enable all the PMU counters. */
493 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) | CCI_PMCR_CEN;
494 writel(val, cci_ctrl_base + CCI_PMCR);
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100495 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100496
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100497}
498
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100499static void cci_pmu_disable(struct pmu *pmu)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100500{
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100501 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
502 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100503 unsigned long flags;
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100504 u32 val;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100505
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100506 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100507
508 /* Disable all the PMU counters. */
509 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN;
510 writel(val, cci_ctrl_base + CCI_PMCR);
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100511 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100512}
513
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100514static void cci_pmu_start(struct perf_event *event, int pmu_flags)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100515{
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100516 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
517 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
518 struct hw_perf_event *hwc = &event->hw;
519 int idx = hwc->idx;
520 unsigned long flags;
521
522 /*
523 * To handle interrupt latency, we always reprogram the period
524 * regardlesss of PERF_EF_RELOAD.
525 */
526 if (pmu_flags & PERF_EF_RELOAD)
527 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
528
529 hwc->state = 0;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100530
531 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
532 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100533 return;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100534 }
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100535
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100536 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
537
538 /* Configure the event to count, unless you are counting cycles */
539 if (idx != CCI_PMU_CYCLE_CNTR_IDX)
540 pmu_set_event(idx, hwc->config_base);
541
542 pmu_event_set_period(event);
543 pmu_enable_counter(idx);
544
545 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100546}
547
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100548static void cci_pmu_stop(struct perf_event *event, int pmu_flags)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100549{
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100550 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
551 struct hw_perf_event *hwc = &event->hw;
552 int idx = hwc->idx;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100553
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100554 if (hwc->state & PERF_HES_STOPPED)
555 return;
556
557 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100558 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100559 return;
560 }
561
562 /*
563 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See
564 * cci_pmu_start()
565 */
566 pmu_disable_counter(idx);
567 pmu_event_update(event);
568 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100569}
570
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100571static int cci_pmu_add(struct perf_event *event, int flags)
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100572{
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100573 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
574 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
575 struct hw_perf_event *hwc = &event->hw;
576 int idx;
577 int err = 0;
578
579 perf_pmu_disable(event->pmu);
580
581 /* If we don't have a space for the counter then finish early. */
582 idx = pmu_get_event_idx(hw_events, event);
583 if (idx < 0) {
584 err = idx;
585 goto out;
586 }
587
588 event->hw.idx = idx;
589 hw_events->events[idx] = event;
590
591 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
592 if (flags & PERF_EF_START)
593 cci_pmu_start(event, PERF_EF_RELOAD);
594
595 /* Propagate our changes to the userspace mapping. */
596 perf_event_update_userpage(event);
597
598out:
599 perf_pmu_enable(event->pmu);
600 return err;
601}
602
603static void cci_pmu_del(struct perf_event *event, int flags)
604{
605 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
606 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
607 struct hw_perf_event *hwc = &event->hw;
608 int idx = hwc->idx;
609
610 cci_pmu_stop(event, PERF_EF_UPDATE);
611 hw_events->events[idx] = NULL;
612 clear_bit(idx, hw_events->used_mask);
613
614 perf_event_update_userpage(event);
615}
616
617static int
Suzuki K. Pouloseb1862192015-03-17 18:15:00 +0000618validate_event(struct pmu *cci_pmu,
619 struct cci_pmu_hw_events *hw_events,
620 struct perf_event *event)
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100621{
622 if (is_software_event(event))
623 return 1;
624
Suzuki K. Pouloseb1862192015-03-17 18:15:00 +0000625 /*
626 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
627 * core perf code won't check that the pmu->ctx == leader->ctx
628 * until after pmu->event_init(event).
629 */
630 if (event->pmu != cci_pmu)
631 return 0;
632
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100633 if (event->state < PERF_EVENT_STATE_OFF)
634 return 1;
635
636 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
637 return 1;
638
639 return pmu_get_event_idx(hw_events, event) >= 0;
640}
641
642static int
643validate_group(struct perf_event *event)
644{
645 struct perf_event *sibling, *leader = event->group_leader;
646 struct cci_pmu_hw_events fake_pmu = {
647 /*
648 * Initialise the fake PMU. We only need to populate the
649 * used_mask for the purposes of validation.
650 */
651 .used_mask = CPU_BITS_NONE,
652 };
653
Suzuki K. Pouloseb1862192015-03-17 18:15:00 +0000654 if (!validate_event(event->pmu, &fake_pmu, leader))
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100655 return -EINVAL;
656
657 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
Suzuki K. Pouloseb1862192015-03-17 18:15:00 +0000658 if (!validate_event(event->pmu, &fake_pmu, sibling))
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100659 return -EINVAL;
660 }
661
Suzuki K. Pouloseb1862192015-03-17 18:15:00 +0000662 if (!validate_event(event->pmu, &fake_pmu, event))
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100663 return -EINVAL;
664
665 return 0;
666}
667
668static int
669__hw_perf_event_init(struct perf_event *event)
670{
671 struct hw_perf_event *hwc = &event->hw;
672 int mapping;
673
674 mapping = pmu_map_event(event);
675
676 if (mapping < 0) {
677 pr_debug("event %x:%llx not supported\n", event->attr.type,
678 event->attr.config);
679 return mapping;
680 }
681
682 /*
683 * We don't assign an index until we actually place the event onto
684 * hardware. Use -1 to signify that we haven't decided where to put it
685 * yet.
686 */
687 hwc->idx = -1;
688 hwc->config_base = 0;
689 hwc->config = 0;
690 hwc->event_base = 0;
691
692 /*
693 * Store the event encoding into the config_base field.
694 */
695 hwc->config_base |= (unsigned long)mapping;
696
697 /*
698 * Limit the sample_period to half of the counter width. That way, the
699 * new counter value is far less likely to overtake the previous one
700 * unless you have some serious IRQ latency issues.
701 */
702 hwc->sample_period = CCI_PMU_CNTR_MASK >> 1;
703 hwc->last_period = hwc->sample_period;
704 local64_set(&hwc->period_left, hwc->sample_period);
705
706 if (event->group_leader != event) {
707 if (validate_group(event) != 0)
708 return -EINVAL;
709 }
710
711 return 0;
712}
713
714static int cci_pmu_event_init(struct perf_event *event)
715{
716 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
717 atomic_t *active_events = &cci_pmu->active_events;
718 int err = 0;
719 int cpu;
720
721 if (event->attr.type != event->pmu->type)
722 return -ENOENT;
723
724 /* Shared by all CPUs, no meaningful state to sample */
725 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
726 return -EOPNOTSUPP;
727
728 /* We have no filtering of any kind */
729 if (event->attr.exclude_user ||
730 event->attr.exclude_kernel ||
731 event->attr.exclude_hv ||
732 event->attr.exclude_idle ||
733 event->attr.exclude_host ||
734 event->attr.exclude_guest)
735 return -EINVAL;
736
737 /*
738 * Following the example set by other "uncore" PMUs, we accept any CPU
739 * and rewrite its affinity dynamically rather than having perf core
740 * handle cpu == -1 and pid == -1 for this case.
741 *
742 * The perf core will pin online CPUs for the duration of this call and
743 * the event being installed into its context, so the PMU's CPU can't
744 * change under our feet.
745 */
746 cpu = cpumask_first(&cci_pmu->cpus);
747 if (event->cpu < 0 || cpu < 0)
748 return -EINVAL;
749 event->cpu = cpu;
750
751 event->destroy = hw_perf_event_destroy;
752 if (!atomic_inc_not_zero(active_events)) {
753 mutex_lock(&cci_pmu->reserve_mutex);
754 if (atomic_read(active_events) == 0)
755 err = cci_pmu_get_hw(cci_pmu);
756 if (!err)
757 atomic_inc(active_events);
758 mutex_unlock(&cci_pmu->reserve_mutex);
759 }
760 if (err)
761 return err;
762
763 err = __hw_perf_event_init(event);
764 if (err)
765 hw_perf_event_destroy(event);
766
767 return err;
768}
769
770static ssize_t pmu_attr_cpumask_show(struct device *dev,
771 struct device_attribute *attr, char *buf)
772{
Tejun Heo660e5ec2015-02-13 14:37:20 -0800773 int n = scnprintf(buf, PAGE_SIZE - 1, "%*pbl",
774 cpumask_pr_args(&pmu->cpus));
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100775 buf[n++] = '\n';
776 buf[n] = '\0';
777 return n;
778}
779
780static DEVICE_ATTR(cpumask, S_IRUGO, pmu_attr_cpumask_show, NULL);
781
782static struct attribute *pmu_attrs[] = {
783 &dev_attr_cpumask.attr,
784 NULL,
785};
786
787static struct attribute_group pmu_attr_group = {
788 .attrs = pmu_attrs,
789};
790
791static const struct attribute_group *pmu_attr_groups[] = {
792 &pmu_attr_group,
793 NULL
794};
795
796static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev)
797{
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000798 char *name = cci_pmu->model->name;
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100799 cci_pmu->pmu = (struct pmu) {
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000800 .name = cci_pmu->model->name,
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100801 .task_ctx_nr = perf_invalid_context,
802 .pmu_enable = cci_pmu_enable,
803 .pmu_disable = cci_pmu_disable,
804 .event_init = cci_pmu_event_init,
805 .add = cci_pmu_add,
806 .del = cci_pmu_del,
807 .start = cci_pmu_start,
808 .stop = cci_pmu_stop,
809 .read = pmu_read,
810 .attr_groups = pmu_attr_groups,
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100811 };
812
813 cci_pmu->plat_device = pdev;
814 cci_pmu->num_events = pmu_get_max_counters();
815
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100816 return perf_pmu_register(&cci_pmu->pmu, name, -1);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100817}
818
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100819static int cci_pmu_cpu_notifier(struct notifier_block *self,
820 unsigned long action, void *hcpu)
821{
822 unsigned int cpu = (long)hcpu;
823 unsigned int target;
824
825 switch (action & ~CPU_TASKS_FROZEN) {
826 case CPU_DOWN_PREPARE:
827 if (!cpumask_test_and_clear_cpu(cpu, &pmu->cpus))
828 break;
829 target = cpumask_any_but(cpu_online_mask, cpu);
830 if (target < 0) // UP, last CPU
831 break;
832 /*
833 * TODO: migrate context once core races on event->ctx have
834 * been fixed.
835 */
836 cpumask_set_cpu(target, &pmu->cpus);
837 default:
838 break;
839 }
840
841 return NOTIFY_OK;
842}
843
844static struct notifier_block cci_pmu_cpu_nb = {
845 .notifier_call = cci_pmu_cpu_notifier,
846 /*
847 * to migrate uncore events, our notifier should be executed
848 * before perf core's notifier.
849 */
850 .priority = CPU_PRI_PERF + 1,
851};
852
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000853static struct cci_pmu_model cci_pmu_models[] = {
854 [CCI_REV_R0] = {
855 .name = "CCI_400",
856 .event_ranges = {
857 [CCI_IF_SLAVE] = {
858 CCI_REV_R0_SLAVE_PORT_MIN_EV,
859 CCI_REV_R0_SLAVE_PORT_MAX_EV,
860 },
861 [CCI_IF_MASTER] = {
862 CCI_REV_R0_MASTER_PORT_MIN_EV,
863 CCI_REV_R0_MASTER_PORT_MAX_EV,
864 },
865 },
866 },
867 [CCI_REV_R1] = {
868 .name = "CCI_400_r1",
869 .event_ranges = {
870 [CCI_IF_SLAVE] = {
871 CCI_REV_R1_SLAVE_PORT_MIN_EV,
872 CCI_REV_R1_SLAVE_PORT_MAX_EV,
873 },
874 [CCI_IF_MASTER] = {
875 CCI_REV_R1_MASTER_PORT_MIN_EV,
876 CCI_REV_R1_MASTER_PORT_MAX_EV,
877 },
878 },
879 },
880};
881
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100882static const struct of_device_id arm_cci_pmu_matches[] = {
883 {
884 .compatible = "arm,cci-400-pmu",
885 },
886 {},
887};
888
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000889static inline const struct cci_pmu_model *get_cci_model(struct platform_device *pdev)
890{
891 const struct of_device_id *match = of_match_node(arm_cci_pmu_matches,
892 pdev->dev.of_node);
893 if (!match)
894 return NULL;
895
896 return probe_cci_model(pdev);
897}
898
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +0000899static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs)
900{
901 int i;
902
903 for (i = 0; i < nr_irqs; i++)
904 if (irq == irqs[i])
905 return true;
906
907 return false;
908}
909
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100910static int cci_pmu_probe(struct platform_device *pdev)
911{
912 struct resource *res;
913 int i, ret, irq;
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000914 const struct cci_pmu_model *model;
915
916 model = get_cci_model(pdev);
917 if (!model) {
918 dev_warn(&pdev->dev, "CCI PMU version not supported\n");
919 return -ENODEV;
920 }
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100921
922 pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);
923 if (!pmu)
924 return -ENOMEM;
925
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000926 pmu->model = model;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100927 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100928 pmu->base = devm_ioremap_resource(&pdev->dev, res);
Wei Yongjunfee4f2c2013-09-22 06:04:23 +0100929 if (IS_ERR(pmu->base))
930 return -ENOMEM;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100931
932 /*
933 * CCI PMU has 5 overflow signals - one per counter; but some may be tied
934 * together to a common interrupt.
935 */
936 pmu->nr_irqs = 0;
937 for (i = 0; i < CCI_PMU_MAX_HW_EVENTS; i++) {
938 irq = platform_get_irq(pdev, i);
939 if (irq < 0)
940 break;
941
942 if (is_duplicate_irq(irq, pmu->irqs, pmu->nr_irqs))
943 continue;
944
945 pmu->irqs[pmu->nr_irqs++] = irq;
946 }
947
948 /*
949 * Ensure that the device tree has as many interrupts as the number
950 * of counters.
951 */
952 if (i < CCI_PMU_MAX_HW_EVENTS) {
953 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n",
954 i, CCI_PMU_MAX_HW_EVENTS);
Wei Yongjunfee4f2c2013-09-22 06:04:23 +0100955 return -EINVAL;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100956 }
957
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100958 raw_spin_lock_init(&pmu->hw_events.pmu_lock);
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100959 mutex_init(&pmu->reserve_mutex);
960 atomic_set(&pmu->active_events, 0);
961 cpumask_set_cpu(smp_processor_id(), &pmu->cpus);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100962
Mark Rutlandc6f85cb2014-06-30 12:20:21 +0100963 ret = register_cpu_notifier(&cci_pmu_cpu_nb);
964 if (ret)
965 return ret;
966
967 ret = cci_pmu_init(pmu, pdev);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100968 if (ret)
Wei Yongjunfee4f2c2013-09-22 06:04:23 +0100969 return ret;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100970
Suzuki K. Poulosefc17c832015-03-18 12:24:39 +0000971 pr_info("ARM %s PMU driver probed", pmu->model->name);
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100972 return 0;
Punit Agrawalb91c8f22013-08-22 14:41:51 +0100973}
974
975static int cci_platform_probe(struct platform_device *pdev)
976{
977 if (!cci_probed())
978 return -ENODEV;
979
980 return of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
981}
982
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +0000983static struct platform_driver cci_pmu_driver = {
984 .driver = {
985 .name = DRIVER_NAME_PMU,
986 .of_match_table = arm_cci_pmu_matches,
987 },
988 .probe = cci_pmu_probe,
989};
990
991static struct platform_driver cci_platform_driver = {
992 .driver = {
993 .name = DRIVER_NAME,
994 .of_match_table = arm_cci_matches,
995 },
996 .probe = cci_platform_probe,
997};
998
999static int __init cci_platform_init(void)
1000{
1001 int ret;
1002
1003 ret = platform_driver_register(&cci_pmu_driver);
1004 if (ret)
1005 return ret;
1006
1007 return platform_driver_register(&cci_platform_driver);
1008}
1009
1010#else /* !CONFIG_HW_PERF_EVENTS */
1011
1012static int __init cci_platform_init(void)
1013{
1014 return 0;
1015}
1016
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001017#endif /* CONFIG_HW_PERF_EVENTS */
1018
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +00001019#define CCI_PORT_CTRL 0x0
1020#define CCI_CTRL_STATUS 0xc
1021
1022#define CCI_ENABLE_SNOOP_REQ 0x1
1023#define CCI_ENABLE_DVM_REQ 0x2
1024#define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ)
1025
1026enum cci_ace_port_type {
1027 ACE_INVALID_PORT = 0x0,
1028 ACE_PORT,
1029 ACE_LITE_PORT,
1030};
1031
1032struct cci_ace_port {
1033 void __iomem *base;
1034 unsigned long phys;
1035 enum cci_ace_port_type type;
1036 struct device_node *dn;
1037};
1038
1039static struct cci_ace_port *ports;
1040static unsigned int nb_cci_ports;
1041
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001042struct cpu_port {
1043 u64 mpidr;
1044 u32 port;
1045};
Nicolas Pitre62158f82013-05-21 23:34:41 -04001046
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001047/*
1048 * Use the port MSB as valid flag, shift can be made dynamic
1049 * by computing number of bits required for port indexes.
1050 * Code disabling CCI cpu ports runs with D-cache invalidated
1051 * and SCTLR bit clear so data accesses must be kept to a minimum
1052 * to improve performance; for now shift is left static to
1053 * avoid one more data access while disabling the CCI port.
1054 */
1055#define PORT_VALID_SHIFT 31
1056#define PORT_VALID (0x1 << PORT_VALID_SHIFT)
1057
1058static inline void init_cpu_port(struct cpu_port *port, u32 index, u64 mpidr)
1059{
1060 port->port = PORT_VALID | index;
1061 port->mpidr = mpidr;
1062}
1063
1064static inline bool cpu_port_is_valid(struct cpu_port *port)
1065{
1066 return !!(port->port & PORT_VALID);
1067}
1068
1069static inline bool cpu_port_match(struct cpu_port *port, u64 mpidr)
1070{
1071 return port->mpidr == (mpidr & MPIDR_HWID_BITMASK);
1072}
1073
1074static struct cpu_port cpu_port[NR_CPUS];
1075
1076/**
1077 * __cci_ace_get_port - Function to retrieve the port index connected to
1078 * a cpu or device.
1079 *
1080 * @dn: device node of the device to look-up
1081 * @type: port type
1082 *
1083 * Return value:
1084 * - CCI port index if success
1085 * - -ENODEV if failure
1086 */
1087static int __cci_ace_get_port(struct device_node *dn, int type)
1088{
1089 int i;
1090 bool ace_match;
1091 struct device_node *cci_portn;
1092
1093 cci_portn = of_parse_phandle(dn, "cci-control-port", 0);
1094 for (i = 0; i < nb_cci_ports; i++) {
1095 ace_match = ports[i].type == type;
1096 if (ace_match && cci_portn == ports[i].dn)
1097 return i;
1098 }
1099 return -ENODEV;
1100}
1101
1102int cci_ace_get_port(struct device_node *dn)
1103{
1104 return __cci_ace_get_port(dn, ACE_LITE_PORT);
1105}
1106EXPORT_SYMBOL_GPL(cci_ace_get_port);
1107
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001108static void cci_ace_init_ports(void)
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001109{
Sudeep KarkadaNagesha78b4d6e2013-06-17 14:51:48 +01001110 int port, cpu;
1111 struct device_node *cpun;
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001112
1113 /*
1114 * Port index look-up speeds up the function disabling ports by CPU,
1115 * since the logical to port index mapping is done once and does
1116 * not change after system boot.
1117 * The stashed index array is initialized for all possible CPUs
1118 * at probe time.
1119 */
Sudeep KarkadaNagesha78b4d6e2013-06-17 14:51:48 +01001120 for_each_possible_cpu(cpu) {
1121 /* too early to use cpu->of_node */
1122 cpun = of_get_cpu_node(cpu, NULL);
1123
1124 if (WARN(!cpun, "Missing cpu device node\n"))
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001125 continue;
1126
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001127 port = __cci_ace_get_port(cpun, ACE_PORT);
1128 if (port < 0)
1129 continue;
1130
1131 init_cpu_port(&cpu_port[cpu], port, cpu_logical_map(cpu));
1132 }
1133
1134 for_each_possible_cpu(cpu) {
1135 WARN(!cpu_port_is_valid(&cpu_port[cpu]),
1136 "CPU %u does not have an associated CCI port\n",
1137 cpu);
1138 }
1139}
1140/*
1141 * Functions to enable/disable a CCI interconnect slave port
1142 *
1143 * They are called by low-level power management code to disable slave
1144 * interfaces snoops and DVM broadcast.
1145 * Since they may execute with cache data allocation disabled and
1146 * after the caches have been cleaned and invalidated the functions provide
1147 * no explicit locking since they may run with D-cache disabled, so normal
1148 * cacheable kernel locks based on ldrex/strex may not work.
1149 * Locking has to be provided by BSP implementations to ensure proper
1150 * operations.
1151 */
1152
1153/**
1154 * cci_port_control() - function to control a CCI port
1155 *
1156 * @port: index of the port to setup
1157 * @enable: if true enables the port, if false disables it
1158 */
1159static void notrace cci_port_control(unsigned int port, bool enable)
1160{
1161 void __iomem *base = ports[port].base;
1162
1163 writel_relaxed(enable ? CCI_ENABLE_REQ : 0, base + CCI_PORT_CTRL);
1164 /*
1165 * This function is called from power down procedures
1166 * and must not execute any instruction that might
1167 * cause the processor to be put in a quiescent state
1168 * (eg wfi). Hence, cpu_relax() can not be added to this
1169 * read loop to optimize power, since it might hide possibly
1170 * disruptive operations.
1171 */
1172 while (readl_relaxed(cci_ctrl_base + CCI_CTRL_STATUS) & 0x1)
1173 ;
1174}
1175
1176/**
1177 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU
1178 * reference
1179 *
1180 * @mpidr: mpidr of the CPU whose CCI port should be disabled
1181 *
1182 * Disabling a CCI port for a CPU implies disabling the CCI port
1183 * controlling that CPU cluster. Code disabling CPU CCI ports
1184 * must make sure that the CPU running the code is the last active CPU
1185 * in the cluster ie all other CPUs are quiescent in a low power state.
1186 *
1187 * Return:
1188 * 0 on success
1189 * -ENODEV on port look-up failure
1190 */
1191int notrace cci_disable_port_by_cpu(u64 mpidr)
1192{
1193 int cpu;
1194 bool is_valid;
1195 for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
1196 is_valid = cpu_port_is_valid(&cpu_port[cpu]);
1197 if (is_valid && cpu_port_match(&cpu_port[cpu], mpidr)) {
1198 cci_port_control(cpu_port[cpu].port, false);
1199 return 0;
1200 }
1201 }
1202 return -ENODEV;
1203}
1204EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu);
1205
1206/**
Nicolas Pitre62158f82013-05-21 23:34:41 -04001207 * cci_enable_port_for_self() - enable a CCI port for calling CPU
1208 *
1209 * Enabling a CCI port for the calling CPU implies enabling the CCI
1210 * port controlling that CPU's cluster. Caller must make sure that the
1211 * CPU running the code is the first active CPU in the cluster and all
1212 * other CPUs are quiescent in a low power state or waiting for this CPU
1213 * to complete the CCI initialization.
1214 *
1215 * Because this is called when the MMU is still off and with no stack,
1216 * the code must be position independent and ideally rely on callee
1217 * clobbered registers only. To achieve this we must code this function
1218 * entirely in assembler.
1219 *
1220 * On success this returns with the proper CCI port enabled. In case of
1221 * any failure this never returns as the inability to enable the CCI is
1222 * fatal and there is no possible recovery at this stage.
1223 */
1224asmlinkage void __naked cci_enable_port_for_self(void)
1225{
1226 asm volatile ("\n"
Arnd Bergmannf4902492013-06-03 15:15:36 +02001227" .arch armv7-a\n"
Nicolas Pitre62158f82013-05-21 23:34:41 -04001228" mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n"
1229" and r0, r0, #"__stringify(MPIDR_HWID_BITMASK)" \n"
1230" adr r1, 5f \n"
1231" ldr r2, [r1] \n"
1232" add r1, r1, r2 @ &cpu_port \n"
1233" add ip, r1, %[sizeof_cpu_port] \n"
1234
1235 /* Loop over the cpu_port array looking for a matching MPIDR */
1236"1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n"
1237" cmp r2, r0 @ compare MPIDR \n"
1238" bne 2f \n"
1239
1240 /* Found a match, now test port validity */
1241" ldr r3, [r1, %[offsetof_cpu_port_port]] \n"
1242" tst r3, #"__stringify(PORT_VALID)" \n"
1243" bne 3f \n"
1244
1245 /* no match, loop with the next cpu_port entry */
1246"2: add r1, r1, %[sizeof_struct_cpu_port] \n"
1247" cmp r1, ip @ done? \n"
1248" blo 1b \n"
1249
1250 /* CCI port not found -- cheaply try to stall this CPU */
1251"cci_port_not_found: \n"
1252" wfi \n"
1253" wfe \n"
1254" b cci_port_not_found \n"
1255
1256 /* Use matched port index to look up the corresponding ports entry */
1257"3: bic r3, r3, #"__stringify(PORT_VALID)" \n"
1258" adr r0, 6f \n"
1259" ldmia r0, {r1, r2} \n"
1260" sub r1, r1, r0 @ virt - phys \n"
1261" ldr r0, [r0, r2] @ *(&ports) \n"
1262" mov r2, %[sizeof_struct_ace_port] \n"
1263" mla r0, r2, r3, r0 @ &ports[index] \n"
1264" sub r0, r0, r1 @ virt_to_phys() \n"
1265
1266 /* Enable the CCI port */
1267" ldr r0, [r0, %[offsetof_port_phys]] \n"
Victor Kamenskyfdb07ae2013-10-15 21:50:34 -07001268" mov r3, %[cci_enable_req]\n"
Nicolas Pitre62158f82013-05-21 23:34:41 -04001269" str r3, [r0, #"__stringify(CCI_PORT_CTRL)"] \n"
1270
1271 /* poll the status reg for completion */
1272" adr r1, 7f \n"
1273" ldr r0, [r1] \n"
1274" ldr r0, [r0, r1] @ cci_ctrl_base \n"
1275"4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS)"] \n"
Victor Kamenskyfdb07ae2013-10-15 21:50:34 -07001276" tst r1, %[cci_control_status_bits] \n"
Nicolas Pitre62158f82013-05-21 23:34:41 -04001277" bne 4b \n"
1278
1279" mov r0, #0 \n"
1280" bx lr \n"
1281
1282" .align 2 \n"
1283"5: .word cpu_port - . \n"
1284"6: .word . \n"
1285" .word ports - 6b \n"
1286"7: .word cci_ctrl_phys - . \n"
1287 : :
1288 [sizeof_cpu_port] "i" (sizeof(cpu_port)),
Victor Kamenskyfdb07ae2013-10-15 21:50:34 -07001289 [cci_enable_req] "i" cpu_to_le32(CCI_ENABLE_REQ),
1290 [cci_control_status_bits] "i" cpu_to_le32(1),
Nicolas Pitre62158f82013-05-21 23:34:41 -04001291#ifndef __ARMEB__
1292 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)),
1293#else
1294 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)+4),
1295#endif
1296 [offsetof_cpu_port_port] "i" (offsetof(struct cpu_port, port)),
1297 [sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)),
1298 [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)),
1299 [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) );
1300
1301 unreachable();
1302}
1303
1304/**
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001305 * __cci_control_port_by_device() - function to control a CCI port by device
1306 * reference
1307 *
1308 * @dn: device node pointer of the device whose CCI port should be
1309 * controlled
1310 * @enable: if true enables the port, if false disables it
1311 *
1312 * Return:
1313 * 0 on success
1314 * -ENODEV on port look-up failure
1315 */
1316int notrace __cci_control_port_by_device(struct device_node *dn, bool enable)
1317{
1318 int port;
1319
1320 if (!dn)
1321 return -ENODEV;
1322
1323 port = __cci_ace_get_port(dn, ACE_LITE_PORT);
1324 if (WARN_ONCE(port < 0, "node %s ACE lite port look-up failure\n",
1325 dn->full_name))
1326 return -ENODEV;
1327 cci_port_control(port, enable);
1328 return 0;
1329}
1330EXPORT_SYMBOL_GPL(__cci_control_port_by_device);
1331
1332/**
1333 * __cci_control_port_by_index() - function to control a CCI port by port index
1334 *
1335 * @port: port index previously retrieved with cci_ace_get_port()
1336 * @enable: if true enables the port, if false disables it
1337 *
1338 * Return:
1339 * 0 on success
1340 * -ENODEV on port index out of range
1341 * -EPERM if operation carried out on an ACE PORT
1342 */
1343int notrace __cci_control_port_by_index(u32 port, bool enable)
1344{
1345 if (port >= nb_cci_ports || ports[port].type == ACE_INVALID_PORT)
1346 return -ENODEV;
1347 /*
1348 * CCI control for ports connected to CPUS is extremely fragile
1349 * and must be made to go through a specific and controlled
1350 * interface (ie cci_disable_port_by_cpu(); control by general purpose
1351 * indexing is therefore disabled for ACE ports.
1352 */
1353 if (ports[port].type == ACE_PORT)
1354 return -EPERM;
1355
1356 cci_port_control(port, enable);
1357 return 0;
1358}
1359EXPORT_SYMBOL_GPL(__cci_control_port_by_index);
1360
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001361static const struct of_device_id arm_cci_ctrl_if_matches[] = {
1362 {.compatible = "arm,cci-400-ctrl-if", },
1363 {},
1364};
1365
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +00001366static int cci_probe_ports(struct device_node *np)
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001367{
1368 struct cci_nb_ports const *cci_config;
1369 int ret, i, nb_ace = 0, nb_ace_lite = 0;
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +00001370 struct device_node *cp;
Nicolas Pitre62158f82013-05-21 23:34:41 -04001371 struct resource res;
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001372 const char *match_str;
1373 bool is_ace;
1374
Abhilash Kesavan896ddd62015-01-10 08:41:35 +05301375
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001376 cci_config = of_match_node(arm_cci_matches, np)->data;
1377 if (!cci_config)
1378 return -ENODEV;
1379
1380 nb_cci_ports = cci_config->nb_ace + cci_config->nb_ace_lite;
1381
Lorenzo Pieralisi7c762032014-01-27 10:50:37 +00001382 ports = kcalloc(nb_cci_ports, sizeof(*ports), GFP_KERNEL);
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001383 if (!ports)
1384 return -ENOMEM;
1385
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001386 for_each_child_of_node(np, cp) {
1387 if (!of_match_node(arm_cci_ctrl_if_matches, cp))
1388 continue;
1389
1390 i = nb_ace + nb_ace_lite;
1391
1392 if (i >= nb_cci_ports)
1393 break;
1394
1395 if (of_property_read_string(cp, "interface-type",
1396 &match_str)) {
1397 WARN(1, "node %s missing interface-type property\n",
1398 cp->full_name);
1399 continue;
1400 }
1401 is_ace = strcmp(match_str, "ace") == 0;
1402 if (!is_ace && strcmp(match_str, "ace-lite")) {
1403 WARN(1, "node %s containing invalid interface-type property, skipping it\n",
1404 cp->full_name);
1405 continue;
1406 }
1407
Nicolas Pitre62158f82013-05-21 23:34:41 -04001408 ret = of_address_to_resource(cp, 0, &res);
1409 if (!ret) {
1410 ports[i].base = ioremap(res.start, resource_size(&res));
1411 ports[i].phys = res.start;
1412 }
1413 if (ret || !ports[i].base) {
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001414 WARN(1, "unable to ioremap CCI port %d\n", i);
1415 continue;
1416 }
1417
1418 if (is_ace) {
1419 if (WARN_ON(nb_ace >= cci_config->nb_ace))
1420 continue;
1421 ports[i].type = ACE_PORT;
1422 ++nb_ace;
1423 } else {
1424 if (WARN_ON(nb_ace_lite >= cci_config->nb_ace_lite))
1425 continue;
1426 ports[i].type = ACE_LITE_PORT;
1427 ++nb_ace_lite;
1428 }
1429 ports[i].dn = cp;
1430 }
1431
1432 /* initialize a stashed array of ACE ports to speed-up look-up */
1433 cci_ace_init_ports();
1434
1435 /*
1436 * Multi-cluster systems may need this data when non-coherent, during
1437 * cluster power-up/power-down. Make sure it reaches main memory.
1438 */
1439 sync_cache_w(&cci_ctrl_base);
Nicolas Pitre62158f82013-05-21 23:34:41 -04001440 sync_cache_w(&cci_ctrl_phys);
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001441 sync_cache_w(&ports);
1442 sync_cache_w(&cpu_port);
1443 __sync_cache_range_w(ports, sizeof(*ports) * nb_cci_ports);
1444 pr_info("ARM CCI driver probed\n");
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +00001445
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001446 return 0;
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +00001447}
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001448
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +00001449static int cci_probe(void)
1450{
1451 int ret;
1452 struct device_node *np;
1453 struct resource res;
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001454
Suzuki K. Poulosef6b9e832015-03-18 12:24:38 +00001455 np = of_find_matching_node(NULL, arm_cci_matches);
1456 if(!np || !of_device_is_available(np))
1457 return -ENODEV;
1458
1459 ret = of_address_to_resource(np, 0, &res);
1460 if (!ret) {
1461 cci_ctrl_base = ioremap(res.start, resource_size(&res));
1462 cci_ctrl_phys = res.start;
1463 }
1464 if (ret || !cci_ctrl_base) {
1465 WARN(1, "unable to ioremap CCI ctrl\n");
1466 return -ENXIO;
1467 }
1468
1469 return cci_probe_ports(np);
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001470}
1471
1472static int cci_init_status = -EAGAIN;
1473static DEFINE_MUTEX(cci_probing);
1474
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001475static int cci_init(void)
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001476{
1477 if (cci_init_status != -EAGAIN)
1478 return cci_init_status;
1479
1480 mutex_lock(&cci_probing);
1481 if (cci_init_status == -EAGAIN)
1482 cci_init_status = cci_probe();
1483 mutex_unlock(&cci_probing);
1484 return cci_init_status;
1485}
1486
1487/*
1488 * To sort out early init calls ordering a helper function is provided to
1489 * check if the CCI driver has beed initialized. Function check if the driver
1490 * has been initialized, if not it calls the init function that probes
1491 * the driver and updates the return value.
1492 */
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001493bool cci_probed(void)
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001494{
1495 return cci_init() == 0;
1496}
1497EXPORT_SYMBOL_GPL(cci_probed);
1498
1499early_initcall(cci_init);
Punit Agrawalb91c8f22013-08-22 14:41:51 +01001500core_initcall(cci_platform_init);
Lorenzo Pieralisied69bdd2012-07-13 15:55:52 +01001501MODULE_LICENSE("GPL");
1502MODULE_DESCRIPTION("ARM CCI support");