blob: 401554074de907dea1bca63158148384319e0afb [file] [log] [blame]
Thomas Gleixner0793a612008-12-04 20:12:29 +01001/*
Ingo Molnar57c0c152009-09-21 12:20:38 +02002 * Performance events:
Thomas Gleixner0793a612008-12-04 20:12:29 +01003 *
Ingo Molnara3084442009-06-11 14:44:26 +02004 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
Ingo Molnare7e7ee22011-05-04 08:42:29 +02005 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
Thomas Gleixner0793a612008-12-04 20:12:29 +01007 *
Ingo Molnar57c0c152009-09-21 12:20:38 +02008 * Data type definitions, declarations, prototypes.
Thomas Gleixner0793a612008-12-04 20:12:29 +01009 *
Ingo Molnara3084442009-06-11 14:44:26 +020010 * Started by: Thomas Gleixner and Ingo Molnar
Thomas Gleixner0793a612008-12-04 20:12:29 +010011 *
Ingo Molnar57c0c152009-09-21 12:20:38 +020012 * For licencing details see kernel-base/COPYING
Thomas Gleixner0793a612008-12-04 20:12:29 +010013 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +020014#ifndef _LINUX_PERF_EVENT_H
15#define _LINUX_PERF_EVENT_H
Thomas Gleixner0793a612008-12-04 20:12:29 +010016
David Howells607ca462012-10-13 10:46:48 +010017#include <uapi/linux/perf_event.h>
Peter Zijlstraa4be7c22009-08-19 11:18:27 +020018
Paul Mackerrasd859e292009-01-17 18:10:22 +110019/*
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110020 * Kernel-internal data types and definitions:
Ingo Molnar9f66a382008-12-10 12:33:23 +010021 */
22
Ingo Molnarcdd6c482009-09-21 12:02:48 +020023#ifdef CONFIG_PERF_EVENTS
24# include <asm/perf_event.h>
Peter Zijlstra7be79232010-06-09 11:57:23 +020025# include <asm/local64.h>
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110026#endif
27
Zhang, Yanmin39447b32010-04-19 13:32:41 +080028struct perf_guest_info_callbacks {
Ingo Molnare7e7ee22011-05-04 08:42:29 +020029 int (*is_in_guest)(void);
30 int (*is_user_mode)(void);
31 unsigned long (*get_guest_ip)(void);
Zhang, Yanmin39447b32010-04-19 13:32:41 +080032};
33
Arnd Bergmann2ff6cfd2009-12-07 17:12:58 +010034#ifdef CONFIG_HAVE_HW_BREAKPOINT
35#include <asm/hw_breakpoint.h>
36#endif
37
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110038#include <linux/list.h>
39#include <linux/mutex.h>
40#include <linux/rculist.h>
41#include <linux/rcupdate.h>
42#include <linux/spinlock.h>
Peter Zijlstrad6d020e2009-03-13 12:21:35 +010043#include <linux/hrtimer.h>
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +020044#include <linux/fs.h>
Peter Zijlstra709e50c2009-06-02 14:13:15 +020045#include <linux/pid_namespace.h>
Peter Zijlstra906010b2009-09-21 16:08:49 +020046#include <linux/workqueue.h>
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +010047#include <linux/ftrace.h>
Peter Zijlstra85cfabb2010-03-11 13:06:56 +010048#include <linux/cpu.h>
Peter Zijlstrae360adb2010-10-14 14:01:34 +080049#include <linux/irq_work.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +010050#include <linux/static_key.h>
Andrew Jones851cf6e2013-08-09 19:51:57 +053051#include <linux/jump_label_ratelimit.h>
Arun Sharma600634972011-07-26 16:09:06 -070052#include <linux/atomic.h>
Jiri Olsa641cc932012-03-15 20:09:14 +010053#include <linux/sysfs.h>
Jiri Olsa40189942012-08-07 15:20:37 +020054#include <linux/perf_regs.h>
Jiri Olsafadfe7b2014-08-01 14:33:02 +020055#include <linux/workqueue.h>
Matt Fleming39bed6c2015-01-23 18:45:40 +000056#include <linux/cgroup.h>
Peter Zijlstrafa588152010-05-18 10:54:20 +020057#include <asm/local.h>
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110058
Peter Zijlstraf9188e02009-06-18 22:20:52 +020059struct perf_callchain_entry {
60 __u64 nr;
61 __u64 ip[PERF_MAX_STACK_DEPTH];
62};
63
Frederic Weisbecker3a43ce62009-08-08 04:26:37 +020064struct perf_raw_record {
65 u32 size;
66 void *data;
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +020067};
68
Stephane Eranianbce38cd2012-02-09 23:20:51 +010069/*
Stephane Eranianbce38cd2012-02-09 23:20:51 +010070 * branch stack layout:
71 * nr: number of taken branches stored in entries[]
72 *
73 * Note that nr can vary from sample to sample
74 * branches (to, from) are stored from most recent
75 * to least recent, i.e., entries[0] contains the most
76 * recent branch.
77 */
Peter Zijlstracaff2be2010-03-03 12:02:30 +010078struct perf_branch_stack {
79 __u64 nr;
80 struct perf_branch_entry entries[0];
81};
82
Paul Mackerrasf3dfd262009-02-26 22:43:46 +110083struct task_struct;
84
Stephane Eranianefc9f052011-06-06 16:57:03 +020085/*
86 * extra PMU register associated with an event
87 */
88struct hw_perf_event_extra {
89 u64 config; /* register value */
90 unsigned int reg; /* register address or index */
91 int alloc; /* extra register already allocated */
92 int idx; /* index in shared_regs->regs[] */
93};
94
Andrew Hunter43b457802013-05-23 11:07:03 -070095struct event_constraint;
96
Thomas Gleixner0793a612008-12-04 20:12:29 +010097/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +020098 * struct hw_perf_event - performance event hardware details:
Thomas Gleixner0793a612008-12-04 20:12:29 +010099 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200100struct hw_perf_event {
101#ifdef CONFIG_PERF_EVENTS
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100102 union {
103 struct { /* hardware */
Ingo Molnara3084442009-06-11 14:44:26 +0200104 u64 config;
Stephane Eranian447a1942010-02-01 14:50:01 +0200105 u64 last_tag;
Ingo Molnara3084442009-06-11 14:44:26 +0200106 unsigned long config_base;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200107 unsigned long event_base;
Vince Weaverc48b6052012-03-01 17:28:14 -0500108 int event_base_rdpmc;
Ingo Molnara3084442009-06-11 14:44:26 +0200109 int idx;
Stephane Eranian447a1942010-02-01 14:50:01 +0200110 int last_cpu;
Stephane Eranian9fac2cf2013-01-24 16:10:27 +0100111 int flags;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100112
Stephane Eranianefc9f052011-06-06 16:57:03 +0200113 struct hw_perf_event_extra extra_reg;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100114 struct hw_perf_event_extra branch_reg;
Andrew Hunter43b457802013-05-23 11:07:03 -0700115
116 struct event_constraint *constraint;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100117 };
Soeren Sandmann721a6692009-09-15 14:33:08 +0200118 struct { /* software */
Ingo Molnara3084442009-06-11 14:44:26 +0200119 struct hrtimer hrtimer;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100120 };
Oleg Nesterovf22c1bb2013-02-02 16:27:52 +0100121 struct { /* tracepoint */
Oleg Nesterovf22c1bb2013-02-02 16:27:52 +0100122 /* for tp_event->class */
123 struct list_head tp_list;
124 };
Matt Fleming4afbb242015-01-23 18:45:44 +0000125 struct { /* intel_cqm */
126 int cqm_state;
127 int cqm_rmid;
128 struct list_head cqm_events_entry;
129 struct list_head cqm_groups_entry;
130 struct list_head cqm_group_entry;
131 };
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200132#ifdef CONFIG_HAVE_HW_BREAKPOINT
Frederic Weisbecker45a73372010-06-23 23:00:37 +0200133 struct { /* breakpoint */
Peter Zijlstrad580ff82010-10-14 17:43:23 +0200134 /*
135 * Crufty hack to avoid the chicken and egg
136 * problem hw_breakpoint has with context
137 * creation and event initalization.
138 */
Oleg Nesterovf22c1bb2013-02-02 16:27:52 +0100139 struct arch_hw_breakpoint info;
140 struct list_head bp_list;
Frederic Weisbecker45a73372010-06-23 23:00:37 +0200141 };
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200142#endif
Peter Zijlstrad6d020e2009-03-13 12:21:35 +0100143 };
Peter Zijlstra50f16a82015-03-05 22:10:19 +0100144 struct task_struct *target;
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200145 int state;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200146 local64_t prev_count;
Peter Zijlstrab23f3322009-06-02 15:13:03 +0200147 u64 sample_period;
Peter Zijlstra9e350de2009-06-10 21:34:59 +0200148 u64 last_period;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200149 local64_t period_left;
Stephane Eraniane050e3f2012-01-26 17:03:19 +0100150 u64 interrupts_seq;
Peter Zijlstra60db5e02009-05-15 15:19:28 +0200151 u64 interrupts;
Peter Zijlstra6a24ed6c2009-06-05 18:01:29 +0200152
Peter Zijlstraabd50712010-01-26 18:50:16 +0100153 u64 freq_time_stamp;
154 u64 freq_count_stamp;
Ingo Molnaree060942008-12-13 09:00:03 +0100155#endif
Thomas Gleixner0793a612008-12-04 20:12:29 +0100156};
157
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200158/*
159 * hw_perf_event::state flags
160 */
161#define PERF_HES_STOPPED 0x01 /* the counter is stopped */
162#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
163#define PERF_HES_ARCH 0x04
164
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200165struct perf_event;
Ingo Molnar621a01e2008-12-11 12:46:46 +0100166
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200167/*
168 * Common implementation detail of pmu::{start,commit,cancel}_txn
169 */
170#define PERF_EVENT_TXN 0x1
Lin Ming6bde9b62010-04-23 13:56:00 +0800171
Ingo Molnar621a01e2008-12-11 12:46:46 +0100172/**
Vince Weaver53b25332014-05-16 17:12:12 -0400173 * pmu::capabilities flags
174 */
175#define PERF_PMU_CAP_NO_INTERRUPT 0x01
Peter Zijlstra34f43922015-02-20 14:05:38 +0100176#define PERF_PMU_CAP_NO_NMI 0x02
Vince Weaver53b25332014-05-16 17:12:12 -0400177
178/**
Robert Richter4aeb0b42009-04-29 12:47:03 +0200179 * struct pmu - generic performance monitoring unit
Ingo Molnar621a01e2008-12-11 12:46:46 +0100180 */
Robert Richter4aeb0b42009-04-29 12:47:03 +0200181struct pmu {
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200182 struct list_head entry;
183
Yan, Zhengc464c762014-03-18 16:56:41 +0800184 struct module *module;
Peter Zijlstraabe43402010-11-17 23:17:37 +0100185 struct device *dev;
Peter Zijlstra0c9d42e2011-11-20 23:30:47 +0100186 const struct attribute_group **attr_groups;
Mischa Jonker03d8e802013-06-04 11:45:48 +0200187 const char *name;
Peter Zijlstra2e80a822010-11-17 23:17:36 +0100188 int type;
189
Vince Weaver53b25332014-05-16 17:12:12 -0400190 /*
191 * various common per-pmu feature flags
192 */
193 int capabilities;
194
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200195 int * __percpu pmu_disable_count;
196 struct perf_cpu_context * __percpu pmu_cpu_context;
Peter Zijlstra8dc85d52010-09-02 16:50:03 +0200197 int task_ctx_nr;
Stephane Eranian62b85632013-04-03 14:21:34 +0200198 int hrtimer_interval_ms;
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200199
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200200 /*
201 * Fully disable/enable this PMU, can be used to protect from the PMI
202 * as well as for lazy/batch writing of the MSRs.
203 */
Peter Zijlstraad5133b2010-06-15 12:22:39 +0200204 void (*pmu_enable) (struct pmu *pmu); /* optional */
205 void (*pmu_disable) (struct pmu *pmu); /* optional */
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200206
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200207 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200208 * Try and initialize the event for this PMU.
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200209 * Should return -ENOENT when the @event doesn't match this PMU.
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200210 */
211 int (*event_init) (struct perf_event *event);
212
Andy Lutomirski1e0fb9e2014-10-24 15:58:10 -0700213 /*
214 * Notification that the event was mapped or unmapped. Called
215 * in the context of the mapping task.
216 */
217 void (*event_mapped) (struct perf_event *event); /*optional*/
218 void (*event_unmapped) (struct perf_event *event); /*optional*/
219
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200220#define PERF_EF_START 0x01 /* start the counter when adding */
221#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
222#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
223
224 /*
225 * Adds/Removes a counter to/from the PMU, can be done inside
226 * a transaction, see the ->*_txn() methods.
227 */
228 int (*add) (struct perf_event *event, int flags);
229 void (*del) (struct perf_event *event, int flags);
230
231 /*
232 * Starts/Stops a counter present on the PMU. The PMI handler
233 * should stop the counter when perf_event_overflow() returns
234 * !0. ->start() will be used to continue.
235 */
236 void (*start) (struct perf_event *event, int flags);
237 void (*stop) (struct perf_event *event, int flags);
238
239 /*
240 * Updates the counter value of the event.
241 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200242 void (*read) (struct perf_event *event);
Lin Ming6bde9b62010-04-23 13:56:00 +0800243
244 /*
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200245 * Group events scheduling is treated as a transaction, add
246 * group events as a whole and perform one schedulability test.
247 * If the test fails, roll back the whole group
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200248 *
249 * Start the transaction, after this ->add() doesn't need to
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200250 * do schedulability tests.
Lin Ming6bde9b62010-04-23 13:56:00 +0800251 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200252 void (*start_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200253 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200254 * If ->start_txn() disabled the ->add() schedulability test
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200255 * then ->commit_txn() is required to perform one. On success
256 * the transaction is closed. On error the transaction is kept
257 * open until ->cancel_txn() is called.
258 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200259 int (*commit_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200260 /*
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200261 * Will cancel the transaction, assumes ->del() is called
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300262 * for each successful ->add() during the transaction.
Peter Zijlstra8d2cacb2010-05-25 17:49:05 +0200263 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200264 void (*cancel_txn) (struct pmu *pmu); /* optional */
Peter Zijlstra35edc2a2011-11-20 20:36:02 +0100265
266 /*
267 * Will return the value for perf_event_mmap_page::index for this event,
268 * if no implementation is provided it will default to: event->hw.idx + 1.
269 */
270 int (*event_idx) (struct perf_event *event); /*optional */
Stephane Eraniand010b332012-02-09 23:21:00 +0100271
272 /*
Yan, Zhengba532502014-11-04 21:55:58 -0500273 * context-switches callback
274 */
275 void (*sched_task) (struct perf_event_context *ctx,
276 bool sched_in);
Yan, Zheng4af57ef2014-11-04 21:56:01 -0500277 /*
278 * PMU specific data size
279 */
280 size_t task_ctx_size;
Yan, Zhengba532502014-11-04 21:55:58 -0500281
Matt Flemingeacd3ec2015-01-23 18:45:41 +0000282
283 /*
284 * Return the count value for a counter.
285 */
286 u64 (*count) (struct perf_event *event); /*optional*/
Ingo Molnar621a01e2008-12-11 12:46:46 +0100287};
288
Thomas Gleixner0793a612008-12-04 20:12:29 +0100289/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200290 * enum perf_event_active_state - the states of a event
Ingo Molnar6a930702008-12-11 15:17:03 +0100291 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200292enum perf_event_active_state {
Jiri Olsa179033b2014-08-07 11:48:26 -0400293 PERF_EVENT_STATE_EXIT = -3,
Ingo Molnar57c0c152009-09-21 12:20:38 +0200294 PERF_EVENT_STATE_ERROR = -2,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200295 PERF_EVENT_STATE_OFF = -1,
296 PERF_EVENT_STATE_INACTIVE = 0,
Ingo Molnar57c0c152009-09-21 12:20:38 +0200297 PERF_EVENT_STATE_ACTIVE = 1,
Ingo Molnar6a930702008-12-11 15:17:03 +0100298};
299
Ingo Molnar9b51f662008-12-12 13:49:45 +0100300struct file;
Peter Zijlstra453f19e2009-11-20 22:19:43 +0100301struct perf_sample_data;
302
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200303typedef void (*perf_overflow_handler_t)(struct perf_event *,
Frederic Weisbeckerb326e952009-12-05 09:44:31 +0100304 struct perf_sample_data *,
305 struct pt_regs *regs);
306
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100307enum perf_group_flag {
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200308 PERF_GROUP_SOFTWARE = 0x1,
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100309};
310
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200311#define SWEVENT_HLIST_BITS 8
312#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
Frederic Weisbecker76e1d902010-04-05 15:35:57 +0200313
314struct swevent_hlist {
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200315 struct hlist_head heads[SWEVENT_HLIST_SIZE];
316 struct rcu_head rcu_head;
Frederic Weisbecker76e1d902010-04-05 15:35:57 +0200317};
318
Peter Zijlstra8a495422010-05-27 15:47:49 +0200319#define PERF_ATTACH_CONTEXT 0x01
320#define PERF_ATTACH_GROUP 0x02
Peter Zijlstrad580ff82010-10-14 17:43:23 +0200321#define PERF_ATTACH_TASK 0x04
Yan, Zheng4af57ef2014-11-04 21:56:01 -0500322#define PERF_ATTACH_TASK_DATA 0x08
Peter Zijlstra8a495422010-05-27 15:47:49 +0200323
Li Zefan877c6852013-03-05 11:38:08 +0800324struct perf_cgroup;
Frederic Weisbecker76369132011-05-19 19:55:04 +0200325struct ring_buffer;
326
Ingo Molnar6a930702008-12-11 15:17:03 +0100327/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200328 * struct perf_event - performance event kernel representation:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100329 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200330struct perf_event {
331#ifdef CONFIG_PERF_EVENTS
Peter Zijlstra98861672013-10-03 16:02:23 +0200332 /*
333 * entry onto perf_event_context::event_list;
334 * modifications require ctx->lock
335 * RCU safe iterations.
336 */
Peter Zijlstra592903c2009-03-13 12:21:36 +0100337 struct list_head event_entry;
Peter Zijlstra98861672013-10-03 16:02:23 +0200338
339 /*
340 * XXX: group_entry and sibling_list should be mutually exclusive;
341 * either you're a sibling on a group, or you're the group leader.
342 * Rework the code to always use the same list element.
343 *
344 * Locked for modification by both ctx->mutex and ctx->lock; holding
345 * either sufficies for read.
346 */
347 struct list_head group_entry;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100348 struct list_head sibling_list;
Peter Zijlstra98861672013-10-03 16:02:23 +0200349
350 /*
351 * We need storage to track the entries in perf_pmu_migrate_context; we
352 * cannot use the event_entry because of RCU and we want to keep the
353 * group in tact which avoids us using the other two entries.
354 */
355 struct list_head migrate_entry;
356
Stephane Eranianf3ae75d2014-01-08 11:15:52 +0100357 struct hlist_node hlist_entry;
358 struct list_head active_entry;
Ingo Molnar0127c3e2009-05-25 22:03:26 +0200359 int nr_siblings;
Frederic Weisbeckerd6f962b2010-01-10 01:25:51 +0100360 int group_flags;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200361 struct perf_event *group_leader;
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200362 struct pmu *pmu;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100363
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200364 enum perf_event_active_state state;
Peter Zijlstra8a495422010-05-27 15:47:49 +0200365 unsigned int attach_state;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200366 local64_t count;
Peter Zijlstraa6e6dea2010-05-21 14:27:58 +0200367 atomic64_t child_count;
Ingo Molnaree060942008-12-13 09:00:03 +0100368
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100369 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200370 * These are the total time in nanoseconds that the event
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100371 * has been enabled (i.e. eligible to run, and the task has
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200372 * been scheduled in, if this is a per-task event)
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100373 * and running (scheduled onto the CPU), respectively.
374 *
375 * They are computed from tstamp_enabled, tstamp_running and
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200376 * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100377 */
378 u64 total_time_enabled;
379 u64 total_time_running;
380
381 /*
382 * These are timestamps used for computing total_time_enabled
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200383 * and total_time_running when the event is in INACTIVE or
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100384 * ACTIVE state, measured in nanoseconds from an arbitrary point
385 * in time.
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200386 * tstamp_enabled: the notional time when the event was enabled
387 * tstamp_running: the notional time when the event was scheduled on
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100388 * tstamp_stopped: in INACTIVE state, the notional time when the
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200389 * event was scheduled off.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100390 */
391 u64 tstamp_enabled;
392 u64 tstamp_running;
393 u64 tstamp_stopped;
394
Stephane Eranianeed01522010-10-26 16:08:01 +0200395 /*
396 * timestamp shadows the actual context timing but it can
397 * be safely used in NMI interrupt context. It reflects the
398 * context time as it was when the event was last scheduled in.
399 *
400 * ctx_time already accounts for ctx->timestamp. Therefore to
401 * compute ctx_time for a sample, simply add perf_clock().
402 */
403 u64 shadow_ctx_time;
404
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200405 struct perf_event_attr attr;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200406 u16 header_size;
Arnaldo Carvalho de Melo6844c092010-12-03 16:36:35 -0200407 u16 id_header_size;
Arnaldo Carvalho de Meloc320c7b2010-10-20 12:50:11 -0200408 u16 read_size;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200409 struct hw_perf_event hw;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100410
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200411 struct perf_event_context *ctx;
Al Viroa6fa9412012-08-20 14:59:25 +0100412 atomic_long_t refcount;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100413
414 /*
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100415 * These accumulate total time (in nanoseconds) that children
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200416 * events have been enabled and running, respectively.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100417 */
418 atomic64_t child_total_time_enabled;
419 atomic64_t child_total_time_running;
420
421 /*
Paul Mackerrasd859e292009-01-17 18:10:22 +1100422 * Protect attach/detach and child_list:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100423 */
Peter Zijlstrafccc7142009-05-23 18:28:56 +0200424 struct mutex child_mutex;
425 struct list_head child_list;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200426 struct perf_event *parent;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100427
428 int oncpu;
429 int cpu;
430
Peter Zijlstra082ff5a2009-05-23 18:29:00 +0200431 struct list_head owner_entry;
432 struct task_struct *owner;
433
Peter Zijlstra7b732a72009-03-23 18:22:10 +0100434 /* mmap bits */
435 struct mutex mmap_mutex;
436 atomic_t mmap_count;
Peter Zijlstra26cb63a2013-05-28 10:55:48 +0200437
Frederic Weisbecker76369132011-05-19 19:55:04 +0200438 struct ring_buffer *rb;
Peter Zijlstra10c6db12011-11-26 02:47:31 +0100439 struct list_head rb_entry;
Peter Zijlstrab69cf532014-03-14 10:50:33 +0100440 unsigned long rcu_batches;
441 int rcu_pending;
Paul Mackerras37d81822009-03-23 18:22:08 +0100442
Peter Zijlstra7b732a72009-03-23 18:22:10 +0100443 /* poll related */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100444 wait_queue_head_t waitq;
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +0200445 struct fasync_struct *fasync;
Peter Zijlstra79f14642009-04-06 11:45:07 +0200446
447 /* delayed work for NMIs and such */
448 int pending_wakeup;
Peter Zijlstra4c9e2542009-04-06 11:45:09 +0200449 int pending_kill;
Peter Zijlstra79f14642009-04-06 11:45:07 +0200450 int pending_disable;
Peter Zijlstrae360adb2010-10-14 14:01:34 +0800451 struct irq_work pending;
Peter Zijlstra592903c2009-03-13 12:21:36 +0100452
Peter Zijlstra79f14642009-04-06 11:45:07 +0200453 atomic_t event_limit;
454
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200455 void (*destroy)(struct perf_event *);
Peter Zijlstra592903c2009-03-13 12:21:36 +0100456 struct rcu_head rcu_head;
Peter Zijlstra709e50c2009-06-02 14:13:15 +0200457
458 struct pid_namespace *ns;
Peter Zijlstra8e5799b2009-06-02 15:08:15 +0200459 u64 id;
Li Zefan6fb29152009-10-15 11:21:42 +0800460
Peter Zijlstra34f43922015-02-20 14:05:38 +0100461 u64 (*clock)(void);
Frederic Weisbeckerb326e952009-12-05 09:44:31 +0100462 perf_overflow_handler_t overflow_handler;
Avi Kivity4dc0da82011-06-29 18:42:35 +0300463 void *overflow_handler_context;
Peter Zijlstra453f19e2009-11-20 22:19:43 +0100464
Li Zefan07b139c2009-12-21 14:27:35 +0800465#ifdef CONFIG_EVENT_TRACING
Peter Zijlstra1c024eca2010-05-19 14:02:22 +0200466 struct ftrace_event_call *tp_event;
Li Zefan6fb29152009-10-15 11:21:42 +0800467 struct event_filter *filter;
Jiri Olsaced39002012-02-15 15:51:52 +0100468#ifdef CONFIG_FUNCTION_TRACER
469 struct ftrace_ops ftrace_ops;
470#endif
Ingo Molnaree060942008-12-13 09:00:03 +0100471#endif
Li Zefan6fb29152009-10-15 11:21:42 +0800472
Stephane Eraniane5d13672011-02-14 11:20:01 +0200473#ifdef CONFIG_CGROUP_PERF
474 struct perf_cgroup *cgrp; /* cgroup event is attach to */
475 int cgrp_defer_enabled;
476#endif
477
Li Zefan6fb29152009-10-15 11:21:42 +0800478#endif /* CONFIG_PERF_EVENTS */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100479};
480
481/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200482 * struct perf_event_context - event context structure
Thomas Gleixner0793a612008-12-04 20:12:29 +0100483 *
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200484 * Used as a container for task events and CPU events as well:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100485 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200486struct perf_event_context {
Peter Zijlstra108b02c2010-09-06 14:32:03 +0200487 struct pmu *pmu;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100488 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200489 * Protect the states of the events in the list,
Paul Mackerrasd859e292009-01-17 18:10:22 +1100490 * nr_active, and the list:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100491 */
Thomas Gleixnere625cce12009-11-17 18:02:06 +0100492 raw_spinlock_t lock;
Paul Mackerrasd859e292009-01-17 18:10:22 +1100493 /*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200494 * Protect the list of events. Locking either mutex or lock
Paul Mackerrasd859e292009-01-17 18:10:22 +1100495 * is sufficient to ensure the list doesn't change; to change
496 * the list you need to lock both the mutex and the spinlock.
497 */
Ingo Molnara3084442009-06-11 14:44:26 +0200498 struct mutex mutex;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100499
Mark Rutland2fde4f92015-01-07 15:01:54 +0000500 struct list_head active_ctx_list;
Frederic Weisbecker889ff012010-01-09 20:04:47 +0100501 struct list_head pinned_groups;
502 struct list_head flexible_groups;
Ingo Molnara3084442009-06-11 14:44:26 +0200503 struct list_head event_list;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200504 int nr_events;
Ingo Molnara3084442009-06-11 14:44:26 +0200505 int nr_active;
506 int is_active;
Peter Zijlstrabfbd3382009-06-24 21:11:59 +0200507 int nr_stat;
Peter Zijlstra0f5a2602011-11-16 14:38:16 +0100508 int nr_freq;
Thomas Gleixnerdddd3372010-11-24 10:05:55 +0100509 int rotate_disable;
Ingo Molnara3084442009-06-11 14:44:26 +0200510 atomic_t refcount;
511 struct task_struct *task;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100512
513 /*
Peter Zijlstra4af49982009-04-06 11:45:10 +0200514 * Context clock, runs when context enabled.
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100515 */
Ingo Molnara3084442009-06-11 14:44:26 +0200516 u64 time;
517 u64 timestamp;
Paul Mackerras564c2b22009-05-22 14:27:22 +1000518
519 /*
520 * These fields let us detect when two contexts have both
521 * been cloned (inherited) from a common ancestor.
522 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200523 struct perf_event_context *parent_ctx;
Ingo Molnara3084442009-06-11 14:44:26 +0200524 u64 parent_gen;
525 u64 generation;
526 int pin_count;
Stephane Eraniand010b332012-02-09 23:21:00 +0100527 int nr_cgroups; /* cgroup evts */
Yan, Zheng4af57ef2014-11-04 21:56:01 -0500528 void *task_ctx_data; /* pmu specific data */
Richard Kennedy28009ce2011-06-07 16:33:38 +0100529 struct rcu_head rcu_head;
Jiri Olsafadfe7b2014-08-01 14:33:02 +0200530
531 struct delayed_work orphans_remove;
532 bool orphans_remove_sched;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100533};
534
Frederic Weisbecker7ae07ea2010-08-14 20:45:13 +0200535/*
536 * Number of contexts where an event can trigger:
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200537 * task, softirq, hardirq, nmi.
Frederic Weisbecker7ae07ea2010-08-14 20:45:13 +0200538 */
539#define PERF_NR_CONTEXTS 4
540
Thomas Gleixner0793a612008-12-04 20:12:29 +0100541/**
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200542 * struct perf_event_cpu_context - per cpu event context structure
Thomas Gleixner0793a612008-12-04 20:12:29 +0100543 */
544struct perf_cpu_context {
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200545 struct perf_event_context ctx;
546 struct perf_event_context *task_ctx;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100547 int active_oncpu;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100548 int exclusive;
Stephane Eranian9e630202013-04-03 14:21:33 +0200549 struct hrtimer hrtimer;
550 ktime_t hrtimer_interval;
Peter Zijlstra3f1f3322012-10-02 15:38:52 +0200551 struct pmu *unique_pmu;
Stephane Eraniane5d13672011-02-14 11:20:01 +0200552 struct perf_cgroup *cgrp;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100553};
554
Markus Metzger5622f292009-09-15 13:00:23 +0200555struct perf_output_handle {
Ingo Molnar57c0c152009-09-21 12:20:38 +0200556 struct perf_event *event;
Frederic Weisbecker76369132011-05-19 19:55:04 +0200557 struct ring_buffer *rb;
Peter Zijlstra6d1acfd2010-05-18 11:12:48 +0200558 unsigned long wakeup;
Peter Zijlstra5d967a82010-05-20 16:46:39 +0200559 unsigned long size;
560 void *addr;
561 int page;
Markus Metzger5622f292009-09-15 13:00:23 +0200562};
563
Matt Fleming39bed6c2015-01-23 18:45:40 +0000564#ifdef CONFIG_CGROUP_PERF
565
566/*
567 * perf_cgroup_info keeps track of time_enabled for a cgroup.
568 * This is a per-cpu dynamically allocated data structure.
569 */
570struct perf_cgroup_info {
571 u64 time;
572 u64 timestamp;
573};
574
575struct perf_cgroup {
576 struct cgroup_subsys_state css;
577 struct perf_cgroup_info __percpu *info;
578};
579
580/*
581 * Must ensure cgroup is pinned (css_get) before calling
582 * this function. In other words, we cannot call this function
583 * if there is no cgroup event for the current CPU context.
584 */
585static inline struct perf_cgroup *
586perf_cgroup_from_task(struct task_struct *task)
587{
588 return container_of(task_css(task, perf_event_cgrp_id),
589 struct perf_cgroup, css);
590}
591#endif /* CONFIG_CGROUP_PERF */
592
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200593#ifdef CONFIG_PERF_EVENTS
Robert Richter829b42d2009-04-29 12:46:59 +0200594
Mischa Jonker03d8e802013-06-04 11:45:48 +0200595extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200596extern void perf_pmu_unregister(struct pmu *pmu);
Ingo Molnar621a01e2008-12-11 12:46:46 +0100597
Matt Fleming3bf101b2010-09-27 20:22:24 +0100598extern int perf_num_counters(void);
Matt Fleming84c79912010-10-03 21:41:13 +0100599extern const char *perf_pmu_name(void);
Jiri Olsaab0cce52012-05-23 13:13:02 +0200600extern void __perf_event_task_sched_in(struct task_struct *prev,
601 struct task_struct *task);
602extern void __perf_event_task_sched_out(struct task_struct *prev,
603 struct task_struct *next);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200604extern int perf_event_init_task(struct task_struct *child);
605extern void perf_event_exit_task(struct task_struct *child);
606extern void perf_event_free_task(struct task_struct *task);
Peter Zijlstra4e231c72010-09-09 21:01:59 +0200607extern void perf_event_delayed_put(struct task_struct *task);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200608extern void perf_event_print_debug(void);
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200609extern void perf_pmu_disable(struct pmu *pmu);
610extern void perf_pmu_enable(struct pmu *pmu);
Yan, Zhengba532502014-11-04 21:55:58 -0500611extern void perf_sched_cb_dec(struct pmu *pmu);
612extern void perf_sched_cb_inc(struct pmu *pmu);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200613extern int perf_event_task_disable(void);
614extern int perf_event_task_enable(void);
Avi Kivity26ca5c12011-06-29 18:42:37 +0300615extern int perf_event_refresh(struct perf_event *event, int refresh);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200616extern void perf_event_update_userpage(struct perf_event *event);
Arjan van de Venfb0459d2009-09-25 12:25:56 +0200617extern int perf_event_release_kernel(struct perf_event *event);
618extern struct perf_event *
619perf_event_create_kernel_counter(struct perf_event_attr *attr,
620 int cpu,
Matt Helsley38a81da2010-09-13 13:01:20 -0700621 struct task_struct *task,
Avi Kivity4dc0da82011-06-29 18:42:35 +0300622 perf_overflow_handler_t callback,
623 void *context);
Yan, Zheng0cda4c02012-06-15 14:31:33 +0800624extern void perf_pmu_migrate_context(struct pmu *pmu,
625 int src_cpu, int dst_cpu);
Peter Zijlstra59ed4462009-11-20 22:19:55 +0100626extern u64 perf_event_read_value(struct perf_event *event,
627 u64 *enabled, u64 *running);
Ingo Molnar5c92d122008-12-11 13:21:10 +0100628
Stephane Eraniand010b332012-02-09 23:21:00 +0100629
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200630struct perf_sample_data {
Peter Zijlstra25657112014-09-24 13:48:42 +0200631 /*
632 * Fields set by perf_sample_data_init(), group so as to
633 * minimize the cachelines touched.
634 */
635 u64 addr;
636 struct perf_raw_record *raw;
637 struct perf_branch_stack *br_stack;
638 u64 period;
639 u64 weight;
640 u64 txn;
641 union perf_mem_data_src data_src;
Markus Metzger5622f292009-09-15 13:00:23 +0200642
Peter Zijlstra25657112014-09-24 13:48:42 +0200643 /*
644 * The other fields, optionally {set,used} by
645 * perf_{prepare,output}_sample().
646 */
647 u64 type;
Markus Metzger5622f292009-09-15 13:00:23 +0200648 u64 ip;
649 struct {
650 u32 pid;
651 u32 tid;
652 } tid_entry;
653 u64 time;
Markus Metzger5622f292009-09-15 13:00:23 +0200654 u64 id;
655 u64 stream_id;
656 struct {
657 u32 cpu;
658 u32 reserved;
659 } cpu_entry;
Markus Metzger5622f292009-09-15 13:00:23 +0200660 struct perf_callchain_entry *callchain;
Andy Lutomirski88a7c262015-01-04 10:36:19 -0800661
662 /*
663 * regs_user may point to task_pt_regs or to regs_user_copy, depending
664 * on arch details.
665 */
Stephane Eranian60e23642014-09-24 13:48:37 +0200666 struct perf_regs regs_user;
Andy Lutomirski88a7c262015-01-04 10:36:19 -0800667 struct pt_regs regs_user_copy;
668
Stephane Eranian60e23642014-09-24 13:48:37 +0200669 struct perf_regs regs_intr;
Jiri Olsac5ebced2012-08-07 15:20:40 +0200670 u64 stack_user_size;
Peter Zijlstra25657112014-09-24 13:48:42 +0200671} ____cacheline_aligned;
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200672
Stephane Eranian770eee12014-08-11 21:27:12 +0200673/* default value for data source */
674#define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\
675 PERF_MEM_S(LVL, NA) |\
676 PERF_MEM_S(SNOOP, NA) |\
677 PERF_MEM_S(LOCK, NA) |\
678 PERF_MEM_S(TLB, NA))
679
Robert Richterfd0d0002012-04-02 20:19:08 +0200680static inline void perf_sample_data_init(struct perf_sample_data *data,
681 u64 addr, u64 period)
Peter Zijlstradc1d628a2010-03-03 15:55:04 +0100682{
Robert Richterfd0d0002012-04-02 20:19:08 +0200683 /* remaining struct members initialized in perf_prepare_sample() */
Peter Zijlstradc1d628a2010-03-03 15:55:04 +0100684 data->addr = addr;
685 data->raw = NULL;
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100686 data->br_stack = NULL;
Jiri Olsa40189942012-08-07 15:20:37 +0200687 data->period = period;
Andi Kleenc3feedf2013-01-24 16:10:28 +0100688 data->weight = 0;
Stephane Eranian770eee12014-08-11 21:27:12 +0200689 data->data_src.val = PERF_MEM_NA;
Andi Kleenfdfbbd02013-09-20 07:40:39 -0700690 data->txn = 0;
Peter Zijlstradc1d628a2010-03-03 15:55:04 +0100691}
692
Markus Metzger5622f292009-09-15 13:00:23 +0200693extern void perf_output_sample(struct perf_output_handle *handle,
694 struct perf_event_header *header,
695 struct perf_sample_data *data,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200696 struct perf_event *event);
Markus Metzger5622f292009-09-15 13:00:23 +0200697extern void perf_prepare_sample(struct perf_event_header *header,
698 struct perf_sample_data *data,
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200699 struct perf_event *event,
Markus Metzger5622f292009-09-15 13:00:23 +0200700 struct pt_regs *regs);
701
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200702extern int perf_event_overflow(struct perf_event *event,
Markus Metzger5622f292009-09-15 13:00:23 +0200703 struct perf_sample_data *data,
704 struct pt_regs *regs);
Peter Zijlstradf1a1322009-06-10 21:02:22 +0200705
Franck Bui-Huu6c7e5502010-11-23 16:21:43 +0100706static inline bool is_sampling_event(struct perf_event *event)
707{
708 return event->attr.sample_period != 0;
709}
710
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100711/*
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200712 * Return 1 for a software event, 0 for a hardware event
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100713 */
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200714static inline int is_software_event(struct perf_event *event)
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100715{
Peter Zijlstra89a1e182010-09-07 17:34:50 +0200716 return event->pmu->task_ctx_nr == perf_sw_context;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100717}
718
Ingo Molnarc5905af2012-02-24 08:31:31 +0100719extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
Peter Zijlstraf29ac752009-06-19 18:27:26 +0200720
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100721extern void ___perf_sw_event(u32, u64, struct pt_regs *, u64);
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200722extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
Peter Zijlstraf29ac752009-06-19 18:27:26 +0200723
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200724#ifndef perf_arch_fetch_caller_regs
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200725static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200726#endif
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100727
728/*
729 * Take a snapshot of the regs. Skip ip and frame pointer to
730 * the nth caller. We only need a few of the regs:
731 * - ip for PERF_SAMPLE_IP
732 * - cs for user_mode() tests
733 * - bp for callchains
734 * - eflags, for future purposes, just in case
735 */
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200736static inline void perf_fetch_caller_regs(struct pt_regs *regs)
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100737{
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100738 memset(regs, 0, sizeof(*regs));
739
Frederic Weisbeckerb0f82b82010-05-20 07:47:21 +0200740 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
Frederic Weisbecker5331d7b2010-03-04 21:15:56 +0100741}
742
Peter Zijlstra7e54a5a2010-10-14 22:32:45 +0200743static __always_inline void
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200744perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
Frederic Weisbeckere49a5bd2010-03-22 19:40:03 +0100745{
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100746 if (static_key_false(&perf_swevent_enabled[event_id]))
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200747 __perf_sw_event(event_id, nr, regs, addr);
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100748}
749
750DECLARE_PER_CPU(struct pt_regs, __perf_regs[4]);
751
752/*
753 * 'Special' version for the scheduler, it hard assumes no recursion,
754 * which is guaranteed by us not actually scheduling inside other swevents
755 * because those disable preemption.
756 */
757static __always_inline void
758perf_sw_event_sched(u32 event_id, u64 nr, u64 addr)
759{
760 if (static_key_false(&perf_swevent_enabled[event_id])) {
761 struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);
762
763 perf_fetch_caller_regs(regs);
764 ___perf_sw_event(event_id, nr, regs, addr);
Frederic Weisbeckere49a5bd2010-03-22 19:40:03 +0100765 }
766}
767
Ingo Molnarc5905af2012-02-24 08:31:31 +0100768extern struct static_key_deferred perf_sched_events;
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100769
Jiri Olsaab0cce52012-05-23 13:13:02 +0200770static inline void perf_event_task_sched_in(struct task_struct *prev,
Stephane Eraniana8d757e2011-08-25 15:58:03 +0200771 struct task_struct *task)
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100772{
Jiri Olsaab0cce52012-05-23 13:13:02 +0200773 if (static_key_false(&perf_sched_events.key))
774 __perf_event_task_sched_in(prev, task);
775}
776
777static inline void perf_event_task_sched_out(struct task_struct *prev,
778 struct task_struct *next)
779{
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100780 perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100781
Ingo Molnarc5905af2012-02-24 08:31:31 +0100782 if (static_key_false(&perf_sched_events.key))
Jiri Olsaab0cce52012-05-23 13:13:02 +0200783 __perf_event_task_sched_out(prev, next);
Peter Zijlstraee6dcfa2010-11-26 13:49:04 +0100784}
785
Matt Flemingeacd3ec2015-01-23 18:45:41 +0000786static inline u64 __perf_event_count(struct perf_event *event)
787{
788 return local64_read(&event->count) + atomic64_read(&event->child_count);
789}
790
Eric B Munson3af9e852010-05-18 15:30:49 +0100791extern void perf_event_mmap(struct vm_area_struct *vma);
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800792extern struct perf_guest_info_callbacks *perf_guest_cbs;
Zhang, Yanmindcf46b92010-04-20 10:13:58 +0800793extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
794extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800795
Peter Zijlstrae041e322014-05-21 17:32:19 +0200796extern void perf_event_exec(void);
Adrian Hunter82b89772014-05-28 11:45:04 +0300797extern void perf_event_comm(struct task_struct *tsk, bool exec);
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200798extern void perf_event_fork(struct task_struct *tsk);
Peter Zijlstra8d1b2d92009-04-08 15:01:30 +0200799
Frederic Weisbecker56962b4442010-06-30 23:03:51 +0200800/* Callchains */
801DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
802
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200803extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
804extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
Frederic Weisbecker56962b4442010-06-30 23:03:51 +0200805
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200806static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
Frederic Weisbecker70791ce2010-06-29 19:34:05 +0200807{
808 if (entry->nr < PERF_MAX_STACK_DEPTH)
809 entry->ip[entry->nr++] = ip;
810}
Peter Zijlstra394ee072009-03-30 19:07:14 +0200811
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200812extern int sysctl_perf_event_paranoid;
813extern int sysctl_perf_event_mlock;
814extern int sysctl_perf_event_sample_rate;
Dave Hansen14c63f12013-06-21 08:51:36 -0700815extern int sysctl_perf_cpu_time_max_percent;
816
817extern void perf_sample_event_took(u64 sample_len_ns);
Peter Zijlstra1ccd1542009-04-09 10:53:45 +0200818
Peter Zijlstra163ec432011-02-16 11:22:34 +0100819extern int perf_proc_update_handler(struct ctl_table *table, int write,
820 void __user *buffer, size_t *lenp,
821 loff_t *ppos);
Dave Hansen14c63f12013-06-21 08:51:36 -0700822extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
823 void __user *buffer, size_t *lenp,
824 loff_t *ppos);
825
Peter Zijlstra163ec432011-02-16 11:22:34 +0100826
Peter Zijlstra320ebf02010-03-02 12:35:37 +0100827static inline bool perf_paranoid_tracepoint_raw(void)
828{
829 return sysctl_perf_event_paranoid > -1;
830}
831
832static inline bool perf_paranoid_cpu(void)
833{
834 return sysctl_perf_event_paranoid > 0;
835}
836
837static inline bool perf_paranoid_kernel(void)
838{
839 return sysctl_perf_event_paranoid > 1;
840}
841
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200842extern void perf_event_init(void);
Peter Zijlstra1c024eca2010-05-19 14:02:22 +0200843extern void perf_tp_event(u64 addr, u64 count, void *record,
844 int entry_size, struct pt_regs *regs,
Andrew Vagine6dab5f2012-07-11 18:14:58 +0400845 struct hlist_head *head, int rctx,
846 struct task_struct *task);
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200847extern void perf_bp_event(struct perf_event *event, void *data);
Ingo Molnar0d905bc2009-05-04 19:13:30 +0200848
Paul Mackerras9d23a902009-05-14 21:48:08 +1000849#ifndef perf_misc_flags
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200850# define perf_misc_flags(regs) \
851 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
852# define perf_instruction_pointer(regs) instruction_pointer(regs)
Paul Mackerras9d23a902009-05-14 21:48:08 +1000853#endif
854
Stephane Eranianbce38cd2012-02-09 23:20:51 +0100855static inline bool has_branch_stack(struct perf_event *event)
856{
857 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
858}
859
Yan, Zhenga46a2302014-11-04 21:56:06 -0500860static inline bool needs_branch_stack(struct perf_event *event)
861{
862 return event->attr.branch_sample_type != 0;
863}
864
Markus Metzger5622f292009-09-15 13:00:23 +0200865extern int perf_output_begin(struct perf_output_handle *handle,
Peter Zijlstraa7ac67e2011-06-27 16:47:16 +0200866 struct perf_event *event, unsigned int size);
Markus Metzger5622f292009-09-15 13:00:23 +0200867extern void perf_output_end(struct perf_output_handle *handle);
Frederic Weisbecker91d77532012-08-07 15:20:38 +0200868extern unsigned int perf_output_copy(struct perf_output_handle *handle,
Markus Metzger5622f292009-09-15 13:00:23 +0200869 const void *buf, unsigned int len);
Jiri Olsa5685e0f2012-08-07 15:20:39 +0200870extern unsigned int perf_output_skip(struct perf_output_handle *handle,
871 unsigned int len);
Peter Zijlstra4ed7c922009-11-23 11:37:29 +0100872extern int perf_swevent_get_recursion_context(void);
873extern void perf_swevent_put_recursion_context(int rctx);
Jiri Olsaab573842013-05-01 17:25:44 +0200874extern u64 perf_swevent_set_period(struct perf_event *event);
Frederic Weisbecker44234ad2009-12-09 09:25:48 +0100875extern void perf_event_enable(struct perf_event *event);
876extern void perf_event_disable(struct perf_event *event);
K.Prasad500ad2d2012-08-02 13:46:35 +0530877extern int __perf_event_disable(void *info);
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200878extern void perf_event_task_tick(void);
Peter Zijlstrae041e322014-05-21 17:32:19 +0200879#else /* !CONFIG_PERF_EVENTS: */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100880static inline void
Jiri Olsaab0cce52012-05-23 13:13:02 +0200881perf_event_task_sched_in(struct task_struct *prev,
882 struct task_struct *task) { }
883static inline void
884perf_event_task_sched_out(struct task_struct *prev,
885 struct task_struct *next) { }
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200886static inline int perf_event_init_task(struct task_struct *child) { return 0; }
887static inline void perf_event_exit_task(struct task_struct *child) { }
888static inline void perf_event_free_task(struct task_struct *task) { }
Peter Zijlstra4e231c72010-09-09 21:01:59 +0200889static inline void perf_event_delayed_put(struct task_struct *task) { }
Ingo Molnar57c0c152009-09-21 12:20:38 +0200890static inline void perf_event_print_debug(void) { }
Ingo Molnar57c0c152009-09-21 12:20:38 +0200891static inline int perf_event_task_disable(void) { return -EINVAL; }
892static inline int perf_event_task_enable(void) { return -EINVAL; }
Avi Kivity26ca5c12011-06-29 18:42:37 +0300893static inline int perf_event_refresh(struct perf_event *event, int refresh)
894{
895 return -EINVAL;
896}
Peter Zijlstra15dbf272009-03-13 12:21:32 +0100897
Peter Zijlstra925d5192009-03-30 19:07:02 +0200898static inline void
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200899perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +0200900static inline void
Peter Zijlstra (Intel)86038c52014-12-16 12:47:34 +0100901perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) { }
902static inline void
Ingo Molnar184f4122010-01-27 08:39:39 +0100903perf_bp_event(struct perf_event *event, void *data) { }
Peter Zijlstra0a4a9392009-03-30 19:07:05 +0200904
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800905static inline int perf_register_guest_info_callbacks
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200906(struct perf_guest_info_callbacks *callbacks) { return 0; }
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800907static inline int perf_unregister_guest_info_callbacks
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200908(struct perf_guest_info_callbacks *callbacks) { return 0; }
Zhang, Yanmin39447b32010-04-19 13:32:41 +0800909
Ingo Molnar57c0c152009-09-21 12:20:38 +0200910static inline void perf_event_mmap(struct vm_area_struct *vma) { }
Peter Zijlstrae041e322014-05-21 17:32:19 +0200911static inline void perf_event_exec(void) { }
Adrian Hunter82b89772014-05-28 11:45:04 +0300912static inline void perf_event_comm(struct task_struct *tsk, bool exec) { }
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200913static inline void perf_event_fork(struct task_struct *tsk) { }
914static inline void perf_event_init(void) { }
Ingo Molnar184f4122010-01-27 08:39:39 +0100915static inline int perf_swevent_get_recursion_context(void) { return -1; }
Peter Zijlstra4ed7c922009-11-23 11:37:29 +0100916static inline void perf_swevent_put_recursion_context(int rctx) { }
Jiri Olsaab573842013-05-01 17:25:44 +0200917static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
Frederic Weisbecker44234ad2009-12-09 09:25:48 +0100918static inline void perf_event_enable(struct perf_event *event) { }
919static inline void perf_event_disable(struct perf_event *event) { }
K.Prasad500ad2d2012-08-02 13:46:35 +0530920static inline int __perf_event_disable(void *info) { return -1; }
Peter Zijlstrae9d2b062010-09-17 11:28:50 +0200921static inline void perf_event_task_tick(void) { }
Thomas Gleixner0793a612008-12-04 20:12:29 +0100922#endif
923
Frederic Weisbecker026249e2013-04-20 15:58:34 +0200924#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL)
925extern bool perf_event_can_stop_tick(void);
926#else
927static inline bool perf_event_can_stop_tick(void) { return true; }
928#endif
929
David Rientjes6c4d3bc2013-03-17 15:49:10 -0700930#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
931extern void perf_restore_debug_store(void);
932#else
Stephane Eranian1d9d8632013-03-15 14:26:07 +0100933static inline void perf_restore_debug_store(void) { }
Thomas Gleixner0793a612008-12-04 20:12:29 +0100934#endif
935
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200936#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
Markus Metzger5622f292009-09-15 13:00:23 +0200937
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100938/*
Viresh Kumar0a0fca92013-06-04 13:10:24 +0530939 * This has to have a higher priority than migration_notifier in sched/core.c.
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100940 */
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200941#define perf_cpu_notifier(fn) \
942do { \
Paul Gortmaker0db06282013-06-19 14:53:51 -0400943 static struct notifier_block fn##_nb = \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200944 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530945 unsigned long cpu = smp_processor_id(); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530946 unsigned long flags; \
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530947 \
948 cpu_notifier_register_begin(); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200949 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530950 (void *)(unsigned long)cpu); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530951 local_irq_save(flags); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200952 fn(&fn##_nb, (unsigned long)CPU_STARTING, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530953 (void *)(unsigned long)cpu); \
Srivatsa S. Bhat6760bca2012-10-16 13:28:10 +0530954 local_irq_restore(flags); \
Ingo Molnare7e7ee22011-05-04 08:42:29 +0200955 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \
Srivatsa S. Bhatc13d38e2012-10-16 13:28:17 +0530956 (void *)(unsigned long)cpu); \
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530957 __register_cpu_notifier(&fn##_nb); \
958 cpu_notifier_register_done(); \
Peter Zijlstra3f6da392010-03-05 13:01:18 +0100959} while (0)
960
Srivatsa S. Bhatf0bdb5e2014-03-11 02:04:39 +0530961/*
962 * Bare-bones version of perf_cpu_notifier(), which doesn't invoke the
963 * callback for already online CPUs.
964 */
965#define __perf_cpu_notifier(fn) \
966do { \
967 static struct notifier_block fn##_nb = \
968 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
969 \
970 __register_cpu_notifier(&fn##_nb); \
971} while (0)
Jiri Olsa641cc932012-03-15 20:09:14 +0100972
Sukadev Bhattiprolu26639602013-01-22 22:24:23 -0800973struct perf_pmu_events_attr {
974 struct device_attribute attr;
975 u64 id;
Stephane Eranian3a54aaa2013-01-24 16:10:26 +0100976 const char *event_str;
Sukadev Bhattiprolu26639602013-01-22 22:24:23 -0800977};
978
Cody P Schaferfd979c02015-01-30 13:45:57 -0800979ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
980 char *page);
981
Sukadev Bhattiprolu26639602013-01-22 22:24:23 -0800982#define PMU_EVENT_ATTR(_name, _var, _id, _show) \
983static struct perf_pmu_events_attr _var = { \
984 .attr = __ATTR(_name, 0444, _show, NULL), \
985 .id = _id, \
986};
987
Cody P Schaferf0405b82015-01-30 13:45:58 -0800988#define PMU_EVENT_ATTR_STRING(_name, _var, _str) \
989static struct perf_pmu_events_attr _var = { \
990 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
991 .id = 0, \
992 .event_str = _str, \
993};
994
Jiri Olsa641cc932012-03-15 20:09:14 +0100995#define PMU_FORMAT_ATTR(_name, _format) \
996static ssize_t \
997_name##_show(struct device *dev, \
998 struct device_attribute *attr, \
999 char *page) \
1000{ \
1001 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
1002 return sprintf(page, _format "\n"); \
1003} \
1004 \
1005static struct device_attribute format_attr_##_name = __ATTR_RO(_name)
1006
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001007#endif /* _LINUX_PERF_EVENT_H */