aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/base/node.c34
-rw-r--r--include/linux/mmzone.h17
-rw-r--r--include/linux/vmstat.h10
-rw-r--r--mm/mempolicy.c6
-rw-r--r--mm/page_alloc.c23
-rw-r--r--mm/vmstat.c73
6 files changed, 73 insertions, 90 deletions
diff --git a/drivers/base/node.c b/drivers/base/node.c
index 772eadac57a..d7de1753e09 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -94,28 +94,6 @@ static SYSDEV_ATTR(meminfo, S_IRUGO, node_read_meminfo, NULL);
static ssize_t node_read_numastat(struct sys_device * dev, char * buf)
{
- unsigned long numa_hit, numa_miss, interleave_hit, numa_foreign;
- unsigned long local_node, other_node;
- int i, cpu;
- pg_data_t *pg = NODE_DATA(dev->id);
- numa_hit = 0;
- numa_miss = 0;
- interleave_hit = 0;
- numa_foreign = 0;
- local_node = 0;
- other_node = 0;
- for (i = 0; i < MAX_NR_ZONES; i++) {
- struct zone *z = &pg->node_zones[i];
- for_each_online_cpu(cpu) {
- struct per_cpu_pageset *ps = zone_pcp(z,cpu);
- numa_hit += ps->numa_hit;
- numa_miss += ps->numa_miss;
- numa_foreign += ps->numa_foreign;
- interleave_hit += ps->interleave_hit;
- local_node += ps->local_node;
- other_node += ps->other_node;
- }
- }
return sprintf(buf,
"numa_hit %lu\n"
"numa_miss %lu\n"
@@ -123,12 +101,12 @@ static ssize_t node_read_numastat(struct sys_device * dev, char * buf)
"interleave_hit %lu\n"
"local_node %lu\n"
"other_node %lu\n",
- numa_hit,
- numa_miss,
- numa_foreign,
- interleave_hit,
- local_node,
- other_node);
+ node_page_state(dev->id, NUMA_HIT),
+ node_page_state(dev->id, NUMA_MISS),
+ node_page_state(dev->id, NUMA_FOREIGN),
+ node_page_state(dev->id, NUMA_INTERLEAVE_HIT),
+ node_page_state(dev->id, NUMA_LOCAL),
+ node_page_state(dev->id, NUMA_OTHER));
}
static SYSDEV_ATTR(numastat, S_IRUGO, node_read_numastat, NULL);
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 2dbeec1d287..27e748eb72b 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -57,6 +57,14 @@ enum zone_stat_item {
NR_WRITEBACK,
NR_UNSTABLE_NFS, /* NFS unstable pages */
NR_BOUNCE,
+#ifdef CONFIG_NUMA
+ NUMA_HIT, /* allocated in intended node */
+ NUMA_MISS, /* allocated in non intended node */
+ NUMA_FOREIGN, /* was intended here, hit elsewhere */
+ NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */
+ NUMA_LOCAL, /* allocation from local node */
+ NUMA_OTHER, /* allocation from other node */
+#endif
NR_VM_ZONE_STAT_ITEMS };
struct per_cpu_pages {
@@ -71,15 +79,6 @@ struct per_cpu_pageset {
#ifdef CONFIG_SMP
s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
#endif
-
-#ifdef CONFIG_NUMA
- unsigned long numa_hit; /* allocated in intended node */
- unsigned long numa_miss; /* allocated in non intended node */
- unsigned long numa_foreign; /* was intended here, hit elsewhere */
- unsigned long interleave_hit; /* interleaver prefered this zone */
- unsigned long local_node; /* allocation from local node */
- unsigned long other_node; /* allocation from other node */
-#endif
} ____cacheline_aligned_in_smp;
#ifdef CONFIG_NUMA
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 5fad1613e7d..16173b63ee6 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -173,9 +173,15 @@ static inline unsigned long node_page_state(int node,
#endif
zone_page_state(&zones[ZONE_DMA], item);
}
+
+extern void zone_statistics(struct zonelist *, struct zone *);
+
#else
+
#define node_page_state(node, item) global_page_state(item)
-#endif
+#define zone_statistics(_zl,_z) do { } while (0)
+
+#endif /* CONFIG_NUMA */
#define __add_zone_page_state(__z, __i, __d) \
__mod_zone_page_state(__z, __i, __d)
@@ -190,6 +196,8 @@ static inline void zap_zone_vm_stats(struct zone *zone)
memset(zone->vm_stat, 0, sizeof(zone->vm_stat));
}
+extern void inc_zone_state(struct zone *, enum zone_stat_item);
+
#ifdef CONFIG_SMP
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
void __inc_zone_page_state(struct page *, enum zone_stat_item);
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 6b9740bbf4c..e07e27e846a 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1209,10 +1209,8 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
page = __alloc_pages(gfp, order, zl);
- if (page && page_zone(page) == zl->zones[0]) {
- zone_pcp(zl->zones[0],get_cpu())->interleave_hit++;
- put_cpu();
- }
+ if (page && page_zone(page) == zl->zones[0])
+ inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
return page;
}
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6aa2c31f513..d61671260f9 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -709,27 +709,6 @@ void drain_local_pages(void)
}
#endif /* CONFIG_PM */
-static void zone_statistics(struct zonelist *zonelist, struct zone *z, int cpu)
-{
-#ifdef CONFIG_NUMA
- pg_data_t *pg = z->zone_pgdat;
- pg_data_t *orig = zonelist->zones[0]->zone_pgdat;
- struct per_cpu_pageset *p;
-
- p = zone_pcp(z, cpu);
- if (pg == orig) {
- p->numa_hit++;
- } else {
- p->numa_miss++;
- zone_pcp(zonelist->zones[0], cpu)->numa_foreign++;
- }
- if (pg == NODE_DATA(numa_node_id()))
- p->local_node++;
- else
- p->other_node++;
-#endif
-}
-
/*
* Free a 0-order page
*/
@@ -827,7 +806,7 @@ again:
}
__mod_page_state_zone(zone, pgalloc, 1 << order);
- zone_statistics(zonelist, zone, cpu);
+ zone_statistics(zonelist, zone);
local_irq_restore(flags);
put_cpu();
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 06a6d105219..ee7f8966625 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -185,9 +185,8 @@ EXPORT_SYMBOL(mod_zone_page_state);
* in between and therefore the atomicity vs. interrupt cannot be exploited
* in a useful way here.
*/
-void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
+static void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
- struct zone *zone = page_zone(page);
s8 *p = diff_pointer(zone, item);
(*p)++;
@@ -197,6 +196,11 @@ void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
*p = 0;
}
}
+
+void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
+{
+ __inc_zone_state(page_zone(page), item);
+}
EXPORT_SYMBOL(__inc_zone_page_state);
void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
@@ -213,22 +217,23 @@ void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
}
EXPORT_SYMBOL(__dec_zone_page_state);
+void inc_zone_state(struct zone *zone, enum zone_stat_item item)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ __inc_zone_state(zone, item);
+ local_irq_restore(flags);
+}
+
void inc_zone_page_state(struct page *page, enum zone_stat_item item)
{
unsigned long flags;
struct zone *zone;
- s8 *p;
zone = page_zone(page);
local_irq_save(flags);
- p = diff_pointer(zone, item);
-
- (*p)++;
-
- if (unlikely(*p > STAT_THRESHOLD)) {
- zone_page_state_add(*p, zone, item);
- *p = 0;
- }
+ __inc_zone_state(zone, item);
local_irq_restore(flags);
}
EXPORT_SYMBOL(inc_zone_page_state);
@@ -297,6 +302,28 @@ EXPORT_SYMBOL(refresh_vm_stats);
#endif
+#ifdef CONFIG_NUMA
+/*
+ * zonelist = the list of zones passed to the allocator
+ * z = the zone from which the allocation occurred.
+ *
+ * Must be called with interrupts disabled.
+ */
+void zone_statistics(struct zonelist *zonelist, struct zone *z)
+{
+ if (z->zone_pgdat == zonelist->zones[0]->zone_pgdat) {
+ __inc_zone_state(z, NUMA_HIT);
+ } else {
+ __inc_zone_state(z, NUMA_MISS);
+ __inc_zone_state(zonelist->zones[0], NUMA_FOREIGN);
+ }
+ if (z->zone_pgdat == NODE_DATA(numa_node_id()))
+ __inc_zone_state(z, NUMA_LOCAL);
+ else
+ __inc_zone_state(z, NUMA_OTHER);
+}
+#endif
+
#ifdef CONFIG_PROC_FS
#include <linux/seq_file.h>
@@ -369,6 +396,15 @@ static char *vmstat_text[] = {
"nr_unstable",
"nr_bounce",
+#ifdef CONFIG_NUMA
+ "numa_hit",
+ "numa_miss",
+ "numa_foreign",
+ "numa_interleave",
+ "numa_local",
+ "numa_other",
+#endif
+
/* Event counters */
"pgpgin",
"pgpgout",
@@ -490,21 +526,6 @@ static int zoneinfo_show(struct seq_file *m, void *arg)
pageset->pcp[j].high,
pageset->pcp[j].batch);
}
-#ifdef CONFIG_NUMA
- seq_printf(m,
- "\n numa_hit: %lu"
- "\n numa_miss: %lu"
- "\n numa_foreign: %lu"
- "\n interleave_hit: %lu"
- "\n local_node: %lu"
- "\n other_node: %lu",
- pageset->numa_hit,
- pageset->numa_miss,
- pageset->numa_foreign,
- pageset->interleave_hit,
- pageset->local_node,
- pageset->other_node);
-#endif
}
seq_printf(m,
"\n all_unreclaimable: %u"
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-26 21:12:05 +0000