blob: 86f6a755af0bbe80c7821755f75a9a54854adfc6 [file] [log] [blame]
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +10001/*
2 * address space "slices" (meta-segments) support
3 *
4 * Copyright (C) 2007 Benjamin Herrenschmidt, IBM Corporation.
5 *
6 * Based on hugetlb implementation
7 *
8 * Copyright (C) 2003 David Gibson, IBM Corporation.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 */
24
25#undef DEBUG
26
27#include <linux/kernel.h>
28#include <linux/mm.h>
29#include <linux/pagemap.h>
30#include <linux/err.h>
31#include <linux/spinlock.h>
Paul Gortmaker4b16f8e2011-07-22 18:24:23 -040032#include <linux/export.h>
Anton Blanchard1217d342014-08-20 08:55:19 +100033#include <linux/hugetlb.h>
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +100034#include <asm/mman.h>
35#include <asm/mmu.h>
36#include <asm/spu.h>
Anton Blanchard1217d342014-08-20 08:55:19 +100037#include <asm/hugetlb.h>
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +100038
Aneesh Kumar K.V78f1dbd2012-09-10 02:52:57 +000039/* some sanity checks */
40#if (PGTABLE_RANGE >> 43) > SLICE_MASK_SIZE
41#error PGTABLE_RANGE exceeds slice_mask high_slices size
42#endif
43
Roel Kluinf7a75f02007-10-16 23:30:25 -070044static DEFINE_SPINLOCK(slice_convert_lock);
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +100045
46
47#ifdef DEBUG
48int _slice_debug = 1;
49
50static void slice_print_mask(const char *label, struct slice_mask mask)
51{
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +000052 char *p, buf[16 + 3 + 64 + 1];
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +100053 int i;
54
55 if (!_slice_debug)
56 return;
57 p = buf;
58 for (i = 0; i < SLICE_NUM_LOW; i++)
59 *(p++) = (mask.low_slices & (1 << i)) ? '1' : '0';
60 *(p++) = ' ';
61 *(p++) = '-';
62 *(p++) = ' ';
63 for (i = 0; i < SLICE_NUM_HIGH; i++)
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +000064 *(p++) = (mask.high_slices & (1ul << i)) ? '1' : '0';
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +100065 *(p++) = 0;
66
67 printk(KERN_DEBUG "%s:%s\n", label, buf);
68}
69
70#define slice_dbg(fmt...) do { if (_slice_debug) pr_debug(fmt); } while(0)
71
72#else
73
74static void slice_print_mask(const char *label, struct slice_mask mask) {}
75#define slice_dbg(fmt...)
76
77#endif
78
79static struct slice_mask slice_range_to_mask(unsigned long start,
80 unsigned long len)
81{
82 unsigned long end = start + len - 1;
83 struct slice_mask ret = { 0, 0 };
84
85 if (start < SLICE_LOW_TOP) {
86 unsigned long mend = min(end, SLICE_LOW_TOP);
87 unsigned long mstart = min(start, SLICE_LOW_TOP);
88
89 ret.low_slices = (1u << (GET_LOW_SLICE_INDEX(mend) + 1))
90 - (1u << GET_LOW_SLICE_INDEX(mstart));
91 }
92
93 if ((start + len) > SLICE_LOW_TOP)
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +000094 ret.high_slices = (1ul << (GET_HIGH_SLICE_INDEX(end) + 1))
95 - (1ul << GET_HIGH_SLICE_INDEX(start));
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +100096
97 return ret;
98}
99
100static int slice_area_is_free(struct mm_struct *mm, unsigned long addr,
101 unsigned long len)
102{
103 struct vm_area_struct *vma;
104
105 if ((mm->task_size - len) < addr)
106 return 0;
107 vma = find_vma(mm, addr);
108 return (!vma || (addr + len) <= vma->vm_start);
109}
110
111static int slice_low_has_vma(struct mm_struct *mm, unsigned long slice)
112{
113 return !slice_area_is_free(mm, slice << SLICE_LOW_SHIFT,
114 1ul << SLICE_LOW_SHIFT);
115}
116
117static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice)
118{
119 unsigned long start = slice << SLICE_HIGH_SHIFT;
120 unsigned long end = start + (1ul << SLICE_HIGH_SHIFT);
121
122 /* Hack, so that each addresses is controlled by exactly one
123 * of the high or low area bitmaps, the first high area starts
124 * at 4GB, not 0 */
125 if (start == 0)
126 start = SLICE_LOW_TOP;
127
128 return !slice_area_is_free(mm, start, end - start);
129}
130
131static struct slice_mask slice_mask_for_free(struct mm_struct *mm)
132{
133 struct slice_mask ret = { 0, 0 };
134 unsigned long i;
135
136 for (i = 0; i < SLICE_NUM_LOW; i++)
137 if (!slice_low_has_vma(mm, i))
138 ret.low_slices |= 1u << i;
139
140 if (mm->task_size <= SLICE_LOW_TOP)
141 return ret;
142
143 for (i = 0; i < SLICE_NUM_HIGH; i++)
144 if (!slice_high_has_vma(mm, i))
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000145 ret.high_slices |= 1ul << i;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000146
147 return ret;
148}
149
150static struct slice_mask slice_mask_for_size(struct mm_struct *mm, int psize)
151{
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000152 unsigned char *hpsizes;
153 int index, mask_index;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000154 struct slice_mask ret = { 0, 0 };
155 unsigned long i;
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000156 u64 lpsizes;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000157
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000158 lpsizes = mm->context.low_slices_psize;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000159 for (i = 0; i < SLICE_NUM_LOW; i++)
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000160 if (((lpsizes >> (i * 4)) & 0xf) == psize)
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000161 ret.low_slices |= 1u << i;
162
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000163 hpsizes = mm->context.high_slices_psize;
164 for (i = 0; i < SLICE_NUM_HIGH; i++) {
165 mask_index = i & 0x1;
166 index = i >> 1;
167 if (((hpsizes[index] >> (mask_index * 4)) & 0xf) == psize)
168 ret.high_slices |= 1ul << i;
169 }
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000170
171 return ret;
172}
173
174static int slice_check_fit(struct slice_mask mask, struct slice_mask available)
175{
176 return (mask.low_slices & available.low_slices) == mask.low_slices &&
177 (mask.high_slices & available.high_slices) == mask.high_slices;
178}
179
180static void slice_flush_segments(void *parm)
181{
182 struct mm_struct *mm = parm;
183 unsigned long flags;
184
185 if (mm != current->active_mm)
186 return;
187
188 /* update the paca copy of the context struct */
189 get_paca()->context = current->active_mm->context;
190
191 local_irq_save(flags);
192 slb_flush_and_rebolt();
193 local_irq_restore(flags);
194}
195
196static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psize)
197{
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000198 int index, mask_index;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000199 /* Write the new slice psize bits */
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000200 unsigned char *hpsizes;
201 u64 lpsizes;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000202 unsigned long i, flags;
203
204 slice_dbg("slice_convert(mm=%p, psize=%d)\n", mm, psize);
205 slice_print_mask(" mask", mask);
206
207 /* We need to use a spinlock here to protect against
208 * concurrent 64k -> 4k demotion ...
209 */
210 spin_lock_irqsave(&slice_convert_lock, flags);
211
212 lpsizes = mm->context.low_slices_psize;
213 for (i = 0; i < SLICE_NUM_LOW; i++)
214 if (mask.low_slices & (1u << i))
215 lpsizes = (lpsizes & ~(0xful << (i * 4))) |
216 (((unsigned long)psize) << (i * 4));
217
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000218 /* Assign the value back */
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000219 mm->context.low_slices_psize = lpsizes;
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000220
221 hpsizes = mm->context.high_slices_psize;
222 for (i = 0; i < SLICE_NUM_HIGH; i++) {
223 mask_index = i & 0x1;
224 index = i >> 1;
225 if (mask.high_slices & (1ul << i))
226 hpsizes[index] = (hpsizes[index] &
227 ~(0xf << (mask_index * 4))) |
228 (((unsigned long)psize) << (mask_index * 4));
229 }
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000230
231 slice_dbg(" lsps=%lx, hsps=%lx\n",
232 mm->context.low_slices_psize,
233 mm->context.high_slices_psize);
234
235 spin_unlock_irqrestore(&slice_convert_lock, flags);
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000236
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000237#ifdef CONFIG_SPU_BASE
238 spu_flush_all_slbs(mm);
239#endif
240}
241
Michel Lespinassefba23692013-04-29 11:53:53 -0700242/*
243 * Compute which slice addr is part of;
244 * set *boundary_addr to the start or end boundary of that slice
245 * (depending on 'end' parameter);
246 * return boolean indicating if the slice is marked as available in the
247 * 'available' slice_mark.
248 */
249static bool slice_scan_available(unsigned long addr,
250 struct slice_mask available,
251 int end,
252 unsigned long *boundary_addr)
253{
254 unsigned long slice;
255 if (addr < SLICE_LOW_TOP) {
256 slice = GET_LOW_SLICE_INDEX(addr);
257 *boundary_addr = (slice + end) << SLICE_LOW_SHIFT;
258 return !!(available.low_slices & (1u << slice));
259 } else {
260 slice = GET_HIGH_SLICE_INDEX(addr);
261 *boundary_addr = (slice + end) ?
262 ((slice + end) << SLICE_HIGH_SHIFT) : SLICE_LOW_TOP;
Anton Blanchard5a049f12013-11-18 14:55:28 +1100263 return !!(available.high_slices & (1ul << slice));
Michel Lespinassefba23692013-04-29 11:53:53 -0700264 }
265}
266
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000267static unsigned long slice_find_area_bottomup(struct mm_struct *mm,
268 unsigned long len,
269 struct slice_mask available,
Michel Lespinasse34d07172013-04-29 11:53:52 -0700270 int psize)
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000271{
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000272 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
Michel Lespinassefba23692013-04-29 11:53:53 -0700273 unsigned long addr, found, next_end;
274 struct vm_unmapped_area_info info;
275
276 info.flags = 0;
277 info.length = len;
278 info.align_mask = PAGE_MASK & ((1ul << pshift) - 1);
279 info.align_offset = 0;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000280
Michel Lespinasse34d07172013-04-29 11:53:52 -0700281 addr = TASK_UNMAPPED_BASE;
Michel Lespinassefba23692013-04-29 11:53:53 -0700282 while (addr < TASK_SIZE) {
283 info.low_limit = addr;
284 if (!slice_scan_available(addr, available, 1, &addr))
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000285 continue;
Michel Lespinassefba23692013-04-29 11:53:53 -0700286
287 next_slice:
288 /*
289 * At this point [info.low_limit; addr) covers
290 * available slices only and ends at a slice boundary.
291 * Check if we need to reduce the range, or if we can
292 * extend it to cover the next available slice.
293 */
294 if (addr >= TASK_SIZE)
295 addr = TASK_SIZE;
296 else if (slice_scan_available(addr, available, 1, &next_end)) {
297 addr = next_end;
298 goto next_slice;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000299 }
Michel Lespinassefba23692013-04-29 11:53:53 -0700300 info.high_limit = addr;
301
302 found = vm_unmapped_area(&info);
303 if (!(found & ~PAGE_MASK))
304 return found;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000305 }
306
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000307 return -ENOMEM;
308}
309
310static unsigned long slice_find_area_topdown(struct mm_struct *mm,
311 unsigned long len,
312 struct slice_mask available,
Michel Lespinasse34d07172013-04-29 11:53:52 -0700313 int psize)
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000314{
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000315 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
Michel Lespinassefba23692013-04-29 11:53:53 -0700316 unsigned long addr, found, prev;
317 struct vm_unmapped_area_info info;
318
319 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
320 info.length = len;
321 info.align_mask = PAGE_MASK & ((1ul << pshift) - 1);
322 info.align_offset = 0;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000323
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000324 addr = mm->mmap_base;
Michel Lespinassefba23692013-04-29 11:53:53 -0700325 while (addr > PAGE_SIZE) {
326 info.high_limit = addr;
327 if (!slice_scan_available(addr - 1, available, 0, &addr))
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000328 continue;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000329
Michel Lespinassefba23692013-04-29 11:53:53 -0700330 prev_slice:
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000331 /*
Michel Lespinassefba23692013-04-29 11:53:53 -0700332 * At this point [addr; info.high_limit) covers
333 * available slices only and starts at a slice boundary.
334 * Check if we need to reduce the range, or if we can
335 * extend it to cover the previous available slice.
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000336 */
Michel Lespinassefba23692013-04-29 11:53:53 -0700337 if (addr < PAGE_SIZE)
338 addr = PAGE_SIZE;
339 else if (slice_scan_available(addr - 1, available, 0, &prev)) {
340 addr = prev;
341 goto prev_slice;
342 }
343 info.low_limit = addr;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000344
Michel Lespinassefba23692013-04-29 11:53:53 -0700345 found = vm_unmapped_area(&info);
346 if (!(found & ~PAGE_MASK))
347 return found;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000348 }
349
350 /*
351 * A failed mmap() very likely causes application failure,
352 * so fall back to the bottom-up function here. This scenario
353 * can happen with large stack limits and large mmap()
354 * allocations.
355 */
Michel Lespinasse34d07172013-04-29 11:53:52 -0700356 return slice_find_area_bottomup(mm, len, available, psize);
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000357}
358
359
360static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
361 struct slice_mask mask, int psize,
Michel Lespinasse34d07172013-04-29 11:53:52 -0700362 int topdown)
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000363{
364 if (topdown)
Michel Lespinasse34d07172013-04-29 11:53:52 -0700365 return slice_find_area_topdown(mm, len, mask, psize);
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000366 else
Michel Lespinasse34d07172013-04-29 11:53:52 -0700367 return slice_find_area_bottomup(mm, len, mask, psize);
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000368}
369
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000370#define or_mask(dst, src) do { \
371 (dst).low_slices |= (src).low_slices; \
372 (dst).high_slices |= (src).high_slices; \
373} while (0)
374
375#define andnot_mask(dst, src) do { \
376 (dst).low_slices &= ~(src).low_slices; \
377 (dst).high_slices &= ~(src).high_slices; \
378} while (0)
379
380#ifdef CONFIG_PPC_64K_PAGES
381#define MMU_PAGE_BASE MMU_PAGE_64K
382#else
383#define MMU_PAGE_BASE MMU_PAGE_4K
384#endif
385
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000386unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
387 unsigned long flags, unsigned int psize,
Michel Lespinasse34d07172013-04-29 11:53:52 -0700388 int topdown)
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000389{
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000390 struct slice_mask mask = {0, 0};
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000391 struct slice_mask good_mask;
392 struct slice_mask potential_mask = {0,0} /* silence stupid warning */;
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000393 struct slice_mask compat_mask = {0, 0};
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000394 int fixed = (flags & MAP_FIXED);
395 int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
396 struct mm_struct *mm = current->mm;
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000397 unsigned long newaddr;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000398
399 /* Sanity checks */
400 BUG_ON(mm->task_size == 0);
401
402 slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize);
Michel Lespinasse34d07172013-04-29 11:53:52 -0700403 slice_dbg(" addr=%lx, len=%lx, flags=%lx, topdown=%d\n",
404 addr, len, flags, topdown);
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000405
406 if (len > mm->task_size)
407 return -ENOMEM;
Benjamin Herrenschmidtd1f5a772007-08-08 15:44:15 +1000408 if (len & ((1ul << pshift) - 1))
409 return -EINVAL;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000410 if (fixed && (addr & ((1ul << pshift) - 1)))
411 return -EINVAL;
412 if (fixed && addr > (mm->task_size - len))
jmarchan@redhat.com19751c02014-01-15 16:27:11 +0100413 return -ENOMEM;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000414
415 /* If hint, make sure it matches our alignment restrictions */
416 if (!fixed && addr) {
417 addr = _ALIGN_UP(addr, 1ul << pshift);
418 slice_dbg(" aligned addr=%lx\n", addr);
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000419 /* Ignore hint if it's too large or overlaps a VMA */
420 if (addr > mm->task_size - len ||
421 !slice_area_is_free(mm, addr, len))
422 addr = 0;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000423 }
424
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000425 /* First make up a "good" mask of slices that have the right size
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000426 * already
427 */
428 good_mask = slice_mask_for_size(mm, psize);
429 slice_print_mask(" good_mask", good_mask);
430
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000431 /*
432 * Here "good" means slices that are already the right page size,
433 * "compat" means slices that have a compatible page size (i.e.
434 * 4k in a 64k pagesize kernel), and "free" means slices without
435 * any VMAs.
436 *
437 * If MAP_FIXED:
438 * check if fits in good | compat => OK
439 * check if fits in good | compat | free => convert free
440 * else bad
441 * If have hint:
442 * check if hint fits in good => OK
443 * check if hint fits in good | free => convert free
444 * Otherwise:
445 * search in good, found => OK
446 * search in good | free, found => convert free
447 * search in good | compat | free, found => convert free.
448 */
449
450#ifdef CONFIG_PPC_64K_PAGES
451 /* If we support combo pages, we can allow 64k pages in 4k slices */
452 if (psize == MMU_PAGE_64K) {
453 compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K);
454 if (fixed)
455 or_mask(good_mask, compat_mask);
456 }
457#endif
458
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000459 /* First check hint if it's valid or if we have MAP_FIXED */
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000460 if (addr != 0 || fixed) {
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000461 /* Build a mask for the requested range */
462 mask = slice_range_to_mask(addr, len);
463 slice_print_mask(" mask", mask);
464
465 /* Check if we fit in the good mask. If we do, we just return,
466 * nothing else to do
467 */
468 if (slice_check_fit(mask, good_mask)) {
469 slice_dbg(" fits good !\n");
470 return addr;
471 }
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000472 } else {
473 /* Now let's see if we can find something in the existing
474 * slices for that size
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000475 */
Michel Lespinasse34d07172013-04-29 11:53:52 -0700476 newaddr = slice_find_area(mm, len, good_mask, psize, topdown);
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000477 if (newaddr != -ENOMEM) {
478 /* Found within the good mask, we don't have to setup,
479 * we thus return directly
480 */
481 slice_dbg(" found area at 0x%lx\n", newaddr);
482 return newaddr;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000483 }
484 }
485
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000486 /* We don't fit in the good mask, check what other slices are
487 * empty and thus can be converted
488 */
489 potential_mask = slice_mask_for_free(mm);
490 or_mask(potential_mask, good_mask);
491 slice_print_mask(" potential", potential_mask);
492
493 if ((addr != 0 || fixed) && slice_check_fit(mask, potential_mask)) {
494 slice_dbg(" fits potential !\n");
495 goto convert;
496 }
497
498 /* If we have MAP_FIXED and failed the above steps, then error out */
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000499 if (fixed)
500 return -EBUSY;
501
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000502 slice_dbg(" search...\n");
503
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000504 /* If we had a hint that didn't work out, see if we can fit
505 * anywhere in the good area.
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000506 */
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000507 if (addr) {
Michel Lespinasse34d07172013-04-29 11:53:52 -0700508 addr = slice_find_area(mm, len, good_mask, psize, topdown);
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000509 if (addr != -ENOMEM) {
510 slice_dbg(" found area at 0x%lx\n", addr);
511 return addr;
512 }
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000513 }
514
515 /* Now let's see if we can find something in the existing slices
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000516 * for that size plus free slices
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000517 */
Michel Lespinasse34d07172013-04-29 11:53:52 -0700518 addr = slice_find_area(mm, len, potential_mask, psize, topdown);
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000519
520#ifdef CONFIG_PPC_64K_PAGES
521 if (addr == -ENOMEM && psize == MMU_PAGE_64K) {
522 /* retry the search with 4k-page slices included */
523 or_mask(potential_mask, compat_mask);
524 addr = slice_find_area(mm, len, potential_mask, psize,
Michel Lespinasse34d07172013-04-29 11:53:52 -0700525 topdown);
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000526 }
527#endif
528
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000529 if (addr == -ENOMEM)
530 return -ENOMEM;
531
532 mask = slice_range_to_mask(addr, len);
533 slice_dbg(" found potential area at 0x%lx\n", addr);
534 slice_print_mask(" mask", mask);
535
536 convert:
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000537 andnot_mask(mask, good_mask);
538 andnot_mask(mask, compat_mask);
539 if (mask.low_slices || mask.high_slices) {
540 slice_convert(mm, mask, psize);
541 if (psize > MMU_PAGE_BASE)
Benjamin Herrenschmidt84c3d4a2008-07-16 11:07:59 +1000542 on_each_cpu(slice_flush_segments, mm, 1);
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000543 }
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000544 return addr;
545
546}
547EXPORT_SYMBOL_GPL(slice_get_unmapped_area);
548
549unsigned long arch_get_unmapped_area(struct file *filp,
550 unsigned long addr,
551 unsigned long len,
552 unsigned long pgoff,
553 unsigned long flags)
554{
555 return slice_get_unmapped_area(addr, len, flags,
Michel Lespinasse34d07172013-04-29 11:53:52 -0700556 current->mm->context.user_psize, 0);
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000557}
558
559unsigned long arch_get_unmapped_area_topdown(struct file *filp,
560 const unsigned long addr0,
561 const unsigned long len,
562 const unsigned long pgoff,
563 const unsigned long flags)
564{
565 return slice_get_unmapped_area(addr0, len, flags,
Michel Lespinasse34d07172013-04-29 11:53:52 -0700566 current->mm->context.user_psize, 1);
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000567}
568
569unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr)
570{
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000571 unsigned char *hpsizes;
572 int index, mask_index;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000573
574 if (addr < SLICE_LOW_TOP) {
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000575 u64 lpsizes;
576 lpsizes = mm->context.low_slices_psize;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000577 index = GET_LOW_SLICE_INDEX(addr);
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000578 return (lpsizes >> (index * 4)) & 0xf;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000579 }
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000580 hpsizes = mm->context.high_slices_psize;
581 index = GET_HIGH_SLICE_INDEX(addr);
582 mask_index = index & 0x1;
583 return (hpsizes[index >> 1] >> (mask_index * 4)) & 0xf;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000584}
585EXPORT_SYMBOL_GPL(get_slice_psize);
586
587/*
588 * This is called by hash_page when it needs to do a lazy conversion of
589 * an address space from real 64K pages to combo 4K pages (typically
590 * when hitting a non cacheable mapping on a processor or hypervisor
591 * that won't allow them for 64K pages).
592 *
593 * This is also called in init_new_context() to change back the user
594 * psize from whatever the parent context had it set to
Stephen Rothwell9dfe5c532007-08-15 16:33:55 +1000595 * N.B. This may be called before mm->context.id has been set.
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000596 *
597 * This function will only change the content of the {low,high)_slice_psize
598 * masks, it will not flush SLBs as this shall be handled lazily by the
599 * caller.
600 */
601void slice_set_user_psize(struct mm_struct *mm, unsigned int psize)
602{
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000603 int index, mask_index;
604 unsigned char *hpsizes;
605 unsigned long flags, lpsizes;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000606 unsigned int old_psize;
607 int i;
608
609 slice_dbg("slice_set_user_psize(mm=%p, psize=%d)\n", mm, psize);
610
611 spin_lock_irqsave(&slice_convert_lock, flags);
612
613 old_psize = mm->context.user_psize;
614 slice_dbg(" old_psize=%d\n", old_psize);
615 if (old_psize == psize)
616 goto bail;
617
618 mm->context.user_psize = psize;
619 wmb();
620
621 lpsizes = mm->context.low_slices_psize;
622 for (i = 0; i < SLICE_NUM_LOW; i++)
623 if (((lpsizes >> (i * 4)) & 0xf) == old_psize)
624 lpsizes = (lpsizes & ~(0xful << (i * 4))) |
625 (((unsigned long)psize) << (i * 4));
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000626 /* Assign the value back */
627 mm->context.low_slices_psize = lpsizes;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000628
629 hpsizes = mm->context.high_slices_psize;
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000630 for (i = 0; i < SLICE_NUM_HIGH; i++) {
631 mask_index = i & 0x1;
632 index = i >> 1;
633 if (((hpsizes[index] >> (mask_index * 4)) & 0xf) == old_psize)
634 hpsizes[index] = (hpsizes[index] &
635 ~(0xf << (mask_index * 4))) |
636 (((unsigned long)psize) << (mask_index * 4));
637 }
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000638
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000639
640
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000641
642 slice_dbg(" lsps=%lx, hsps=%lx\n",
643 mm->context.low_slices_psize,
644 mm->context.high_slices_psize);
645
646 bail:
647 spin_unlock_irqrestore(&slice_convert_lock, flags);
648}
649
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000650void slice_set_psize(struct mm_struct *mm, unsigned long address,
651 unsigned int psize)
652{
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000653 unsigned char *hpsizes;
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000654 unsigned long i, flags;
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000655 u64 *lpsizes;
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000656
657 spin_lock_irqsave(&slice_convert_lock, flags);
658 if (address < SLICE_LOW_TOP) {
659 i = GET_LOW_SLICE_INDEX(address);
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000660 lpsizes = &mm->context.low_slices_psize;
661 *lpsizes = (*lpsizes & ~(0xful << (i * 4))) |
662 ((unsigned long) psize << (i * 4));
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000663 } else {
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000664 int index, mask_index;
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000665 i = GET_HIGH_SLICE_INDEX(address);
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000666 hpsizes = mm->context.high_slices_psize;
667 mask_index = i & 0x1;
668 index = i >> 1;
669 hpsizes[index] = (hpsizes[index] &
670 ~(0xf << (mask_index * 4))) |
671 (((unsigned long)psize) << (mask_index * 4));
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000672 }
Aneesh Kumar K.V7aa07272012-09-10 02:52:52 +0000673
Paul Mackerras3a8247c2008-06-18 15:29:12 +1000674 spin_unlock_irqrestore(&slice_convert_lock, flags);
675
676#ifdef CONFIG_SPU_BASE
677 spu_flush_all_slbs(mm);
678#endif
679}
680
681void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
682 unsigned long len, unsigned int psize)
683{
684 struct slice_mask mask = slice_range_to_mask(start, len);
685
686 slice_convert(mm, mask, psize);
687}
688
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000689/*
Adam Buchbinder48fc7f72012-09-19 21:48:00 -0400690 * is_hugepage_only_range() is used by generic code to verify whether
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000691 * a normal mmap mapping (non hugetlbfs) is valid on a given area.
692 *
693 * until the generic code provides a more generic hook and/or starts
694 * calling arch get_unmapped_area for MAP_FIXED (which our implementation
695 * here knows how to deal with), we hijack it to keep standard mappings
696 * away from us.
697 *
698 * because of that generic code limitation, MAP_FIXED mapping cannot
699 * "convert" back a slice with no VMAs to the standard page size, only
700 * get_unmapped_area() can. It would be possible to fix it here but I
701 * prefer working on fixing the generic code instead.
702 *
703 * WARNING: This will not work if hugetlbfs isn't enabled since the
704 * generic code will redefine that function as 0 in that. This is ok
705 * for now as we only use slices with hugetlbfs enabled. This should
706 * be fixed as the generic code gets fixed.
707 */
708int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
709 unsigned long len)
710{
711 struct slice_mask mask, available;
Dave Kleikamp9ba0fdb2009-01-14 09:09:34 +0000712 unsigned int psize = mm->context.user_psize;
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000713
714 mask = slice_range_to_mask(addr, len);
Dave Kleikamp9ba0fdb2009-01-14 09:09:34 +0000715 available = slice_mask_for_size(mm, psize);
716#ifdef CONFIG_PPC_64K_PAGES
717 /* We need to account for 4k slices too */
718 if (psize == MMU_PAGE_64K) {
719 struct slice_mask compat_mask;
720 compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K);
721 or_mask(available, compat_mask);
722 }
723#endif
Benjamin Herrenschmidtd0f13e32007-05-08 16:27:27 +1000724
725#if 0 /* too verbose */
726 slice_dbg("is_hugepage_only_range(mm=%p, addr=%lx, len=%lx)\n",
727 mm, addr, len);
728 slice_print_mask(" mask", mask);
729 slice_print_mask(" available", available);
730#endif
731 return !slice_check_fit(mask, available);
732}
733