/* * Copyright (c) 2012, ARM Limited. All rights reserved. * * Redistribution and use in source and binary forms, with * or without modification, are permitted provided that the * following conditions are met: * * Redistributions of source code must retain the above * copyright notice, this list of conditions and the * following disclaimer. * * Redistributions in binary form must reproduce the * above copyright notice, this list of conditions and * the following disclaimer in the documentation * and/or other materials provided with the distribution. * * Neither the name of ARM nor the names of its * contributors may be used to endorse or promote products * derived from this software without specific prior written * permission. */ /* ---------------------------------------------------------------------------- * i n c l u d e s * --------------------------------------------------------------------------*/ #include "hyp_types.h" #include "hyp_vmmap.h" #include "misc.h" #include "events.h" #include "virt_helpers.h" typedef struct { unsigned va; unsigned pa; unsigned long long attrs; unsigned long long *pt_addr; } four_kb_pt_desc; /* ---------------------------------------------------------------------------- * d e f i n e s * --------------------------------------------------------------------------*/ #define LEVEL1 0x1 #define LEVEL2 0x2 #define HYP_PA_START 0x00000000 /* Flat mapping */ #define HYP_PA_END 0xFFFFFFFF #define HYP_VA_START HYP_PA_START #define HYP_VA_END HYP_PA_END /* * First level pagetables to cover 512GB. * Only first 4GB used */ unsigned long long hyp_l1_pagetable[512] __attribute__ ((aligned(4096))); /* * Second level pagetables to cover each GB. * Arranged contiguously for ease */ unsigned long long hyp_l2_pagetable[4][512] __attribute__ ((aligned(16384))); /* * Allocate one Level 3 page table which will * create a 4K SO ordered page. */ unsigned long long hyp_l3_so_pt[512] __attribute__ ((aligned(4096))); /* * Allocate space for 4 contiguous level 2 page * tables which will cover the 32 bit address * space. Align it to the 16K boundary. */ unsigned long long stage2_l2_pagetable[4][512] __attribute__ ((aligned(16384))); /* * Allocate one Level 3 page table which will * route guestOS physical cpu interface accesses * to the virtual cpu interface. Align it to the * 4K boundary. */ unsigned long long stage2_l3_cpuif_pt[512] __attribute__ ((aligned(4096))); /* * Allocate one Level 3 page table which will * create a 4K SO ordered page. */ unsigned long long stage2_l3_so_pt[512] __attribute__ ((aligned(4096))); /* * Allocate one Level 3 page table which will create a 4K page for KFSCB. Align * it to the 4K boundary. */ unsigned long long stage2_l3_kfscb_pt[512] __attribute__ ((aligned(4096))); #define ENABLE 0x1 #define DISABLE 0x0 /* HMAIR attributes relevant to us */ #define HMAIR_INNER_WB_RWA_MEM 0x0f #define HMAIR_OUTER_WB_RWA_MEM 0xf0 #define HMAIR_DEVICE_MEM 0x04 #define HMAIR_SO_MEM 0x00 #define IDX0(x) ((x) << 0) #define IDX1(x) ((x) << 8) #define IDX2(x) ((x) << 16) /* Memory attributes index for HMAIR0 */ #define HMAIR0_DEVICE_MEM_ATTR_IDX (0x0 << 2) #define HMAIR0_NORMAL_MEM_ATTR_IDX (0x1 << 2) #define HMAIR0_SO_MEM_ATTR_IDX (0x2 << 2) #define NS_BIT (1 << 5) /* Access permissions */ #define AP(x) ((x) << 6) /* HAP permissions */ #define HAP_RO 0x1 #define HAP_RW 0x3 /* Simplified Access permissions */ #define KERN_RO 0x2 #define KERN_RW 0x0 /* HTCR/VTCR fields */ #define EAE(x) ((unsigned) x << 31) #define T0SZ(x) (x << 0) #define IRGN0(x) (x << 8) #define ORGN0(x) (x << 10) #define SH0(x) (x << 12) #define CR_C_WBWA 0x1 #define CR_OUTER_SH 0x2 #define CR_INNER_SH 0x3 #define CR_ADDR_SPC_4GB 0x0 /* HSCTLR fields */ #define MMU(x) (x << 0) #define ALIGNMENT(x) (x << 1) #define DCACHE(x) (x << 2) #define ICACHE(x) (x << 12) /* * BUG: * Dcache clean by MVA ops added to ensure that main memory is updated prior to * the first page table walk upon entry into NS world. This is potentially an AEM * bug as the descriptors should be picked from the cache itself since the VTCR * marks PTWs as cacheable. * It would be better to collect the writes and then perform the clean rather then * picking them up individually. */ /* * Map the physical cpu interface to the virtual * cpu interface for OS use. */ static void CreateL3PageTable(four_kb_pt_desc * l3_mapping, unsigned level, unsigned long long *base_pt_addr) { unsigned one_gb_index = l3_mapping->pa >> 30; unsigned two_mb_index = l3_mapping->pa >> 21; unsigned four_kb_index = 0; unsigned pa_4k_index = 0; unsigned long long l1_desc = 0; unsigned long long *l2_desc = 0; unsigned long long old_attrs = 0; unsigned long long *l1_pt_addr = 0; unsigned long long *l2_pt_addr = 0; unsigned long long *l3_pt_addr = l3_mapping->pt_addr; /* * Indices calculated above are relative to the GB or MB they * belong to rather than an offset of 0x0. e.g. for the 2mb index * index = (address >> 21) - ( x ) */ /* Calculate the level 2 page table descriptor */ if (level == 1) { l1_pt_addr = base_pt_addr; l1_desc = l1_pt_addr[one_gb_index]; l2_pt_addr = (unsigned long long *)((unsigned)((&l1_desc)[0] & 0xfffff000UL)); l2_desc = &l2_pt_addr[two_mb_index - (512 * one_gb_index)]; } else { l2_pt_addr = &base_pt_addr[one_gb_index << 9]; l2_desc = &l2_pt_addr[two_mb_index - (512 * one_gb_index)]; } /* Preserve the old attributes */ old_attrs = *l2_desc & 0xfff0000000000fffULL; /* Replace block mapping with table mapping */ *l2_desc = (unsigned long long)l3_pt_addr | TABLE_MAPPING; /* Create a flat mapping for all 4k descriptors to begin with */ for (four_kb_index = 0; four_kb_index < 512; four_kb_index++) { l3_pt_addr[four_kb_index] = (((two_mb_index << 9) + four_kb_index) << 12) | old_attrs | VALID_MAPPING; } pa_4k_index = ((l3_mapping->pa << 11) >> 11) >> 12; /* * Replace the existing descriptor with new mapping and attributes */ l3_pt_addr[pa_4k_index] = l3_mapping->va | l3_mapping->attrs; return; } /* * Add an 4KB mapping to an existing L3 page table. */ static void Add4KMapping(four_kb_pt_desc * l3_mapping) { unsigned pa_4k_index = ((l3_mapping->pa << 11) >> 11) >> 12; unsigned long long *l3_pt_addr = l3_mapping->pt_addr; /* * Replace the existing descriptor with new mapping and attributes */ l3_pt_addr[pa_4k_index] = l3_mapping->va | l3_mapping->attrs; return; } void CreateHypModePageTables(void) { unsigned num_l1_descs = 0, num_l2_descs = 0; unsigned l1_index, l2_index; unsigned long long l2_attrs = 0; four_kb_pt_desc l3_desc; /* Create the pagetables */ num_l1_descs = ((HYP_PA_END - HYP_PA_START) >> 30) + 1; num_l2_descs = ((HYP_PA_END - HYP_PA_START) >> 21) + 1; /* Only the first 4GB are valid translations */ for (l1_index = 0; l1_index < num_l1_descs; l1_index++) { hyp_l1_pagetable[l1_index] = (unsigned long long)&hyp_l2_pagetable[l1_index][0] | TABLE_MAPPING; for (l2_index = 0; l2_index < num_l2_descs / num_l1_descs; l2_index++) { if ((l2_index + (l1_index << 9)) < 32) { /* 0-64M(Secure ROM/NOR Flash):Block mapping with RO access, Inner shareable, Inner/Outer WBWA */ l2_attrs = BLOCK_MAPPING | HMAIR0_NORMAL_MEM_ATTR_IDX | NS_BIT | SHAREABILITY(0x3) | ACCESS_FLAG | AP(KERN_RO); ((unsigned *)&l2_attrs)[1] |= XN; } else if ((l2_index + (l1_index << 9)) < 64) /* 64-128M(Secure RAM) : Block mapping with RW access, Inner shareable, Inner/Outer WBWA */ l2_attrs = BLOCK_MAPPING | HMAIR0_NORMAL_MEM_ATTR_IDX | NS_BIT | SHAREABILITY(0x3) | ACCESS_FLAG | AP(KERN_RW); else if ((l2_index + (l1_index << 9)) < 1024) { /* 128-2048M (Peripherals) : Block mapping of Device memory */ l2_attrs = BLOCK_MAPPING | HMAIR0_DEVICE_MEM_ATTR_IDX | NS_BIT | SHAREABILITY(0x3) | ACCESS_FLAG | AP(KERN_RW); ((unsigned *)&l2_attrs)[1] |= XN; } else /* 2-4GB (RAM) : Block mapping with RW access, Inner shareable, Inner/Outer WBWA */ l2_attrs = BLOCK_MAPPING | HMAIR0_NORMAL_MEM_ATTR_IDX | NS_BIT | SHAREABILITY(0x3) | ACCESS_FLAG | AP(KERN_RW); hyp_l2_pagetable[l1_index][l2_index] = ((l2_index + (l1_index << 9)) << 21) | l2_attrs; } } /* * Create a mapping for a device page to be used * for Locks, Events & anything that is shared when both * the clusters are executing at the same time. */ l3_desc.va = (unsigned)&BL_DV_PAGE$$Base; l3_desc.pa = (unsigned)&BL_DV_PAGE$$Base; l3_desc.attrs = ACCESS_FLAG | HMAIR0_DEVICE_MEM_ATTR_IDX | SHAREABILITY(0x3) | AP(KERN_RW) | VALID_MAPPING; l3_desc.pt_addr = hyp_l3_so_pt; CreateL3PageTable(&l3_desc, LEVEL1, (unsigned long long *)hyp_l1_pagetable); return; } void EnableHypModePageTables(void) { /* Update the HTTBR */ write_httbr((unsigned long long)hyp_l1_pagetable); /* * Setup the HMAIR0 register. * [7:0] = Device memory * [15:8] = Normal memory, Inner and outer cacheable, WBWA */ write_hmair0(IDX2(HMAIR_SO_MEM) | IDX1(HMAIR_INNER_WB_RWA_MEM | HMAIR_OUTER_WB_RWA_MEM) | IDX0(HMAIR_DEVICE_MEM)); /* * Set the HTCR. * Pagetables are Normal memory, Inner/Outer shareable, Inner/Outer WBWA */ write_htcr(EAE(ENABLE) | SH0(CR_INNER_SH) | ORGN0(CR_C_WBWA) | IRGN0(CR_C_WBWA) | T0SZ(CR_ADDR_SPC_4GB)); /* Enable the Hyp MMU */ write_hsctlr(ICACHE(ENABLE) | DCACHE(ENABLE) | ALIGNMENT(ENABLE) | MMU(ENABLE)); return; } void Create2ndStagePageTables(void) { unsigned two_mb_index = 0; unsigned one_gb_index = 0; unsigned long long level2_desc = 0; four_kb_pt_desc l3_desc = { 0 }; /* * Create the flat mapped 2nd stage page tables. * This should be done only once. The remaining * cpus can share the mappings and wait while * this is being done. */ for (one_gb_index = 0; one_gb_index < 4; one_gb_index++) for (two_mb_index = 0; two_mb_index < 512; two_mb_index++) { if ((two_mb_index + (one_gb_index << 9)) < 32) /* 0-64M (Secure ROM/NOR Flash) : Block mapping with RO access, Inner shareable, Inner/Outer WBWA */ level2_desc = ACCESS_FLAG | SHAREABILITY(0x3) | AP(HAP_RO) | MEM_ATTR(0xf) | BLOCK_MAPPING; else if ((two_mb_index + (one_gb_index << 9)) < 64) /* 64-128M (Secure RAM) : Block mapping with RW access, Inner shareable, Inner/Outer WBWA */ level2_desc = ACCESS_FLAG | SHAREABILITY(0x3) | AP(HAP_RW) | MEM_ATTR(0xf) | BLOCK_MAPPING; else if ((two_mb_index + (one_gb_index << 9)) < 1024) /* 128-2048M (Peripherals) : Block mapping of Device memory */ level2_desc = ACCESS_FLAG | SHAREABILITY(0x3) | AP(HAP_RW) | MEM_ATTR(0x1) | BLOCK_MAPPING; else /* 2-4GB (RAM) : Block mapping with RW access, Inner shareable, Inner/Outer WBWA */ level2_desc = ACCESS_FLAG | SHAREABILITY(0x3) | AP(HAP_RW) | MEM_ATTR(0xf) | BLOCK_MAPPING; stage2_l2_pagetable[one_gb_index][two_mb_index] = (two_mb_index + (512 * one_gb_index) << 21) | level2_desc; } /* First 4KB Mapping PCPUIF to the VCPUIF for the payload software */ l3_desc.va = VGIC_VM_PHY_BASE; l3_desc.pa = GIC_IC_PHY_BASE; l3_desc.attrs = ACCESS_FLAG | SHAREABILITY(0x3) | ACCESS_PERM(0x3) | MEM_ATTR(0x1) | VALID_MAPPING; l3_desc.pt_addr = stage2_l3_cpuif_pt; CreateL3PageTable(&l3_desc, LEVEL2, (unsigned long long *)stage2_l2_pagetable); /* Second 4KB Mapping PCPUIF to the VCPUIF for the payload software */ l3_desc.va = VGIC_VM_PHY_BASE + 0x1000; l3_desc.pa = GIC_IC_PHY_BASE + 0x1000; l3_desc.attrs = ACCESS_FLAG | SHAREABILITY(0x3) | ACCESS_PERM(0x3) | MEM_ATTR(0x1) | VALID_MAPPING; l3_desc.pt_addr = stage2_l3_cpuif_pt; Add4KMapping(&l3_desc); /* 4KB Mapping for trapping GIC Distributor accesses */ l3_desc.va = GIC_ID_PHY_BASE; l3_desc.pa = GIC_ID_PHY_BASE; l3_desc.attrs = ACCESS_FLAG | SHAREABILITY(0x3) | ACCESS_PERM(0x3) | MEM_ATTR(0x1); l3_desc.pt_addr = stage2_l3_cpuif_pt; Add4KMapping(&l3_desc); /* * Create a mapping for a device page to be used * for Locks, Events & anything that is shared when both * the clusters are executing at the same time. */ l3_desc.va = (unsigned)&BL_DV_PAGE$$Base; l3_desc.pa = (unsigned)&BL_DV_PAGE$$Base; l3_desc.attrs = ACCESS_FLAG | SHAREABILITY(0x3) | ACCESS_PERM(0x3) | MEM_ATTR(0x1) | VALID_MAPPING; l3_desc.pt_addr = stage2_l3_so_pt; CreateL3PageTable(&l3_desc, LEVEL2, (unsigned long long *)stage2_l2_pagetable); /* 4KB Mapping for the KFSCB for the payload software */ l3_desc.va = KFSCB_BASE; l3_desc.pa = KFSCB_BASE; l3_desc.attrs = ACCESS_FLAG | SHAREABILITY(0x3) | ACCESS_PERM(0x3) | MEM_ATTR(0x1); l3_desc.pt_addr = stage2_l3_kfscb_pt; CreateL3PageTable(&l3_desc, LEVEL2, (unsigned long long *)stage2_l2_pagetable); return; } void Enable2ndStagePageTables(void) { /* * Set the VTCR to: * Normal memory outer shareable, Device memory shareable * Outer and Inner WBWA * Start at level 2 * Size of addressed region is 4GB (16k worth of page tables) */ write_vtcr(SH0(CR_INNER_SH) | ORGN0(CR_C_WBWA) | IRGN0(CR_C_WBWA)); /* Address is already aligned to 16k or 2*14 */ write_vttbr((unsigned long long)stage2_l2_pagetable); write_hcr(read_hcr() | HCR_VM); /* * TODO: We do not need a synchronization barrier here as we * are not yet executing out of NS PL0 & PL1 and there will be * a barrier at some point before that. */ return; } void SetupVirtExtPageTables(void) { unsigned cpu_id = read_cpuid(); unsigned first_cpu = find_first_cpu(); unsigned cluster_id = read_clusterid(); unsigned abs_cpuid = 0; unsigned cpu_mask = 0; unsigned num_cpus = 0; if (!switcher) abs_cpuid = abs_cpuid(cpu_id, cluster_id); /* * First cpu creates the pagetables after * a cold reset. Reused by all cpus across * warm resets. */ if (switcher) { /* * While switching its possible that the host cluster * is brought out of reset first. Hence, the first * cpu of whichever cluster reaches here does the * pagetable setup */ if (cpu_id == first_cpu) { CreateHypModePageTables(); Create2ndStagePageTables(); /* Send the event to all the cpus on this cluster */ cpu_mask = (((1 << num_secondaries()) - 1) << 1) | 0x1; set_events(VIRT_PGT_DONE, cpu_mask); } wait_for_event(VIRT_PGT_DONE, cpu_id); reset_event(VIRT_PGT_DONE, cpu_id); } else { /* * Any cluster can do the initialisation as long as * only one of them does it. */ if (cpu_id == first_cpu && cluster_id == host_cluster) { CreateHypModePageTables(); Create2ndStagePageTables(); /* Send the event to all the cpus on this cluster */ num_cpus = CLUSTER_CPU_COUNT(cluster_id) + CLUSTER_CPU_COUNT(!cluster_id); cpu_mask = (1 << num_cpus) - 1; set_events(VIRT_PGT_DONE, cpu_mask); } wait_for_event(VIRT_PGT_DONE, abs_cpuid); reset_event(VIRT_PGT_DONE, abs_cpuid); } return; }