4 * ARM7 Virtual Memory Manager
5 * - arch/arm7/mm_virt.c
14 #define AP_KRW_ONLY 1 // Kernel page
15 #define AP_KRO_ONLY 5 // Kernel RO page
16 #define AP_RW_BOTH 3 // Standard RW
17 #define AP_RO_BOTH 6 // COW Page
18 #define AP_RO_USER 2 // User RO Page
19 #define PADDR_MASK_LVL1 0xFFFFFC00
22 extern Uint32 kernel_table0[];
36 //#define FRACTAL(table1, addr) ((table1)[ (0xFF8/4*1024) + ((addr)>>20)])
37 #define FRACTAL(table1, addr) ((table1)[ (0xFF8/4*1024) + ((addr)>>22)])
38 #define USRFRACTAL(addr) (*((Uint32*)(0x7FDFF000) + ((addr)>>22)))
39 #define TLBIALL() __asm__ __volatile__ ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0))
40 #define TLBIMVA(addr) __asm__ __volatile__ ("mcr p15, 0, %0, c8, c7, 1" : : "r" (addr))
43 void MM_int_GetTables(tVAddr VAddr, Uint32 **Table0, Uint32 **Table1);
44 int MM_int_AllocateCoarse(tVAddr VAddr, int Domain);
45 int MM_int_SetPageInfo(tVAddr VAddr, tMM_PageInfo *pi);
46 int MM_int_GetPageInfo(tVAddr VAddr, tMM_PageInfo *pi);
47 tVAddr MM_NewUserStack(void);
48 tPAddr MM_AllocateZero(tVAddr VAddr);
49 tPAddr MM_AllocateRootTable(void);
50 void MM_int_CloneTable(Uint32 *DestEnt, int Table);
51 tPAddr MM_Clone(void);
52 tVAddr MM_NewKStack(int bGlobal);
53 void MM_int_DumpTableEnt(tVAddr Start, size_t Len, tMM_PageInfo *Info);
54 //void MM_DumpTables(tVAddr Start, tVAddr End);
60 int MM_InitialiseVirtual(void)
65 void MM_int_GetTables(tVAddr VAddr, Uint32 **Table0, Uint32 **Table1)
67 if(VAddr & 0x80000000) {
68 *Table0 = (void*)&kernel_table0; // Level 0
69 *Table1 = (void*)MM_TABLE1KERN; // Level 1
72 *Table0 = (void*)MM_TABLE0USER;
73 *Table1 = (void*)MM_TABLE1USER;
77 int MM_int_AllocateCoarse(tVAddr VAddr, int Domain)
79 Uint32 *table0, *table1;
83 ENTER("xVAddr iDomain", VAddr, Domain);
85 MM_int_GetTables(VAddr, &table0, &table1);
87 VAddr &= ~(0x400000-1); // 4MiB per "block", 1 Page
89 desc = &table0[ VAddr>>20];
90 LOG("desc = %p", desc);
92 // table0: 4 bytes = 1 MiB
94 LOG("desc[0] = %x", desc[0]);
95 LOG("desc[1] = %x", desc[1]);
96 LOG("desc[2] = %x", desc[2]);
97 LOG("desc[3] = %x", desc[3]);
99 if( (desc[0] & 3) != 0 || (desc[1] & 3) != 0
100 || (desc[2] & 3) != 0 || (desc[3] & 3) != 0 )
107 paddr = MM_AllocPhys();
115 *desc = paddr | (Domain << 5) | 1;
116 desc[1] = desc[0] + 0x400;
117 desc[2] = desc[0] + 0x800;
118 desc[3] = desc[0] + 0xC00;
120 if( VAddr < 0x80000000 ) {
121 // Log("USRFRACTAL(%p) = %p", VAddr, &USRFRACTAL(VAddr));
122 USRFRACTAL(VAddr) = paddr | 3;
125 // Log("FRACTAL(%p) = %p", VAddr, &FRACTAL(table1, VAddr));
126 FRACTAL(table1, VAddr) = paddr | 3;
136 int MM_int_SetPageInfo(tVAddr VAddr, tMM_PageInfo *pi)
138 Uint32 *table0, *table1;
141 ENTER("pVAddr ppi", VAddr, pi);
143 MM_int_GetTables(VAddr, &table0, &table1);
145 desc = &table0[ VAddr >> 20 ];
146 LOG("desc = %p", desc);
150 case 12: // Small Page
151 case 16: // Large Page
153 if( (*desc & 3) == 0 ) {
154 MM_int_AllocateCoarse( VAddr, pi->Domain );
156 desc = &table1[ VAddr >> 12 ];
157 LOG("desc (2) = %p", desc);
161 // - Error if overwriting a large page
162 if( (*desc & 3) == 1 ) LEAVE_RET('i', 1);
163 if( pi->PhysAddr == 0 ) {
169 *desc = (pi->PhysAddr & 0xFFFFF000) | 2;
170 if(!pi->bExecutable) *desc |= 1; // XN
171 if(!pi->bGlobal) *desc |= 1 << 11; // NG
172 if( pi->bShared) *desc |= 1 << 10; // S
173 *desc |= (pi->AP & 3) << 4; // AP
174 *desc |= ((pi->AP >> 2) & 1) << 9; // APX
175 TLBIMVA(VAddr & 0xFFFFF000);
183 Log_Warning("MMVirt", "TODO: Implement large pages in MM_int_SetPageInfo");
186 case 20: // Section or unmapped
187 Warning("TODO: Implement sections");
189 case 24: // Supersection
190 // Error if not aligned
191 if( VAddr & 0xFFFFFF ) {
195 if( (*desc & 3) == 0 || ((*desc & 3) == 2 && (*desc & (1 << 18))) )
197 if( pi->PhysAddr == 0 ) {
199 // TODO: Apply to all entries
204 *desc = pi->PhysAddr & 0xFF000000;
205 // *desc |= ((pi->PhysAddr >> 32) & 0xF) << 20;
206 // *desc |= ((pi->PhysAddr >> 36) & 0x7) << 5;
207 *desc |= 2 | (1 << 18);
208 // TODO: Apply to all entries
221 int MM_int_GetPageInfo(tVAddr VAddr, tMM_PageInfo *pi)
223 Uint32 *table0, *table1;
226 // LogF("MM_int_GetPageInfo: VAddr=%p, pi=%p\n", VAddr, pi);
228 MM_int_GetTables(VAddr, &table0, &table1);
230 desc = table0[ VAddr >> 20 ];
232 // if( VAddr > 0x90000000)
233 // LOG("table0 desc(%p) = %x", &table0[ VAddr >> 20 ], desc);
249 // 1: Coarse page table
251 // Domain from top level table
252 pi->Domain = (desc >> 5) & 7;
254 desc = table1[ VAddr >> 12 ];
255 // LOG("table1 desc(%p) = %x", &table1[ VAddr >> 12 ], desc);
262 // 1: Large Page (64KiB)
265 pi->PhysAddr = desc & 0xFFFF0000;
266 pi->AP = ((desc >> 4) & 3) | (((desc >> 9) & 1) << 2);
267 pi->bExecutable = !(desc & 0x8000);
268 pi->bShared = (desc >> 10) & 1;
274 pi->PhysAddr = desc & 0xFFFFF000;
275 pi->bExecutable = !(desc & 1);
276 pi->bGlobal = !(desc >> 11);
277 pi->bShared = (desc >> 10) & 1;
278 pi->AP = ((desc >> 4) & 3) | (((desc >> 9) & 1) << 2);
283 // 2: Section (or Supersection)
285 if( desc & (1 << 18) ) {
287 pi->PhysAddr = desc & 0xFF000000;
288 pi->PhysAddr |= (Uint64)((desc >> 20) & 0xF) << 32;
289 pi->PhysAddr |= (Uint64)((desc >> 5) & 0x7) << 36;
291 pi->Domain = 0; // Supersections default to zero
292 pi->AP = ((desc >> 10) & 3) | (((desc >> 15) & 1) << 2);
297 pi->PhysAddr = desc & 0xFFF80000;
299 pi->Domain = (desc >> 5) & 7;
300 pi->AP = ((desc >> 10) & 3) | (((desc >> 15) & 1) << 2);
303 // 3: Reserved (invalid)
314 tPAddr MM_GetPhysAddr(tVAddr VAddr)
317 if( MM_int_GetPageInfo(VAddr, &pi) )
319 return pi.PhysAddr | (VAddr & ((1 << pi.Size)-1));
322 Uint MM_GetFlags(tVAddr VAddr)
327 if( MM_int_GetPageInfo(VAddr, &pi) )
337 ret |= MM_PFLAG_KERNEL;
340 ret |= MM_PFLAG_KERNEL|MM_PFLAG_RO;
352 if( pi.bExecutable ) ret |= MM_PFLAG_EXEC;
356 void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask)
361 if( MM_int_GetPageInfo(VAddr, &pi) )
364 curFlags = MM_GetPhysAddr(VAddr);
365 if( (curFlags & Mask) == Flags )
370 if( curFlags & MM_PFLAG_COW )
374 switch(curFlags & (MM_PFLAG_KERNEL|MM_PFLAG_RO) )
377 pi.AP = AP_RW_BOTH; break;
378 case MM_PFLAG_KERNEL:
379 pi.AP = AP_KRW_ONLY; break;
381 pi.AP = AP_RO_USER; break;
382 case MM_PFLAG_KERNEL|MM_PFLAG_RO:
383 pi.AP = AP_KRO_ONLY; break;
387 pi.bExecutable = !!(curFlags & MM_PFLAG_EXEC);
389 MM_int_SetPageInfo(VAddr, &pi);
392 int MM_Map(tVAddr VAddr, tPAddr PAddr)
394 tMM_PageInfo pi = {0};
396 Log("MM_Map %P=>%p", PAddr, VAddr);
401 if(VAddr < USER_STACK_TOP)
404 pi.AP = AP_KRW_ONLY; // Kernel Read/Write
406 if( MM_int_SetPageInfo(VAddr, &pi) ) {
407 MM_DerefPhys(pi.PhysAddr);
413 tPAddr MM_Allocate(tVAddr VAddr)
415 tMM_PageInfo pi = {0};
417 ENTER("pVAddr", VAddr);
419 pi.PhysAddr = MM_AllocPhys();
420 if( pi.PhysAddr == 0 ) LEAVE_RET('i', 0);
422 if(VAddr < USER_STACK_TOP)
427 if( MM_int_SetPageInfo(VAddr, &pi) ) {
428 MM_DerefPhys(pi.PhysAddr);
432 LEAVE('x', pi.PhysAddr);
436 tPAddr MM_AllocateZero(tVAddr VAddr)
438 if( !giMM_ZeroPage ) {
439 giMM_ZeroPage = MM_Allocate(VAddr);
440 MM_RefPhys(giMM_ZeroPage);
441 memset((void*)VAddr, 0, PAGE_SIZE);
444 MM_RefPhys(giMM_ZeroPage);
445 MM_Map(VAddr, giMM_ZeroPage);
447 MM_SetFlags(VAddr, MM_PFLAG_COW, MM_PFLAG_COW);
448 return giMM_ZeroPage;
451 void MM_Deallocate(tVAddr VAddr)
455 if( MM_int_GetPageInfo(VAddr, &pi) ) return ;
457 if( pi.PhysAddr == 0 ) return;
458 MM_DerefPhys(pi.PhysAddr);
463 MM_int_SetPageInfo(VAddr, &pi);
466 tPAddr MM_AllocateRootTable(void)
470 ret = MM_AllocPhysRange(2, -1);
473 MM_DerefPhys(ret+0x1000);
474 ret = MM_AllocPhysRange(3, -1);
478 // Log("MM_AllocateRootTable: Second try not aligned, %P", ret);
481 MM_DerefPhys(ret + 0x2000);
482 // Log("MM_AllocateRootTable: Second try aligned, %P", ret);
486 // Log("MM_AllocateRootTable: Got it in one, %P", ret);
490 void MM_int_CloneTable(Uint32 *DestEnt, int Table)
494 Uint32 *cur = (void*)MM_TABLE0USER;
495 // Uint32 *cur = &FRACTAL(MM_TABLE1USER,0);
498 table = MM_AllocPhys();
501 tmp_map = (void*)MM_MapTemp(table);
503 for( i = 0; i < 1024; i ++ )
507 case 0: tmp_map[i] = 0; break;
510 Log_Error("MMVirt", "TODO: Support large pages in MM_int_CloneTable");
517 if( (cur[Table*256] & 0x230) == 0x030 )
518 cur[Table*256+i] |= 0x200; // Set to full RO (Full RO=COW, User RO = RO)
519 tmp_map[i] = cur[Table*256+i];
524 DestEnt[0] = table + 0*0x400 + 1;
525 DestEnt[1] = table + 1*0x400 + 1;
526 DestEnt[2] = table + 2*0x400 + 1;
527 DestEnt[3] = table + 3*0x400 + 1;
530 tPAddr MM_Clone(void)
533 Uint32 *new_lvl1_1, *new_lvl1_2, *cur;
537 ret = MM_AllocateRootTable();
539 cur = (void*)MM_TABLE0USER;
540 new_lvl1_1 = (void*)MM_MapTemp(ret);
541 new_lvl1_2 = (void*)MM_MapTemp(ret+0x1000);
542 tmp_map = new_lvl1_1;
543 for( i = 0; i < 0x800-4; i ++ )
547 tmp_map = &new_lvl1_2[-0x400];
550 case 0: tmp_map[i] = 0; break;
552 MM_int_CloneTable(&tmp_map[i], i);
553 i += 3; // Tables are alocated in blocks of 4
557 Log_Error("MMVirt", "TODO: Support Sections/Supersections in MM_Clone (i=%i)", i);
563 // Allocate Fractal table
566 tPAddr tmp = MM_AllocPhys();
567 Uint32 *table = (void*)MM_MapTemp(tmp);
569 register Uint32 __SP asm("sp");
571 // Map table to last 4MiB of user space
572 new_lvl1_2[0x3FC] = tmp + 0*0x400 + 1;
573 new_lvl1_2[0x3FD] = tmp + 1*0x400 + 1;
574 new_lvl1_2[0x3FE] = tmp + 2*0x400 + 1;
575 new_lvl1_2[0x3FF] = tmp + 3*0x400 + 1;
577 tmp_map = new_lvl1_1;
578 for( j = 0; j < 512; j ++ )
581 tmp_map = &new_lvl1_2[-0x400];
582 if( (tmp_map[j*4] & 3) == 1 )
584 table[j] = tmp_map[j*4] & PADDR_MASK_LVL1;// 0xFFFFFC00;
585 table[j] |= 0x813; // nG, Kernel Only, Small page, XN
591 table[j++] = (ret + 0x0000) | 0x813;
592 table[j++] = (ret + 0x1000) | 0x813;
594 for( ; j < 1024; j ++ )
597 // Get kernel stack bottom
598 sp = __SP & ~(MM_KSTACK_SIZE-1);
599 j = (sp / 0x1000) % 1024;
600 num = MM_KSTACK_SIZE/0x1000;
603 for(; num--; j ++, sp += 0x1000)
608 page = MM_AllocPhys();
609 table[j] = page | 0x813;
611 tmp_page = (void*)MM_MapTemp(page);
612 memcpy(tmp_page, (void*)sp, 0x1000);
613 MM_FreeTemp( (tVAddr) tmp_page );
616 MM_FreeTemp( (tVAddr)table );
619 MM_FreeTemp( (tVAddr)new_lvl1_1 );
620 MM_FreeTemp( (tVAddr)new_lvl1_2 );
625 tPAddr MM_ClearUser(void)
627 // TODO: Implement ClearUser
631 tVAddr MM_MapTemp(tPAddr PAddr)
636 for( ret = MM_TMPMAP_BASE; ret < MM_TMPMAP_END - PAGE_SIZE; ret += PAGE_SIZE )
638 if( MM_int_GetPageInfo(ret, &pi) == 0 )
641 // Log("MapTemp %P at %p", PAddr, ret);
642 MM_RefPhys(PAddr); // Counter the MM_Deallocate in FreeTemp
647 Log_Warning("MMVirt", "MM_MapTemp: All slots taken");
651 void MM_FreeTemp(tVAddr VAddr)
653 // TODO: Implement FreeTemp
654 if( VAddr < MM_TMPMAP_BASE || VAddr >= MM_TMPMAP_END ) {
655 Log_Warning("MMVirt", "MM_FreeTemp: Passed an addr not from MM_MapTemp (%p)", VAddr);
659 MM_Deallocate(VAddr);
662 tVAddr MM_MapHWPages(tPAddr PAddr, Uint NPages)
668 ENTER("xPAddr iNPages", PAddr, NPages);
670 // Scan for a location
671 for( ret = MM_HWMAP_BASE; ret < MM_HWMAP_END - NPages * PAGE_SIZE; ret += PAGE_SIZE )
673 // LOG("checking %p", ret);
674 // Check if there is `NPages` free pages
675 for( i = 0; i < NPages; i ++ )
677 if( MM_int_GetPageInfo(ret + i*PAGE_SIZE, &pi) == 0 )
680 // Nope, jump to after the used page found and try again
681 // LOG("i = %i, ==? %i", i, NPages);
683 ret += i * PAGE_SIZE;
688 for( i = 0; i < NPages; i ++ )
689 MM_Map(ret+i*PAGE_SIZE, PAddr+i*PAGE_SIZE);
694 Log_Warning("MMVirt", "MM_MapHWPages: No space for a %i page block", NPages);
699 tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PAddr)
704 phys = MM_AllocPhysRange(Pages, MaxBits);
706 Log_Warning("MMVirt", "No space left for a %i page block (MM_AllocDMA)", Pages);
710 ret = MM_MapHWPages(phys, Pages);
716 void MM_UnmapHWPages(tVAddr Vaddr, Uint Number)
718 Log_Error("MMVirt", "TODO: Implement MM_UnmapHWPages");
721 tVAddr MM_NewKStack(int bShared)
723 tVAddr min_addr, max_addr;
727 min_addr = MM_GLOBALSTACKS;
728 max_addr = MM_GLOBALSTACKS_END;
731 min_addr = MM_KSTACK_BASE;
732 max_addr = MM_KSTACK_END;
735 // Locate a free slot
736 for( addr = min_addr; addr < max_addr; addr += MM_KSTACK_SIZE )
739 if( MM_int_GetPageInfo(addr+MM_KSTACK_SIZE-PAGE_SIZE, &pi) ) break;
742 // Check for an error
743 if(addr >= max_addr) {
748 for( ofs = PAGE_SIZE; ofs < MM_KSTACK_SIZE; ofs += PAGE_SIZE )
750 if( MM_Allocate(addr + ofs) == 0 )
755 MM_Deallocate(addr + ofs);
757 Log_Warning("MMVirt", "MM_NewKStack: Unable to allocate");
764 tVAddr MM_NewUserStack(void)
768 addr = USER_STACK_TOP - USER_STACK_SIZE;
769 if( MM_GetPhysAddr(addr + PAGE_SIZE) ) {
770 Log_Error("MMVirt", "Unable to create initial user stack, addr %p taken",
777 for( ofs = PAGE_SIZE; ofs < USER_STACK_SIZE; ofs += PAGE_SIZE )
780 if(ofs >= USER_STACK_SIZE - USER_STACK_COMM)
781 rv = MM_Allocate(addr + ofs);
783 rv = MM_AllocateZero(addr + ofs);
789 MM_Deallocate(addr + ofs);
791 Log_Warning("MMVirt", "MM_NewUserStack: Unable to allocate");
794 MM_SetFlags(addr+ofs, 0, MM_PFLAG_KERNEL);
796 Log("Return %p", addr + ofs);
797 MM_DumpTables(0, 0x80000000);
801 void MM_int_DumpTableEnt(tVAddr Start, size_t Len, tMM_PageInfo *Info)
803 if( giMM_ZeroPage && Info->PhysAddr == giMM_ZeroPage )
805 Log("%p => %8s - 0x%7x %i %x",
807 Info->Domain, Info->AP
812 Log("%p => %8x - 0x%7x %i %x",
813 Start, Info->PhysAddr-Len, Len,
814 Info->Domain, Info->AP
819 void MM_DumpTables(tVAddr Start, tVAddr End)
821 tVAddr range_start = 0, addr;
822 tMM_PageInfo pi, pi_old;
823 int i = 0, inRange=0;
827 Log("Page Table Dump:");
829 for( addr = Start; i == 0 || (addr && addr < End); i = 1 )
831 // Log("addr = %p", addr);
832 int rv = MM_int_GetPageInfo(addr, &pi);
834 || pi.Size != pi_old.Size
835 || pi.Domain != pi_old.Domain
836 || pi.AP != pi_old.AP
837 || pi_old.PhysAddr != pi.PhysAddr )
840 MM_int_DumpTableEnt(range_start, addr - range_start, &pi_old);
842 addr &= ~((1 << pi.Size)-1);
847 // Handle the zero page
848 if( !giMM_ZeroPage || pi_old.Size != 12 || pi_old.PhysAddr != giMM_ZeroPage )
849 pi_old.PhysAddr += 1 << pi_old.Size;
850 addr += 1 << pi_old.Size;
854 MM_int_DumpTableEnt(range_start, addr - range_start, &pi);