X-Git-Url: https://git.ucc.asn.au/?a=blobdiff_plain;f=KernelLand%2FKernel%2Farch%2Farmv7%2Fmm_virt.c;h=cc67497705f2a77fd75fc0647c3a9a8a68944238;hb=802762ae7efd39786c0e134ed0bdfe8100d97230;hp=90655b631af109e4390ab070de0defcd34210d4b;hpb=51ab5f489bc356940c95cc936fd0508e8f07ea97;p=tpg%2Facess2.git diff --git a/KernelLand/Kernel/arch/armv7/mm_virt.c b/KernelLand/Kernel/arch/armv7/mm_virt.c index 90655b63..cc674977 100644 --- a/KernelLand/Kernel/arch/armv7/mm_virt.c +++ b/KernelLand/Kernel/arch/armv7/mm_virt.c @@ -10,6 +10,7 @@ #include #define TRACE_MAPS 0 +#define TRACE_COW 0 #define AP_KRW_ONLY 1 // Kernel page #define AP_KRO_ONLY 5 // Kernel RO page @@ -18,6 +19,13 @@ #define AP_RO_USER 2 // User RO Page #define PADDR_MASK_LVL1 0xFFFFFC00 +const char * const caAPValueNames[] = { + "AP_NOACCESS", "AP_KRW_ONLY", + "AP_RO_USER", "AP_RW_BOTH", + "AP_???_4", "AP_KRO_ONLY", + "AP_???_6", "AP_RO_BOTH" +}; + // === IMPORTS === extern Uint32 kernel_table0[]; @@ -46,14 +54,14 @@ void MM_int_GetTables(tVAddr VAddr, Uint32 **Table0, Uint32 **Table1); int MM_int_SetPageInfo(tVAddr VAddr, tMM_PageInfo *pi); int MM_int_GetPageInfo(tVAddr VAddr, tMM_PageInfo *pi); tVAddr MM_NewUserStack(void); -tPAddr MM_AllocateZero(tVAddr VAddr); +//tPAddr MM_AllocateZero(volatile void *VAddr); tPAddr MM_AllocateRootTable(void); void MM_int_CloneTable(Uint32 *DestEnt, int Table); -tPAddr MM_Clone(void); +tPAddr MM_Clone(int ClearUser); tVAddr MM_NewKStack(int bGlobal); void MM_int_DumpTableEnt(tVAddr Start, size_t Len, tMM_PageInfo *Info); //void MM_DumpTables(tVAddr Start, tVAddr End); -void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch); +void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch, Uint32 SystemLR, Uint32 UserLR); // === GLOBALS === tPAddr giMM_ZeroPage; @@ -320,20 +328,20 @@ int MM_int_GetPageInfo(tVAddr VAddr, tMM_PageInfo *pi) } // --- Exports --- -tPAddr MM_GetPhysAddr(tVAddr VAddr) +tPAddr MM_GetPhysAddr(volatile const void *Ptr) { tMM_PageInfo pi; - if( MM_int_GetPageInfo(VAddr, &pi) ) + if( MM_int_GetPageInfo((tVAddr)Ptr, &pi) ) return 0; - return pi.PhysAddr | (VAddr & ((1 << pi.Size)-1)); + return pi.PhysAddr | ((tVAddr)Ptr & ((1 << pi.Size)-1)); } -Uint MM_GetFlags(tVAddr VAddr) +Uint MM_GetFlags(const volatile void *VAddr) { tMM_PageInfo pi; int ret; - if( MM_int_GetPageInfo(VAddr, &pi) ) + if( MM_int_GetPageInfo((tVAddr)VAddr, &pi) ) return 0; ret = 0; @@ -362,12 +370,12 @@ Uint MM_GetFlags(tVAddr VAddr) return ret; } -void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) +void MM_SetFlags(volatile void *VAddr, Uint Flags, Uint Mask) { tMM_PageInfo pi; Uint curFlags; - if( MM_int_GetPageInfo(VAddr, &pi) ) + if( MM_int_GetPageInfo((tVAddr)VAddr, &pi) ) return ; curFlags = MM_GetFlags(VAddr); @@ -395,7 +403,7 @@ void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) pi.bExecutable = !!(curFlags & MM_PFLAG_EXEC); - MM_int_SetPageInfo(VAddr, &pi); + MM_int_SetPageInfo((tVAddr)VAddr, &pi); } int MM_IsValidBuffer(tVAddr Addr, size_t Size) @@ -406,7 +414,7 @@ int MM_IsValidBuffer(tVAddr Addr, size_t Size) Size += Addr & (PAGE_SIZE-1); Addr &= ~(PAGE_SIZE-1); - if( MM_int_GetPageInfo(Addr, &pi) ) return 0; + if( MM_int_GetPageInfo((tVAddr)Addr, &pi) ) return 0; Addr += PAGE_SIZE; if(pi.AP != AP_KRW_ONLY && pi.AP != AP_KRO_ONLY) @@ -425,28 +433,27 @@ int MM_IsValidBuffer(tVAddr Addr, size_t Size) return 1; } -int MM_Map(tVAddr VAddr, tPAddr PAddr) +int MM_Map(volatile void *VAddr, tPAddr PAddr) { tMM_PageInfo pi = {0}; #if TRACE_MAPS Log("MM_Map %P=>%p", PAddr, VAddr); #endif + // TODO: Double check that an address isn't being clobbered + pi.PhysAddr = PAddr; pi.Size = 12; - if(VAddr < USER_STACK_TOP) - pi.AP = AP_RW_BOTH; - else - pi.AP = AP_KRW_ONLY; // Kernel Read/Write + pi.AP = ( (tVAddr)VAddr < USER_STACK_TOP ? AP_RW_BOTH : AP_KRW_ONLY ); pi.bExecutable = 1; - if( MM_int_SetPageInfo(VAddr, &pi) ) { + if( MM_int_SetPageInfo( (tVAddr)VAddr, &pi) ) { // MM_DerefPhys(pi.PhysAddr); return 0; } return pi.PhysAddr; } -tPAddr MM_Allocate(tVAddr VAddr) +tPAddr MM_Allocate(volatile void *VAddr) { tMM_PageInfo pi = {0}; @@ -455,12 +462,10 @@ tPAddr MM_Allocate(tVAddr VAddr) pi.PhysAddr = MM_AllocPhys(); if( pi.PhysAddr == 0 ) LEAVE_RET('i', 0); pi.Size = 12; - if(VAddr < USER_STACK_TOP) - pi.AP = AP_RW_BOTH; - else - pi.AP = AP_KRW_ONLY; + pi.AP = ( (tVAddr)VAddr < USER_STACK_TOP ? AP_RW_BOTH : AP_KRW_ONLY ); pi.bExecutable = 0; - if( MM_int_SetPageInfo(VAddr, &pi) ) { + if( MM_int_SetPageInfo( (tVAddr)VAddr, &pi ) ) + { MM_DerefPhys(pi.PhysAddr); LEAVE('i', 0); return 0; @@ -469,7 +474,7 @@ tPAddr MM_Allocate(tVAddr VAddr) return pi.PhysAddr; } -tPAddr MM_AllocateZero(tVAddr VAddr) +void MM_AllocateZero(volatile void *VAddr) { if( !giMM_ZeroPage ) { giMM_ZeroPage = MM_Allocate(VAddr); @@ -481,21 +486,20 @@ tPAddr MM_AllocateZero(tVAddr VAddr) MM_Map(VAddr, giMM_ZeroPage); } MM_SetFlags(VAddr, MM_PFLAG_COW, MM_PFLAG_COW); - return giMM_ZeroPage; } -void MM_Deallocate(tVAddr VAddr) +void MM_Deallocate(volatile void *VAddr) { tMM_PageInfo pi; - if( MM_int_GetPageInfo(VAddr, &pi) ) return ; + if( MM_int_GetPageInfo((tVAddr)VAddr, &pi) ) return ; if( pi.PhysAddr == 0 ) return; MM_DerefPhys(pi.PhysAddr); pi.PhysAddr = 0; pi.AP = 0; pi.bExecutable = 0; - MM_int_SetPageInfo(VAddr, &pi); + MM_int_SetPageInfo((tVAddr)VAddr, &pi); } tPAddr MM_AllocateRootTable(void) @@ -535,7 +539,7 @@ void MM_int_CloneTable(Uint32 *DestEnt, int Table) cur += 256*Table; - tmp_map = (void*)MM_MapTemp(table); + tmp_map = MM_MapTemp(table); for( i = 0; i < 1024; i ++ ) { @@ -559,10 +563,10 @@ void MM_int_CloneTable(Uint32 *DestEnt, int Table) tPAddr newpage; newpage = MM_AllocPhys(); src = (void*)( (Table*256+i)*0x1000 ); - dst = (void*)MM_MapTemp(newpage); + dst = MM_MapTemp(newpage); // Debug("Taking a copy of kernel page %p (%P)", src, cur[i] & ~0xFFF); memcpy(dst, src, PAGE_SIZE); - MM_FreeTemp( (tVAddr)dst ); + MM_FreeTemp( dst ); tmp_map[i] = newpage | (cur[i] & 0xFFF); } else @@ -575,7 +579,7 @@ void MM_int_CloneTable(Uint32 *DestEnt, int Table) break; } } - MM_FreeTemp( (tVAddr) tmp_map ); + MM_FreeTemp( tmp_map ); DestEnt[0] = table + 0*0x400 + 1; DestEnt[1] = table + 1*0x400 + 1; @@ -583,52 +587,50 @@ void MM_int_CloneTable(Uint32 *DestEnt, int Table) DestEnt[3] = table + 3*0x400 + 1; } -tPAddr MM_Clone(void) +tPAddr MM_Clone(int EmptyUser) { - tPAddr ret; - Uint32 *new_lvl1_1, *new_lvl1_2, *cur; Uint32 *tmp_map; - int i; // MM_DumpTables(0, KERNEL_BASE); - ret = MM_AllocateRootTable(); + tPAddr ret = MM_AllocateRootTable(); - cur = (void*)MM_TABLE0USER; - new_lvl1_1 = (void*)MM_MapTemp(ret); - new_lvl1_2 = (void*)MM_MapTemp(ret+0x1000); - tmp_map = new_lvl1_1; - for( i = 0; i < 0x800-4; i ++ ) + Uint32 *new_lvl1_1 = MM_MapTemp(ret); + Uint32 *new_lvl1_2 = MM_MapTemp(ret+0x1000); + + if( !EmptyUser ) { - // HACK! Ignore the original identity mapping - if( i == 0 && Threads_GetTID() == 0 ) { - tmp_map[0] = 0; - continue; - } - if( i == 0x400 ) - tmp_map = &new_lvl1_2[-0x400]; - switch( cur[i] & 3 ) + Uint32 *cur = (void*)MM_TABLE0USER; + tmp_map = new_lvl1_1; + for( int i = 0; i < 0x800-4; i ++ ) { - case 0: tmp_map[i] = 0; break; - case 1: - MM_int_CloneTable(&tmp_map[i], i); - i += 3; // Tables are alocated in blocks of 4 - break; - case 2: - case 3: - Log_Error("MMVirt", "TODO: Support Sections/Supersections in MM_Clone (i=%i)", i); - tmp_map[i] = 0; - break; + // HACK! Ignore the original identity mapping + if( i == 0 && Threads_GetTID() == 0 ) { + tmp_map[0] = 0; + continue; + } + if( i == 0x400 ) + tmp_map = &new_lvl1_2[-0x400]; + switch( cur[i] & 3 ) + { + case 0: tmp_map[i] = 0; break; + case 1: + MM_int_CloneTable(&tmp_map[i], i); + i += 3; // Tables are alocated in blocks of 4 + break; + case 2: + case 3: + Log_Error("MMVirt", "TODO: Support Sections/Supersections in MM_Clone (i=%i)", i); + tmp_map[i] = 0; + break; + } } } // Allocate Fractal table { - int j, num; tPAddr tmp = MM_AllocPhys(); - Uint32 *table = (void*)MM_MapTemp(tmp); - Uint32 sp; - register Uint32 __SP asm("sp"); + Uint32 *table = MM_MapTemp(tmp); // Map table to last 4MiB of user space new_lvl1_2[0x3FC] = tmp + 0*0x400 + 1; @@ -637,7 +639,8 @@ tPAddr MM_Clone(void) new_lvl1_2[0x3FF] = tmp + 3*0x400 + 1; tmp_map = new_lvl1_1; - for( j = 0; j < 512; j ++ ) + int j = 0; + for( ; j < 512; j ++ ) { if( j == 256 ) tmp_map = &new_lvl1_2[-0x400]; @@ -652,19 +655,17 @@ tPAddr MM_Clone(void) // Fractal table[j++] = (ret + 0x0000) | 0x813; table[j++] = (ret + 0x1000) | 0x813; - // Nuke the rest - for( ; j < 1024; j ++ ) + // Clear the rest of the table + for( ; j < 1024; j ++ ) table[j] = 0; // Get kernel stack bottom - sp = __SP & ~(MM_KSTACK_SIZE-1); + register Uint32 __SP asm("sp"); + Uint32 sp = __SP & ~(MM_KSTACK_SIZE-1); j = (sp / 0x1000) % 1024; - num = MM_KSTACK_SIZE/0x1000; -// Log("num = %i, sp = %p, j = %i", num, sp, j); - // Copy stack pages - for(; num--; j ++, sp += 0x1000) + for(int num = MM_KSTACK_SIZE/PAGE_SIZE; num--; j ++, sp += PAGE_SIZE) { tVAddr page; void *tmp_page; @@ -673,16 +674,16 @@ tPAddr MM_Clone(void) // Log("page = %P", page); table[j] = page | 0x813; - tmp_page = (void*)MM_MapTemp(page); + tmp_page = MM_MapTemp(page); memcpy(tmp_page, (void*)sp, 0x1000); - MM_FreeTemp( (tVAddr) tmp_page ); + MM_FreeTemp( tmp_page ); } - MM_FreeTemp( (tVAddr)table ); + MM_FreeTemp( table ); } - MM_FreeTemp( (tVAddr)new_lvl1_1 ); - MM_FreeTemp( (tVAddr)new_lvl1_2 ); + MM_FreeTemp( new_lvl1_1 ); + MM_FreeTemp( new_lvl1_2 ); // Log("MM_Clone: ret = %P", ret); @@ -764,37 +765,37 @@ void MM_ClearUser(void) // MM_DumpTables(0, 0x80000000); } -tVAddr MM_MapTemp(tPAddr PAddr) +void *MM_MapTemp(tPAddr PAddr) { - tVAddr ret; - tMM_PageInfo pi; - - for( ret = MM_TMPMAP_BASE; ret < MM_TMPMAP_END - PAGE_SIZE; ret += PAGE_SIZE ) + for( tVAddr ret = MM_TMPMAP_BASE; ret < MM_TMPMAP_END - PAGE_SIZE; ret += PAGE_SIZE ) { + tMM_PageInfo pi; + if( MM_int_GetPageInfo(ret, &pi) == 0 ) continue; // Log("MapTemp %P at %p by %p", PAddr, ret, __builtin_return_address(0)); MM_RefPhys(PAddr); // Counter the MM_Deallocate in FreeTemp - MM_Map(ret, PAddr); + MM_Map( (void*)ret, PAddr ); - return ret; + return (void*)ret; } Log_Warning("MMVirt", "MM_MapTemp: All slots taken"); return 0; } -void MM_FreeTemp(tVAddr VAddr) +void MM_FreeTemp(void *Ptr) { + tVAddr VAddr = (tVAddr)Ptr; if( VAddr < MM_TMPMAP_BASE || VAddr >= MM_TMPMAP_END ) { Log_Warning("MMVirt", "MM_FreeTemp: Passed an addr not from MM_MapTemp (%p)", VAddr); return ; } - MM_Deallocate(VAddr); + MM_Deallocate(Ptr); } -tVAddr MM_MapHWPages(tPAddr PAddr, Uint NPages) +void *MM_MapHWPages(tPAddr PAddr, Uint NPages) { tVAddr ret; int i; @@ -821,20 +822,20 @@ tVAddr MM_MapHWPages(tPAddr PAddr, Uint NPages) // Map the pages for( i = 0; i < NPages; i ++ ) - MM_Map(ret+i*PAGE_SIZE, PAddr+i*PAGE_SIZE); + MM_Map( (tPage*)ret + i, PAddr+i*PAGE_SIZE); // and return LEAVE('p', ret); - return ret; + return (void*)ret; } Log_Warning("MMVirt", "MM_MapHWPages: No space for a %i page block", NPages); LEAVE('p', 0); return 0; } -tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PAddr) +void *MM_AllocDMA(int Pages, int MaxBits, tPAddr *PAddr) { tPAddr phys; - tVAddr ret; + void *ret; phys = MM_AllocPhysRange(Pages, MaxBits); if(!phys) { @@ -843,12 +844,17 @@ tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PAddr) } ret = MM_MapHWPages(phys, Pages); - *PAddr = phys; + if( !ret ) { + MM_DerefPhys(phys); + return NULL; + } + if( PAddr ) + *PAddr = phys; return ret; } -void MM_UnmapHWPages(tVAddr Vaddr, Uint Number) +void MM_UnmapHWPages(volatile void *VAddr, Uint Number) { Log_Error("MMVirt", "TODO: Implement MM_UnmapHWPages"); } @@ -880,14 +886,15 @@ tVAddr MM_NewKStack(int bShared) } // 1 guard page + tPage *pageptr = (void*)(addr + PAGE_SIZE); for( ofs = PAGE_SIZE; ofs < MM_KSTACK_SIZE; ofs += PAGE_SIZE ) { - if( MM_Allocate(addr + ofs) == 0 ) + if( MM_Allocate( pageptr ) == 0 ) { while(ofs) { ofs -= PAGE_SIZE; - MM_Deallocate(addr + ofs); + MM_Deallocate( pageptr-- ); } Log_Warning("MMVirt", "MM_NewKStack: Unable to allocate"); return 0; @@ -901,34 +908,37 @@ tVAddr MM_NewUserStack(void) tVAddr addr, ofs; addr = USER_STACK_TOP - USER_STACK_SIZE; - if( MM_GetPhysAddr(addr + PAGE_SIZE) ) { + if( MM_GetPhysAddr( (void*)(addr + PAGE_SIZE) ) ) { Log_Error("MMVirt", "Unable to create initial user stack, addr %p taken", addr + PAGE_SIZE ); + MM_DumpTables(0,KERNEL_BASE); return 0; } // 1 guard page - for( ofs = PAGE_SIZE; ofs < USER_STACK_SIZE; ofs += PAGE_SIZE ) + tPage *pageptr = (void*)addr; + for( ofs = PAGE_SIZE; ofs < USER_STACK_SIZE; ofs += PAGE_SIZE, pageptr ++ ) { - tPAddr rv; - if(ofs >= USER_STACK_SIZE - USER_STACK_COMM) - rv = MM_Allocate(addr + ofs); - else - rv = MM_AllocateZero(addr + ofs); - if(rv == 0) - { - while(ofs) + if(ofs >= USER_STACK_SIZE - USER_STACK_COMM) { + tPAddr rv = MM_Allocate(pageptr); + if(rv == 0) { - ofs -= PAGE_SIZE; - MM_Deallocate(addr + ofs); + while(ofs) + { + ofs -= PAGE_SIZE; + MM_Deallocate(pageptr --); + } + Log_Warning("MMVirt", "MM_NewUserStack: Unable to allocate"); + return 0; } - Log_Warning("MMVirt", "MM_NewUserStack: Unable to allocate"); - return 0; } - MM_SetFlags(addr+ofs, 0, MM_PFLAG_KERNEL); + else { + MM_AllocateZero(pageptr); + } + MM_SetFlags(pageptr, 0, MM_PFLAG_KERNEL); } -// Log("Return %p", addr + ofs); + Log("Return %p", addr + ofs); // MM_DumpTables(0, 0x80000000); return addr + ofs; } @@ -937,18 +947,23 @@ void MM_int_DumpTableEnt(tVAddr Start, size_t Len, tMM_PageInfo *Info) { if( giMM_ZeroPage && Info->PhysAddr == giMM_ZeroPage ) { - Debug("%p => %8s - 0x%7x %i %x %s", + Debug("0x%08x => %8s - 0x%7x D%i %x %s %s", Start, "ZERO", Len, Info->Domain, Info->AP, - Info->bGlobal ? "G" : "nG" + Info->bExecutable ? " X" : "nX", + Info->bGlobal ? " G" : "nG" ); } else { - Debug("%p => %8x - 0x%7x %i %x %s", + void *node; + MM_GetPageNode(Info->PhysAddr, &node); + Debug("0x%08x => %8x - 0x%7x D%i %x %s %s %p", Start, Info->PhysAddr-Len, Len, Info->Domain, Info->AP, - Info->bGlobal ? "G" : "nG" + Info->bExecutable ? " X" : "nX", + Info->bGlobal ? " G" : "nG", + node ); } } @@ -995,7 +1010,7 @@ void MM_DumpTables(tVAddr Start, tVAddr End) } // NOTE: Runs in abort context, not much difference, just a smaller stack -void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch) +void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch, Uint32 SystemLR, Uint32 UserLR) { int rv; tMM_PageInfo pi; @@ -1040,10 +1055,10 @@ void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch) Log_Error("MMVirt", "Unable to allocate new page for COW"); for(;;); } - dst = (void*)MM_MapTemp(newpage); + dst = MM_MapTemp(newpage); src = (void*)(Addr & ~(PAGE_SIZE-1)); memcpy( dst, src, PAGE_SIZE ); - MM_FreeTemp( (tVAddr)dst ); + MM_FreeTemp( dst ); #if TRACE_COW Log_Notice("MMVirt", "COW %p caused by %p, %P duped to %P (RefCnt(%i)--)", Addr, PC, @@ -1069,6 +1084,33 @@ void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch) Log_Error("MMVirt", "Code at %p accessed %p (DFSR = 0x%x)%s", PC, Addr, DFSR, (bPrefetch ? " - Prefetch" : "") ); + Log_Error("MMVirt", "- User LR = 0x%x, System LR = 0x%x", UserLR, SystemLR); + const char * const dfsr_errors[] = { + /* 00000 */ "-", "Alignment Fault", + /* 00010 */ "Debug event", "Access Flag (Section)", + /* 00100 */ "Instr Cache Maint", "Translation (Section)", + /* 00110 */ "Access Flag (Page)", "Translation (Page)", + /* 01000 */ "Sync. External abort", "Domain (Section)", + /* 01010 */ "-", "Domain (Page)", + /* 01100 */ "Table Walk sync ext (lvl 1)", "Permission (Section)", + /* 01110 */ "Table Walk sync ext (lvl 2)", "Permission (Page)", + // 0b10000 + /* 10000 */ "-", "-", + /* 10010 */ "-", "-", + /* 10100 */ "IMPL (Lockdown)", "-", + /* 10110 */ "Async. Extern. Abort", "-", + /* 11000 */ "Mem. access async pairity error", "Mem. access async pairity error", + /* 11010 */ "IMPL (Coprocessor abort)", "-", + /* 11100 */ "Table Walk Sync parity (lvl 1)", "-", + /* 11110 */ "Table Walk Sync parity (lvl 2)", "-" + }; + int errcode = (DFSR & 0xF) | (((DFSR >> 10) & 1) << 4); + Log_Error("MMVirt", "- Errcode 0b%05b", errcode); + Log_Error("MMVirt", "- Dom %i %s %s", + (DFSR >> 4) & 0xF, (DFSR & 0x800 ? "Write": "Read"), + dfsr_errors[errcode] + ); + Log_Error("MMVirt", "- AP=%i(%s) %s", pi.AP, caAPValueNames[pi.AP], pi.bExecutable ? " Executable":""); if( Addr < 0x80000000 ) MM_DumpTables(0, 0x80000000); else