#include <hal_proc.h>
#define TRACE_MAPS 0
+#define TRACE_COW 0
#define AP_KRW_ONLY 1 // Kernel page
#define AP_KRO_ONLY 5 // Kernel RO page
#define AP_RO_USER 2 // User RO Page
#define PADDR_MASK_LVL1 0xFFFFFC00
+const char * const caAPValueNames[] = {
+ "AP_NOACCESS", "AP_KRW_ONLY",
+ "AP_RO_USER", "AP_RW_BOTH",
+ "AP_???_4", "AP_KRO_ONLY",
+ "AP_???_6", "AP_RO_BOTH"
+};
+
// === IMPORTS ===
extern Uint32 kernel_table0[];
int MM_int_SetPageInfo(tVAddr VAddr, tMM_PageInfo *pi);
int MM_int_GetPageInfo(tVAddr VAddr, tMM_PageInfo *pi);
tVAddr MM_NewUserStack(void);
-tPAddr MM_AllocateZero(tVAddr VAddr);
+//tPAddr MM_AllocateZero(volatile void *VAddr);
tPAddr MM_AllocateRootTable(void);
void MM_int_CloneTable(Uint32 *DestEnt, int Table);
-tPAddr MM_Clone(void);
+tPAddr MM_Clone(int ClearUser);
tVAddr MM_NewKStack(int bGlobal);
void MM_int_DumpTableEnt(tVAddr Start, size_t Len, tMM_PageInfo *Info);
//void MM_DumpTables(tVAddr Start, tVAddr End);
-void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch);
+void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch, Uint32 SystemLR, Uint32 UserLR);
// === GLOBALS ===
tPAddr giMM_ZeroPage;
}
// --- Exports ---
-tPAddr MM_GetPhysAddr(tVAddr VAddr)
+tPAddr MM_GetPhysAddr(volatile const void *Ptr)
{
tMM_PageInfo pi;
- if( MM_int_GetPageInfo(VAddr, &pi) )
+ if( MM_int_GetPageInfo((tVAddr)Ptr, &pi) )
return 0;
- return pi.PhysAddr | (VAddr & ((1 << pi.Size)-1));
+ return pi.PhysAddr | ((tVAddr)Ptr & ((1 << pi.Size)-1));
}
-Uint MM_GetFlags(tVAddr VAddr)
+Uint MM_GetFlags(const volatile void *VAddr)
{
tMM_PageInfo pi;
int ret;
- if( MM_int_GetPageInfo(VAddr, &pi) )
+ if( MM_int_GetPageInfo((tVAddr)VAddr, &pi) )
return 0;
ret = 0;
return ret;
}
-void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask)
+void MM_SetFlags(volatile void *VAddr, Uint Flags, Uint Mask)
{
tMM_PageInfo pi;
Uint curFlags;
- if( MM_int_GetPageInfo(VAddr, &pi) )
+ if( MM_int_GetPageInfo((tVAddr)VAddr, &pi) )
return ;
curFlags = MM_GetFlags(VAddr);
pi.bExecutable = !!(curFlags & MM_PFLAG_EXEC);
- MM_int_SetPageInfo(VAddr, &pi);
+ MM_int_SetPageInfo((tVAddr)VAddr, &pi);
}
int MM_IsValidBuffer(tVAddr Addr, size_t Size)
Size += Addr & (PAGE_SIZE-1);
Addr &= ~(PAGE_SIZE-1);
- if( MM_int_GetPageInfo(Addr, &pi) ) return 0;
+ if( MM_int_GetPageInfo((tVAddr)Addr, &pi) ) return 0;
Addr += PAGE_SIZE;
if(pi.AP != AP_KRW_ONLY && pi.AP != AP_KRO_ONLY)
return 1;
}
-int MM_Map(tVAddr VAddr, tPAddr PAddr)
+int MM_Map(volatile void *VAddr, tPAddr PAddr)
{
tMM_PageInfo pi = {0};
#if TRACE_MAPS
Log("MM_Map %P=>%p", PAddr, VAddr);
#endif
+ // TODO: Double check that an address isn't being clobbered
+
pi.PhysAddr = PAddr;
pi.Size = 12;
- if(VAddr < USER_STACK_TOP)
- pi.AP = AP_RW_BOTH;
- else
- pi.AP = AP_KRW_ONLY; // Kernel Read/Write
+ pi.AP = ( (tVAddr)VAddr < USER_STACK_TOP ? AP_RW_BOTH : AP_KRW_ONLY );
pi.bExecutable = 1;
- if( MM_int_SetPageInfo(VAddr, &pi) ) {
+ if( MM_int_SetPageInfo( (tVAddr)VAddr, &pi) ) {
// MM_DerefPhys(pi.PhysAddr);
return 0;
}
return pi.PhysAddr;
}
-tPAddr MM_Allocate(tVAddr VAddr)
+tPAddr MM_Allocate(volatile void *VAddr)
{
tMM_PageInfo pi = {0};
pi.PhysAddr = MM_AllocPhys();
if( pi.PhysAddr == 0 ) LEAVE_RET('i', 0);
pi.Size = 12;
- if(VAddr < USER_STACK_TOP)
- pi.AP = AP_RW_BOTH;
- else
- pi.AP = AP_KRW_ONLY;
+ pi.AP = ( (tVAddr)VAddr < USER_STACK_TOP ? AP_RW_BOTH : AP_KRW_ONLY );
pi.bExecutable = 0;
- if( MM_int_SetPageInfo(VAddr, &pi) ) {
+ if( MM_int_SetPageInfo( (tVAddr)VAddr, &pi ) )
+ {
MM_DerefPhys(pi.PhysAddr);
LEAVE('i', 0);
return 0;
return pi.PhysAddr;
}
-tPAddr MM_AllocateZero(tVAddr VAddr)
+void MM_AllocateZero(volatile void *VAddr)
{
if( !giMM_ZeroPage ) {
giMM_ZeroPage = MM_Allocate(VAddr);
MM_Map(VAddr, giMM_ZeroPage);
}
MM_SetFlags(VAddr, MM_PFLAG_COW, MM_PFLAG_COW);
- return giMM_ZeroPage;
}
-void MM_Deallocate(tVAddr VAddr)
+void MM_Deallocate(volatile void *VAddr)
{
tMM_PageInfo pi;
- if( MM_int_GetPageInfo(VAddr, &pi) ) return ;
+ if( MM_int_GetPageInfo((tVAddr)VAddr, &pi) ) return ;
if( pi.PhysAddr == 0 ) return;
MM_DerefPhys(pi.PhysAddr);
pi.PhysAddr = 0;
pi.AP = 0;
pi.bExecutable = 0;
- MM_int_SetPageInfo(VAddr, &pi);
+ MM_int_SetPageInfo((tVAddr)VAddr, &pi);
}
tPAddr MM_AllocateRootTable(void)
cur += 256*Table;
- tmp_map = (void*)MM_MapTemp(table);
+ tmp_map = MM_MapTemp(table);
for( i = 0; i < 1024; i ++ )
{
tPAddr newpage;
newpage = MM_AllocPhys();
src = (void*)( (Table*256+i)*0x1000 );
- dst = (void*)MM_MapTemp(newpage);
+ dst = MM_MapTemp(newpage);
// Debug("Taking a copy of kernel page %p (%P)", src, cur[i] & ~0xFFF);
memcpy(dst, src, PAGE_SIZE);
- MM_FreeTemp( (tVAddr)dst );
+ MM_FreeTemp( dst );
tmp_map[i] = newpage | (cur[i] & 0xFFF);
}
else
break;
}
}
- MM_FreeTemp( (tVAddr) tmp_map );
+ MM_FreeTemp( tmp_map );
DestEnt[0] = table + 0*0x400 + 1;
DestEnt[1] = table + 1*0x400 + 1;
DestEnt[3] = table + 3*0x400 + 1;
}
-tPAddr MM_Clone(void)
+tPAddr MM_Clone(int EmptyUser)
{
- tPAddr ret;
- Uint32 *new_lvl1_1, *new_lvl1_2, *cur;
Uint32 *tmp_map;
- int i;
// MM_DumpTables(0, KERNEL_BASE);
- ret = MM_AllocateRootTable();
+ tPAddr ret = MM_AllocateRootTable();
- cur = (void*)MM_TABLE0USER;
- new_lvl1_1 = (void*)MM_MapTemp(ret);
- new_lvl1_2 = (void*)MM_MapTemp(ret+0x1000);
- tmp_map = new_lvl1_1;
- for( i = 0; i < 0x800-4; i ++ )
+ Uint32 *new_lvl1_1 = MM_MapTemp(ret);
+ Uint32 *new_lvl1_2 = MM_MapTemp(ret+0x1000);
+
+ if( !EmptyUser )
{
- // HACK! Ignore the original identity mapping
- if( i == 0 && Threads_GetTID() == 0 ) {
- tmp_map[0] = 0;
- continue;
- }
- if( i == 0x400 )
- tmp_map = &new_lvl1_2[-0x400];
- switch( cur[i] & 3 )
+ Uint32 *cur = (void*)MM_TABLE0USER;
+ tmp_map = new_lvl1_1;
+ for( int i = 0; i < 0x800-4; i ++ )
{
- case 0: tmp_map[i] = 0; break;
- case 1:
- MM_int_CloneTable(&tmp_map[i], i);
- i += 3; // Tables are alocated in blocks of 4
- break;
- case 2:
- case 3:
- Log_Error("MMVirt", "TODO: Support Sections/Supersections in MM_Clone (i=%i)", i);
- tmp_map[i] = 0;
- break;
+ // HACK! Ignore the original identity mapping
+ if( i == 0 && Threads_GetTID() == 0 ) {
+ tmp_map[0] = 0;
+ continue;
+ }
+ if( i == 0x400 )
+ tmp_map = &new_lvl1_2[-0x400];
+ switch( cur[i] & 3 )
+ {
+ case 0: tmp_map[i] = 0; break;
+ case 1:
+ MM_int_CloneTable(&tmp_map[i], i);
+ i += 3; // Tables are alocated in blocks of 4
+ break;
+ case 2:
+ case 3:
+ Log_Error("MMVirt", "TODO: Support Sections/Supersections in MM_Clone (i=%i)", i);
+ tmp_map[i] = 0;
+ break;
+ }
}
}
// Allocate Fractal table
{
- int j, num;
tPAddr tmp = MM_AllocPhys();
- Uint32 *table = (void*)MM_MapTemp(tmp);
- Uint32 sp;
- register Uint32 __SP asm("sp");
+ Uint32 *table = MM_MapTemp(tmp);
// Map table to last 4MiB of user space
new_lvl1_2[0x3FC] = tmp + 0*0x400 + 1;
new_lvl1_2[0x3FF] = tmp + 3*0x400 + 1;
tmp_map = new_lvl1_1;
- for( j = 0; j < 512; j ++ )
+ int j = 0;
+ for( ; j < 512; j ++ )
{
if( j == 256 )
tmp_map = &new_lvl1_2[-0x400];
// Fractal
table[j++] = (ret + 0x0000) | 0x813;
table[j++] = (ret + 0x1000) | 0x813;
- // Nuke the rest
- for( ; j < 1024; j ++ )
+ // Clear the rest of the table
+ for( ; j < 1024; j ++ )
table[j] = 0;
// Get kernel stack bottom
- sp = __SP & ~(MM_KSTACK_SIZE-1);
+ register Uint32 __SP asm("sp");
+ Uint32 sp = __SP & ~(MM_KSTACK_SIZE-1);
j = (sp / 0x1000) % 1024;
- num = MM_KSTACK_SIZE/0x1000;
-// Log("num = %i, sp = %p, j = %i", num, sp, j);
-
// Copy stack pages
- for(; num--; j ++, sp += 0x1000)
+ for(int num = MM_KSTACK_SIZE/PAGE_SIZE; num--; j ++, sp += PAGE_SIZE)
{
tVAddr page;
void *tmp_page;
// Log("page = %P", page);
table[j] = page | 0x813;
- tmp_page = (void*)MM_MapTemp(page);
+ tmp_page = MM_MapTemp(page);
memcpy(tmp_page, (void*)sp, 0x1000);
- MM_FreeTemp( (tVAddr) tmp_page );
+ MM_FreeTemp( tmp_page );
}
- MM_FreeTemp( (tVAddr)table );
+ MM_FreeTemp( table );
}
- MM_FreeTemp( (tVAddr)new_lvl1_1 );
- MM_FreeTemp( (tVAddr)new_lvl1_2 );
+ MM_FreeTemp( new_lvl1_1 );
+ MM_FreeTemp( new_lvl1_2 );
// Log("MM_Clone: ret = %P", ret);
// MM_DumpTables(0, 0x80000000);
}
-tVAddr MM_MapTemp(tPAddr PAddr)
+void *MM_MapTemp(tPAddr PAddr)
{
- tVAddr ret;
- tMM_PageInfo pi;
-
- for( ret = MM_TMPMAP_BASE; ret < MM_TMPMAP_END - PAGE_SIZE; ret += PAGE_SIZE )
+ for( tVAddr ret = MM_TMPMAP_BASE; ret < MM_TMPMAP_END - PAGE_SIZE; ret += PAGE_SIZE )
{
+ tMM_PageInfo pi;
+
if( MM_int_GetPageInfo(ret, &pi) == 0 )
continue;
// Log("MapTemp %P at %p by %p", PAddr, ret, __builtin_return_address(0));
MM_RefPhys(PAddr); // Counter the MM_Deallocate in FreeTemp
- MM_Map(ret, PAddr);
+ MM_Map( (void*)ret, PAddr );
- return ret;
+ return (void*)ret;
}
Log_Warning("MMVirt", "MM_MapTemp: All slots taken");
return 0;
}
-void MM_FreeTemp(tVAddr VAddr)
+void MM_FreeTemp(void *Ptr)
{
+ tVAddr VAddr = (tVAddr)Ptr;
if( VAddr < MM_TMPMAP_BASE || VAddr >= MM_TMPMAP_END ) {
Log_Warning("MMVirt", "MM_FreeTemp: Passed an addr not from MM_MapTemp (%p)", VAddr);
return ;
}
- MM_Deallocate(VAddr);
+ MM_Deallocate(Ptr);
}
-tVAddr MM_MapHWPages(tPAddr PAddr, Uint NPages)
+void *MM_MapHWPages(tPAddr PAddr, Uint NPages)
{
tVAddr ret;
int i;
// Map the pages
for( i = 0; i < NPages; i ++ )
- MM_Map(ret+i*PAGE_SIZE, PAddr+i*PAGE_SIZE);
+ MM_Map( (tPage*)ret + i, PAddr+i*PAGE_SIZE);
// and return
LEAVE('p', ret);
- return ret;
+ return (void*)ret;
}
Log_Warning("MMVirt", "MM_MapHWPages: No space for a %i page block", NPages);
LEAVE('p', 0);
return 0;
}
-tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PAddr)
+void *MM_AllocDMA(int Pages, int MaxBits, tPAddr *PAddr)
{
tPAddr phys;
- tVAddr ret;
+ void *ret;
phys = MM_AllocPhysRange(Pages, MaxBits);
if(!phys) {
}
ret = MM_MapHWPages(phys, Pages);
- *PAddr = phys;
+ if( !ret ) {
+ MM_DerefPhys(phys);
+ return NULL;
+ }
+ if( PAddr )
+ *PAddr = phys;
return ret;
}
-void MM_UnmapHWPages(tVAddr Vaddr, Uint Number)
+void MM_UnmapHWPages(volatile void *VAddr, Uint Number)
{
Log_Error("MMVirt", "TODO: Implement MM_UnmapHWPages");
}
}
// 1 guard page
+ tPage *pageptr = (void*)(addr + PAGE_SIZE);
for( ofs = PAGE_SIZE; ofs < MM_KSTACK_SIZE; ofs += PAGE_SIZE )
{
- if( MM_Allocate(addr + ofs) == 0 )
+ if( MM_Allocate( pageptr ) == 0 )
{
while(ofs)
{
ofs -= PAGE_SIZE;
- MM_Deallocate(addr + ofs);
+ MM_Deallocate( pageptr-- );
}
Log_Warning("MMVirt", "MM_NewKStack: Unable to allocate");
return 0;
tVAddr addr, ofs;
addr = USER_STACK_TOP - USER_STACK_SIZE;
- if( MM_GetPhysAddr(addr + PAGE_SIZE) ) {
+ if( MM_GetPhysAddr( (void*)(addr + PAGE_SIZE) ) ) {
Log_Error("MMVirt", "Unable to create initial user stack, addr %p taken",
addr + PAGE_SIZE
);
+ MM_DumpTables(0,KERNEL_BASE);
return 0;
}
// 1 guard page
- for( ofs = PAGE_SIZE; ofs < USER_STACK_SIZE; ofs += PAGE_SIZE )
+ tPage *pageptr = (void*)addr;
+ for( ofs = PAGE_SIZE; ofs < USER_STACK_SIZE; ofs += PAGE_SIZE, pageptr ++ )
{
- tPAddr rv;
- if(ofs >= USER_STACK_SIZE - USER_STACK_COMM)
- rv = MM_Allocate(addr + ofs);
- else
- rv = MM_AllocateZero(addr + ofs);
- if(rv == 0)
- {
- while(ofs)
+ if(ofs >= USER_STACK_SIZE - USER_STACK_COMM) {
+ tPAddr rv = MM_Allocate(pageptr);
+ if(rv == 0)
{
- ofs -= PAGE_SIZE;
- MM_Deallocate(addr + ofs);
+ while(ofs)
+ {
+ ofs -= PAGE_SIZE;
+ MM_Deallocate(pageptr --);
+ }
+ Log_Warning("MMVirt", "MM_NewUserStack: Unable to allocate");
+ return 0;
}
- Log_Warning("MMVirt", "MM_NewUserStack: Unable to allocate");
- return 0;
}
- MM_SetFlags(addr+ofs, 0, MM_PFLAG_KERNEL);
+ else {
+ MM_AllocateZero(pageptr);
+ }
+ MM_SetFlags(pageptr, 0, MM_PFLAG_KERNEL);
}
-// Log("Return %p", addr + ofs);
+ Log("Return %p", addr + ofs);
// MM_DumpTables(0, 0x80000000);
return addr + ofs;
}
{
if( giMM_ZeroPage && Info->PhysAddr == giMM_ZeroPage )
{
- Debug("%p => %8s - 0x%7x %i %x %s",
+ Debug("0x%08x => %8s - 0x%7x D%i %x %s %s",
Start, "ZERO", Len,
Info->Domain, Info->AP,
- Info->bGlobal ? "G" : "nG"
+ Info->bExecutable ? " X" : "nX",
+ Info->bGlobal ? " G" : "nG"
);
}
else
{
- Debug("%p => %8x - 0x%7x %i %x %s",
+ void *node;
+ MM_GetPageNode(Info->PhysAddr, &node);
+ Debug("0x%08x => %8x - 0x%7x D%i %x %s %s %p",
Start, Info->PhysAddr-Len, Len,
Info->Domain, Info->AP,
- Info->bGlobal ? "G" : "nG"
+ Info->bExecutable ? " X" : "nX",
+ Info->bGlobal ? " G" : "nG",
+ node
);
}
}
}
// NOTE: Runs in abort context, not much difference, just a smaller stack
-void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch)
+void MM_PageFault(Uint32 PC, Uint32 Addr, Uint32 DFSR, int bPrefetch, Uint32 SystemLR, Uint32 UserLR)
{
int rv;
tMM_PageInfo pi;
Log_Error("MMVirt", "Unable to allocate new page for COW");
for(;;);
}
- dst = (void*)MM_MapTemp(newpage);
+ dst = MM_MapTemp(newpage);
src = (void*)(Addr & ~(PAGE_SIZE-1));
memcpy( dst, src, PAGE_SIZE );
- MM_FreeTemp( (tVAddr)dst );
+ MM_FreeTemp( dst );
#if TRACE_COW
Log_Notice("MMVirt", "COW %p caused by %p, %P duped to %P (RefCnt(%i)--)", Addr, PC,
Log_Error("MMVirt", "Code at %p accessed %p (DFSR = 0x%x)%s", PC, Addr, DFSR,
(bPrefetch ? " - Prefetch" : "")
);
+ Log_Error("MMVirt", "- User LR = 0x%x, System LR = 0x%x", UserLR, SystemLR);
+ const char * const dfsr_errors[] = {
+ /* 00000 */ "-", "Alignment Fault",
+ /* 00010 */ "Debug event", "Access Flag (Section)",
+ /* 00100 */ "Instr Cache Maint", "Translation (Section)",
+ /* 00110 */ "Access Flag (Page)", "Translation (Page)",
+ /* 01000 */ "Sync. External abort", "Domain (Section)",
+ /* 01010 */ "-", "Domain (Page)",
+ /* 01100 */ "Table Walk sync ext (lvl 1)", "Permission (Section)",
+ /* 01110 */ "Table Walk sync ext (lvl 2)", "Permission (Page)",
+ // 0b10000
+ /* 10000 */ "-", "-",
+ /* 10010 */ "-", "-",
+ /* 10100 */ "IMPL (Lockdown)", "-",
+ /* 10110 */ "Async. Extern. Abort", "-",
+ /* 11000 */ "Mem. access async pairity error", "Mem. access async pairity error",
+ /* 11010 */ "IMPL (Coprocessor abort)", "-",
+ /* 11100 */ "Table Walk Sync parity (lvl 1)", "-",
+ /* 11110 */ "Table Walk Sync parity (lvl 2)", "-"
+ };
+ int errcode = (DFSR & 0xF) | (((DFSR >> 10) & 1) << 4);
+ Log_Error("MMVirt", "- Errcode 0b%05b", errcode);
+ Log_Error("MMVirt", "- Dom %i %s %s",
+ (DFSR >> 4) & 0xF, (DFSR & 0x800 ? "Write": "Read"),
+ dfsr_errors[errcode]
+ );
+ Log_Error("MMVirt", "- AP=%i(%s) %s", pi.AP, caAPValueNames[pi.AP], pi.bExecutable ? " Executable":"");
if( Addr < 0x80000000 )
MM_DumpTables(0, 0x80000000);
else