X-Git-Url: https://git.ucc.asn.au/?a=blobdiff_plain;f=Kernel%2Farch%2Fx86%2Fmm_virt.c;h=bd7b1dff66adb4474783db55a1d589fcf7d80857;hb=a2495c6ea4f4cab16b5d339ae511428e92e89e73;hp=6fa491b8c4762ecc33f6fc27dedb7589bd1ead36;hpb=05e8ae6b8fd960b2d9d3dafb34263fe7bd1cb90b;p=tpg%2Facess2.git diff --git a/Kernel/arch/x86/mm_virt.c b/Kernel/arch/x86/mm_virt.c index 6fa491b8..bd7b1dff 100644 --- a/Kernel/arch/x86/mm_virt.c +++ b/Kernel/arch/x86/mm_virt.c @@ -9,71 +9,121 @@ * 0xFE - Unused * 0xFF - System Calls / Kernel's User Code */ -#include +#define DEBUG 0 +#define SANITY 1 +#include +#include #include #include +#include +#include -#define KERNEL_STACKS 0xF0000000 -#define KERNEL_STACK_SIZE 0x00002000 -#define KERNEL_STACK_END 0xFD000000 -#define PAGE_TABLE_ADDR 0xFD000000 -#define PAGE_DIR_ADDR 0xFD3F4000 -#define PAGE_CR3_ADDR 0xFD3F4FD0 -#define TMP_CR3_ADDR 0xFD3F4FD4 // Part of core instead of temp -#define TMP_DIR_ADDR 0xFD3F5000 // Same -#define TMP_TABLE_ADDR 0xFD400000 -#define HW_MAP_ADDR 0xFD800000 -#define HW_MAP_MAX 0xFEFF0000 +#define TAB 22 + +#define WORKER_STACKS 0x00100000 // Thread0 Only! +#define WORKER_STACK_SIZE MM_KERNEL_STACK_SIZE +#define WORKER_STACKS_END 0xB0000000 +#define NUM_WORKER_STACKS ((WORKER_STACKS_END-WORKER_STACKS)/WORKER_STACK_SIZE) + +#define PAE_PAGE_TABLE_ADDR 0xFC000000 // 16 MiB +#define PAE_PAGE_DIR_ADDR 0xFCFC0000 // 16 KiB +#define PAE_PAGE_PDPT_ADDR 0xFCFC3F00 // 32 bytes +#define PAE_TMP_PDPT_ADDR 0xFCFC3F20 // 32 bytes +#define PAE_TMP_DIR_ADDR 0xFCFE0000 // 16 KiB +#define PAE_TMP_TABLE_ADDR 0xFD000000 // 16 MiB + +#define PAGE_TABLE_ADDR 0xFC000000 +#define PAGE_DIR_ADDR 0xFC3F0000 +#define PAGE_CR3_ADDR 0xFC3F0FC0 +#define TMP_CR3_ADDR 0xFC3F0FC4 // Part of core instead of temp +#define TMP_DIR_ADDR 0xFC3F1000 // Same +#define TMP_TABLE_ADDR 0xFC400000 + +#define HW_MAP_ADDR 0xFE000000 +#define HW_MAP_MAX 0xFFEF0000 #define NUM_HW_PAGES ((HW_MAP_MAX-HW_MAP_ADDR)/0x1000) -#define TEMP_MAP_ADDR 0xFEFF0000 // Allows 16 "temp" pages +#define TEMP_MAP_ADDR 0xFFEF0000 // Allows 16 "temp" pages #define NUM_TEMP_PAGES 16 - -#define USE_COW 1 +#define LAST_BLOCK_ADDR 0xFFFF0000 // Free space for kernel provided user code/ *(-1) protection #define PF_PRESENT 0x1 #define PF_WRITE 0x2 #define PF_USER 0x4 +#define PF_GLOBAL 0x80 #define PF_COW 0x200 -#define PF_PAGED 0x400 +#define PF_NOPAGE 0x400 #define INVLPG(addr) __asm__ __volatile__ ("invlpg (%0)"::"r"(addr)) +#define GET_TEMP_MAPPING(cr3) do { \ + __ASM__("cli"); \ + __AtomicTestSetLoop( (Uint *)gpTmpCR3, cr3 | 3 ); \ +} while(0) +#define REL_TEMP_MAPPING() do { \ + *gpTmpCR3 = 0; \ + __ASM__("sti"); \ +} while(0) + +typedef Uint32 tTabEnt; + // === IMPORTS === +extern char _UsertextEnd[], _UsertextBase[]; extern Uint32 gaInitPageDir[1024]; extern Uint32 gaInitPageTable[1024]; +extern void Threads_SegFault(tVAddr Addr); +extern void Error_Backtrace(Uint eip, Uint ebp); // === PROTOTYPES === -void MM_PreinitVirtual(); -void MM_InstallVirtual(); -void MM_PageFault(Uint Addr, Uint ErrorCode, tRegs *Regs); -void MM_DumpTables(tVAddr Start, tVAddr End); -tPAddr MM_DuplicatePage(Uint VAddr); +void MM_PreinitVirtual(void); +void MM_InstallVirtual(void); +void MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs); +//void MM_DumpTables(tVAddr Start, tVAddr End); +//void MM_ClearUser(void); +tPAddr MM_DuplicatePage(tVAddr VAddr); // === GLOBALS === -tPAddr *gaPageTable = (void*)PAGE_TABLE_ADDR; -tPAddr *gaPageDir = (void*)PAGE_DIR_ADDR; -tPAddr *gaPageCR3 = (void*)PAGE_CR3_ADDR; -tPAddr *gaTmpTable = (void*)TMP_TABLE_ADDR; -tPAddr *gaTmpDir = (void*)TMP_DIR_ADDR; -tPAddr *gTmpCR3 = (void*)TMP_CR3_ADDR; - int gilTempMappings = 0; +#define gaPageTable ((tTabEnt*)PAGE_TABLE_ADDR) +#define gaPageDir ((tTabEnt*)PAGE_DIR_ADDR) +#define gaTmpTable ((tTabEnt*)TMP_TABLE_ADDR) +#define gaTmpDir ((tTabEnt*)TMP_DIR_ADDR) +#define gpPageCR3 ((tTabEnt*)PAGE_CR3_ADDR) +#define gpTmpCR3 ((tTabEnt*)TMP_CR3_ADDR) + +#define gaPAE_PageTable ((tTabEnt*)PAE_PAGE_TABLE_ADDR) +#define gaPAE_PageDir ((tTabEnt*)PAE_PAGE_DIR_ADDR) +#define gaPAE_MainPDPT ((tTabEnt*)PAE_PAGE_PDPT_ADDR) +#define gaPAE_TmpTable ((tTabEnt*)PAE_TMP_DIR_ADDR) +#define gaPAE_TmpDir ((tTabEnt*)PAE_TMP_DIR_ADDR) +#define gaPAE_TmpPDPT ((tTabEnt*)PAE_TMP_PDPT_ADDR) + int gbUsePAE = 0; +tMutex glTempMappings; +tMutex glTempFractal; +Uint32 gWorkerStacks[(NUM_WORKER_STACKS+31)/32]; + int giLastUsedWorker = 0; +struct sPageInfo { + void *Node; + tVAddr Base; + Uint64 Offset; + int Length; + int Flags; +} *gaMappedRegions; // sizeof = 24 bytes // === CODE === /** - * \fn void MM_PreinitVirtual() + * \fn void MM_PreinitVirtual(void) * \brief Maps the fractal mappings */ -void MM_PreinitVirtual() +void MM_PreinitVirtual(void) { - gaInitPageDir[ 0 ] = 0; - gaInitPageDir[ PAGE_TABLE_ADDR >> 22 ] = ((Uint)&gaInitPageDir - KERNEL_BASE) | 3; + gaInitPageDir[ PAGE_TABLE_ADDR >> 22 ] = ((tTabEnt)&gaInitPageDir - KERNEL_BASE) | 3; + INVLPG( PAGE_TABLE_ADDR ); } /** - * \fn void MM_InstallVirtual() + * \fn void MM_InstallVirtual(void) * \brief Sets up the constant page mappings */ -void MM_InstallVirtual() +void MM_InstallVirtual(void) { int i; @@ -82,7 +132,7 @@ void MM_InstallVirtual() { if( gaPageDir[ i ] ) continue; // Skip stack tables, they are process unique - if( i > KERNEL_STACKS >> 22 && i < KERNEL_STACK_END >> 22) { + if( i > MM_KERNEL_STACKS >> 22 && i < MM_KERNEL_STACKS_END >> 22) { gaPageDir[ i ] = 0; continue; } @@ -91,84 +141,170 @@ void MM_InstallVirtual() INVLPG( &gaPageTable[i*1024] ); memset( &gaPageTable[i*1024], 0, 0x1000 ); } + + // Unset kernel on the User Text pages + for( i = ((tVAddr)&_UsertextEnd-(tVAddr)&_UsertextBase+0xFFF)/4096; i--; ) { + MM_SetFlags( (tVAddr)&_UsertextBase + i*4096, 0, MM_PFLAG_KERNEL ); + } + + *gpTmpCR3 = 0; } /** - * \fn void MM_PageFault(Uint Addr, Uint ErrorCode, tRegs *Regs) + * \brief Cleans up the SMP required mappings + */ +void MM_FinishVirtualInit(void) +{ + gaInitPageDir[ 0 ] = 0; +} + +/** + * \fn void MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs) * \brief Called on a page fault */ -void MM_PageFault(Uint Addr, Uint ErrorCode, tRegs *Regs) +void MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs) { - ENTER("xAddr bErrorCode", Addr, ErrorCode); + //ENTER("xAddr bErrorCode", Addr, ErrorCode); // -- Check for COW -- - if( gaPageDir [Addr>>22] & PF_PRESENT - && gaPageTable[Addr>>12] & PF_PRESENT + if( gaPageDir [Addr>>22] & PF_PRESENT && gaPageTable[Addr>>12] & PF_PRESENT && gaPageTable[Addr>>12] & PF_COW ) { tPAddr paddr; - paddr = MM_DuplicatePage( Addr ); - MM_DerefPhys( gaPageTable[Addr>>12] & ~0xFFF ); - gaPageTable[Addr>>12] &= PF_USER; - gaPageTable[Addr>>12] |= paddr|PF_PRESENT|PF_WRITE; + if(MM_GetRefCount( gaPageTable[Addr>>12] & ~0xFFF ) == 1) + { + gaPageTable[Addr>>12] &= ~PF_COW; + gaPageTable[Addr>>12] |= PF_PRESENT|PF_WRITE; + } + else + { + //Log("MM_PageFault: COW - MM_DuplicatePage(0x%x)", Addr); + paddr = MM_DuplicatePage( Addr ); + MM_DerefPhys( gaPageTable[Addr>>12] & ~0xFFF ); + gaPageTable[Addr>>12] &= PF_USER; + gaPageTable[Addr>>12] |= paddr|PF_PRESENT|PF_WRITE; + } + +// Log_Debug("MMVirt", "COW for %p (%P)", Addr, gaPageTable[Addr>>12]); + INVLPG( Addr & ~0xFFF ); - LEAVE('-'); return; } + + // Disable instruction tracing + __ASM__("pushf; andw $0xFEFF, 0(%esp); popf"); + Proc_GetCurThread()->bInstrTrace = 0; + + // If it was a user, tell the thread handler + if(ErrorCode & 4) { + Log_Warning("MMVirt", "User %s %s memory%s", + (ErrorCode&2?"write to":"read from"), + (ErrorCode&1?"bad/locked":"non-present"), + (ErrorCode&16?" (Instruction Fetch)":"") + ); + Log_Warning("MMVirt", "Instruction %04x:%08x accessed %p", Regs->cs, Regs->eip, Addr); + __ASM__("sti"); // Restart IRQs + #if 1 + Error_Backtrace(Regs->eip, Regs->ebp); + #endif + Threads_SegFault(Addr); + return ; + } + + Debug_KernelPanic(); // -- Check Error Code -- if(ErrorCode & 8) Warning("Reserved Bits Trashed!"); else { - Warning("%s %s %s memory%s", - (ErrorCode&4?"User":"Kernel"), + Warning("Kernel %s %s memory%s", (ErrorCode&2?"write to":"read from"), (ErrorCode&1?"bad/locked":"non-present"), (ErrorCode&16?" (Instruction Fetch)":"") ); } + Log("CPU %i - Code at %p accessed %p", GetCPUNum(), Regs->eip, Addr); + // Print Stack Backtrace + Error_Backtrace(Regs->eip, Regs->ebp); + + #if 0 Log("gaPageDir[0x%x] = 0x%x", Addr>>22, gaPageDir[Addr>>22]); if( gaPageDir[Addr>>22] & PF_PRESENT ) Log("gaPageTable[0x%x] = 0x%x", Addr>>12, gaPageTable[Addr>>12]); + #endif + //MM_DumpTables(0, -1); - MM_DumpTables(0, -1); + // Register Dump + Log("EAX %08x ECX %08x EDX %08x EBX %08x", Regs->eax, Regs->ecx, Regs->edx, Regs->ebx); + Log("ESP %08x EBP %08x ESI %08x EDI %08x", Regs->esp, Regs->ebp, Regs->esi, Regs->edi); + //Log("SS:ESP %04x:%08x", Regs->ss, Regs->esp); + Log("CS:EIP %04x:%08x", Regs->cs, Regs->eip); + Log("DS %04x ES %04x FS %04x GS %04x", Regs->ds, Regs->es, Regs->fs, Regs->gs); + { + Uint dr0, dr1; + __ASM__ ("mov %%dr0, %0":"=r"(dr0):); + __ASM__ ("mov %%dr1, %0":"=r"(dr1):); + Log("DR0 %08x DR1 %08x", dr0, dr1); + } - Panic("Page Fault at 0x%x\n", Regs->eip); - LEAVE('-'); + Panic("Page Fault at 0x%x (Accessed 0x%x)", Regs->eip, Addr); } /** - * \fn void MM_DumpTables(Uint Start, Uint End) + * \fn void MM_DumpTables(tVAddr Start, tVAddr End) * \brief Dumps the layout of the page tables */ void MM_DumpTables(tVAddr Start, tVAddr End) { tVAddr rangeStart = 0; tPAddr expected = 0; + void *expected_node = NULL, *tmpnode = NULL; tVAddr curPos; Uint page; - const tPAddr MASK = ~0xF98; + const tPAddr MASK = ~0xF78; Start >>= 12; End >>= 12; + + #if 0 + Log("Directory Entries:"); + for(page = Start >> 10; + page < (End >> 10)+1; + page ++) + { + if(gaPageDir[page]) + { + Log(" 0x%08x-0x%08x :: 0x%08x", + page<<22, ((page+1)<<22)-1, + gaPageDir[page]&~0xFFF + ); + } + } + #endif + + Log("Table Entries:"); for(page = Start, curPos = Start<<12; page < End; curPos += 0x1000, page++) { if( !(gaPageDir[curPos>>22] & PF_PRESENT) || !(gaPageTable[page] & PF_PRESENT) - || (gaPageTable[page] & MASK) != expected) + || (gaPageTable[page] & MASK) != expected + || (tmpnode=NULL,MM_GetPageNode(expected, &tmpnode), tmpnode != expected_node)) { if(expected) { - Log("0x%08x-0x%08x => 0x%08x-0x%08x (%s%s%s%s)", - rangeStart, curPos - 1, - gaPageTable[rangeStart>>12] & ~0xFFF, - (expected & ~0xFFF) - 1, - (expected & PF_PAGED ? "p" : "-"), - (expected & PF_COW ? "C" : "-"), - (expected & PF_USER ? "U" : "-"), - (expected & PF_WRITE ? "W" : "-") + tPAddr orig = gaPageTable[rangeStart>>12]; + Log(" 0x%08x => 0x%08x - 0x%08x (%s%s%s%s%s) %p", + rangeStart, + orig & ~0xFFF, + curPos - rangeStart, + (orig & PF_NOPAGE ? "P" : "-"), + (orig & PF_COW ? "C" : "-"), + (orig & PF_GLOBAL ? "G" : "-"), + (orig & PF_USER ? "U" : "-"), + (orig & PF_WRITE ? "W" : "-"), + expected_node ); expected = 0; } @@ -176,36 +312,49 @@ void MM_DumpTables(tVAddr Start, tVAddr End) if( !(gaPageTable[curPos>>12] & PF_PRESENT) ) continue; expected = (gaPageTable[page] & MASK); + MM_GetPageNode(expected, &expected_node); rangeStart = curPos; } if(expected) expected += 0x1000; } if(expected) { - Log("0x%08x-0x%08x => 0x%08x-0x%08x (%s%s%s%s)", - rangeStart, curPos - 1, - gaPageTable[rangeStart>>12] & ~0xFFF, - (expected & ~0xFFF) - 1, - (expected & PF_PAGED ? "p" : "-"), - (expected & PF_COW ? "C" : "-"), - (expected & PF_USER ? "U" : "-"), - (expected & PF_WRITE ? "W" : "-") + tPAddr orig = gaPageTable[rangeStart>>12]; + Log("0x%08x => 0x%08x - 0x%08x (%s%s%s%s%s) %p", + rangeStart, + orig & ~0xFFF, + curPos - rangeStart, + (orig & PF_NOPAGE ? "p" : "-"), + (orig & PF_COW ? "C" : "-"), + (orig & PF_GLOBAL ? "G" : "-"), + (orig & PF_USER ? "U" : "-"), + (orig & PF_WRITE ? "W" : "-"), + expected_node ); expected = 0; } } /** - * \fn tPAddr MM_Allocate(Uint VAddr) + * \fn tPAddr MM_Allocate(tVAddr VAddr) */ -tPAddr MM_Allocate(Uint VAddr) +tPAddr MM_Allocate(tVAddr VAddr) { + tPAddr paddr; + //ENTER("xVAddr", VAddr); + //__ASM__("xchg %bx,%bx"); // Check if the directory is mapped if( gaPageDir[ VAddr >> 22 ] == 0 ) { // Allocate directory - gaPageDir[ VAddr >> 22 ] = MM_AllocPhys() | 3; - // Mark as user + paddr = MM_AllocPhys(); + if( paddr == 0 ) { + Warning("MM_Allocate - Out of Memory (Called by %p)", __builtin_return_address(0)); + //LEAVE('i',0); + return 0; + } + // Map and mark as user (if needed) + gaPageDir[ VAddr >> 22 ] = paddr | 3; if(VAddr < MM_USER_MAX) gaPageDir[ VAddr >> 22 ] |= PF_USER; INVLPG( &gaPageDir[ VAddr >> 22 ] ); @@ -213,24 +362,35 @@ tPAddr MM_Allocate(Uint VAddr) } // Check if the page is already allocated else if( gaPageTable[ VAddr >> 12 ] != 0 ) { - Warning("MM_Allocate - Allocating to used address"); + Warning("MM_Allocate - Allocating to used address (%p)", VAddr); + //LEAVE('X', gaPageTable[ VAddr >> 12 ] & ~0xFFF); return gaPageTable[ VAddr >> 12 ] & ~0xFFF; } // Allocate - gaPageTable[ VAddr >> 12 ] = MM_AllocPhys() | 3; + paddr = MM_AllocPhys(); + //LOG("paddr = 0x%llx", paddr); + if( paddr == 0 ) { + Warning("MM_Allocate - Out of Memory when allocating at %p (Called by %p)", + VAddr, __builtin_return_address(0)); + //LEAVE('i',0); + return 0; + } + // Map + gaPageTable[ VAddr >> 12 ] = paddr | 3; // Mark as user if(VAddr < MM_USER_MAX) gaPageTable[ VAddr >> 12 ] |= PF_USER; // Invalidate Cache for address INVLPG( VAddr & ~0xFFF ); - return gaPageTable[ VAddr >> 12 ] & ~0xFFF; + //LEAVE('X', paddr); + return paddr; } /** - * \fn void MM_Deallocate(Uint VAddr) + * \fn void MM_Deallocate(tVAddr VAddr) */ -void MM_Deallocate(Uint VAddr) +void MM_Deallocate(tVAddr VAddr) { if( gaPageDir[ VAddr >> 22 ] == 0 ) { Warning("MM_Deallocate - Directory not mapped"); @@ -249,10 +409,10 @@ void MM_Deallocate(Uint VAddr) } /** - * \fn tPAddr MM_GetPhysAddr(Uint Addr) + * \fn tPAddr MM_GetPhysAddr(tVAddr Addr) * \brief Checks if the passed address is accesable */ -tPAddr MM_GetPhysAddr(Uint Addr) +tPAddr MM_GetPhysAddr(tVAddr Addr) { if( !(gaPageDir[Addr >> 22] & 1) ) return 0; @@ -267,19 +427,19 @@ tPAddr MM_GetPhysAddr(Uint Addr) */ void MM_SetCR3(Uint CR3) { - __asm__ __volatile__ ("mov %0, %%cr3"::"r"(CR3)); + __ASM__("mov %0, %%cr3"::"r"(CR3)); } /** - * \fn int MM_Map(Uint VAddr, tPAddr PAddr) + * \fn int MM_Map(tVAddr VAddr, tPAddr PAddr) * \brief Map a physical page to a virtual one */ -int MM_Map(Uint VAddr, tPAddr PAddr) +int MM_Map(tVAddr VAddr, tPAddr PAddr) { //ENTER("xVAddr xPAddr", VAddr, PAddr); // Sanity check if( PAddr & 0xFFF || VAddr & 0xFFF ) { - Warning("MM_Map - Physical or Virtual Addresses are not aligned"); + Log_Warning("MM_Virt", "MM_Map - Physical or Virtual Addresses are not aligned"); //LEAVE('i', 0); return 0; } @@ -290,7 +450,10 @@ int MM_Map(Uint VAddr, tPAddr PAddr) // Check if the directory is mapped if( gaPageDir[ VAddr >> 22 ] == 0 ) { - gaPageDir[ VAddr >> 22 ] = MM_AllocPhys() | 3; + tPAddr tmp = MM_AllocPhys(); + if( tmp == 0 ) + return 0; + gaPageDir[ VAddr >> 22 ] = tmp | 3; // Mark as user if(VAddr < MM_USER_MAX) gaPageDir[ VAddr >> 22 ] |= PF_USER; @@ -324,14 +487,12 @@ int MM_Map(Uint VAddr, tPAddr PAddr) } /** - * \fn Uint MM_ClearUser() * \brief Clear user's address space */ -Uint MM_ClearUser() +void MM_ClearUser(void) { Uint i, j; - // Copy Directories for( i = 0; i < (MM_USER_MAX>>22); i ++ ) { // Check if directory is not allocated @@ -340,7 +501,7 @@ Uint MM_ClearUser() continue; } - + // Deallocate tables for( j = 0; j < 1024; j ++ ) { if( gaPageTable[i*1024+j] & 1 ) @@ -348,65 +509,124 @@ Uint MM_ClearUser() gaPageTable[i*1024+j] = 0; } + // Deallocate directory MM_DerefPhys( gaPageDir[i] & ~0xFFF ); + gaPageDir[i] = 0; + INVLPG( &gaPageTable[i*1024] ); } + INVLPG( gaPageDir ); +} + +/** + * \brief Deallocate an address space + */ +void MM_ClearSpace(Uint32 CR3) +{ + int i, j; + if(CR3 == (*gpPageCR3 & ~0xFFF)) { + Log_Error("MMVirt", "Can't clear current address space"); + return ; + } + + if( MM_GetRefCount(CR3) > 1 ) { + MM_DerefPhys(CR3); + Log_Log("MMVirt", "CR3 %P is still referenced, not cleaning (but dereferenced)", CR3); + return ; + } + + Log_Debug("MMVirt", "Clearing out address space 0x%x from 0x%x", CR3, *gpPageCR3); - return *gTmpCR3; + GET_TEMP_MAPPING(CR3); + INVLPG( gaTmpDir ); + + for( i = 0; i < 1024; i ++ ) + { + Uint32 *table = &gaTmpTable[i*1024]; + if( !(gaTmpDir[i] & PF_PRESENT) ) + continue ; + + INVLPG( table ); + + if( i < 768 || (i > MM_KERNEL_STACKS >> 22 && i < MM_KERNEL_STACKS_END >> 22) ) + { + for( j = 0; j < 1024; j ++ ) + { + if( !(table[j] & 1) ) + continue; + MM_DerefPhys( table[j] & ~0xFFF ); + } + } + + if( i != (PAGE_TABLE_ADDR >> 22) ) + { + MM_DerefPhys( gaTmpDir[i] & ~0xFFF ); + } + } + + + MM_DerefPhys( CR3 ); + + REL_TEMP_MAPPING(); } /** - * \fn Uint MM_Clone() + * \fn tPAddr MM_Clone(void) * \brief Clone the current address space */ -Uint MM_Clone() +tPAddr MM_Clone(int bNoUserCopy) { Uint i, j; - Uint kStackBase = gCurrentThread->KernelStack - KERNEL_STACK_SIZE; + tPAddr ret; + Uint page = 0; + tVAddr kStackBase = Proc_GetCurThread()->KernelStack - MM_KERNEL_STACK_SIZE; void *tmp; - //ENTER(""); - // Create Directory Table - *gTmpCR3 = MM_AllocPhys() | 3; + ret = MM_AllocPhys(); + if( ret == 0 ) { + return 0; + } + + // Map + GET_TEMP_MAPPING( ret ); INVLPG( gaTmpDir ); - //LOG("Allocated Directory (%x)", *gTmpCR3); memsetd( gaTmpDir, 0, 1024 ); - // Copy Tables - for(i=0;i<768;i++) - { - // Check if table is allocated - if( !(gaPageDir[i] & PF_PRESENT) ) { - gaTmpDir[i] = 0; - continue; - } - - // Allocate new table - gaTmpDir[i] = MM_AllocPhys() | (gaPageDir[i] & 7); - INVLPG( &gaTmpTable[i*1024] ); - // Fill - for( j = 0; j < 1024; j ++ ) + if( Threads_GetPID() != 0 && !bNoUserCopy ) + { + // Copy Tables + for( i = 0; i < 768; i ++) { - if( !(gaPageTable[i*1024+j] & PF_PRESENT) ) { - gaTmpTable[i*1024+j] = 0; + // Check if table is allocated + if( !(gaPageDir[i] & PF_PRESENT) ) { + gaTmpDir[i] = 0; + page += 1024; continue; } - #if USE_COW - // Refrence old page - MM_RefPhys( gaPageTable[i*1024+j] & ~0xFFF ); - // Add to new table - if(gaPageTable[i*1024+j] & PF_WRITE) { - gaTmpTable[i*1024+j] = (gaPageTable[i*1024+j] & ~PF_WRITE) | PF_COW; - gaPageTable[i*1024+j] = (gaPageTable[i*1024+j] & ~PF_WRITE) | PF_COW; + // Allocate new table + gaTmpDir[i] = MM_AllocPhys() | (gaPageDir[i] & 7); + INVLPG( &gaTmpTable[page] ); + // Fill + for( j = 0; j < 1024; j ++, page++ ) + { + if( !(gaPageTable[page] & PF_PRESENT) ) { + gaTmpTable[page] = 0; + continue; + } + + // Refrence old page + MM_RefPhys( gaPageTable[page] & ~0xFFF ); + // Add to new table + if(gaPageTable[page] & PF_WRITE) { + gaTmpTable[page] = (gaPageTable[page] & ~PF_WRITE) | PF_COW; + gaPageTable[page] = (gaPageTable[page] & ~PF_WRITE) | PF_COW; + INVLPG( page << 12 ); + } + else + gaTmpTable[page] = gaPageTable[page]; } - else - gaTmpTable[i*1024+j] = gaPageTable[i*1024+j]; - LOG("gaTmpTable[0x%x] = 0x%x", i*1024+j, gaTmpTable[i*1024+j]); - #else - gaTmpTable[i*1024+j] = MM_DuplicatePage( (i*1024+j)<<12 ) | (gaPageTable[i*1024+j]&7); - #endif } } @@ -415,9 +635,13 @@ Uint MM_Clone() { // Fractal if( i == (PAGE_TABLE_ADDR >> 22) ) { - gaTmpDir[ PAGE_TABLE_ADDR >> 22 ] = *gTmpCR3; + gaTmpDir[ PAGE_TABLE_ADDR >> 22 ] = *gpTmpCR3; continue; } + if( i == (TMP_TABLE_ADDR >> 22) ) { + gaTmpDir[ TMP_TABLE_ADDR >> 22 ] = 0; + continue ; + } if( gaPageDir[i] == 0 ) { gaTmpDir[i] = 0; @@ -430,9 +654,7 @@ Uint MM_Clone() } // Allocate kernel stack - for(i = KERNEL_STACKS >> 22; - i < KERNEL_STACK_END >> 22; - i ++ ) + for(i = MM_KERNEL_STACKS >> 22; i < MM_KERNEL_STACKS_END >> 22; i ++ ) { // Check if directory is allocated if( (gaPageDir[i] & 1) == 0 ) { @@ -459,7 +681,7 @@ Uint MM_Clone() } // We don't care about other kernel stacks - if( ((i*1024+j)*4096 & ~(KERNEL_STACK_SIZE-1)) != kStackBase ) { + if( ((i*1024+j)*4096 & ~(MM_KERNEL_STACK_SIZE-1)) != kStackBase ) { gaTmpTable[i*1024+j] = 0; continue; } @@ -475,37 +697,125 @@ Uint MM_Clone() } } - //LEAVE('x', *gTmpCR3 & ~0xFFF); - return *gTmpCR3 & ~0xFFF; + REL_TEMP_MAPPING(); + + //LEAVE('x', ret); + return ret; } /** - * \fn Uint MM_NewKStack() + * \fn tVAddr MM_NewKStack(void) * \brief Create a new kernel stack */ -Uint MM_NewKStack() +tVAddr MM_NewKStack(void) { - Uint base = KERNEL_STACKS; + tVAddr base; Uint i; - for(;base= NUM_WORKER_STACKS) { + Warning("Uh-oh! Out of worker stacks"); + return 0; + } + + // It's ours now! + gWorkerStacks[base/32] |= (1 << base); + // Make life easier for later calls + giLastUsedWorker = base; + // We have one + base = WORKER_STACKS + base * WORKER_STACK_SIZE; + //Log(" MM_NewWorkerStack: base = 0x%x", base); + + // Set the temp fractals to TID0's address space + GET_TEMP_MAPPING( ((Uint)gaInitPageDir - KERNEL_BASE) ); + INVLPG( gaTmpDir ); + + // Check if the directory is mapped (we are assuming that the stacks + // will fit neatly in a directory) + //Log(" MM_NewWorkerStack: gaTmpDir[ 0x%x ] = 0x%x", base>>22, gaTmpDir[ base >> 22 ]); + if(gaTmpDir[ base >> 22 ] == 0) { + gaTmpDir[ base >> 22 ] = MM_AllocPhys() | 3; + INVLPG( &gaTmpTable[ (base>>12) & ~0x3FF ] ); + } + + // Mapping Time! + for( addr = 0; addr < WORKER_STACK_SIZE; addr += 0x1000 ) + { + page = MM_AllocPhys(); + gaTmpTable[ (base + addr) >> 12 ] = page | 3; + } + + // Release temporary fractal + REL_TEMP_MAPPING(); + + // NOTE: Max of 1 page + // `page` is the last allocated page from the previious for loop + tmpPage = MM_MapTemp( page ); + memcpy( (void*)( tmpPage + (0x1000 - ContentsSize) ), StackContents, ContentsSize); + MM_FreeTemp(tmpPage); + + //Log("MM_NewWorkerStack: RETURN 0x%x", base); + return base + WORKER_STACK_SIZE; +} + +/** + * \fn void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) * \brief Sets the flags on a page */ -void MM_SetFlags(Uint VAddr, Uint Flags, Uint Mask) +void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) { - tPAddr *ent; + tTabEnt *ent; if( !(gaPageDir[VAddr >> 22] & 1) ) return ; if( !(gaPageTable[VAddr >> 12] & 1) ) return ; @@ -514,15 +824,25 @@ void MM_SetFlags(Uint VAddr, Uint Flags, Uint Mask) // Read-Only if( Mask & MM_PFLAG_RO ) { - if( Flags & MM_PFLAG_RO ) *ent &= ~PF_WRITE; - else *ent |= PF_WRITE; + if( Flags & MM_PFLAG_RO ) { + *ent &= ~PF_WRITE; + } + else { + gaPageDir[VAddr >> 22] |= PF_WRITE; + *ent |= PF_WRITE; + } } // Kernel if( Mask & MM_PFLAG_KERNEL ) { - if( Flags & MM_PFLAG_KERNEL ) *ent &= ~PF_USER; - else *ent |= PF_USER; + if( Flags & MM_PFLAG_KERNEL ) { + *ent &= ~PF_USER; + } + else { + gaPageDir[VAddr >> 22] |= PF_USER; + *ent |= PF_USER; + } } // Copy-On-Write @@ -537,18 +857,86 @@ void MM_SetFlags(Uint VAddr, Uint Flags, Uint Mask) *ent |= PF_WRITE; } } + + //Log("MM_SetFlags: *ent = 0x%08x, gaPageDir[%i] = 0x%08x", + // *ent, VAddr >> 22, gaPageDir[VAddr >> 22]); +} + +/** + * \brief Get the flags on a page + */ +Uint MM_GetFlags(tVAddr VAddr) +{ + tTabEnt *ent; + Uint ret = 0; + + // Validity Check + if( !(gaPageDir[VAddr >> 22] & 1) ) return 0; + if( !(gaPageTable[VAddr >> 12] & 1) ) return 0; + + ent = &gaPageTable[VAddr >> 12]; + + // Read-Only + if( !(*ent & PF_WRITE) ) ret |= MM_PFLAG_RO; + // Kernel + if( !(*ent & PF_USER) ) ret |= MM_PFLAG_KERNEL; + // Copy-On-Write + if( *ent & PF_COW ) ret |= MM_PFLAG_COW; + + return ret; } /** - * \fn tPAddr MM_DuplicatePage(Uint VAddr) + * \brief Check if the provided buffer is valid + * \return Boolean valid + */ +int MM_IsValidBuffer(tVAddr Addr, size_t Size) +{ + int bIsUser; + int dir, tab; + + Size += Addr & (PAGE_SIZE-1); + Addr &= ~(PAGE_SIZE-1); + + dir = Addr >> 22; + tab = Addr >> 12; + +// Debug("Addr = %p, Size = 0x%x, dir = %i, tab = %i", Addr, Size, dir, tab); + + if( !(gaPageDir[dir] & 1) ) return 0; + if( !(gaPageTable[tab] & 1) ) return 0; + + bIsUser = !!(gaPageTable[tab] & PF_USER); + + while( Size >= PAGE_SIZE ) + { + if( (tab & 1023) == 0 ) + { + dir ++; + if( !(gaPageDir[dir] & 1) ) return 0; + } + + if( !(gaPageTable[tab] & 1) ) return 0; + if( bIsUser && !(gaPageTable[tab] & PF_USER) ) return 0; + + tab ++; + Size -= PAGE_SIZE; + } + return 1; +} + +/** + * \fn tPAddr MM_DuplicatePage(tVAddr VAddr) * \brief Duplicates a virtual page to a physical one */ -tPAddr MM_DuplicatePage(Uint VAddr) +tPAddr MM_DuplicatePage(tVAddr VAddr) { tPAddr ret; Uint temp; int wasRO = 0; + //ENTER("xVAddr", VAddr); + // Check if mapped if( !(gaPageDir [VAddr >> 22] & PF_PRESENT) ) return 0; if( !(gaPageTable[VAddr >> 12] & PF_PRESENT) ) return 0; @@ -558,6 +946,9 @@ tPAddr MM_DuplicatePage(Uint VAddr) // Allocate new page ret = MM_AllocPhys(); + if( !ret ) { + return 0; + } // Write-lock the page (to keep data constistent), saving its R/W state wasRO = (gaPageTable[VAddr >> 12] & PF_WRITE ? 0 : 1); @@ -573,6 +964,7 @@ tPAddr MM_DuplicatePage(Uint VAddr) if(!wasRO) gaPageTable[VAddr >> 12] |= PF_WRITE; INVLPG(VAddr); + //LEAVE('X', ret); return ret; } @@ -581,7 +973,7 @@ tPAddr MM_DuplicatePage(Uint VAddr) * \brief Create a temporary memory mapping * \todo Show Luigi Barone (C Lecturer) and see what he thinks */ -Uint MM_MapTemp(tPAddr PAddr) +tVAddr MM_MapTemp(tPAddr PAddr) { int i; @@ -589,11 +981,11 @@ Uint MM_MapTemp(tPAddr PAddr) PAddr &= ~0xFFF; - //LOG("gilTempMappings = %i", gilTempMappings); + //LOG("glTempMappings = %i", glTempMappings); for(;;) { - LOCK( &gilTempMappings ); + Mutex_Acquire( &glTempMappings ); for( i = 0; i < NUM_TEMP_PAGES; i ++ ) { @@ -603,19 +995,19 @@ Uint MM_MapTemp(tPAddr PAddr) gaPageTable[ (TEMP_MAP_ADDR >> 12) + i ] = PAddr | 3; INVLPG( TEMP_MAP_ADDR + (i << 12) ); //LEAVE('p', TEMP_MAP_ADDR + (i << 12)); - RELEASE( &gilTempMappings ); + Mutex_Release( &glTempMappings ); return TEMP_MAP_ADDR + (i << 12); } - RELEASE( &gilTempMappings ); - Proc_Yield(); + Mutex_Release( &glTempMappings ); + Threads_Yield(); // TODO: Use a sleep queue here instead } } /** - * \fn void MM_FreeTemp(Uint PAddr) + * \fn void MM_FreeTemp(tVAddr PAddr) * \brief Free's a temp mapping */ -void MM_FreeTemp(Uint VAddr) +void MM_FreeTemp(tVAddr VAddr) { int i = VAddr >> 12; //ENTER("xVAddr", VAddr); @@ -627,10 +1019,10 @@ void MM_FreeTemp(Uint VAddr) } /** - * \fn Uint MM_MapHWPage(tPAddr PAddr, Uint Number) + * \fn tVAddr MM_MapHWPages(tPAddr PAddr, Uint Number) * \brief Allocates a contigous number of pages */ -Uint MM_MapHWPage(tPAddr PAddr, Uint Number) +tVAddr MM_MapHWPages(tPAddr PAddr, Uint Number) { int i, j; @@ -665,31 +1057,95 @@ Uint MM_MapHWPage(tPAddr PAddr, Uint Number) } /** - * \fn void MM_UnmapHWPage(Uint VAddr, Uint Number) + * \fn tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr) + * \brief Allocates DMA physical memory + * \param Pages Number of pages required + * \param MaxBits Maximum number of bits the physical address can have + * \param PhysAddr Pointer to the location to place the physical address allocated + * \return Virtual address allocate + */ +tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr) +{ + tPAddr maxCheck = (1 << MaxBits); + tPAddr phys; + tVAddr ret; + + ENTER("iPages iMaxBits pPhysAddr", Pages, MaxBits, PhysAddr); + + // Sanity Check + if(MaxBits < 12 || !PhysAddr) { + LEAVE('i', 0); + return 0; + } + + // Bound + if(MaxBits >= PHYS_BITS) maxCheck = -1; + + // Fast Allocate + if(Pages == 1 && MaxBits >= PHYS_BITS) + { + phys = MM_AllocPhys(); + if( !phys ) { + *PhysAddr = 0; + LEAVE_RET('i', 0); + } + *PhysAddr = phys; + ret = MM_MapHWPages(phys, 1); + if(ret == 0) { + MM_DerefPhys(phys); + LEAVE('i', 0); + return 0; + } + LEAVE('x', ret); + return ret; + } + + // Slow Allocate + phys = MM_AllocPhysRange(Pages, MaxBits); + // - Was it allocated? + if(phys == 0) { + LEAVE('i', 0); + return 0; + } + + // Allocated successfully, now map + ret = MM_MapHWPages(phys, Pages); + if( ret == 0 ) { + // If it didn't map, free then return 0 + for(;Pages--;phys+=0x1000) + MM_DerefPhys(phys); + LEAVE('i', 0); + return 0; + } + + *PhysAddr = phys; + LEAVE('x', ret); + return ret; +} + +/** + * \fn void MM_UnmapHWPages(tVAddr VAddr, Uint Number) * \brief Unmap a hardware page */ -void MM_UnmapHWPage(Uint VAddr, Uint Number) +void MM_UnmapHWPages(tVAddr VAddr, Uint Number) { int i, j; + + //Log_Debug("VirtMem", "MM_UnmapHWPages: (VAddr=0x%08x, Number=%i)", VAddr, Number); + // Sanity Check - if(VAddr < HW_MAP_ADDR || VAddr-Number*0x1000 > HW_MAP_MAX) return; + if(VAddr < HW_MAP_ADDR || VAddr+Number*0x1000 > HW_MAP_MAX) return; i = VAddr >> 12; - LOCK( &gilTempMappings ); // Temp and HW share a directory, so they share a lock + Mutex_Acquire( &glTempMappings ); // Temp and HW share a directory, so they share a lock for( j = 0; j < Number; j++ ) { - MM_DerefPhys( gaPageTable[ (HW_MAP_ADDR >> 12) + i + j ] ); - gaPageTable[ (HW_MAP_ADDR >> 12) + i + j ] = 0; + MM_DerefPhys( gaPageTable[ i + j ] & ~0xFFF ); + gaPageTable[ i + j ] = 0; } - RELEASE( &gilTempMappings ); + Mutex_Release( &glTempMappings ); } -// --- EXPORTS --- -EXPORT(MM_GetPhysAddr); -EXPORT(MM_Map); -//EXPORT(MM_Unmap); -EXPORT(MM_MapHWPage); -EXPORT(MM_UnmapHWPage);