X-Git-Url: https://git.ucc.asn.au/?a=blobdiff_plain;f=KernelLand%2FKernel%2Farch%2Fx86_64%2Fmm_virt.c;h=4e2968fd80e70e99e9a3beb5b7a48b79fbd53a4f;hb=af67042b8d7f05fe76583aef1afb2022b519a1aa;hp=89aeaa15be9072ae6a3851f827248c9be74aaecb;hpb=51ab5f489bc356940c95cc936fd0508e8f07ea97;p=tpg%2Facess2.git diff --git a/KernelLand/Kernel/arch/x86_64/mm_virt.c b/KernelLand/Kernel/arch/x86_64/mm_virt.c index 89aeaa15..4e2968fd 100644 --- a/KernelLand/Kernel/arch/x86_64/mm_virt.c +++ b/KernelLand/Kernel/arch/x86_64/mm_virt.c @@ -70,7 +70,7 @@ void MM_int_ClonePageEnt( Uint64 *Ent, void *NextLevel, tVAddr Addr, int bTable void MM_int_DumpTablesEnt(tVAddr RangeStart, size_t Length, tPAddr Expected); //void MM_DumpTables(tVAddr Start, tVAddr End); int MM_GetPageEntryPtr(tVAddr Addr, BOOL bTemp, BOOL bAllocate, BOOL bLargePage, tPAddr **Pointer); - int MM_MapEx(tVAddr VAddr, tPAddr PAddr, BOOL bTemp, BOOL bLarge); + int MM_MapEx(volatile void *VAddr, tPAddr PAddr, BOOL bTemp, BOOL bLarge); // int MM_Map(tVAddr VAddr, tPAddr PAddr); void MM_Unmap(tVAddr VAddr); void MM_int_ClearTableLevel(tVAddr VAddr, int LevelBits, int MaxEnts); @@ -79,6 +79,7 @@ void MM_int_ClearTableLevel(tVAddr VAddr, int LevelBits, int MaxEnts); // === GLOBALS === tMutex glMM_TempFractalLock; +tShortSpinlock glMM_ZeroPage; tPAddr gMM_ZeroPage; // === CODE === @@ -128,9 +129,9 @@ void MM_int_ClonePageEnt( Uint64 *Ent, void *NextLevel, tVAddr Addr, int bTable ASSERT(paddr != curpage); - tmp = (void*)MM_MapTemp(paddr); + tmp = MM_MapTemp(paddr); memcpy( tmp, NextLevel, 0x1000 ); - MM_FreeTemp( (tVAddr)tmp ); + MM_FreeTemp( tmp ); #if TRACE_COW Log_Debug("MMVirt", "COW ent at %p (%p) from %P to %P", Ent, NextLevel, curpage, paddr); @@ -269,9 +270,9 @@ void MM_int_DumpTablesEnt(tVAddr RangeStart, size_t Length, tPAddr Expected) #define CANOICAL(addr) ((addr)&0x800000000000?(addr)|0xFFFF000000000000:(addr)) LogF("%016llx => ", CANOICAL(RangeStart)); // LogF("%6llx %6llx %6llx %016llx => ", -// MM_GetPhysAddr( (tVAddr)&PAGEDIRPTR(RangeStart>>30) ), -// MM_GetPhysAddr( (tVAddr)&PAGEDIR(RangeStart>>21) ), -// MM_GetPhysAddr( (tVAddr)&PAGETABLE(RangeStart>>12) ), +// MM_GetPhysAddr( &PAGEDIRPTR(RangeStart>>30) ), +// MM_GetPhysAddr( &PAGEDIR(RangeStart>>21) ), +// MM_GetPhysAddr( &PAGETABLE(RangeStart>>12) ), // CANOICAL(RangeStart) // ); if( gMM_ZeroPage && (PAGETABLE(RangeStart>>12) & PADDR_MASK) == gMM_ZeroPage ) @@ -340,8 +341,8 @@ void MM_DumpTables(tVAddr Start, tVAddr End) expected |= expected_pml4 & PF_NX; expected |= expected_pdp & PF_NX; expected |= expected_pd & PF_NX; - Log("expected (pml4 = %x, pdp = %x, pd = %x)", - expected_pml4, expected_pdp, expected_pd); +// Log("expected (pml4 = %x, pdp = %x, pd = %x)", +// expected_pml4, expected_pdp, expected_pd); // Dump MM_int_DumpTablesEnt( rangeStart, curPos - rangeStart, expected ); expected = CHANGEABLE_BITS; @@ -470,7 +471,7 @@ int MM_GetPageEntryPtr(tVAddr Addr, BOOL bTemp, BOOL bAllocate, BOOL bLargePage, * \param bTemp Use tempoary mappings * \param bLarge Treat as a large page */ -int MM_MapEx(tVAddr VAddr, tPAddr PAddr, BOOL bTemp, BOOL bLarge) +int MM_MapEx(volatile void *VAddr, tPAddr PAddr, BOOL bTemp, BOOL bLarge) { tPAddr *ent; int rv; @@ -478,16 +479,15 @@ int MM_MapEx(tVAddr VAddr, tPAddr PAddr, BOOL bTemp, BOOL bLarge) ENTER("pVAddr PPAddr", VAddr, PAddr); // Get page pointer (Allow allocating) - rv = MM_GetPageEntryPtr(VAddr, bTemp, 1, bLarge, &ent); + rv = MM_GetPageEntryPtr( (tVAddr)VAddr, bTemp, 1, bLarge, &ent); if(rv < 0) LEAVE_RET('i', 0); if( *ent & 1 ) LEAVE_RET('i', 0); *ent = PAddr | 3; - if( VAddr < 0x800000000000 ) + if( (tVAddr)VAddr < USER_MAX ) *ent |= PF_USER; - INVLPG( VAddr ); LEAVE('i', 1); @@ -499,7 +499,7 @@ int MM_MapEx(tVAddr VAddr, tPAddr PAddr, BOOL bTemp, BOOL bLarge) * \param VAddr Target virtual address * \param PAddr Physical address of page */ -int MM_Map(tVAddr VAddr, tPAddr PAddr) +int MM_Map(volatile void *VAddr, tPAddr PAddr) { return MM_MapEx(VAddr, PAddr, 0, 0); } @@ -516,21 +516,22 @@ void MM_Unmap(tVAddr VAddr) // Check Page Dir if( !(PAGEDIR(VAddr >> 21) & 1) ) return ; - PAGETABLE(VAddr >> PTAB_SHIFT) = 0; + tPAddr *ent = &PAGETABLE(VAddr >> PTAB_SHIFT); + *ent = 0; INVLPG( VAddr ); } /** * \brief Allocate a block of memory at the specified virtual address */ -tPAddr MM_Allocate(tVAddr VAddr) +tPAddr MM_Allocate(volatile void *VAddr) { tPAddr ret; - ENTER("xVAddr", VAddr); + ENTER("pVAddr", VAddr); // Ensure the tables are allocated before the page (keeps things neat) - MM_GetPageEntryPtr(VAddr, 0, 1, 0, NULL); + MM_GetPageEntryPtr( (tVAddr)VAddr, 0, 1, 0, NULL ); // Allocate the page ret = MM_AllocPhys(); @@ -549,37 +550,39 @@ tPAddr MM_Allocate(tVAddr VAddr) return ret; } -tPAddr MM_AllocateZero(tVAddr VAddr) +void MM_AllocateZero(volatile void *VAddr) { - tPAddr ret = gMM_ZeroPage; - - MM_GetPageEntryPtr(VAddr, 0, 1, 0, NULL); + // Ensure dir is populated + MM_GetPageEntryPtr((tVAddr)VAddr, 0, 1, 0, NULL); - if(!gMM_ZeroPage) { - ret = gMM_ZeroPage = MM_AllocPhys(); - MM_RefPhys(ret); // Don't free this please - MM_Map(VAddr, ret); - memset((void*)VAddr, 0, 0x1000); + if(!gMM_ZeroPage) + { + SHORTLOCK(&glMM_ZeroPage); + if( !gMM_ZeroPage ) + { + gMM_ZeroPage = MM_AllocPhys(); + MM_Map(VAddr, gMM_ZeroPage); + memset((void*)VAddr, 0, PAGE_SIZE); + } + SHORTREL(&glMM_ZeroPage); } - else { - MM_Map(VAddr, ret); + else + { + MM_Map(VAddr, gMM_ZeroPage); } - MM_RefPhys(ret); // Refernce for this map + MM_RefPhys(gMM_ZeroPage); // Refernce for this map MM_SetFlags(VAddr, MM_PFLAG_COW, MM_PFLAG_COW); - return ret; } /** * \brief Deallocate a page at a virtual address */ -void MM_Deallocate(tVAddr VAddr) +void MM_Deallocate(volatile void *VAddr) { - tPAddr phys; - - phys = MM_GetPhysAddr(VAddr); + tPAddr phys = MM_GetPhysAddr( VAddr ); if(!phys) return ; - MM_Unmap(VAddr); + MM_Unmap((tVAddr)VAddr); MM_DerefPhys(phys); } @@ -609,8 +612,9 @@ int MM_GetPageEntry(tVAddr Addr, tPAddr *Phys, Uint *Flags) /** * \brief Get the physical address of a virtual location */ -tPAddr MM_GetPhysAddr(tVAddr Addr) +tPAddr MM_GetPhysAddr(volatile const void *Ptr) { + tVAddr Addr = (tVAddr)Ptr; tPAddr *ptr; int ret; @@ -625,13 +629,13 @@ tPAddr MM_GetPhysAddr(tVAddr Addr) /** * \brief Sets the flags on a page */ -void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) +void MM_SetFlags(volatile void *VAddr, Uint Flags, Uint Mask) { tPAddr *ent; int rv; // Get pointer - rv = MM_GetPageEntryPtr(VAddr, 0, 0, 0, &ent); + rv = MM_GetPageEntryPtr( (tVAddr)VAddr, 0, 0, 0, &ent); if(rv < 0) return ; // Ensure the entry is valid @@ -665,7 +669,6 @@ void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) if( Flags & MM_PFLAG_COW ) { *ent &= ~PF_WRITE; *ent |= PF_COW; - INVLPG_ALL(); } else { *ent &= ~PF_COW; @@ -688,12 +691,12 @@ void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) /** * \brief Get the flags applied to a page */ -Uint MM_GetFlags(tVAddr VAddr) +Uint MM_GetFlags(volatile const void *VAddr) { tPAddr *ent; int rv, ret = 0; - rv = MM_GetPageEntryPtr(VAddr, 0, 0, 0, &ent); + rv = MM_GetPageEntryPtr((tVAddr)VAddr, 0, 0, 0, &ent); if(rv < 0) return 0; if( !(*ent & 1) ) return 0; @@ -721,43 +724,73 @@ int MM_IsValidBuffer(tVAddr Addr, size_t Size) Size += Addr & (PAGE_SIZE-1); Addr &= ~(PAGE_SIZE-1); - Addr &= ((1UL << 48)-1); // Clap to address space + // NC addr + if( ((Addr >> 47) & 1) != ((Addr>>48) == 0xFFFF)) + return 0; + Addr &= ((1UL << 48)-1); // Clamp to address space pml4 = Addr >> 39; pdp = Addr >> 30; dir = Addr >> 21; tab = Addr >> 12; - if( !(PAGEMAPLVL4(pml4) & 1) ) return 0; - if( !(PAGEDIRPTR(pdp) & 1) ) return 0; - if( !(PAGEDIR(dir) & 1) ) return 0; - if( !(PAGETABLE(tab) & 1) ) return 0; + if( !(PAGEMAPLVL4(pml4) & 1) ) { + Log_Debug("MMVirt", "PML4E %i NP", pml4); + return 0; + } + if( !(PAGEDIRPTR(pdp) & 1) ) { + Log_Debug("MMVirt", "PDPE %i NP", pdp); + return 0; + } + if( !(PAGEDIR(dir) & 1) ) { + Log_Debug("MMVirt", "PDE %i NP", dir); + return 0; + } + if( !(PAGETABLE(tab) & 1) ) { + Log_Debug("MMVirt", "PTE %i NP", tab); + return 0; + } bIsUser = !!(PAGETABLE(tab) & PF_USER); while( Size >= PAGE_SIZE ) { + tab ++; + Size -= PAGE_SIZE; + if( (tab & 511) == 0 ) { dir ++; - if( ((dir >> 9) & 511) == 0 ) + if( (dir & 511) == 0 ) { pdp ++; - if( ((pdp >> 18) & 511) == 0 ) + if( (pdp & 511) == 0 ) { pml4 ++; - if( !(PAGEMAPLVL4(pml4) & 1) ) return 0; + if( !(PAGEMAPLVL4(pml4) & 1) ) { + Log_Debug("MMVirt", "IsValidBuffer - PML4E %x NP, Size=%x", pml4, Size); + return 0; + } } - if( !(PAGEDIRPTR(pdp) & 1) ) return 0; + if( !(PAGEDIRPTR(pdp) & 1) ) { + Log_Debug("MMVirt", "IsValidBuffer - PDPE %x NP", pdp); + return 0; + } + } + if( !(PAGEDIR(dir) & 1) ) { + Log_Debug("MMVirt", "IsValidBuffer - PDE %x NP", dir); + return 0; } - if( !(PAGEDIR(dir) & 1) ) return 0; } - if( !(PAGETABLE(tab) & 1) ) return 0; - if( bIsUser && !(PAGETABLE(tab) & PF_USER) ) return 0; - - tab ++; - Size -= PAGE_SIZE; + if( !(PAGETABLE(tab) & 1) ) { + Log_Debug("MMVirt", "IsValidBuffer - PTE %x NP", tab); + return 0; + } + if( bIsUser && !(PAGETABLE(tab) & PF_USER) ) { + Log_Debug("MMVirt", "IsValidBuffer - PTE %x Not user", tab); + return 0; + } } return 1; } @@ -766,27 +799,27 @@ int MM_IsValidBuffer(tVAddr Addr, size_t Size) /** * \brief Map a range of hardware pages */ -tVAddr MM_MapHWPages(tPAddr PAddr, Uint Number) +void *MM_MapHWPages(tPAddr PAddr, Uint Number) { - tVAddr ret; - int num; - //TODO: Add speedups (memory of first possible free) - for( ret = MM_HWMAP_BASE; ret < MM_HWMAP_TOP; ret += 0x1000 ) + for( tPage *ret = (void*)MM_HWMAP_BASE; ret < (tPage*)MM_HWMAP_TOP; ret ++ ) { - for( num = Number; num -- && ret < MM_HWMAP_TOP; ret += 0x1000 ) + // Check if this region has already been used + int num; + for( num = Number; num -- && ret < (tPage*)MM_HWMAP_TOP; ret ++ ) { - if( MM_GetPhysAddr(ret) != 0 ) break; + if( MM_GetPhysAddr( ret ) != 0 ) + break; } if( num >= 0 ) continue; // Log_Debug("MMVirt", "Mapping %i pages to %p (base %P)", Number, ret-Number*0x1000, PAddr); + // Map backwards (because `ret` is at the top of the region atm) PAddr += 0x1000 * Number; - while( Number -- ) { - ret -= 0x1000; + ret --; PAddr -= 0x1000; MM_Map(ret, PAddr); MM_RefPhys(PAddr); @@ -802,14 +835,15 @@ tVAddr MM_MapHWPages(tPAddr PAddr, Uint Number) /** * \brief Free a range of hardware pages */ -void MM_UnmapHWPages(tVAddr VAddr, Uint Number) +void MM_UnmapHWPages(volatile void *VAddr, Uint Number) { // Log_KernelPanic("MM", "TODO: Implement MM_UnmapHWPages"); + tPage *page = (void*)VAddr; while( Number -- ) { - MM_DerefPhys( MM_GetPhysAddr(VAddr) ); - MM_Unmap(VAddr); - VAddr += 0x1000; + MM_DerefPhys( MM_GetPhysAddr(page) ); + MM_Unmap((tVAddr)page); + page ++; } } @@ -822,10 +856,10 @@ void MM_UnmapHWPages(tVAddr VAddr, Uint Number) * \param PhysAddr Pointer to the location to place the physical address allocated * \return Virtual address allocate */ -tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr) +void *MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr) { tPAddr phys; - tVAddr ret; + void *ret; // Sanity Check if(MaxBits < 12 || !PhysAddr) return 0; @@ -847,6 +881,7 @@ tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr) // Allocated successfully, now map ret = MM_MapHWPages(phys, Pages); + *PhysAddr = phys; // MapHWPages references the pages, so deref them back down to 1 for(;Pages--;phys+=0x1000) MM_DerefPhys(phys); @@ -855,12 +890,11 @@ tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr) return 0; } - *PhysAddr = phys; return ret; } // --- Tempory Mappings --- -tVAddr MM_MapTemp(tPAddr PAddr) +void *MM_MapTemp(tPAddr PAddr) { const int max_slots = (MM_TMPMAP_END - MM_TMPMAP_BASE) / PAGE_SIZE; tVAddr ret = MM_TMPMAP_BASE; @@ -879,24 +913,22 @@ tVAddr MM_MapTemp(tPAddr PAddr) *ent = PAddr | 3; MM_RefPhys(PAddr); INVLPG(ret); - return ret; + return (void*)ret; } return 0; } -void MM_FreeTemp(tVAddr VAddr) +void MM_FreeTemp(void *Ptr) { - MM_Deallocate(VAddr); - return ; + MM_Deallocate(Ptr); } // --- Address Space Clone -- -tPAddr MM_Clone(void) +tPAddr MM_Clone(int bNoUserCopy) { tPAddr ret; int i; - tVAddr kstackbase; // #1 Create a copy of the PML4 ret = MM_AllocPhys(); @@ -908,7 +940,7 @@ tPAddr MM_Clone(void) INVLPG_ALL(); // #3 Set Copy-On-Write to all user pages - if( Threads_GetPID() != 0 ) + if( Threads_GetPID() != 0 && !bNoUserCopy ) { for( i = 0; i < 256; i ++) { @@ -959,21 +991,22 @@ tPAddr MM_Clone(void) // #6 Create kernel stack // tThread->KernelStack is the top // There is 1 guard page below the stack - kstackbase = Proc_GetCurThread()->KernelStack - KERNEL_STACK_SIZE; + tPage *kstackbase = (void*)( Proc_GetCurThread()->KernelStack - KERNEL_STACK_SIZE ); // Clone stack TMPMAPLVL4(MM_KSTACK_BASE >> PML4_SHIFT) = 0; - for( i = 1; i < KERNEL_STACK_SIZE/0x1000; i ++ ) + for( i = 1; i < KERNEL_STACK_SIZE/PAGE_SIZE; i ++ ) { tPAddr phys = MM_AllocPhys(); - tVAddr tmpmapping; - MM_MapEx(kstackbase+i*0x1000, phys, 1, 0); + void *tmpmapping; + MM_MapEx(kstackbase + i, phys, 1, 0); tmpmapping = MM_MapTemp(phys); - if( MM_GetPhysAddr( kstackbase+i*0x1000 ) ) - memcpy((void*)tmpmapping, (void*)(kstackbase+i*0x1000), 0x1000); + // If the current thread's stack is shorter than the new one, zero + if( MM_GetPhysAddr( kstackbase + i ) ) + memcpy(tmpmapping, kstackbase + i, 0x1000); else - memset((void*)tmpmapping, 0, 0x1000); + memset(tmpmapping, 0, 0x1000); // if( i == 0xF ) // Debug_HexDump("MM_Clone: *tmpmapping = ", (void*)tmpmapping, 0x1000); MM_FreeTemp(tmpmapping); @@ -1018,7 +1051,7 @@ void MM_int_ClearTableLevel(tVAddr VAddr, int LevelBits, int MaxEnts) void MM_ClearUser(void) { - MM_int_ClearTableLevel(0, 39, 256); + MM_int_ClearTableLevel(0, 39, 256); } tVAddr MM_NewWorkerStack(void *StackData, size_t StackSize) @@ -1055,8 +1088,9 @@ tVAddr MM_NewWorkerStack(void *StackData, size_t StackSize) Log_Error("MM", "MM_NewWorkerStack - Unable to allocate page"); return 0; } - MM_MapEx(ret + i*0x1000, phys, 1, 0); - MM_SetFlags(ret + i*0x1000, MM_PFLAG_KERNEL|MM_PFLAG_RO, MM_PFLAG_KERNEL); + MM_MapEx( (void*)(ret + i*0x1000), phys, 1, 0); + // XXX: ... this doesn't change the correct address space + MM_SetFlags( (void*)(ret + i*0x1000), MM_PFLAG_KERNEL|MM_PFLAG_RO, MM_PFLAG_KERNEL); } // Copy data @@ -1064,12 +1098,10 @@ tVAddr MM_NewWorkerStack(void *StackData, size_t StackSize) Log_Error("MM", "MM_NewWorkerStack: StackSize(0x%x) > 0x1000, cbf handling", StackSize); } else { - tVAddr tmp_addr, dest; + void *tmp_addr, *dest; tmp_addr = MM_MapTemp(phys); - dest = tmp_addr + (0x1000 - StackSize); - memcpy( (void*)dest, StackData, StackSize ); - Log_Debug("MM", "MM_NewWorkerStack: %p->%p %i bytes (i=%i)", StackData, dest, StackSize, i); - Log_Debug("MM", "MM_NewWorkerStack: ret = %p", ret); + dest = (char*)tmp_addr + (0x1000 - StackSize); + memcpy( dest, StackData, StackSize ); MM_FreeTemp(tmp_addr); } @@ -1088,17 +1120,17 @@ tVAddr MM_NewKStack(void) Uint i; for( ; base < MM_KSTACK_TOP; base += KERNEL_STACK_SIZE ) { - if(MM_GetPhysAddr(base+KERNEL_STACK_SIZE-0x1000) != 0) + if(MM_GetPhysAddr( (void*)(base+KERNEL_STACK_SIZE-0x1000) ) != 0) continue; //Log("MM_NewKStack: Found one at %p", base + KERNEL_STACK_SIZE); for( i = 0x1000; i < KERNEL_STACK_SIZE; i += 0x1000) { - if( !MM_Allocate(base+i) ) + if( !MM_Allocate( (void*)(base+i) ) ) { Log_Warning("MM", "MM_NewKStack - Allocation failed"); for( i -= 0x1000; i; i -= 0x1000) - MM_Deallocate(base+i); + MM_Deallocate((void*)(base+i)); return 0; } }