X-Git-Url: https://git.ucc.asn.au/?a=blobdiff_plain;f=Kernel%2Farch%2Fx86_64%2Fmm_virt.c;h=b13c47cb08aa6f90b52e58d1fa51177000315bb1;hb=9e5348670e92edab24c034bd69793200eeb21847;hp=b4b26ec19ada9d004a108d7cd5c9e567d5f57e02;hpb=e71a426f7cc3cd98e43ca8f529ee8e1e13395885;p=tpg%2Facess2.git diff --git a/Kernel/arch/x86_64/mm_virt.c b/Kernel/arch/x86_64/mm_virt.c index b4b26ec1..b13c47cb 100644 --- a/Kernel/arch/x86_64/mm_virt.c +++ b/Kernel/arch/x86_64/mm_virt.c @@ -3,11 +3,14 @@ * * Virtual Memory Manager */ +#define DEBUG 0 #include #include #include // === CONSTANTS === +#define PHYS_BITS 52 // TODO: Move out + #define PML4_SHIFT 39 #define PDP_SHIFT 30 #define PDIR_SHIFT 21 @@ -19,9 +22,10 @@ #define PDP_MASK (((Uint)1 << 18)-1) #define PML4_MASK (((Uint)1 << 9)-1) -#define PF_PRESENT 0x1 -#define PF_WRITE 0x2 -#define PF_USER 0x4 +#define PF_PRESENT 0x001 +#define PF_WRITE 0x002 +#define PF_USER 0x004 +#define PF_LARGE 0x000 #define PF_COW 0x200 #define PF_PAGED 0x400 #define PF_NX 0x80000000##00000000 @@ -32,20 +36,38 @@ #define PAGEDIRPTR(idx) PAGEDIR((MM_FRACTAL_BASE>>21)+((idx)&PDP_MASK)) #define PAGEMAPLVL4(idx) PAGEDIRPTR((MM_FRACTAL_BASE>>30)+((idx)&PML4_MASK)) +#define TMPTABLE(idx) (*((tPAddr*)MM_TMPFRAC_BASE+((idx)&PAGE_MASK))) +#define TMPDIR(idx) TMPTABLE((MM_FRACTAL_BASE>>12)+((idx)&TABLE_MASK)) +#define TMPDIRPTR(idx) TMPDIR((MM_FRACTAL_BASE>>21)+((idx)&PDP_MASK)) +#define TMPMAPLVL4(idx) TMPDIRPTR((MM_FRACTAL_BASE>>30)+((idx)&PML4_MASK)) +#define TMPCR3() PAGETABLE(MM_TMPFRAC_BASE>>12) + #define INVLPG(__addr) __asm__ __volatile__ ("invlpg (%0)"::"r"(__addr)); +// === CONSTS === +//tPAddr * const gaPageTable = MM_FRACTAL_BASE; + +// === EXTERNS === +extern tPAddr gInitialPML4[512]; + // === PROTOTYPES === void MM_InitVirt(void); -void MM_FinishVirtualInit(void); +//void MM_FinishVirtualInit(void); void MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs); void MM_DumpTables(tVAddr Start, tVAddr End); - int MM_Map(tVAddr VAddr, tPAddr PAddr); + int MM_GetPageEntryPtr(tVAddr Addr, BOOL bTemp, BOOL bAllocate, BOOL bLargePage, tPAddr **Pointer); +// int MM_Map(tVAddr VAddr, tPAddr PAddr); +void MM_Unmap(tVAddr VAddr); +void MM_ClearUser(void); + int MM_GetPageEntry(tVAddr Addr, tPAddr *Phys, Uint *Flags); // === GLOBALS === +tMutex glMM_TempFractalLock; // === CODE === void MM_InitVirt(void) { + MM_DumpTables(0, -1L); } void MM_FinishVirtualInit(void) @@ -57,7 +79,7 @@ void MM_FinishVirtualInit(void) */ void MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs) { - // TODO: Copy on Write + // TODO: Implement Copy-on-Write #if 0 if( gaPageDir [Addr>>22] & PF_PRESENT && gaPageTable[Addr>>12] & PF_PRESENT @@ -118,6 +140,10 @@ void MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs) // Error_Backtrace(Regs->RIP, Regs->RBP); MM_DumpTables(0, -1); + + __asm__ __volatile__ ("cli"); + for( ;; ) + HALT(); } /** @@ -132,11 +158,12 @@ void MM_DumpTables(tVAddr Start, tVAddr End) tVAddr curPos; Uint page; + Log("Table Entries: (%p to %p)", Start, End); + End &= (1L << 48) - 1; Start >>= 12; End >>= 12; - Log("Table Entries:"); for(page = Start, curPos = Start<<12; page < End; curPos += 0x1000, page++) @@ -144,6 +171,11 @@ void MM_DumpTables(tVAddr Start, tVAddr End) if( curPos == 0x800000000000L ) curPos = 0xFFFF800000000000L; + //Debug("&PAGEMAPLVL4(%i page>>27) = %p", page>>27, &PAGEMAPLVL4(page>>27)); + //Debug("&PAGEDIRPTR(%i page>>18) = %p", page>>18, &PAGEDIRPTR(page>>18)); + //Debug("&PAGEDIR(%i page>>9) = %p", page>>9, &PAGEDIR(page>>9)); + //Debug("&PAGETABLE(%i page) = %p", page, &PAGETABLE(page)); + // End of a range if( !(PAGEMAPLVL4(page>>27) & PF_PRESENT) @@ -167,16 +199,19 @@ void MM_DumpTables(tVAddr Start, tVAddr End) if( !(PAGEMAPLVL4(page>>27) & PF_PRESENT) ) { page += (1 << 27) - 1; curPos += (1L << 39) - 0x1000; + //Debug("pml4 ent unset (page = 0x%x now)", page); continue; } if( !(PAGEDIRPTR(page>>18) & PF_PRESENT) ) { page += (1 << 18) - 1; curPos += (1L << 30) - 0x1000; + //Debug("pdp ent unset (page = 0x%x now)", page); continue; } if( !(PAGEDIR(page>>9) & PF_PRESENT) ) { page += (1 << 9) - 1; curPos += (1L << 21) - 0x1000; + //Debug("pd ent unset (page = 0x%x now)", page); continue; } if( !(PAGETABLE(page) & PF_PRESENT) ) continue; @@ -203,58 +238,94 @@ void MM_DumpTables(tVAddr Start, tVAddr End) } /** - * \brief Map a physical page to a virtual one + * \brief Get a pointer to a page entry + * \param Addr Virtual Address + * \param bTemp Use the Temporary fractal mapping + * \param bAllocate Allocate entries + * \param bLargePage Request a large page + * \param Pointer Location to place the calculated pointer + * \return Page size, or -ve on error */ -int MM_Map(tVAddr VAddr, tPAddr PAddr) +int MM_GetPageEntryPtr(tVAddr Addr, BOOL bTemp, BOOL bAllocate, BOOL bLargePage, tPAddr **Pointer) { + tPAddr *pmlevels[4]; tPAddr tmp; + const int ADDR_SIZES[] = {39, 30, 21, 12}; + const int nADDR_SIZES = sizeof(ADDR_SIZES)/sizeof(ADDR_SIZES[0]); + int i; - Log("MM_Map: (VAddr=0x%x, PAddr=0x%x)", VAddr, PAddr); - - // Check PML4 - //Log(" MM_Map: &PAGEMAPLVL4(%x) = %x", VAddr >> 39, &PAGEMAPLVL4(VAddr >> 39)); - //Log(" MM_Map: &PAGEDIRPTR(%x) = %x", VAddr >> 30, &PAGEDIRPTR(VAddr >> 30)); - //Log(" MM_Map: &PAGEDIR(%x) = %x", VAddr >> 21, &PAGEDIR(VAddr >> 21)); - //Log(" MM_Map: &PAGETABLE(%x) = %x", VAddr >> 12, &PAGETABLE(VAddr >> 12)); - //Log(" MM_Map: &PAGETABLE(0) = %x", &PAGETABLE(0)); - if( !(PAGEMAPLVL4(VAddr >> 39) & 1) ) + if( bTemp ) + pmlevels[3] = (void*)MM_TMPFRAC_BASE; // Temporary Page Table + else + pmlevels[3] = (void*)MM_FRACTAL_BASE; // Page Table + pmlevels[2] = &pmlevels[3][(MM_FRACTAL_BASE>>12)&PAGE_MASK]; // PDIR + pmlevels[1] = &pmlevels[2][(MM_FRACTAL_BASE>>21)&TABLE_MASK]; // PDPT + pmlevels[0] = &pmlevels[1][(MM_FRACTAL_BASE>>30)&PDP_MASK]; // PML4 +// Log("pmlevels = {%p, %p, %p, %p}", +// MM_FRACTAL_BASE>>30, MM_FRACTAL_BASE>>21, MM_FRACTAL_BASE>>12, MM_FRACTAL_BASE); +// Log("pmlevels = {%p, %p, %p, %p}", +// pmlevels[0], pmlevels[1], pmlevels[2], pmlevels[3]); + + // Mask address + Addr &= (1ULL << 48)-1; + + for( i = 0; i < nADDR_SIZES-1; i ++ ) { - tmp = MM_AllocPhys(); - if(!tmp) return 0; - PAGEMAPLVL4(VAddr >> 39) = tmp | 3; - INVLPG( &PAGEDIRPTR( (VAddr>>39)<<9 ) ); - memset( &PAGEDIRPTR( (VAddr>>39)<<9 ), 0, 4096 ); + + // Check for a large page + if( (Addr & ((1ULL << ADDR_SIZES[i])-1)) == 0 && bLargePage ) + { + if(Pointer) *Pointer = &pmlevels[i][Addr >> ADDR_SIZES[i]]; + return ADDR_SIZES[i]; + } +// Log("&pmlevels[%i][0x%llx (>> %i)] = %p", i, Addr >> ADDR_SIZES[i], ADDR_SIZES[i], +// &pmlevels[i][Addr >> ADDR_SIZES[i]]); + // Allocate an entry if required + if( !(pmlevels[i][Addr >> ADDR_SIZES[i]] & 1) ) + { + if( !bAllocate ) return -4; // If allocation is not requested, error + tmp = MM_AllocPhys(); + if(!tmp) return -2; + pmlevels[i][Addr >> ADDR_SIZES[i]] = tmp | 3; + INVLPG( &pmlevels[i+1][ (Addr>>ADDR_SIZES[i])<<9 ] ); + memset( &pmlevels[i+1][ (Addr>>ADDR_SIZES[i])<<9 ], 0, 0x1000 ); + } + else if( pmlevels[i][Addr >> ADDR_SIZES[i]] & PF_LARGE ) + { + if( (Addr & ((1ULL << ADDR_SIZES[i])-1)) != 0 ) + return -3; // Alignment + if(Pointer) *Pointer = &pmlevels[i][Addr >> ADDR_SIZES[i]]; + return ADDR_SIZES[i]; // Large page warning + } } - // Check PDP - if( !(PAGEDIRPTR(VAddr >> 30) & 1) ) - { - tmp = MM_AllocPhys(); - if(!tmp) return 0; - PAGEDIRPTR(VAddr >> 30) = tmp | 3; - INVLPG( &PAGEDIR( (VAddr>>30)<<9 ) ); - memset( &PAGEDIR( (VAddr>>30)<<9 ), 0, 0x1000 ); - } + // And, set the page table entry + if(Pointer) *Pointer = &pmlevels[i][Addr >> ADDR_SIZES[i]]; + return ADDR_SIZES[i]; +} + +/** + * \brief Map a physical page to a virtual one + */ +int MM_Map(tVAddr VAddr, tPAddr PAddr) +{ + tPAddr *ent; + int rv; - // Check Page Dir - if( !(PAGEDIR(VAddr >> 21) & 1) ) - { - tmp = MM_AllocPhys(); - if(!tmp) return 0; - PAGEDIR(VAddr >> 21) = tmp | 3; - INVLPG( &PAGETABLE( (VAddr>>21)<<9 ) ); - memset( &PAGETABLE( (VAddr>>21)<<9 ), 0, 4096 ); - } + ENTER("xVAddr xPAddr", VAddr, PAddr); - // Check if this virtual address is already mapped - if( PAGETABLE(VAddr >> PTAB_SHIFT) & 1 ) - return 0; + // Get page pointer (Allow allocating) + rv = MM_GetPageEntryPtr(VAddr, 0, 1, 0, &ent); + if(rv < 0) LEAVE_RET('i', 0); - PAGETABLE(VAddr >> PTAB_SHIFT) = PAddr | 3; + if( *ent & 1 ) + LEAVE_RET('i', 0); - INVLPG( VAddr ); - Log("MM_Map: RETURN 1"); + *ent = PAddr | 3; + INVLPG( VAddr ); + + LEAVE('i', 1); return 1; } @@ -283,12 +354,18 @@ tPAddr MM_Allocate(tVAddr VAddr) ENTER("xVAddr", VAddr); + // NOTE: This is hack, but I like my dumps to be neat + #if 1 + MM_GetPageEntryPtr(VAddr, 0, 1, 0, NULL); + #elif 1 if( !MM_Map(VAddr, 0) ) // Make sure things are allocated { Warning("MM_Allocate: Unable to map, tables did not initialise"); LEAVE('i', 0); return 0; } + MM_Unmap(VAddr); + #endif ret = MM_AllocPhys(); LOG("ret = %x", ret); @@ -309,6 +386,9 @@ tPAddr MM_Allocate(tVAddr VAddr) return ret; } +/** + * \brief Deallocate a page at a virtual address + */ void MM_Deallocate(tVAddr VAddr) { tPAddr phys; @@ -321,26 +401,40 @@ void MM_Deallocate(tVAddr VAddr) MM_DerefPhys(phys); } +/** + * \brief Get the page table entry of a virtual address + * \param Addr Virtual Address + * \param Phys Location to put the physical address + * \param Flags Flags on the entry (set to zero if unmapped) + * \return Size of the entry (in address bits) - 12 = 4KiB page + */ +int MM_GetPageEntry(tVAddr Addr, tPAddr *Phys, Uint *Flags) +{ + tPAddr *ptr; + int ret; + + if(!Phys || !Flags) return 0; + + ret = MM_GetPageEntryPtr(Addr, 0, 0, 0, &ptr); + if( ret < 0 ) return 0; + + *Phys = *ptr & ~0xFFF; + *Flags = *ptr & 0xFFF; + return ret; +} + /** * \brief Get the physical address of a virtual location */ tPAddr MM_GetPhysAddr(tVAddr Addr) { - Log("MM_GetPhysAddr: (Addr=0x%x)", Addr); - if( !(PAGEMAPLVL4(Addr >> 39) & 1) ) - return 0; - Log(" MM_GetPhysAddr: PDP Valid"); - if( !(PAGEDIRPTR(Addr >> 30) & 1) ) - return 0; - Log(" MM_GetPhysAddr: PD Valid"); - if( !(PAGEDIR(Addr >> 21) & 1) ) - return 0; - Log(" MM_GetPhysAddr: PT Valid"); - if( !(PAGETABLE(Addr >> PTAB_SHIFT) & 1) ) - return 0; - Log(" MM_GetPhysAddr: Page Valid"); + tPAddr *ptr; + int ret; - return (PAGETABLE(Addr >> PTAB_SHIFT) & ~0xFFF) | (Addr & 0xFFF); + ret = MM_GetPageEntryPtr(Addr, 0, 0, 0, &ptr); + if( ret < 0 ) return 0; + + return (*ptr & ~0xFFF) | (Addr & 0xFFF); } /** @@ -349,19 +443,14 @@ tPAddr MM_GetPhysAddr(tVAddr Addr) void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) { tPAddr *ent; + int rv; - // Validity Check - if( !(PAGEMAPLVL4(VAddr >> 39) & 1) ) - return ; - if( !(PAGEDIRPTR(VAddr >> 30) & 1) ) - return ; - if( !(PAGEDIR(VAddr >> 21) & 1) ) - return ; - if( !(PAGETABLE(VAddr >> 12) & 1) ) - return ; + // Get pointer + rv = MM_GetPageEntryPtr(VAddr, 0, 0, 0, &ent); + if(rv < 0) return ; - // Set Flags - ent = &PAGETABLE(VAddr >> 12); + // Ensure the entry is valid + if( !(*ent & 1) ) return ; // Read-Only if( Mask & MM_PFLAG_RO ) @@ -416,20 +505,12 @@ void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask) Uint MM_GetFlags(tVAddr VAddr) { tPAddr *ent; - Uint ret = 0; + int rv, ret = 0; - // Validity Check - if( !(PAGEMAPLVL4(VAddr >> 39) & 1) ) - return 0; - if( !(PAGEDIRPTR(VAddr >> 30) & 1) ) - return 0; - if( !(PAGEDIR(VAddr >> 21) & 1) ) - return 0; - if( !(PAGETABLE(VAddr >> 12) & 1) ) - return 0; + rv = MM_GetPageEntryPtr(VAddr, 0, 0, 0, &ent); + if(rv < 0) return 0; - // Set Flags - ent = &PAGETABLE(VAddr >> 12); + if( !(*ent & 1) ) return 0; // Read-Only if( !(*ent & PF_WRITE) ) ret |= MM_PFLAG_RO; @@ -449,6 +530,28 @@ Uint MM_GetFlags(tVAddr VAddr) */ tVAddr MM_MapHWPages(tPAddr PAddr, Uint Number) { + tVAddr ret; + int num; + + //TODO: Add speedups (memory of first possible free) + for( ret = MM_HWMAP_BASE; ret < MM_HWMAP_TOP; ret += 0x1000 ) + { + for( num = Number; num -- && ret < MM_HWMAP_TOP; ret += 0x1000 ) + { + if( MM_GetPhysAddr(ret) != 0 ) break; + } + if( num >= 0 ) continue; + + while( Number -- ) + { + ret -= 0x1000; + MM_Map(ret, PAddr); + PAddr += 0x1000; + } + + return ret; + } + Log_KernelPanic("MM", "TODO: Implement MM_MapHWPages"); return 0; } @@ -458,7 +561,60 @@ tVAddr MM_MapHWPages(tPAddr PAddr, Uint Number) */ void MM_UnmapHWPages(tVAddr VAddr, Uint Number) { - Log_KernelPanic("MM", "TODO: Implement MM_UnmapHWPages"); +// Log_KernelPanic("MM", "TODO: Implement MM_UnmapHWPages"); + while( Number -- ) + { + MM_Unmap(VAddr); + VAddr += 0x1000; + } +} + + +/** + * \fn tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr) + * \brief Allocates DMA physical memory + * \param Pages Number of pages required + * \param MaxBits Maximum number of bits the physical address can have + * \param PhysAddr Pointer to the location to place the physical address allocated + * \return Virtual address allocate + */ +tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr) +{ + tPAddr phys; + tVAddr ret; + + // Sanity Check + if(MaxBits < 12 || !PhysAddr) return 0; + + // Fast Allocate + if(Pages == 1 && MaxBits >= PHYS_BITS) + { + phys = MM_AllocPhys(); + *PhysAddr = phys; + ret = MM_MapHWPages(phys, 1); + if(ret == 0) { + MM_DerefPhys(phys); + return 0; + } + return ret; + } + + // Slow Allocate + phys = MM_AllocPhysRange(Pages, MaxBits); + // - Was it allocated? + if(phys == 0) return 0; + + // Allocated successfully, now map + ret = MM_MapHWPages(phys, Pages); + if( ret == 0 ) { + // If it didn't map, free then return 0 + for(;Pages--;phys+=0x1000) + MM_DerefPhys(phys); + return 0; + } + + *PhysAddr = phys; + return ret; } // --- Tempory Mappings --- @@ -485,8 +641,19 @@ tPAddr MM_Clone(void) if(!ret) return 0; // #2 Alter the fractal pointer + Mutex_Acquire(&glMM_TempFractalLock); + TMPCR3() = ret | 3; + + INVLPG(TMPMAPLVL4(0)); + memcpy(&TMPMAPLVL4(0), &PAGEMAPLVL4(0), 0x1000); + + Log_KernelPanic("MM", "TODO: Implement MM_Clone"); + // #3 Set Copy-On-Write to all user pages // #4 Return + TMPCR3() = 0; + INVLPG(TMPMAPLVL4(0)); + Mutex_Release(&glMM_TempFractalLock); return 0; } @@ -545,10 +712,63 @@ void MM_ClearUser(void) tVAddr MM_NewWorkerStack(void) { + tVAddr ret; + int i; + + Log_KernelPanic("MM", "TODO: Implement MM_NewWorkerStack"); + + // #1 Set temp fractal to PID0 + Mutex_Acquire(&glMM_TempFractalLock); + TMPCR3() = ((tPAddr)gInitialPML4 - KERNEL_BASE) | 3; + + // #2 Scan for a free stack addresss < 2^47 + for(ret = 0x100000; ret < (1ULL << 47); ret += KERNEL_STACK_SIZE) + { + if( MM_GetPhysAddr(ret) == 0 ) break; + } + if( ret >= (1ULL << 47) ) { + Mutex_Release(&glMM_TempFractalLock); + return 0; + } + + // #3 Map all save the last page in the range + // - This acts as as guard page, and doesn't cost us anything. + for( i = 0; i < KERNEL_STACK_SIZE/0x1000 - 1; i ++ ) + { +// MM_MapTemp + } + + Mutex_Release(&glMM_TempFractalLock); + return 0; } +/** + * \brief Allocate a new kernel stack + */ tVAddr MM_NewKStack(void) { + tVAddr base = MM_KSTACK_BASE; + Uint i; + for( ; base < MM_KSTACK_TOP; base += KERNEL_STACK_SIZE ) + { + if(MM_GetPhysAddr(base) != 0) + continue; + + //Log("MM_NewKStack: Found one at %p", base + KERNEL_STACK_SIZE); + for( i = 0; i < KERNEL_STACK_SIZE; i += 0x1000) + { + if( !MM_Allocate(base+i) ) + { + Log_Warning("MM", "MM_NewKStack - Allocation failed"); + for( i -= 0x1000; i; i -= 0x1000) + MM_Deallocate(base+i); + return 0; + } + } + + return base + KERNEL_STACK_SIZE; + } + Log_Warning("MM", "MM_NewKStack - No address space left\n"); return 0; }