X-Git-Url: https://git.ucc.asn.au/?a=blobdiff_plain;ds=sidebyside;f=Kernel%2Farch%2Fx86_64%2Fmm_phys.c;h=c2c215b3b99603d96146a1044d4451cedfbc2919;hb=e31829ecc2b8ae2338745f4ed393748704a81531;hp=e2c2cf6ecabcc39f56a353e419de195b18518d2f;hpb=2af4884a872632edd582f632c65506af68fb0efa;p=tpg%2Facess2.git diff --git a/Kernel/arch/x86_64/mm_phys.c b/Kernel/arch/x86_64/mm_phys.c index e2c2cf6e..c2c215b3 100644 --- a/Kernel/arch/x86_64/mm_phys.c +++ b/Kernel/arch/x86_64/mm_phys.c @@ -3,10 +3,13 @@ * * Physical Memory Manager */ +#define DEBUG 0 #include #include #include +#define TRACE_REF 0 + enum eMMPhys_Ranges { MM_PHYS_16BIT, // Does anything need this? @@ -18,23 +21,32 @@ enum eMMPhys_Ranges }; // === IMPORTS === -extern void gKernelBase; -extern void gKernelEnd; +extern char gKernelBase[]; +extern char gKernelEnd[]; // === PROTOTYPES === void MM_InitPhys_Multiboot(tMBoot_Info *MBoot); -tPAddr MM_AllocPhysRange(int Num, int Bits); -tPAddr MM_AllocPhys(void); -void MM_RefPhys(tPAddr PAddr); -void MM_DerefPhys(tPAddr PAddr); +//tPAddr MM_AllocPhysRange(int Num, int Bits); +//tPAddr MM_AllocPhys(void); +//void MM_RefPhys(tPAddr PAddr); +//void MM_DerefPhys(tPAddr PAddr); int MM_int_GetRangeID( tPAddr Addr ); +// === MACROS === +#define PAGE_ALLOC_TEST(__page) (gaMainBitmap[(__page)>>6] & (1ULL << ((__page)&63))) +#define PAGE_ALLOC_SET(__page) do{gaMainBitmap[(__page)>>6] |= (1ULL << ((__page)&63));}while(0) +#define PAGE_ALLOC_CLEAR(__page) do{gaMainBitmap[(__page)>>6] &= ~(1ULL << ((__page)&63));}while(0) +//#define PAGE_MULTIREF_TEST(__page) (gaMultiBitmap[(__page)>>6] & (1ULL << ((__page)&63))) +//#define PAGE_MULTIREF_SET(__page) do{gaMultiBitmap[(__page)>>6] |= 1ULL << ((__page)&63);}while(0) +//#define PAGE_MULTIREF_CLEAR(__page) do{gaMultiBitmap[(__page)>>6] &= ~(1ULL << ((__page)&63));}while(0) + // === GLOBALS === -tSpinlock glPhysicalPages; -Uint64 *gaSuperBitmap; // 1 bit = 64 Pages, 16 MiB Per Word -Uint64 *gaMainBitmap; // 1 bit = 1 Page, 256 KiB per Word -Uint64 *gaMultiBitmap; // Each bit means that the page is being used multiple times +tMutex glPhysicalPages; +Uint64 *gaSuperBitmap = (void*)MM_PAGE_SUPBMP; // 1 bit = 64 Pages, 16 MiB per Word +Uint64 *gaMainBitmap = (void*)MM_PAGE_BITMAP; // 1 bit = 1 Page, 256 KiB per Word +Uint64 *gaMultiBitmap = (void*)MM_PAGE_DBLBMP; // Each bit means that the page is being used multiple times Uint32 *gaiPageReferences = (void*)MM_PAGE_COUNTS; // Reference Counts +void **gapPageNodes = (void*)MM_PAGE_NODES; // Reference Counts tPAddr giFirstFreePage; // First possibly free page Uint64 giPhysRangeFree[NUM_MM_PHYS_RANGES]; // Number of free pages in each range Uint64 giPhysRangeFirst[NUM_MM_PHYS_RANGES]; // First free page in each range @@ -61,18 +73,18 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) tVAddr vaddr; tPAddr paddr, firstFreePage; - Log("MM_InitPhys_Multiboot: (MBoot=%p)", MBoot); + ENTER("pMBoot=%p", MBoot); // Scan the physical memory map // Looking for the top of physical memory mmapStart = (void *)( KERNEL_BASE | MBoot->MMapAddr ); - Log(" MM_InitPhys_Multiboot: mmapStart = %p", mmapStart); + LOG("mmapStart = %p", mmapStart); ent = mmapStart; while( (Uint)ent < (Uint)mmapStart + MBoot->MMapLength ) { // Adjust for the size of the entry ent->Size += 4; - Log(" MM_InitPhys_Multiboot: ent={Type:%i,Base:0x%x,Length:%x", + LOG("ent={Type:%i,Base:0x%x,Length:%x", ent->Type, ent->Base, ent->Length); // If entry is RAM and is above `maxAddr`, change `maxAddr` @@ -92,7 +104,7 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) // Goodie, goodie gumdrops giMaxPhysPage = maxAddr >> 12; } - Log(" MM_InitPhys_Multiboot: giMaxPhysPage = 0x%x", giMaxPhysPage); + LOG("giMaxPhysPage = 0x%x", giMaxPhysPage); // Find a contigous section of memory to hold it in // - Starting from the end of the kernel @@ -100,8 +112,7 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) superPages = ((giMaxPhysPage+64*8-1)/(64*8) + 0xFFF) >> 12; numPages = (giMaxPhysPage + 7) / 8; numPages = (numPages + 0xFFF) >> 12; - Log(" MM_InitPhys_Multiboot: numPages = %i, superPages = %i", - numPages, superPages); + LOG("numPages = %i, superPages = %i", numPages, superPages); if(maxAddr == 0) { int todo = numPages*2 + superPages; @@ -153,7 +164,7 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) if( ent->Base + ent->Size < (tPAddr)&gKernelBase ) continue; - Log(" MM_InitPhys_Multiboot: %x <= %x && %x > %x", + LOG("%x <= %x && %x > %x", ent->Base, (tPAddr)&gKernelBase, ent->Base + ent->Size, (tPAddr)&gKernelEnd - KERNEL_BASE ); @@ -172,7 +183,7 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) paddr = ent->Base; } - Log(" MM_InitPhys_Multiboot: paddr=0x%x, avail=%i", paddr, avail); + Log("MM_InitPhys_Multiboot: paddr=0x%x, avail=0x%x pg", paddr, avail); // Map while( todo && avail --) @@ -206,14 +217,15 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) // Save the current value of paddr to simplify the allocation later firstFreePage = paddr; - Log(" MM_InitPhys_Multiboot: Clearing multi bitmap"); + LOG("Clearing multi bitmap"); // Fill the bitmaps - memset(gaMultiBitmap, 0, numPages<<12); + memset(gaMultiBitmap, 0, (numPages<<12)/8); // - initialise to one, then clear the avaliable areas - memset(gaMainBitmap, -1, numPages<<12); - Log(" MM_InitPhys_Multiboot: Setting main bitmap"); + memset(gaMainBitmap, -1, (numPages<<12)/8); + memset(gaSuperBitmap, -1, (numPages<<12)/(8*64)); + LOG("Setting main bitmap"); // - Clear all Type=1 areas - Log(" MM_InitPhys_Multiboot: Clearing valid regions"); + LOG("Clearing valid regions"); for( ent = mmapStart; (Uint)ent < (Uint)mmapStart + MBoot->MMapLength; @@ -228,14 +240,14 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) size = ent->Size >> 12; if(base & 63) { - Uint64 val = -1 << (base & 63); + Uint64 val = -1LL << (base & 63); gaMainBitmap[base / 64] &= ~val; size -= (base & 63); base += 64 - (base & 63); } memset( &gaMainBitmap[base / 64], 0, size/8 ); if( size & 7 ) { - Uint64 val = -1 << (size & 7); + Uint64 val = -1LL << (size & 7); val <<= (size/8)&7; gaMainBitmap[base / 64] &= ~val; } @@ -246,10 +258,10 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) size = (size + (base & 63) + 63) >> 6; base = base >> 6; if(base & 63) { - Uint64 val = -1 << (base & 63); + Uint64 val = -1LL << (base & 63); gaSuperBitmap[base / 64] &= ~val; - size -= (base & 63); - base += 64 - (base & 63); +// size -= (base & 63); +// base += 64 - (base & 63); } } @@ -258,7 +270,7 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) size = firstFreePage >> 12; memset( &gaMainBitmap[base / 64], -1, size/8 ); if( size & 7 ) { - Uint64 val = -1 << (size & 7); + Uint64 val = -1LL << (size & 7); val <<= (size/8)&7; gaMainBitmap[base / 64] |= val; } @@ -268,16 +280,16 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) if(gaiStaticAllocPages[i] != 0) continue; gaMainBitmap[ gaiStaticAllocPages[i] >> (12+6) ] - &= ~(1 << ((gaiStaticAllocPages[i]>>12)&63)); + &= ~(1LL << ((gaiStaticAllocPages[i]>>12)&63)); } // Fill the super bitmap - Log(" MM_InitPhys_Multiboot: Filling super bitmap"); + LOG("Filling super bitmap"); memset(gaSuperBitmap, 0, superPages<<12); for( base = 0; base < (size+63)/64; base ++) { - if( gaMainBitmap[ base ] == -1 ) - gaSuperBitmap[ base/64 ] |= 1 << (base&63); + if( gaMainBitmap[ base ] + 1 == 0 ) + gaSuperBitmap[ base/64 ] |= 1LL << (base&63); } // Set free page counts @@ -285,7 +297,7 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) { int rangeID; // Skip allocated - if( gaMainBitmap[ base >> 6 ] & (1 << (base&63)) ) continue; + if( gaMainBitmap[ base >> 6 ] & (1LL << (base&63)) ) continue; // Get range ID rangeID = MM_int_GetRangeID( base << 12 ); @@ -300,57 +312,59 @@ void MM_InitPhys_Multiboot(tMBoot_Info *MBoot) // updated anymore, hence will be correct) giPhysRangeLast[ rangeID ] = base; } + + LEAVE('-'); } /** * \brief Allocate a contiguous range of physical pages with a maximum - * bit size of \a Bits - * \param Num Number of pages to allocate - * \param Bits Maximum size of the physical address - * \note If \a Bits is <= 0, any sized address is used (with preference + * bit size of \a MaxBits + * \param Pages Number of pages to allocate + * \param MaxBits Maximum size of the physical address + * \note If \a MaxBits is <= 0, any sized address is used (with preference * to higher addresses) */ -tPAddr MM_AllocPhysRange(int Num, int Bits) +tPAddr MM_AllocPhysRange(int Pages, int MaxBits) { - tPAddr addr; + tPAddr addr, ret; int rangeID; int nFree = 0, i; - Log("MM_AllocPhysRange: (Num=%i,Bits=%i)", Num, Bits); + ENTER("iPages iBits", Pages, MaxBits); - if( Bits <= 0 || Bits >= 64 ) // Speedup for the common case + if( MaxBits <= 0 || MaxBits >= 64 ) // Speedup for the common case rangeID = MM_PHYS_MAX; else - rangeID = MM_int_GetRangeID( (1 << Bits) -1 ); + rangeID = MM_int_GetRangeID( (1LL << MaxBits) - 1 ); - Log(" MM_AllocPhysRange: rangeID = %i", rangeID); + LOG("rangeID = %i", rangeID); - LOCK(&glPhysicalPages); - Log(" MM_AllocPhysRange: i has lock"); + Mutex_Acquire(&glPhysicalPages); // Check if the range actually has any free pages while(giPhysRangeFree[rangeID] == 0 && rangeID) rangeID --; - Log(" MM_AllocPhysRange: rangeID = %i", rangeID); + LOG("rangeID = %i", rangeID); // What the? Oh, man. No free pages if(giPhysRangeFree[rangeID] == 0) { - RELEASE(&glPhysicalPages); + Mutex_Release(&glPhysicalPages); // TODO: Page out // ATM. Just Warning Warning(" MM_AllocPhysRange: Out of free pages"); Log_Warning("Arch", "Out of memory (unable to fulfil request for %i pages), zero remaining", - Num + Pages ); + LEAVE('i', 0); return 0; } // Check if there is enough in the range - if(giPhysRangeFree[rangeID] >= Num) + if(giPhysRangeFree[rangeID] >= Pages) { - Log(" MM_AllocPhysRange: {%i,0x%x -> 0x%x}", + LOG("{%i,0x%x -> 0x%x}", giPhysRangeFree[rangeID], giPhysRangeFirst[rangeID], giPhysRangeLast[rangeID] ); @@ -362,81 +376,93 @@ tPAddr MM_AllocPhysRange(int Num, int Bits) { //Log(" MM_AllocPhysRange: addr = 0x%x", addr); // Check the super bitmap - if( gaSuperBitmap[addr >> (6+6)] == -1 ) { - Log(" MM_AllocPhysRange: nFree = %i = 0 (super) (0x%x)", nFree, addr); + if( gaSuperBitmap[addr >> (6+6)] + 1 == 0 ) { + LOG("nFree = %i = 0 (super) (0x%x)", nFree, addr); nFree = 0; - addr += 1 << (6+6); - addr &= (1 << (6+6)) - 1; + addr += 1LL << (6+6); + addr &= ~0xFFF; // (1LL << 6+6) - 1 continue; } // Check page block (64 pages) - if( gaSuperBitmap[addr >> (6+6)] & (1 << (addr>>6)&63)) { - Log(" MM_AllocPhysRange: nFree = %i = 0 (main) (0x%x)", nFree, addr); + if( gaMainBitmap[addr >> 6] + 1 == 0) { + LOG("nFree = %i = 0 (main) (0x%x)", nFree, addr); nFree = 0; - addr += 1 << (12+6); - addr &= (1 << (12+6)) - 1; + addr += 1LL << (6); + addr &= ~0x3F; continue; } // Check individual page - if( gaMainBitmap[addr >> 6] & (1 << (addr & 63)) ) { - Log(" MM_AllocPhysRange: nFree = %i = 0 (page) (0x%x)", nFree, addr); + if( gaMainBitmap[addr >> 6] & (1LL << (addr & 63)) ) { + LOG("nFree = %i = 0 (page) (0x%x)", nFree, addr); nFree = 0; addr ++; continue; } nFree ++; addr ++; - Log(" MM_AllocPhysRange: nFree(%i) == %i (0x%x)", nFree, Num, addr); - if(nFree == Num) + LOG("nFree(%i) == %i (0x%x)", nFree, Pages, addr); + if(nFree == Pages) break; } - Log(" MM_AllocPhysRange: nFree = %i", nFree); + LOG("nFree = %i", nFree); // If we don't find a contiguous block, nFree will not be equal // to Num, so we set it to zero and do the expensive lookup. - if(nFree != Num) nFree = 0; + if(nFree != Pages) nFree = 0; } if( !nFree ) { // Oops. ok, let's do an expensive check (scan down the list // until a free range is found) - nFree = 1; - addr = giPhysRangeLast[ rangeID ]; - // TODO - RELEASE(&glPhysicalPages); +// nFree = 1; +// addr = giPhysRangeLast[ rangeID ]; + // TODO: Expensive Check + Mutex_Release(&glPhysicalPages); // TODO: Page out // ATM. Just Warning - Warning(" MM_AllocPhysRange: Out of memory (unable to fulfil request for %i pages)", Num); + Warning(" MM_AllocPhysRange: Out of memory (unable to fulfil request for %i pages)", Pages); Log_Warning("Arch", "Out of memory (unable to fulfil request for %i pages)", - Num + Pages ); + LEAVE('i', 0); return 0; } - Log(" MM_AllocPhysRange: nFree = %i, addr = 0x%08x", nFree, addr); + LOG("nFree = %i, addr = 0x%08x", nFree, addr); // Mark pages as allocated - addr -= Num; - for( i = 0; i < Num; i++ ) + addr -= Pages; + for( i = 0; i < Pages; i++, addr++ ) { - gaMainBitmap[addr >> 6] |= 1 << (addr & 63); - rangeID = MM_int_GetRangeID(addr); + gaMainBitmap[addr >> 6] |= 1LL << (addr & 63); + if( MM_GetPhysAddr( (tVAddr)&gaiPageReferences[addr] ) ) + gaiPageReferences[addr] = 1; +// Log("page %P refcount = %i", MM_GetRefCount(addr<<12)); + rangeID = MM_int_GetRangeID(addr << 12); giPhysRangeFree[ rangeID ] --; + LOG("%x == %x", addr, giPhysRangeFirst[ rangeID ]); + if(addr == giPhysRangeFirst[ rangeID ]) + giPhysRangeFirst[ rangeID ] += 1; } + addr -= Pages; + ret = addr; // Save the return address // Update super bitmap - Num += addr & (64-1); + Pages += addr & (64-1); addr &= ~(64-1); - Num = (Num + (64-1)) & ~(64-1); - for( i = 0; i < Num/64; i++ ) + Pages = (Pages + (64-1)) & ~(64-1); + for( i = 0; i < Pages/64; i++ ) { - if( gaMainBitmap[ addr >> 6 ] == -1 ) - gaSuperBitmap[addr>>12] |= 1 << ((addr >> 6) & 64); + if( gaMainBitmap[ addr >> 6 ] + 1 == 0 ) + gaSuperBitmap[addr>>12] |= 1LL << ((addr >> 6) & 63); } - RELEASE(&glPhysicalPages); - Log("MM_AllocPhysRange: RETURN %x", addr << 12); - return addr << 12; + Mutex_Release(&glPhysicalPages); + #if TRACE_REF + Log("MM_AllocPhysRange: ret = %P (Ref %i)", ret << 12, MM_GetRefCount(ret<<12)); + #endif + LEAVE('x', ret << 12); + return ret << 12; } /** @@ -452,7 +478,7 @@ tPAddr MM_AllocPhys(void) if( gaiStaticAllocPages[i] ) { tPAddr ret = gaiStaticAllocPages[i]; gaiStaticAllocPages[i] = 0; - Log("MM_AllocPhys: Return %x, static alloc %i", ret, i); + Log("MM_AllocPhys: Return %P, static alloc %i", ret, i); return ret; } } @@ -467,21 +493,43 @@ void MM_RefPhys(tPAddr PAddr) { Uint64 page = PAddr >> 12; - if( PAddr >> 12 > giMaxPhysPage ) return ; + if( page > giMaxPhysPage ) return ; - if( gaMainBitmap[ page >> 6 ] & (1 << (page&63)) ) + if( PAGE_ALLOC_TEST(page) ) { - // Reference again - gaMultiBitmap[ page >> 6 ] |= 1 << (page&63); - gaiPageReferences[ page ] ++; + tVAddr ref_base = ((tVAddr)&gaiPageReferences[ page ]) & ~0xFFF; + // Allocate reference page + if( !MM_GetPhysAddr(ref_base) ) + { + const int pages_per_refpage = PAGE_SIZE/sizeof(gaiPageReferences[0]); + int i; + int page_base = page / pages_per_refpage * pages_per_refpage; + if( !MM_Allocate( ref_base ) ) { + Log_Error("Arch", "Out of memory when allocating reference count page"); + return ; + } + // Fill block + Log("Allocated references for %P-%P", page_base << 12, (page_base+pages_per_refpage)<<12); + for( i = 0; i < pages_per_refpage; i ++ ) { + int pg = page_base + i; + gaiPageReferences[pg] = !!PAGE_ALLOC_TEST(pg); + } + } + gaiPageReferences[page] ++; } else { // Allocate - gaMainBitmap[page >> 6] |= 1 << (page&63); - if( gaMainBitmap[page >> 6 ] == -1 ) - gaSuperBitmap[page>> 12] |= 1 << ((page >> 6) & 63); + PAGE_ALLOC_SET(page); + if( gaMainBitmap[page >> 6] + 1 == 0 ) + gaSuperBitmap[page>> 12] |= 1LL << ((page >> 6) & 63); + if( MM_GetPhysAddr( (tVAddr)&gaiPageReferences[page] ) ) + gaiPageReferences[page] = 1; } + + #if TRACE_REF + Log("MM_RefPhys: %P referenced (%i)", page << 12, MM_GetRefCount(page << 12)); + #endif } /** @@ -493,27 +541,52 @@ void MM_DerefPhys(tPAddr PAddr) if( PAddr >> 12 > giMaxPhysPage ) return ; - if( gaMultiBitmap[ page >> 6 ] & (1 << (page&63)) ) { + if( MM_GetPhysAddr( (tVAddr) &gaiPageReferences[page] ) ) + { gaiPageReferences[ page ] --; - if( gaiPageReferences[ page ] == 1 ) - gaMultiBitmap[ page >> 6 ] &= ~(1 << (page&63)); if( gaiPageReferences[ page ] == 0 ) - gaMainBitmap[ page >> 6 ] &= ~(1 << (page&63)); + PAGE_ALLOC_CLEAR(page); } else - gaMainBitmap[ page >> 6 ] &= ~(1 << (page&63)); + PAGE_ALLOC_CLEAR(page); - // TODO: Update free counts - if( !(gaMainBitmap[ page >> 6 ] & (1 << (page&63))) ) + // Update the free counts if the page was freed + if( !PAGE_ALLOC_TEST(page) ) { int rangeID; rangeID = MM_int_GetRangeID( PAddr ); giPhysRangeFree[ rangeID ] ++; + if( giPhysRangeFirst[rangeID] > page ) + giPhysRangeFirst[rangeID] = page; + if( giPhysRangeLast[rangeID] < page ) + giPhysRangeLast[rangeID] = page; } - if(gaMainBitmap[ page >> 6 ] == 0) { - gaSuperBitmap[page >> 12] &= ~(1 << ((page >> 6) & 63)); + // If the bitmap entry is not -1, unset the bit in the super bitmap + if(gaMainBitmap[ page >> 6 ] + 1 != 0 ) { + gaSuperBitmap[page >> 12] &= ~(1LL << ((page >> 6) & 63)); } + + #if TRACE_REF + Log("Page %P dereferenced (%i)", page << 12, MM_GetRefCount(page << 12)); + #endif +} + +int MM_GetRefCount( tPAddr PAddr ) +{ + PAddr >>= 12; + + if( PAddr > giMaxPhysPage ) return 0; + + if( MM_GetPhysAddr( (tVAddr)&gaiPageReferences[PAddr] ) ) { + return gaiPageReferences[PAddr]; + } + + if( PAGE_ALLOC_TEST(PAddr) ) + { + return 1; + } + return 0; } /** @@ -534,3 +607,35 @@ int MM_int_GetRangeID( tPAddr Addr ) else return MM_PHYS_16BIT; } + +int MM_SetPageNode(tPAddr PAddr, void *Node) +{ + tPAddr page = PAddr >> 12; + tVAddr node_page = ((tVAddr)&gapPageNodes[page]) & ~(PAGE_SIZE-1); + +// if( !MM_GetRefCount(PAddr) ) return 1; + + if( !MM_GetPhysAddr(node_page) ) { + if( !MM_Allocate(node_page) ) + return -1; + memset( (void*)node_page, 0, PAGE_SIZE ); + } + + gapPageNodes[page] = Node; + return 0; +} + +int MM_GetPageNode(tPAddr PAddr, void **Node) +{ +// if( !MM_GetRefCount(PAddr) ) return 1; + PAddr >>= 12; + + if( !MM_GetPhysAddr( (tVAddr)&gapPageNodes[PAddr] ) ) { + *Node = NULL; + return 0; + } + + *Node = gapPageNodes[PAddr]; + return 0; +} +