4 * Physical Memory Manager
15 MM_PHYS_16BIT, // Does anything need this?
16 MM_PHYS_20BIT, // Real-Mode
17 MM_PHYS_24BIT, // ISA DMA
18 MM_PHYS_32BIT, // x86 Hardware
19 MM_PHYS_MAX, // Doesn't care
24 extern char gKernelBase[];
25 extern char gKernelEnd[];
28 void MM_InitPhys_Multiboot(tMBoot_Info *MBoot);
29 //tPAddr MM_AllocPhysRange(int Num, int Bits);
30 //tPAddr MM_AllocPhys(void);
31 //void MM_RefPhys(tPAddr PAddr);
32 //void MM_DerefPhys(tPAddr PAddr);
33 int MM_int_GetRangeID( tPAddr Addr );
36 #define PAGE_ALLOC_TEST(__page) (gaMainBitmap[(__page)>>6] & (1ULL << ((__page)&63)))
37 #define PAGE_ALLOC_SET(__page) do{gaMainBitmap[(__page)>>6] |= (1ULL << ((__page)&63));}while(0)
38 #define PAGE_ALLOC_CLEAR(__page) do{gaMainBitmap[(__page)>>6] &= ~(1ULL << ((__page)&63));}while(0)
39 //#define PAGE_MULTIREF_TEST(__page) (gaMultiBitmap[(__page)>>6] & (1ULL << ((__page)&63)))
40 //#define PAGE_MULTIREF_SET(__page) do{gaMultiBitmap[(__page)>>6] |= 1ULL << ((__page)&63);}while(0)
41 //#define PAGE_MULTIREF_CLEAR(__page) do{gaMultiBitmap[(__page)>>6] &= ~(1ULL << ((__page)&63));}while(0)
44 tMutex glPhysicalPages;
45 Uint64 *gaSuperBitmap = (void*)MM_PAGE_SUPBMP; // 1 bit = 64 Pages, 16 MiB per Word
46 Uint64 *gaMainBitmap = (void*)MM_PAGE_BITMAP; // 1 bit = 1 Page, 256 KiB per Word
47 Uint64 *gaMultiBitmap = (void*)MM_PAGE_DBLBMP; // Each bit means that the page is being used multiple times
48 Uint32 *gaiPageReferences = (void*)MM_PAGE_COUNTS; // Reference Counts
49 void **gapPageNodes = (void*)MM_PAGE_NODES; // Reference Counts
50 tPAddr giFirstFreePage; // First possibly free page
51 Uint64 giPhysRangeFree[NUM_MM_PHYS_RANGES]; // Number of free pages in each range
52 Uint64 giPhysRangeFirst[NUM_MM_PHYS_RANGES]; // First free page in each range
53 Uint64 giPhysRangeLast[NUM_MM_PHYS_RANGES]; // Last free page in each range
54 Uint64 giMaxPhysPage = 0; // Maximum Physical page
55 // Only used in init, allows the init code to provide pages for use by
56 // the allocator before the bitmaps exist.
57 // 3 entries because the are three calls to MM_AllocPhys in MM_Map
58 #define NUM_STATIC_ALLOC 3
59 tPAddr gaiStaticAllocPages[NUM_STATIC_ALLOC] = {0};
63 * \brief Initialise the physical memory map using a Multiboot 1 map
65 void MM_InitPhys_Multiboot(tMBoot_Info *MBoot)
67 tMBoot_MMapEnt *mmapStart;
70 int numPages, superPages;
74 tPAddr paddr, firstFreePage;
76 ENTER("pMBoot=%p", MBoot);
78 // Scan the physical memory map
79 // Looking for the top of physical memory
80 mmapStart = (void *)( KERNEL_BASE | MBoot->MMapAddr );
81 LOG("mmapStart = %p", mmapStart);
83 while( (Uint)ent < (Uint)mmapStart + MBoot->MMapLength )
85 // Adjust for the size of the entry
87 LOG("ent={Type:%i,Base:0x%x,Length:%x",
88 ent->Type, ent->Base, ent->Length);
90 // If entry is RAM and is above `maxAddr`, change `maxAddr`
91 if(ent->Type == 1 && ent->Base + ent->Length > maxAddr)
92 maxAddr = ent->Base + ent->Length;
95 ent = (tMBoot_MMapEnt *)( (Uint)ent + ent->Size );
98 // Did we find a valid end?
100 // No, darn, let's just use the HighMem hack
101 giMaxPhysPage = (MBoot->HighMem >> 2) + 256; // HighMem is a kByte value
104 // Goodie, goodie gumdrops
105 giMaxPhysPage = maxAddr >> 12;
107 LOG("giMaxPhysPage = 0x%x", giMaxPhysPage);
109 // Find a contigous section of memory to hold it in
110 // - Starting from the end of the kernel
111 // - We also need a region for the super bitmap
112 superPages = ((giMaxPhysPage+64*8-1)/(64*8) + 0xFFF) >> 12;
113 numPages = (giMaxPhysPage + 7) / 8;
114 numPages = (numPages + 0xFFF) >> 12;
115 LOG("numPages = %i, superPages = %i", numPages, superPages);
118 int todo = numPages*2 + superPages;
119 // Ok, naieve allocation, just put it after the kernel
120 // - Allocated Bitmap
121 vaddr = MM_PAGE_BITMAP;
122 paddr = (tPAddr)&gKernelEnd - KERNEL_BASE;
126 for( i = 0; i < NUM_STATIC_ALLOC; i++) {
127 if(gaiStaticAllocPages[i] != 0) continue;
128 gaiStaticAllocPages[i] = paddr;
132 MM_Map(vaddr, paddr);
138 if( todo == numPages + superPages )
139 vaddr = MM_PAGE_DBLBMP;
140 if( todo == superPages )
141 vaddr = MM_PAGE_SUPBMP;
144 // Scan for a nice range
147 int todo = numPages*2 + superPages;
149 vaddr = MM_PAGE_BITMAP;
153 (Uint)ent < (Uint)mmapStart + MBoot->MMapLength;
154 ent = (tMBoot_MMapEnt *)( (Uint)ent + ent->Size )
163 // Let's not put it below the kernel, shall we?
164 if( ent->Base + ent->Size < (tPAddr)&gKernelBase )
167 LOG("%x <= %x && %x > %x",
168 ent->Base, (tPAddr)&gKernelBase,
169 ent->Base + ent->Size, (tPAddr)&gKernelEnd - KERNEL_BASE
171 // Check if the kernel is in this range
172 if( ent->Base <= (tPAddr)&gKernelBase
173 && ent->Base + ent->Length > (tPAddr)&gKernelEnd - KERNEL_BASE )
175 avail = ent->Length >> 12;
176 avail -= ((tPAddr)&gKernelEnd - KERNEL_BASE - ent->Base) >> 12;
177 paddr = (tPAddr)&gKernelEnd - KERNEL_BASE;
179 // No? then we can use all of the block
182 avail = ent->Length >> 12;
186 Log("MM_InitPhys_Multiboot: paddr=0x%x, avail=0x%x pg", paddr, avail);
189 while( todo && avail --)
191 // Static Allocations
192 for( i = 0; i < NUM_STATIC_ALLOC && avail; i++) {
193 if(gaiStaticAllocPages[i] != 0) continue;
194 gaiStaticAllocPages[i] = paddr;
201 MM_Map(vaddr, paddr);
206 // Alter the destination address when needed
207 if(todo == superPages+numPages)
208 vaddr = MM_PAGE_DBLBMP;
209 if(todo == superPages)
210 vaddr = MM_PAGE_SUPBMP;
213 // Fast quit if there's nothing left to allocate
217 // Save the current value of paddr to simplify the allocation later
218 firstFreePage = paddr;
220 LOG("Clearing multi bitmap");
222 memset(gaMultiBitmap, 0, (numPages<<12)/8);
223 // - initialise to one, then clear the avaliable areas
224 memset(gaMainBitmap, -1, (numPages<<12)/8);
225 memset(gaSuperBitmap, -1, (numPages<<12)/(8*64));
226 LOG("Setting main bitmap");
227 // - Clear all Type=1 areas
228 LOG("Clearing valid regions");
231 (Uint)ent < (Uint)mmapStart + MBoot->MMapLength;
232 ent = (tMBoot_MMapEnt *)( (Uint)ent + ent->Size )
235 // Check if the type is RAM
236 if(ent->Type != 1) continue;
239 base = ent->Base >> 12;
240 size = ent->Size >> 12;
243 Uint64 val = -1LL << (base & 63);
244 gaMainBitmap[base / 64] &= ~val;
246 base += 64 - (base & 63);
248 memset( &gaMainBitmap[base / 64], 0, size/8 );
250 Uint64 val = -1LL << (size & 7);
252 gaMainBitmap[base / 64] &= ~val;
256 base = ent->Base >> 12;
257 size = ent->Size >> 12;
258 size = (size + (base & 63) + 63) >> 6;
261 Uint64 val = -1LL << (base & 63);
262 gaSuperBitmap[base / 64] &= ~val;
263 // size -= (base & 63);
264 // base += 64 - (base & 63);
268 // Reference the used pages
269 base = (tPAddr)&gKernelBase >> 12;
270 size = firstFreePage >> 12;
271 memset( &gaMainBitmap[base / 64], -1, size/8 );
273 Uint64 val = -1LL << (size & 7);
275 gaMainBitmap[base / 64] |= val;
278 // Free the unused static allocs
279 for( i = 0; i < NUM_STATIC_ALLOC; i++) {
280 if(gaiStaticAllocPages[i] != 0)
282 gaMainBitmap[ gaiStaticAllocPages[i] >> (12+6) ]
283 &= ~(1LL << ((gaiStaticAllocPages[i]>>12)&63));
286 // Fill the super bitmap
287 LOG("Filling super bitmap");
288 memset(gaSuperBitmap, 0, superPages<<12);
289 for( base = 0; base < (size+63)/64; base ++)
291 if( gaMainBitmap[ base ] + 1 == 0 )
292 gaSuperBitmap[ base/64 ] |= 1LL << (base&63);
295 // Set free page counts
296 for( base = 1; base < giMaxPhysPage; base ++ )
300 if( gaMainBitmap[ base >> 6 ] & (1LL << (base&63)) ) continue;
303 rangeID = MM_int_GetRangeID( base << 12 );
305 // Increment free page count
306 giPhysRangeFree[ rangeID ] ++;
308 // Check for first free page in range
309 if(giPhysRangeFirst[ rangeID ] == 0)
310 giPhysRangeFirst[ rangeID ] = base;
311 // Set last (when the last free page is reached, this won't be
312 // updated anymore, hence will be correct)
313 giPhysRangeLast[ rangeID ] = base;
319 void MM_DumpStatistics(void)
321 // TODO: Statistics for x86_64 PMM
325 * \brief Allocate a contiguous range of physical pages with a maximum
326 * bit size of \a MaxBits
327 * \param Pages Number of pages to allocate
328 * \param MaxBits Maximum size of the physical address
329 * \note If \a MaxBits is <= 0, any sized address is used (with preference
330 * to higher addresses)
332 tPAddr MM_AllocPhysRange(int Pages, int MaxBits)
338 ENTER("iPages iBits", Pages, MaxBits);
340 if( MaxBits <= 0 || MaxBits >= 64 ) // Speedup for the common case
341 rangeID = MM_PHYS_MAX;
343 rangeID = MM_int_GetRangeID( (1LL << MaxBits) - 1 );
345 LOG("rangeID = %i", rangeID);
347 Mutex_Acquire(&glPhysicalPages);
349 // Check if the range actually has any free pages
350 while(giPhysRangeFree[rangeID] == 0 && rangeID)
353 LOG("rangeID = %i", rangeID);
355 // What the? Oh, man. No free pages
356 if(giPhysRangeFree[rangeID] == 0) {
357 Mutex_Release(&glPhysicalPages);
360 Warning(" MM_AllocPhysRange: Out of free pages");
362 "Out of memory (unable to fulfil request for %i pages), zero remaining",
369 // Check if there is enough in the range
370 if(giPhysRangeFree[rangeID] >= Pages)
372 LOG("{%i,0x%x -> 0x%x}",
373 giPhysRangeFree[rangeID],
374 giPhysRangeFirst[rangeID], giPhysRangeLast[rangeID]
376 // Do a cheap scan, scanning upwards from the first free page in
379 addr = giPhysRangeFirst[ rangeID ];
380 while( addr <= giPhysRangeLast[ rangeID ] )
382 //Log(" MM_AllocPhysRange: addr = 0x%x", addr);
383 // Check the super bitmap
384 if( gaSuperBitmap[addr >> (6+6)] + 1 == 0 ) {
385 LOG("nFree = %i = 0 (super) (0x%x)", nFree, addr);
387 addr += 1LL << (6+6);
388 addr &= ~0xFFF; // (1LL << 6+6) - 1
391 // Check page block (64 pages)
392 if( gaMainBitmap[addr >> 6] + 1 == 0) {
393 LOG("nFree = %i = 0 (main) (0x%x)", nFree, addr);
399 // Check individual page
400 if( gaMainBitmap[addr >> 6] & (1LL << (addr & 63)) ) {
401 LOG("nFree = %i = 0 (page) (0x%x)", nFree, addr);
408 LOG("nFree(%i) == %i (0x%x)", nFree, Pages, addr);
412 LOG("nFree = %i", nFree);
413 // If we don't find a contiguous block, nFree will not be equal
414 // to Num, so we set it to zero and do the expensive lookup.
415 if(nFree != Pages) nFree = 0;
420 // Oops. ok, let's do an expensive check (scan down the list
421 // until a free range is found)
423 // addr = giPhysRangeLast[ rangeID ];
424 // TODO: Expensive Check
425 Mutex_Release(&glPhysicalPages);
428 Warning(" MM_AllocPhysRange: Out of memory (unable to fulfil request for %i pages)", Pages);
430 "Out of memory (unable to fulfil request for %i pages)",
436 LOG("nFree = %i, addr = 0x%08x", nFree, addr);
438 // Mark pages as allocated
440 for( i = 0; i < Pages; i++, addr++ )
442 gaMainBitmap[addr >> 6] |= 1LL << (addr & 63);
443 if( MM_GetPhysAddr( (tVAddr)&gaiPageReferences[addr] ) )
444 gaiPageReferences[addr] = 1;
445 // Log("page %P refcount = %i", MM_GetRefCount(addr<<12));
446 rangeID = MM_int_GetRangeID(addr << 12);
447 giPhysRangeFree[ rangeID ] --;
448 LOG("%x == %x", addr, giPhysRangeFirst[ rangeID ]);
449 if(addr == giPhysRangeFirst[ rangeID ])
450 giPhysRangeFirst[ rangeID ] += 1;
453 ret = addr; // Save the return address
455 // Update super bitmap
456 Pages += addr & (64-1);
458 Pages = (Pages + (64-1)) & ~(64-1);
459 for( i = 0; i < Pages/64; i++ )
461 if( gaMainBitmap[ addr >> 6 ] + 1 == 0 )
462 gaSuperBitmap[addr>>12] |= 1LL << ((addr >> 6) & 63);
465 Mutex_Release(&glPhysicalPages);
467 Log("MM_AllocPhysRange: ret = %P (Ref %i)", ret << 12, MM_GetRefCount(ret<<12));
469 LEAVE('x', ret << 12);
474 * \brief Allocate a single physical page, with no preference as to address
477 tPAddr MM_AllocPhys(void)
481 // Hack to allow allocation during setup
482 for(i = 0; i < NUM_STATIC_ALLOC; i++) {
483 if( gaiStaticAllocPages[i] ) {
484 tPAddr ret = gaiStaticAllocPages[i];
485 gaiStaticAllocPages[i] = 0;
486 Log("MM_AllocPhys: Return %P, static alloc %i", ret, i);
491 return MM_AllocPhysRange(1, -1);
495 * \brief Reference a physical page
497 void MM_RefPhys(tPAddr PAddr)
499 Uint64 page = PAddr >> 12;
501 if( page > giMaxPhysPage ) return ;
503 if( PAGE_ALLOC_TEST(page) )
505 tVAddr ref_base = ((tVAddr)&gaiPageReferences[ page ]) & ~0xFFF;
506 // Allocate reference page
507 if( !MM_GetPhysAddr(ref_base) )
509 const int pages_per_refpage = PAGE_SIZE/sizeof(gaiPageReferences[0]);
511 int page_base = page / pages_per_refpage * pages_per_refpage;
512 if( !MM_Allocate( ref_base ) ) {
513 Log_Error("Arch", "Out of memory when allocating reference count page");
517 Log("Allocated references for %P-%P", page_base << 12, (page_base+pages_per_refpage)<<12);
518 for( i = 0; i < pages_per_refpage; i ++ ) {
519 int pg = page_base + i;
520 gaiPageReferences[pg] = !!PAGE_ALLOC_TEST(pg);
523 gaiPageReferences[page] ++;
528 PAGE_ALLOC_SET(page);
529 if( gaMainBitmap[page >> 6] + 1 == 0 )
530 gaSuperBitmap[page>> 12] |= 1LL << ((page >> 6) & 63);
531 if( MM_GetPhysAddr( (tVAddr)&gaiPageReferences[page] ) )
532 gaiPageReferences[page] = 1;
536 Log("MM_RefPhys: %P referenced (%i)", page << 12, MM_GetRefCount(page << 12));
541 * \brief Dereference a physical page
543 void MM_DerefPhys(tPAddr PAddr)
545 Uint64 page = PAddr >> 12;
547 if( PAddr >> 12 > giMaxPhysPage ) return ;
549 if( MM_GetPhysAddr( (tVAddr) &gaiPageReferences[page] ) )
551 gaiPageReferences[ page ] --;
552 if( gaiPageReferences[ page ] == 0 )
553 PAGE_ALLOC_CLEAR(page);
556 PAGE_ALLOC_CLEAR(page);
558 // Update the free counts if the page was freed
559 if( !PAGE_ALLOC_TEST(page) )
562 rangeID = MM_int_GetRangeID( PAddr );
563 giPhysRangeFree[ rangeID ] ++;
564 if( giPhysRangeFirst[rangeID] > page )
565 giPhysRangeFirst[rangeID] = page;
566 if( giPhysRangeLast[rangeID] < page )
567 giPhysRangeLast[rangeID] = page;
570 // If the bitmap entry is not -1, unset the bit in the super bitmap
571 if(gaMainBitmap[ page >> 6 ] + 1 != 0 ) {
572 gaSuperBitmap[page >> 12] &= ~(1LL << ((page >> 6) & 63));
576 Log("Page %P dereferenced (%i)", page << 12, MM_GetRefCount(page << 12));
580 int MM_GetRefCount( tPAddr PAddr )
584 if( PAddr > giMaxPhysPage ) return 0;
586 if( MM_GetPhysAddr( (tVAddr)&gaiPageReferences[PAddr] ) ) {
587 return gaiPageReferences[PAddr];
590 if( PAGE_ALLOC_TEST(PAddr) )
598 * \brief Takes a physical address and returns the ID of its range
599 * \param Addr Physical address of page
600 * \return Range ID from eMMPhys_Ranges
602 int MM_int_GetRangeID( tPAddr Addr )
607 return MM_PHYS_32BIT;
609 return MM_PHYS_24BIT;
611 return MM_PHYS_20BIT;
613 return MM_PHYS_16BIT;
616 int MM_SetPageNode(tPAddr PAddr, void *Node)
618 tPAddr page = PAddr >> 12;
619 tVAddr node_page = ((tVAddr)&gapPageNodes[page]) & ~(PAGE_SIZE-1);
621 // if( !MM_GetRefCount(PAddr) ) return 1;
623 if( !MM_GetPhysAddr(node_page) ) {
624 if( !MM_Allocate(node_page) )
626 memset( (void*)node_page, 0, PAGE_SIZE );
629 gapPageNodes[page] = Node;
633 int MM_GetPageNode(tPAddr PAddr, void **Node)
635 // if( !MM_GetRefCount(PAddr) ) return 1;
638 if( !MM_GetPhysAddr( (tVAddr)&gapPageNodes[PAddr] ) ) {
643 *Node = gapPageNodes[PAddr];