3 * - Physical memory manager
11 #define TRACE_ALLOCS 0 // Print trace messages on AllocPhys/DerefPhys
13 static const int addrClasses[] = {0,16,20,24,32,64};
14 static const int numAddrClasses = sizeof(addrClasses)/sizeof(addrClasses[0]);
17 extern char gKernelEnd[];
18 extern void Proc_PrintBacktrace(void);
21 void MM_Install(tMBoot_Info *MBoot);
22 //tPAddr MM_AllocPhys(void);
23 //tPAddr MM_AllocPhysRange(int Pages, int MaxBits);
24 //void MM_RefPhys(tPAddr PAddr);
25 //void MM_DerefPhys(tPAddr PAddr);
26 // int MM_GetRefCount(tPAddr PAddr);
30 Uint64 giPhysAlloc = 0; // Number of allocated pages
31 Uint64 giPageCount = 0; // Total number of pages
32 Uint64 giLastPossibleFree = 0; // Last possible free page (before all pages are used)
33 Uint64 giTotalMemorySize = 0; // Total number of allocatable pages
35 Uint32 gaSuperBitmap[1024]; // Blocks of 1024 Pages
36 Uint32 gaPageBitmap[1024*1024/32]; // Individual pages
37 int *gaPageReferences;
38 void **gaPageNodes = (void*)MM_PAGENODE_BASE;
39 #define REFENT_PER_PAGE (0x1000/sizeof(gaPageReferences[0]))
42 void MM_Install(tMBoot_Info *MBoot)
44 Uint kernelPages, num;
50 // --- Find largest address
51 MBoot->MMapAddr |= KERNEL_BASE;
52 ent = (void *)( MBoot->MMapAddr );
53 while( (Uint)ent < MBoot->MMapAddr + MBoot->MMapLength )
58 // If entry is RAM and is above `maxAddr`, change `maxAddr`
61 if(ent->Base + ent->Length > maxAddr)
62 maxAddr = ent->Base + ent->Length;
63 giTotalMemorySize += ent->Length >> 12;
66 ent = (tMBoot_MMapEnt *)( (Uint)ent + ent->Size );
70 giPageCount = (MBoot->HighMem >> 2) + 256; // HighMem is a kByte value
73 giPageCount = maxAddr >> 12;
75 giLastPossibleFree = giPageCount - 1;
77 memsetd(gaPageBitmap, 0xFFFFFFFF, giPageCount/32);
79 // Set up allocateable space
80 ent = (void *)( MBoot->MMapAddr );
81 while( (Uint)ent < MBoot->MMapAddr + MBoot->MMapLength )
83 memsetd( &gaPageBitmap[ent->Base/(4096*32)], 0, ent->Length/(4096*32) );
84 ent = (tMBoot_MMapEnt *)( (Uint)ent + ent->Size );
87 // Get used page count (Kernel)
88 kernelPages = (Uint)&gKernelEnd - KERNEL_BASE - 0x100000;
89 kernelPages += 0xFFF; // Page Align
91 giPhysAlloc += kernelPages; // Add to used count
95 memsetd( &gaPageBitmap[0x100000/(4096*32)], -1, num );
96 gaPageBitmap[ 0x100000/(4096*32) + num ] = (1 << (kernelPages & 31)) - 1;
98 // Fill Superpage bitmap
99 num = kernelPages/(32*32);
100 memsetd( &gaSuperBitmap[0x100000/(4096*32*32)], -1, num );
101 gaSuperBitmap[ 0x100000/(4096*32*32) + num ] = (1 << ((kernelPages / 32) & 31)) - 1;
103 // Mark Multiboot's pages as taken
105 MM_RefPhys( (Uint)MBoot - KERNEL_BASE );
107 for(i = (MBoot->ModuleCount*sizeof(tMBoot_Module)+0xFFF)>12; i--; )
108 MM_RefPhys( MBoot->Modules + (i << 12) );
110 mods = (void*)(MBoot->Modules + KERNEL_BASE);
111 for(i = 0; i < MBoot->ModuleCount; i++)
113 num = (mods[i].End - mods[i].Start + 0xFFF) >> 12;
115 MM_RefPhys( (mods[i].Start & ~0xFFF) + (num<<12) );
118 gaPageReferences = (void*)MM_REFCOUNT_BASE;
120 Log_Log("PMem", "Physical memory set up (%lli pages of ~%lli MiB used)",
121 giPhysAlloc, (giTotalMemorySize*4)/1024
125 void MM_DumpStatistics(void)
128 for( i = 1; i < numAddrClasses; i ++ )
130 int first = (i == 1 ? 0 : (1UL << (addrClasses[i-1] - 12)));
131 int last = (1UL << (addrClasses[i] - 12)) - 1;
136 if( last > giPageCount )
139 int total = last - first + 1;
141 for( pg = first; pg < last; pg ++ )
143 if( !MM_GetPhysAddr(&gaPageReferences[pg]) || gaPageReferences[pg] == 0 ) {
147 totalRefs += gaPageReferences[pg];
148 if(gaPageReferences[pg] > 1)
152 int nUsed = (total - nFree);
153 Log_Log("MMPhys", "%ipbit - %i/%i used, %i reused, %i average reference count",
154 addrClasses[i], nUsed, total, nMultiRef,
155 nMultiRef ? (totalRefs-(nUsed - nMultiRef)) / nMultiRef : 0
158 if( last == giPageCount )
161 Log_Log("MMPhys", "%lli/%lli total pages used, 0 - %i possible free range",
162 giPhysAlloc, giTotalMemorySize, giLastPossibleFree);
166 * \fn tPAddr MM_AllocPhys(void)
167 * \brief Allocates a physical page from the general pool
169 tPAddr MM_AllocPhys(void)
177 Mutex_Acquire( &glPhysAlloc );
184 for( i = numAddrClasses; i -- > 1; )
186 first = 1UL << (addrClasses[i-1] - 12);
187 last = (1UL << (addrClasses[i] - 12)) - 1;
188 // Range is above the last free page
189 if( first > giLastPossibleFree )
191 // Last possible free page is in the range
192 if( last > giLastPossibleFree )
193 last = giLastPossibleFree;
196 for( indx = first; indx < last; )
198 if( gaSuperBitmap[indx>>10] == -1 ) {
203 if( gaPageBitmap[indx>>5] == -1 ) {
208 if( gaPageBitmap[indx>>5] & (1 << (indx&31)) ) {
214 if( indx < last ) break;
216 giLastPossibleFree = first; // Well, we couldn't find any in this range
219 if( i <= 1 ) indx = -1;
224 LOG("giLastPossibleFree = %i", giLastPossibleFree);
225 for( indx = giLastPossibleFree; indx >= 0; )
227 if( gaSuperBitmap[indx>>10] == -1 ) {
232 if( gaPageBitmap[indx>>5] == -1 ) {
237 if( gaPageBitmap[indx>>5] & (1 << (indx&31)) ) {
244 giLastPossibleFree = indx;
245 LOG("indx = %i", indx);
247 c = giLastPossibleFree % 32;
248 b = (giLastPossibleFree / 32) % 32;
249 a = giLastPossibleFree / 1024;
251 LOG("a=%i,b=%i,c=%i", a, b, c);
252 for( ; gaSuperBitmap[a] == -1 && a >= 0; a-- );
254 Mutex_Release( &glPhysAlloc );
255 Warning("MM_AllocPhys - OUT OF MEMORY (Called by %p) - %lli/%lli used",
256 __builtin_return_address(0), giPhysAlloc, giPageCount);
260 for( ; gaSuperBitmap[a] & (1<<b); b-- );
261 for( ; gaPageBitmap[a*32+b] & (1<<c); c-- );
262 LOG("a=%i,b=%i,c=%i", a, b, c);
263 indx = (a << 10) | (b << 5) | c;
265 giLastPossibleFree = indx;
269 Mutex_Release( &glPhysAlloc );
270 Warning("MM_AllocPhys - OUT OF MEMORY (Called by %p) - %lli/%lli used (indx = %x)",
271 __builtin_return_address(0), giPhysAlloc, giPageCount, indx);
272 Log_Debug("PMem", "giLastPossibleFree = %lli", giLastPossibleFree);
277 if( indx > 0xFFFFF ) {
278 Panic("The fuck? Too many pages! (indx = 0x%x)", indx);
281 if( indx >= giPageCount ) {
282 Mutex_Release( &glPhysAlloc );
283 Log_Error("PMem", "MM_AllocPhys - indx(%i) > giPageCount(%i)", indx, giPageCount);
289 if( MM_GetPhysAddr( (tVAddr)&gaPageReferences[indx] ) )
290 gaPageReferences[indx] = 1;
291 gaPageBitmap[ indx>>5 ] |= 1 << (indx&31);
299 if(gaPageBitmap[ indx>>5 ] == -1) {
300 gaSuperBitmap[indx>>10] |= 1 << ((indx>>5)&31);
304 Mutex_Release( &glPhysAlloc );
309 Log_Debug("PMem", "MM_AllocPhys: RETURN %P (%i free)", ret, giPageCount-giPhysAlloc);
310 Proc_PrintBacktrace();
317 * \fn tPAddr MM_AllocPhysRange(int Pages, int MaxBits)
318 * \brief Allocate a range of physical pages
319 * \param Pages Number of pages to allocate
320 * \param MaxBits Maximum number of address bits to use
322 tPAddr MM_AllocPhysRange(int Pages, int MaxBits)
328 ENTER("iPages iMaxBits", Pages, MaxBits);
335 if(MaxBits > PHYS_BITS) MaxBits = PHYS_BITS;
338 Mutex_Acquire( &glPhysAlloc );
340 // Set up search state
341 if( giLastPossibleFree > ((tPAddr)1 << (MaxBits-12)) ) {
342 sidx = (tPAddr)1 << (MaxBits-12);
345 sidx = giLastPossibleFree;
353 LOG("a=%i, b=%i, idx=%i, sidx=%i", a, b, idx, sidx);
356 for( ; gaSuperBitmap[a] == -1 && a --; ) b = 31;
358 Mutex_Release( &glPhysAlloc );
359 Warning("MM_AllocPhysRange - OUT OF MEMORY (Called by %p)", __builtin_return_address(0));
364 for( ; gaSuperBitmap[a] & (1 << b); b-- ) sidx = 31;
367 for( ; gaPageBitmap[idx] & (1 << sidx); sidx-- )
368 LOG("gaPageBitmap[%i] = 0x%08x", idx, gaPageBitmap[idx]);
370 LOG("idx = %i, sidx = %i", idx, sidx);
375 // Check if the gap is large enough
382 if( gaPageBitmap[idx] == -1 ) {
388 if( gaPageBitmap[idx] & (1 << sidx) ) {
390 if(sidx < 0) { sidx = 31; idx --; }
398 // Check if it is a free range
399 for( i = 0; i < Pages; i++ )
402 if( gaPageBitmap[idx] & (1 << sidx) )
406 if(sidx < 0) { sidx = 31; idx --; }
414 // Check if an address was found
416 Mutex_Release( &glPhysAlloc );
417 Warning("MM_AllocPhysRange - OUT OF MEMORY (Called by %p)", __builtin_return_address(0));
423 for( i = 0; i < Pages; i++ )
425 if( MM_GetPhysAddr( (tVAddr)&gaPageReferences[idx*32+sidx] ) )
426 gaPageReferences[idx*32+sidx] = 1;
427 gaPageBitmap[ idx ] |= 1 << sidx;
430 if(sidx == 32) { sidx = 0; idx ++; }
434 ret = (idx << 17) | (sidx << 12);
437 if(gaPageBitmap[ idx ] == -1) gaSuperBitmap[idx/32] |= 1 << (idx%32);
440 Mutex_Release( &glPhysAlloc );
444 Log_Debug("PMem", "MM_AllocPhysRange: RETURN 0x%llx-0x%llx (%i free)",
445 ret, ret + (1<<Pages)-1, giPageCount-giPhysAlloc);
451 * \fn void MM_RefPhys(tPAddr PAddr)
453 void MM_RefPhys(tPAddr PAddr)
458 // We don't care about non-ram pages
459 if(PAddr >= giPageCount) return;
462 Mutex_Acquire( &glPhysAlloc );
464 // Reference the page
465 if( gaPageReferences )
467 if( MM_GetPhysAddr( (tVAddr)&gaPageReferences[PAddr] ) == 0 )
470 tVAddr addr = ((tVAddr)&gaPageReferences[PAddr]) & ~0xFFF;
471 // Log_Debug("PMem", "MM_RefPhys: Allocating info for %X", PAddr);
472 Mutex_Release( &glPhysAlloc );
473 if( MM_Allocate( addr ) == 0 ) {
474 Log_KernelPanic("PMem",
475 "MM_RefPhys: Out of physical memory allocating info for %X",
479 Mutex_Acquire( &glPhysAlloc );
481 base = PAddr & ~(1024-1);
482 for( i = 0; i < 1024; i ++ ) {
483 gaPageReferences[base + i] = (gaPageBitmap[(base+i)/32] & (1 << (base+i)%32)) ? 1 : 0;
486 gaPageReferences[ PAddr ] ++;
489 // If not already used
490 if( !(gaPageBitmap[ PAddr / 32 ] & 1 << (PAddr&31)) ) {
493 gaPageBitmap[ PAddr / 32 ] |= 1 << (PAddr&31);
497 if(gaPageBitmap[ PAddr / 32 ] == -1)
498 gaSuperBitmap[PAddr/1024] |= 1 << ((PAddr/32)&31);
501 Mutex_Release( &glPhysAlloc );
505 * \fn void MM_DerefPhys(tPAddr PAddr)
506 * \brief Dereferences a physical page
508 void MM_DerefPhys(tPAddr PAddr)
513 // We don't care about non-ram pages
514 if(PAddr >= giPageCount) return;
516 // Check if it is freed
517 if( !(gaPageBitmap[PAddr / 32] & (1 << PAddr%32)) ) {
518 Log_Warning("MMVirt", "MM_DerefPhys - Non-referenced memory dereferenced");
523 Mutex_Acquire( &glPhysAlloc );
525 if( giLastPossibleFree < PAddr )
526 giLastPossibleFree = PAddr;
529 if( !MM_GetPhysAddr( (tVAddr)&gaPageReferences[PAddr] ) || (-- gaPageReferences[PAddr]) == 0 )
532 Log_Debug("PMem", "MM_DerefPhys: Free'd %P (%i free)", PAddr<<12, giPageCount-giPhysAlloc);
533 Proc_PrintBacktrace();
535 //LOG("Freed 0x%x by %p\n", PAddr<<12, __builtin_return_address(0));
537 gaPageBitmap[ PAddr / 32 ] &= ~(1 << (PAddr&31));
538 if(gaPageBitmap[ PAddr / 32 ] == 0)
539 gaSuperBitmap[ PAddr >> 10 ] &= ~(1 << ((PAddr >> 5)&31));
541 if( MM_GetPhysAddr( (tVAddr) &gaPageNodes[PAddr] ) )
543 gaPageNodes[PAddr] = NULL;
544 // TODO: Free Node Page when fully unused
549 Mutex_Release( &glPhysAlloc );
553 * \fn int MM_GetRefCount(tPAddr Addr)
555 int MM_GetRefCount(tPAddr PAddr)
560 // We don't care about non-ram pages
561 if(PAddr >= giPageCount) return -1;
563 if( MM_GetPhysAddr( (tVAddr)&gaPageReferences[PAddr] ) == 0 )
564 return (gaPageBitmap[PAddr / 32] & (1 << PAddr%32)) ? 1 : 0;
566 // Check if it is freed
567 return gaPageReferences[ PAddr ];
570 int MM_SetPageNode(tPAddr PAddr, void *Node)
574 if( MM_GetRefCount(PAddr) == 0 ) return 1;
578 block_addr = (tVAddr) &gaPageNodes[PAddr];
579 block_addr &= ~(PAGE_SIZE-1);
581 if( !MM_GetPhysAddr( block_addr ) )
583 if( !MM_Allocate( block_addr ) ) {
584 Log_Warning("PMem", "Unable to allocate Node page");
587 memset( (void*)block_addr, 0, PAGE_SIZE );
590 gaPageNodes[PAddr] = Node;
591 // Log("gaPageNodes[0x%x] = %p", PAddr, Node);
595 int MM_GetPageNode(tPAddr PAddr, void **Node)
597 if( MM_GetRefCount(PAddr) == 0 ) return 1;
600 if( !MM_GetPhysAddr( (tVAddr) &gaPageNodes[PAddr] ) ) {
604 *Node = gaPageNodes[PAddr];