3 * - Physical memory manager
12 #define TRACE_ALLOCS 0 // Print trace messages on AllocPhys/DerefPhys
14 static const int addrClasses[] = {0,16,20,24,32,64};
15 static const int numAddrClasses = sizeof(addrClasses)/sizeof(addrClasses[0]);
18 extern void Proc_PrintBacktrace(void);
21 void MM_Install(int NPMemRanges, tPMemMapEnt *PMemRanges);
22 //tPAddr MM_AllocPhys(void);
23 //tPAddr MM_AllocPhysRange(int Pages, int MaxBits);
24 //void MM_RefPhys(tPAddr PAddr);
25 //void MM_DerefPhys(tPAddr PAddr);
26 // int MM_GetRefCount(tPAddr PAddr);
30 Uint64 giPhysAlloc = 0; // Number of allocated pages
31 Uint64 giPageCount = 0; // Total number of pages
32 Uint64 giLastPossibleFree = 0; // Last possible free page (before all pages are used)
33 Uint64 giTotalMemorySize = 0; // Total number of allocatable pages
35 Uint32 gaSuperBitmap[1024]; // Blocks of 1024 Pages
36 Uint32 gaPageBitmap[1024*1024/32]; // Individual pages
37 int *gaPageReferences;
38 void **gaPageNodes = (void*)MM_PAGENODE_BASE;
39 #define REFENT_PER_PAGE (0x1000/sizeof(gaPageReferences[0]))
42 void MM_Install(int NPMemRanges, tPMemMapEnt *PMemRanges)
47 // --- Find largest address
48 for( i = 0; i < NPMemRanges; i ++ )
50 tPMemMapEnt *ent = &PMemRanges[i];
51 // If entry is RAM and is above `maxAddr`, change `maxAddr`
52 if(ent->Type == PMEMTYPE_FREE || ent->Type == PMEMTYPE_USED)
54 if(ent->Start + ent->Length > maxAddr)
55 maxAddr = ent->Start + ent->Length;
56 giTotalMemorySize += ent->Length >> 12;
60 giPageCount = maxAddr >> 12;
61 giLastPossibleFree = giPageCount - 1;
63 memsetd(gaPageBitmap, 0xFFFFFFFF, giPageCount/32);
65 // Set up allocateable space
66 for( i = 0; i < NPMemRanges; i ++ )
68 tPMemMapEnt *ent = &PMemRanges[i];
69 if( ent->Type == PMEMTYPE_FREE )
71 Uint64 startpg = ent->Start / PAGE_SIZE;
72 Uint64 pgcount = ent->Length / PAGE_SIZE;
73 while( startpg % 32 && pgcount ) {
74 gaPageBitmap[startpg/32] &= ~(1U << (startpg%32));
78 memsetd( &gaPageBitmap[startpg/32], 0, pgcount/32 );
79 startpg += pgcount - pgcount%32;
80 pgcount -= pgcount - pgcount%32;
82 gaPageBitmap[startpg/32] &= ~(1U << (startpg%32));
87 else if( ent->Type == PMEMTYPE_USED )
89 giPhysAlloc += ent->Length / PAGE_SIZE;
93 // Fill Superpage bitmap
94 // - A set bit means that there are no free pages in this block of 32
95 for( i = 0; i < (giPageCount+31)/32; i ++ )
97 if( gaPageBitmap[i] + 1 == 0 ) {
98 gaSuperBitmap[i/32] |= (1 << i%32);
102 gaPageReferences = (void*)MM_REFCOUNT_BASE;
104 Log_Log("PMem", "Physical memory set up (%lli pages of ~%lli MiB used)",
105 giPhysAlloc, (giTotalMemorySize*PAGE_SIZE)/(1024*1024)
109 void MM_DumpStatistics(void)
112 for( i = 1; i < numAddrClasses; i ++ )
114 int first = (i == 1 ? 0 : (1UL << (addrClasses[i-1] - 12)));
115 int last = (1UL << (addrClasses[i] - 12)) - 1;
120 if( last > giPageCount )
123 int total = last - first + 1;
125 for( pg = first; pg < last; pg ++ )
127 if( !MM_GetPhysAddr(&gaPageReferences[pg]) || gaPageReferences[pg] == 0 ) {
131 totalRefs += gaPageReferences[pg];
132 if(gaPageReferences[pg] > 1)
136 int nUsed = (total - nFree);
137 Log_Log("MMPhys", "%ipbit - %i/%i used, %i reused, %i average reference count",
138 addrClasses[i], nUsed, total, nMultiRef,
139 nMultiRef ? (totalRefs-(nUsed - nMultiRef)) / nMultiRef : 0
142 if( last == giPageCount )
145 Log_Log("MMPhys", "%lli/%lli total pages used, 0 - %i possible free range",
146 giPhysAlloc, giTotalMemorySize, giLastPossibleFree);
150 * \fn tPAddr MM_AllocPhys(void)
151 * \brief Allocates a physical page from the general pool
153 tPAddr MM_AllocPhys(void)
161 Mutex_Acquire( &glPhysAlloc );
168 for( i = numAddrClasses; i -- > 1; )
170 first = 1UL << (addrClasses[i-1] - 12);
171 last = (1UL << (addrClasses[i] - 12)) - 1;
172 // Range is above the last free page
173 if( first > giLastPossibleFree )
175 // Last possible free page is in the range
176 if( last > giLastPossibleFree )
177 last = giLastPossibleFree;
180 for( indx = first; indx < last; )
182 if( gaSuperBitmap[indx>>10] == -1 ) {
187 if( gaPageBitmap[indx>>5] == -1 ) {
192 if( gaPageBitmap[indx>>5] & (1 << (indx&31)) ) {
198 if( indx < last ) break;
200 giLastPossibleFree = first; // Well, we couldn't find any in this range
203 if( i <= 1 ) indx = -1;
208 LOG("giLastPossibleFree = %i", giLastPossibleFree);
209 for( indx = giLastPossibleFree; indx >= 0; )
211 if( gaSuperBitmap[indx>>10] == -1 ) {
216 if( gaPageBitmap[indx>>5] == -1 ) {
221 if( gaPageBitmap[indx>>5] & (1 << (indx&31)) ) {
228 giLastPossibleFree = indx;
229 LOG("indx = %i", indx);
231 c = giLastPossibleFree % 32;
232 b = (giLastPossibleFree / 32) % 32;
233 a = giLastPossibleFree / 1024;
235 LOG("a=%i,b=%i,c=%i", a, b, c);
236 for( ; gaSuperBitmap[a] == -1 && a >= 0; a-- );
238 Mutex_Release( &glPhysAlloc );
239 Warning("MM_AllocPhys - OUT OF MEMORY (Called by %p) - %lli/%lli used",
240 __builtin_return_address(0), giPhysAlloc, giPageCount);
244 for( ; gaSuperBitmap[a] & (1<<b); b-- );
245 for( ; gaPageBitmap[a*32+b] & (1<<c); c-- );
246 LOG("a=%i,b=%i,c=%i", a, b, c);
247 indx = (a << 10) | (b << 5) | c;
249 giLastPossibleFree = indx;
253 Mutex_Release( &glPhysAlloc );
254 Warning("MM_AllocPhys - OUT OF MEMORY (Called by %p) - %lli/%lli used (indx = %x)",
255 __builtin_return_address(0), giPhysAlloc, giPageCount, indx);
256 Log_Debug("PMem", "giLastPossibleFree = %lli", giLastPossibleFree);
261 if( indx > 0xFFFFF ) {
262 Panic("The fuck? Too many pages! (indx = 0x%x)", indx);
265 if( indx >= giPageCount ) {
266 Mutex_Release( &glPhysAlloc );
267 Log_Error("PMem", "MM_AllocPhys - indx(%i) > giPageCount(%i)", indx, giPageCount);
273 if( MM_GetPhysAddr( (tVAddr)&gaPageReferences[indx] ) )
274 gaPageReferences[indx] = 1;
275 gaPageBitmap[ indx>>5 ] |= 1 << (indx&31);
283 if(gaPageBitmap[ indx>>5 ] == -1) {
284 gaSuperBitmap[indx>>10] |= 1 << ((indx>>5)&31);
288 Mutex_Release( &glPhysAlloc );
293 Log_Debug("PMem", "MM_AllocPhys: RETURN %P (%i free)", ret, giPageCount-giPhysAlloc);
294 Proc_PrintBacktrace();
301 * \fn tPAddr MM_AllocPhysRange(int Pages, int MaxBits)
302 * \brief Allocate a range of physical pages
303 * \param Pages Number of pages to allocate
304 * \param MaxBits Maximum number of address bits to use
306 tPAddr MM_AllocPhysRange(int Pages, int MaxBits)
312 ENTER("iPages iMaxBits", Pages, MaxBits);
319 if(MaxBits > PHYS_BITS) MaxBits = PHYS_BITS;
322 Mutex_Acquire( &glPhysAlloc );
324 // Set up search state
325 if( giLastPossibleFree > ((tPAddr)1 << (MaxBits-12)) ) {
326 sidx = (tPAddr)1 << (MaxBits-12);
329 sidx = giLastPossibleFree;
337 LOG("a=%i, b=%i, idx=%i, sidx=%i", a, b, idx, sidx);
340 for( ; gaSuperBitmap[a] == -1 && a --; ) b = 31;
342 Mutex_Release( &glPhysAlloc );
343 Warning("MM_AllocPhysRange - OUT OF MEMORY (Called by %p)", __builtin_return_address(0));
348 for( ; gaSuperBitmap[a] & (1 << b); b-- ) sidx = 31;
351 for( ; gaPageBitmap[idx] & (1 << sidx); sidx-- )
352 LOG("gaPageBitmap[%i] = 0x%08x", idx, gaPageBitmap[idx]);
354 LOG("idx = %i, sidx = %i", idx, sidx);
359 // Check if the gap is large enough
366 if( gaPageBitmap[idx] == -1 ) {
372 if( gaPageBitmap[idx] & (1 << sidx) ) {
374 if(sidx < 0) { sidx = 31; idx --; }
382 // Check if it is a free range
383 for( i = 0; i < Pages; i++ )
386 if( gaPageBitmap[idx] & (1 << sidx) )
390 if(sidx < 0) { sidx = 31; idx --; }
398 // Check if an address was found
400 Mutex_Release( &glPhysAlloc );
401 Warning("MM_AllocPhysRange - OUT OF MEMORY (Called by %p)", __builtin_return_address(0));
407 for( i = 0; i < Pages; i++ )
409 if( MM_GetPhysAddr( (tVAddr)&gaPageReferences[idx*32+sidx] ) )
410 gaPageReferences[idx*32+sidx] = 1;
411 gaPageBitmap[ idx ] |= 1 << sidx;
414 if(sidx == 32) { sidx = 0; idx ++; }
418 ret = (idx << 17) | (sidx << 12);
421 if(gaPageBitmap[ idx ] == -1) gaSuperBitmap[idx/32] |= 1 << (idx%32);
424 Mutex_Release( &glPhysAlloc );
428 Log_Debug("PMem", "MM_AllocPhysRange: RETURN 0x%llx-0x%llx (%i free)",
429 ret, ret + (1<<Pages)-1, giPageCount-giPhysAlloc);
435 * \fn void MM_RefPhys(tPAddr PAddr)
437 void MM_RefPhys(tPAddr PAddr)
442 // We don't care about non-ram pages
443 if(PAddr >= giPageCount) return;
446 Mutex_Acquire( &glPhysAlloc );
448 // Reference the page
449 if( gaPageReferences )
451 if( MM_GetPhysAddr( (tVAddr)&gaPageReferences[PAddr] ) == 0 )
454 tVAddr addr = ((tVAddr)&gaPageReferences[PAddr]) & ~0xFFF;
455 // Log_Debug("PMem", "MM_RefPhys: Allocating info for %X", PAddr);
456 Mutex_Release( &glPhysAlloc );
457 if( MM_Allocate( addr ) == 0 ) {
458 Log_KernelPanic("PMem",
459 "MM_RefPhys: Out of physical memory allocating info for %X",
463 Mutex_Acquire( &glPhysAlloc );
465 base = PAddr & ~(1024-1);
466 for( i = 0; i < 1024; i ++ ) {
467 gaPageReferences[base + i] = (gaPageBitmap[(base+i)/32] & (1 << (base+i)%32)) ? 1 : 0;
470 gaPageReferences[ PAddr ] ++;
473 // If not already used
474 if( !(gaPageBitmap[ PAddr / 32 ] & 1 << (PAddr&31)) ) {
477 gaPageBitmap[ PAddr / 32 ] |= 1 << (PAddr&31);
481 if(gaPageBitmap[ PAddr / 32 ] == -1)
482 gaSuperBitmap[PAddr/1024] |= 1 << ((PAddr/32)&31);
485 Mutex_Release( &glPhysAlloc );
489 * \fn void MM_DerefPhys(tPAddr PAddr)
490 * \brief Dereferences a physical page
492 void MM_DerefPhys(tPAddr PAddr)
497 // We don't care about non-ram pages
498 if(PAddr >= giPageCount) return;
500 // Check if it is freed
501 if( !(gaPageBitmap[PAddr / 32] & (1 << PAddr%32)) ) {
502 Log_Warning("MMVirt", "MM_DerefPhys - Non-referenced memory dereferenced");
507 Mutex_Acquire( &glPhysAlloc );
509 if( giLastPossibleFree < PAddr )
510 giLastPossibleFree = PAddr;
513 if( !MM_GetPhysAddr( (tVAddr)&gaPageReferences[PAddr] ) || (-- gaPageReferences[PAddr]) == 0 )
516 Log_Debug("PMem", "MM_DerefPhys: Free'd %P (%i free)", PAddr<<12, giPageCount-giPhysAlloc);
517 Proc_PrintBacktrace();
519 //LOG("Freed 0x%x by %p\n", PAddr<<12, __builtin_return_address(0));
521 gaPageBitmap[ PAddr / 32 ] &= ~(1 << (PAddr&31));
522 if(gaPageBitmap[ PAddr / 32 ] == 0)
523 gaSuperBitmap[ PAddr >> 10 ] &= ~(1 << ((PAddr >> 5)&31));
525 if( MM_GetPhysAddr( (tVAddr) &gaPageNodes[PAddr] ) )
527 gaPageNodes[PAddr] = NULL;
528 // TODO: Free Node Page when fully unused
533 Mutex_Release( &glPhysAlloc );
537 * \fn int MM_GetRefCount(tPAddr Addr)
539 int MM_GetRefCount(tPAddr PAddr)
544 // We don't care about non-ram pages
545 if(PAddr >= giPageCount) return -1;
547 if( MM_GetPhysAddr( (tVAddr)&gaPageReferences[PAddr] ) == 0 )
548 return (gaPageBitmap[PAddr / 32] & (1 << PAddr%32)) ? 1 : 0;
550 // Check if it is freed
551 return gaPageReferences[ PAddr ];
554 int MM_SetPageNode(tPAddr PAddr, void *Node)
558 if( MM_GetRefCount(PAddr) == 0 ) return 1;
562 block_addr = (tVAddr) &gaPageNodes[PAddr];
563 block_addr &= ~(PAGE_SIZE-1);
565 if( !MM_GetPhysAddr( block_addr ) )
567 if( !MM_Allocate( block_addr ) ) {
568 Log_Warning("PMem", "Unable to allocate Node page");
571 memset( (void*)block_addr, 0, PAGE_SIZE );
574 gaPageNodes[PAddr] = Node;
575 // Log("gaPageNodes[0x%x] = %p", PAddr, Node);
579 int MM_GetPageNode(tPAddr PAddr, void **Node)
581 if( MM_GetRefCount(PAddr) == 0 ) return 1;
584 if( !MM_GetPhysAddr( (tVAddr) &gaPageNodes[PAddr] ) ) {
588 *Node = gaPageNodes[PAddr];