3 * - By John Hodge (thePowersGang)
6 * - Dynamic memory allocation
13 #define WARNINGS 1 // Warn and dump on heap errors
14 #define DEBUG_TRACE 0 // Enable tracing of allocations
15 #define VERBOSE_DUMP 0 // Set to 1 to enable a verbose dump when heap errors are encountered
18 #define HEAP_INIT_SIZE 0x8000 // 32 KiB
19 #define MIN_SIZE (sizeof(void*))*8 // 8 Machine Words
21 #define COMPACT_HEAP 0 // Use 4 byte header?
24 //#define MAGIC_FOOT 0x2ACE5505
25 //#define MAGIC_FREE 0xACE55000
26 //#define MAGIC_USED 0x862B0505 // MAGIC_FOOT ^ MAGIC_FREE
27 #define MAGIC_FOOT 0x544F4F46 // 'FOOT'
28 #define MAGIC_FREE 0x45455246 // 'FREE'
29 #define MAGIC_USED 0x44455355 // 'USED'
32 void Heap_Install(void);
33 void *Heap_Extend(size_t Bytes);
34 void *Heap_Merge(tHeapHead *Head);
35 //void *Heap_Allocate(const char *File, int Line, size_t Bytes);
36 //void *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
37 //void *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
38 //void Heap_Deallocate(const char *File, int Line, void *Ptr);
39 void Heap_Dump(int bVerbose);
40 void Heap_Stats(void);
44 tHeapHead *gHeapStart;
48 void Heap_Install(void)
50 gHeapStart = (void*)MM_KHEAP_BASE;
51 gHeapEnd = gHeapStart;
52 Heap_Extend(HEAP_INIT_SIZE);
55 static inline tHeapHead *Heap_NextHead(tHeapHead *Head) {
56 return (void*)( (char*)Head + Head->Size );
58 static inline tHeapFoot *Heap_ThisFoot(tHeapHead *Head) {
59 return (void*)( (char*)Head + Head->Size - sizeof(tHeapFoot) );
61 static inline tHeapFoot *Heap_PrevFoot(tHeapHead *Head) {
62 //ASSERT(Head != gHeapStart);
63 return (void*)( (char*)Head - sizeof(tHeapFoot) );
67 * \brief Extend the size of the heap
69 void *Heap_Extend(size_t Bytes)
71 Debug("Heap_Extend(0x%x)", Bytes);
74 if( gHeapEnd == (tHeapHead*)MM_KHEAP_MAX ) {
75 Log_Error("Heap", "Heap limit reached (%p)", (void*)MM_KHEAP_MAX);
80 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
84 const Uint pages = (Bytes + 0xFFF) >> 12;
85 tHeapHead *new_end = (void*)( (tVAddr)gHeapEnd + (pages << 12) );
87 if( new_end > (tHeapHead*)MM_KHEAP_MAX )
89 Log_Error("Heap", "Heap limit exceeded (%p)", (void*)new_end);
90 // TODO: Clip allocation to available space, and have caller check returned block
94 // Heap expands in pages
95 for( Uint i = 0; i < pages; i ++ )
97 if( !MM_Allocate( (tVAddr)gHeapEnd+(i<<12) ) )
99 Warning("OOM - Heap_Extend (%i bytes)");
106 tHeapHead *head = gHeapEnd;
110 head->Size = (Bytes+0xFFF)&~0xFFF;
111 head->Magic = MAGIC_FREE;
112 tHeapFoot *foot = Heap_ThisFoot(head);
114 foot->Magic = MAGIC_FOOT;
116 return Heap_Merge(head); // Merge with previous block
120 * \brief Merges two adjacent heap blocks
122 void *Heap_Merge(tHeapHead *Head)
128 //Log("Heap_Merge: (Head=%p)", Head);
130 thisFoot = Heap_ThisFoot(Head);
131 if( (void*)thisFoot > (void*)gHeapEnd )
135 foot = Heap_PrevFoot(Head);
136 if( ((Uint)foot < (Uint)gHeapEnd && (Uint)foot > (Uint)gHeapStart)
137 && foot->Head->Magic == MAGIC_FREE) {
138 foot->Head->Size += Head->Size; // Increase size
139 thisFoot->Head = foot->Head; // Change backlink
140 Head->Magic = 0; // Clear old head
142 Head = foot->Head; // Save new head address
143 foot->Head = NULL; // Clear central footer
147 // Merge Right (Upwards)
148 head = Heap_NextHead(Head);
149 if((Uint)head < (Uint)gHeapEnd && head->Magic == MAGIC_FREE)
151 Head->Size += head->Size;
152 foot = Heap_ThisFoot(Head);
153 foot->Head = Head; // Update Backlink
154 thisFoot->Head = NULL; // Clear old footer
156 head->Size = 0; // Clear old header
160 // Return new address
165 * \param File Allocating source file
166 * \param Line Source line
167 * \param __Bytes Size of region to allocate
169 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
172 tHeapFoot *foot, *newfoot;
173 tHeapHead *best = NULL;
174 Uint bestSize = UINT_MAX; // Speed hack
178 return NULL; // TODO: Return a known un-mapped range.
184 Bytes = __Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot);
185 Bytes = 1UUL << LOG2(__Bytes);
187 Bytes = (__Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
191 Mutex_Acquire(&glHeap);
194 for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
198 if( head->Size != 1UUL << LOG2(head->Size) ) {
200 if( head->Size & (MIN_SIZE-1) ) {
202 Mutex_Release(&glHeap); // Release spinlock
204 Log_Warning("Heap", "Size of heap address %p is invalid"
205 " - not aligned (0x%x) [at paddr 0x%x]",
206 head, head->Size, MM_GetPhysAddr(&head->Size));
207 Heap_Dump(VERBOSE_DUMP);
211 if( head->Size < MIN_SIZE ) {
212 Mutex_Release(&glHeap);
213 Log_Warning("Heap", "Size of heap address %p is invalid"
214 " - Too small (0x%x) [at paddr 0x%x]",
215 head, head->Size, MM_GetPhysAddr(&head->Size));
216 Heap_Dump(VERBOSE_DUMP);
219 if( head->Size > (2<<30) ) {
220 Mutex_Release(&glHeap);
221 Log_Warning("Heap", "Size of heap address %p is invalid"
222 " - Over 2GiB (0x%x) [at paddr 0x%x]",
223 head, head->Size, MM_GetPhysAddr(&head->Size));
224 Heap_Dump(VERBOSE_DUMP);
228 // Check if allocated
229 if(head->Magic == MAGIC_USED) continue;
231 if(head->Magic != MAGIC_FREE) {
232 Mutex_Release(&glHeap); // Release spinlock
234 Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
235 head, &head->Magic, head->Magic);
236 Heap_Dump(VERBOSE_DUMP);
242 if(head->Size < Bytes) continue;
245 if(head->Size == Bytes) {
246 head->Magic = MAGIC_USED;
249 head->ValidSize = __Bytes;
250 head->AllocateTime = now();
251 Mutex_Release(&glHeap); // Release spinlock
253 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
254 head->Data, head->Size, File, Line);
262 bestSize = head->Size;
265 // or check if the block is the best size
266 if(bestSize > head->Size) {
268 bestSize = head->Size;
273 // If no block large enough is found, create one
276 best = Heap_Extend( Bytes );
279 Warning("OOM when allocating 0x%x bytes", Bytes);
280 Mutex_Release(&glHeap); // Release spinlock
284 if(best->Size == Bytes) {
285 best->Magic = MAGIC_USED; // Mark block as used
288 best->ValidSize = __Bytes;
289 best->AllocateTime = now();
290 Mutex_Release(&glHeap); // Release spinlock
292 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
293 best->Data, best->Size, File, Line);
300 // - Save size for new block
301 size_t newsize = best->Size - Bytes;
302 // - Allocate beginning of old block
303 best->Size = Bytes; // Update size in old header
304 best->ValidSize = __Bytes;
305 best->Magic = MAGIC_USED; // Mark block as used
308 best->AllocateTime = now();
309 // - Create a new foot on old block
310 newfoot = Heap_ThisFoot(best);
311 newfoot->Head = best; // Create new footer
312 newfoot->Magic = MAGIC_FOOT;
313 // - Create a new header after resized old
314 newhead = Heap_NextHead(best);
315 newhead->Size = newsize;
316 newhead->Magic = MAGIC_FREE;
317 newhead->ValidSize = 0;
318 newhead->File = NULL;
321 foot = Heap_ThisFoot(newhead);
322 foot->Head = newhead;
324 Mutex_Release(&glHeap); // Release spinlock
326 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
327 best->Data, best->Size, File, Line);
333 * \brief Free an allocated memory block
335 void Heap_Deallocate(const char *File, int Line, void *Ptr)
337 // INVLPTR is returned from Heap_Allocate when the allocation
339 if( Ptr == INVLPTR ) return;
342 if( (tVAddr)Ptr % sizeof(void*) != 0 ) {
343 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
348 if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
350 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
351 __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
355 // Check memory block - Header
356 tHeapHead *head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
357 if(head->Magic == MAGIC_FREE) {
358 Log_Warning("Heap", "free - Passed a freed block (%p) by %s:%i (was freed by %s:%i)",
360 head->File, head->Line);
363 if(head->Magic != MAGIC_USED) {
364 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
365 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
369 // Check memory block - Footer
370 tHeapFoot *foot = Heap_ThisFoot(head);
371 if(foot->Head != head) {
372 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
373 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
376 if(foot->Magic != MAGIC_FOOT) {
377 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)",
378 head, &foot->Magic, foot->Magic);
379 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
384 Log_Debug("Heap", "free: %p freed by %s:%i (%i old)",
385 Ptr, File, Line, now()-head->AllocateTime);
389 Mutex_Acquire( &glHeap );
392 head->Magic = MAGIC_FREE;
400 Mutex_Release( &glHeap );
404 * \brief Increase/Decrease the size of an allocation
405 * \param File Calling File
406 * \param Line Calling Line
407 * \param __ptr Old memory
408 * \param __size New Size
410 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
412 tHeapHead *head = (void*)( (Uint)__ptr-sizeof(tHeapHead) );
415 Uint newSize = (__size + sizeof(tHeapFoot)+sizeof(tHeapHead)+MIN_SIZE-1)&~(MIN_SIZE-1);
417 // Check for reallocating NULL
418 if(__ptr == NULL) return Heap_Allocate(File, Line, __size);
420 // Check if resize is needed
421 if(newSize <= head->Size) return __ptr;
423 // Check if next block is free
424 nextHead = Heap_NextHead(head);
426 // Extend into next block
427 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
429 Uint size = nextHead->Size + head->Size;
430 foot = Heap_ThisFoot(nextHead);
432 if(size == newSize) {
433 head->Size = newSize;
434 head->ValidSize = __size;
442 // Create a new heap block
443 // - Update old with new information
444 head->Size = newSize;
445 head->File = File; // Kinda counts as a new allocation
447 head->ValidSize = __size;
448 // - Create new footer
449 foot = Heap_ThisFoot(head);
451 foot->Magic = MAGIC_FOOT;
452 // - Create new header
453 nextHead = Heap_NextHead(head);
454 nextHead->Size = size - newSize;
455 nextHead->Magic = MAGIC_FREE;
456 // - Update old footer
457 foot->Head = nextHead;
462 foot = Heap_PrevFoot(head);
463 nextHead = foot->Head;
464 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
466 Uint size = nextHead->Size + head->Size;
467 // Inexact fit, split things up
470 // TODO: Handle splitting of downwards blocks
471 Warning("[Heap ] TODO: Space efficient realloc when new size is smaller");
476 // Set 1st (new/lower) header
477 nextHead->Magic = MAGIC_USED;
478 nextHead->Size = newSize;
479 nextHead->File = File;
480 nextHead->Line = Line;
481 nextHead->ValidSize = __size;
482 // Get 2nd (old) footer
483 foot = Heap_ThisFoot(nextHead);
484 foot->Head = nextHead;
485 // Save old data size
486 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
491 memmove(nextHead->Data, __ptr, oldDataSize);
493 return nextHead->Data;
497 nextHead = Heap_Allocate( File, Line, __size );
498 if(!nextHead) return NULL;
500 nextHead->File = File;
501 nextHead->Line = Line;
502 nextHead->ValidSize = __size;
507 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
512 return nextHead->Data;
516 * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
517 * \brief Allocate and Zero a buffer in memory
518 * \param File Allocating file
519 * \param Line Line of allocation
520 * \param Bytes Size of the allocation
522 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
524 void *ret = Heap_Allocate(File, Line, Bytes);
525 if(ret == NULL) return NULL;
527 memset( ret, 0, Bytes );
533 * \fn int Heap_IsHeapAddr(void *Ptr)
534 * \brief Checks if an address is a heap pointer
536 int Heap_IsHeapAddr(void *Ptr)
539 if((Uint)Ptr < (Uint)gHeapStart) return 0;
540 if((Uint)Ptr > (Uint)gHeapEnd) return 0;
541 if((Uint)Ptr & (sizeof(Uint)-1)) return 0;
543 head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
544 if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
552 void Heap_Validate(void)
554 // Call dump non-verbosely.
555 // - If a heap error is detected, it'll print
559 void Heap_Dump(int bVerbose)
561 tHeapHead *head, *badHead;
562 tHeapFoot *foot = NULL;
563 static int in_heap_dump;
565 if( in_heap_dump ) return;
570 while( (Uint)head < (Uint)gHeapEnd )
572 foot = Heap_ThisFoot(head);
576 Log_Log("Heap", "%p (%P): 0x%08x (%i) %4C",
577 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
578 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
580 Log_Log("Heap", "%sowned by %s:%i",
581 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
585 // Sanity Check Header
586 if(head->Size == 0) {
587 Log_Warning("Heap", "HALTED - Size is zero");
590 if(head->Size & (MIN_SIZE-1)) {
591 Log_Warning("Heap", "HALTED - Size is malaligned");
594 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
595 Log_Warning("Heap", "HALTED - Head Magic is Bad");
600 if(foot->Magic != MAGIC_FOOT) {
601 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
604 if(head != foot->Head) {
605 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
614 // All OK? Go to next
615 head = foot->NextHead;
618 // If the heap is valid, ok!
619 if( (tVAddr)head == (tVAddr)gHeapEnd ) {
624 // Check for a bad return
625 if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
630 // If not verbose, we need to dump the failing block
633 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
634 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
636 Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
638 Log_Log("Heap", "%sowned by %s:%i",
639 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
648 foot = Heap_PrevFoot(gHeapEnd);
649 Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
651 while( (tVAddr)head >= (tVAddr)badHead )
653 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
654 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
655 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
657 Log_Log("Heap", "%sowned by %s:%i",
658 (head->Magic!=MAGIC_USED?"was ":""),
659 head->File, head->Line);
662 // Sanity Check Header
663 if(head->Size == 0) {
664 Log_Warning("Heap", "HALTED - Size is zero");
667 if(head->Size & (MIN_SIZE-1)) {
668 Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
671 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
672 Log_Warning("Heap", "HALTED - Head Magic is Bad");
677 if(foot->Magic != MAGIC_FOOT) {
678 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
681 if(head != foot->Head) {
682 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
686 if(head == badHead) break;
688 foot = Heap_PrevFoot(head);
690 Log_Debug("Heap", "head=%p", head);
693 Panic("Heap_Dump - Heap is corrupted, kernel panic!");
696 void Heap_Stats(void)
702 int maxAlloc=0, minAlloc=-1;
703 int avgAlloc, frag, overhead;
705 for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
708 totalBytes += head->Size;
709 if( head->Magic == MAGIC_FREE )
712 freeBytes += head->Size;
714 else if( head->Magic == MAGIC_USED) {
715 if(maxAlloc < head->Size) maxAlloc = head->Size;
716 if(minAlloc == -1 || minAlloc > head->Size)
717 minAlloc = head->Size;
720 Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
724 // Print the block info?
726 if( head->Magic == MAGIC_FREE )
727 Log_Debug("Heap", "%p (%P) - 0x%x free",
728 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
730 Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
731 head->Data, MM_GetPhysAddr(&head->Data), head->Size,
732 head->ValidSize, head->File, head->Line,
733 now() - head->AllocateTime
738 Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
739 Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
741 frag = (nFree-1)*10000/nBlocks;
744 Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
748 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
750 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
753 Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
754 avgAlloc, overhead/100, overhead%100
756 Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
759 // Scan and get distribution
766 } sizeCounts[nBlocks];
769 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
771 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
773 for( i = 0; i < nBlocks; i ++ ) {
774 if( sizeCounts[i].Size == 0 )
776 if( sizeCounts[i].Size == head->Size )
779 // Should never reach this part (in a non-concurrent case)
780 if( i == nBlocks ) continue;
781 sizeCounts[i].Size = head->Size;
782 sizeCounts[i].Count ++;
784 Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
785 head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
787 Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
788 sizeCounts[i].Size, sizeCounts[i].Count);
792 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
794 Log("Heap_Stats: 0x%x - %i blocks",
795 sizeCounts[i].Size, sizeCounts[i].Count
803 EXPORT(Heap_Allocate);
804 EXPORT(Heap_AllocateZero);
805 EXPORT(Heap_Reallocate);
806 EXPORT(Heap_Deallocate);
807 EXPORT(Heap_IsHeapAddr);