3 * - By John Hodge (thePowersGang)
6 * - Dynamic memory allocation
12 #include <debug_hooks.h>
14 #define WARNINGS 1 // Warn and dump on heap errors
15 #define DEBUG_TRACE 0 // Enable tracing of allocations
16 #define VERBOSE_DUMP 0 // Set to 1 to enable a verbose dump when heap errors are encountered
17 #define VALIDATE_ON_ALLOC 1 // Set to 1 to enable validation of the heap on all malloc() calls
20 #define HEAP_INIT_SIZE 0x8000 // 32 KiB
21 #define MIN_SIZE (sizeof(void*))*8 // 8 Machine Words
23 #define COMPACT_HEAP 0 // Use 4 byte header?
26 //#define MAGIC_FOOT 0x2ACE5505
27 //#define MAGIC_FREE 0xACE55000
28 //#define MAGIC_USED 0x862B0505 // MAGIC_FOOT ^ MAGIC_FREE
29 #define MAGIC_FOOT 0x544F4F46 // 'FOOT'
30 #define MAGIC_FREE 0x45455246 // 'FREE'
31 #define MAGIC_USED 0x44455355 // 'USED'
34 void Heap_Install(void);
35 void *Heap_Extend(size_t Bytes);
36 void *Heap_Merge(tHeapHead *Head);
37 //void *Heap_Allocate(const char *File, int Line, size_t Bytes);
38 //void *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
39 //void *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
40 //void Heap_Deallocate(const char *File, int Line, void *Ptr);
41 //void Heap_Dump(void);
42 void Heap_ValidateDump(int bVerbose);
43 //void Heap_Stats(void);
47 tHeapHead *gHeapStart;
51 void Heap_Install(void)
53 gHeapStart = (void*)MM_KHEAP_BASE;
54 gHeapEnd = gHeapStart;
55 Heap_Extend(HEAP_INIT_SIZE);
58 static inline tHeapHead *Heap_NextHead(tHeapHead *Head) {
59 return (void*)( (char*)Head + Head->Size );
61 static inline tHeapFoot *Heap_ThisFoot(tHeapHead *Head) {
62 return (void*)( (char*)Head + Head->Size - sizeof(tHeapFoot) );
64 static inline tHeapFoot *Heap_PrevFoot(tHeapHead *Head) {
65 //ASSERT(Head != gHeapStart);
66 return (void*)( (char*)Head - sizeof(tHeapFoot) );
70 * \brief Extend the size of the heap
72 void *Heap_Extend(size_t Bytes)
74 Debug("Heap_Extend(0x%x)", Bytes);
77 if( gHeapEnd == (tHeapHead*)MM_KHEAP_MAX ) {
78 Log_Error("Heap", "Heap limit reached (%p)", (void*)MM_KHEAP_MAX);
83 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
87 const Uint pages = (Bytes + 0xFFF) >> 12;
88 tHeapHead *new_end = (void*)( (tVAddr)gHeapEnd + (pages << 12) );
90 if( new_end > (tHeapHead*)MM_KHEAP_MAX )
92 Log_Error("Heap", "Heap limit exceeded (%p)", (void*)new_end);
93 // TODO: Clip allocation to available space, and have caller check returned block
97 // Heap expands in pages
98 for( Uint i = 0; i < pages; i ++ )
100 if( !MM_Allocate( (tVAddr)gHeapEnd+(i<<12) ) )
102 Warning("OOM - Heap_Extend (%i bytes)");
109 tHeapHead *head = gHeapEnd;
113 head->Size = (Bytes+0xFFF)&~0xFFF;
114 head->Magic = MAGIC_FREE;
115 tHeapFoot *foot = Heap_ThisFoot(head);
117 foot->Magic = MAGIC_FOOT;
119 return Heap_Merge(head); // Merge with previous block
123 * \brief Merges two adjacent heap blocks
125 void *Heap_Merge(tHeapHead *Head)
131 //Log("Heap_Merge: (Head=%p)", Head);
133 thisFoot = Heap_ThisFoot(Head);
134 if( (void*)thisFoot > (void*)gHeapEnd )
138 foot = Heap_PrevFoot(Head);
139 if( ((Uint)foot < (Uint)gHeapEnd && (Uint)foot > (Uint)gHeapStart)
140 && foot->Head->Magic == MAGIC_FREE) {
141 foot->Head->Size += Head->Size; // Increase size
142 thisFoot->Head = foot->Head; // Change backlink
143 Head->Magic = 0; // Clear old head
145 Head = foot->Head; // Save new head address
146 foot->Head = NULL; // Clear central footer
150 // Merge Right (Upwards)
151 head = Heap_NextHead(Head);
152 if((Uint)head < (Uint)gHeapEnd && head->Magic == MAGIC_FREE)
154 Head->Size += head->Size;
155 foot = Heap_ThisFoot(Head);
156 foot->Head = Head; // Update Backlink
157 thisFoot->Head = NULL; // Clear old footer
159 head->Size = 0; // Clear old header
163 // Return new address
168 * \param File Allocating source file
169 * \param Line Source line
170 * \param __Bytes Size of region to allocate
172 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
175 tHeapFoot *foot, *newfoot;
176 tHeapHead *best = NULL;
177 Uint bestSize = UINT_MAX; // Speed hack
181 return NULL; // TODO: Return a known un-mapped range.
185 #if VALIDATE_ON_ALLOC
191 Bytes = __Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot);
192 Bytes = 1UUL << LOG2(__Bytes);
194 Bytes = (__Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
198 Mutex_Acquire(&glHeap);
201 for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
205 if( head->Size != 1UUL << LOG2(head->Size) ) {
207 if( head->Size & (MIN_SIZE-1) ) {
209 Mutex_Release(&glHeap); // Release spinlock
211 Log_Warning("Heap", "Size of heap address %p is invalid"
212 " - not aligned (0x%x) [at paddr 0x%x]",
213 head, head->Size, MM_GetPhysAddr(&head->Size));
214 Heap_ValidateDump(VERBOSE_DUMP);
218 if( head->Size < MIN_SIZE ) {
219 Mutex_Release(&glHeap);
220 Log_Warning("Heap", "Size of heap address %p is invalid"
221 " - Too small (0x%x) [at paddr 0x%x]",
222 head, head->Size, MM_GetPhysAddr(&head->Size));
223 Heap_ValidateDump(VERBOSE_DUMP);
226 if( head->Size > (2<<30) ) {
227 Mutex_Release(&glHeap);
228 Log_Warning("Heap", "Size of heap address %p is invalid"
229 " - Over 2GiB (0x%x) [at paddr 0x%x]",
230 head, head->Size, MM_GetPhysAddr(&head->Size));
231 Heap_ValidateDump(VERBOSE_DUMP);
235 // Check if allocated
236 if(head->Magic == MAGIC_USED) continue;
238 if(head->Magic != MAGIC_FREE) {
239 Mutex_Release(&glHeap); // Release spinlock
241 Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
242 head, &head->Magic, head->Magic);
243 Heap_ValidateDump(VERBOSE_DUMP);
249 if(head->Size < Bytes) continue;
252 if(head->Size == Bytes) {
253 head->Magic = MAGIC_USED;
256 head->ValidSize = __Bytes;
257 head->AllocateTime = now();
258 Mutex_Release(&glHeap); // Release spinlock
260 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
261 head->Data, head->Size, File, Line);
269 bestSize = head->Size;
272 // or check if the block is the best size
273 if(bestSize > head->Size) {
275 bestSize = head->Size;
280 // If no block large enough is found, create one
283 best = Heap_Extend( Bytes );
286 Warning("OOM when allocating 0x%x bytes", Bytes);
287 Mutex_Release(&glHeap); // Release spinlock
291 if(best->Size == Bytes) {
292 best->Magic = MAGIC_USED; // Mark block as used
295 best->ValidSize = __Bytes;
296 best->AllocateTime = now();
297 Mutex_Release(&glHeap); // Release spinlock
299 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
300 best->Data, best->Size, File, Line);
307 // - Save size for new block
308 size_t newsize = best->Size - Bytes;
309 // - Allocate beginning of old block
310 best->Size = Bytes; // Update size in old header
311 best->ValidSize = __Bytes;
312 best->Magic = MAGIC_USED; // Mark block as used
315 best->AllocateTime = now();
316 // - Create a new foot on old block
317 newfoot = Heap_ThisFoot(best);
318 newfoot->Head = best; // Create new footer
319 newfoot->Magic = MAGIC_FOOT;
320 // - Create a new header after resized old
321 newhead = Heap_NextHead(best);
322 newhead->Size = newsize;
323 newhead->Magic = MAGIC_FREE;
324 newhead->ValidSize = 0;
325 newhead->File = NULL;
328 foot = Heap_ThisFoot(newhead);
329 foot->Head = newhead;
331 Mutex_Release(&glHeap); // Release spinlock
333 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
334 best->Data, best->Size, File, Line);
340 * \brief Free an allocated memory block
342 void Heap_Deallocate(const char *File, int Line, void *Ptr)
344 // INVLPTR is returned from Heap_Allocate when the allocation
346 if( Ptr == INVLPTR ) return;
349 if( (tVAddr)Ptr % sizeof(void*) != 0 ) {
350 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
355 if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
357 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
358 __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
362 // Check memory block - Header
363 tHeapHead *head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
364 if(head->Magic == MAGIC_FREE) {
365 Log_Warning("Heap", "free - Passed a freed block (%p) by %s:%i (was freed by %s:%i)",
367 head->File, head->Line);
368 Proc_PrintBacktrace();
371 if(head->Magic != MAGIC_USED) {
372 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
373 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
377 // Check memory block - Footer
378 tHeapFoot *foot = Heap_ThisFoot(head);
379 if(foot->Head != head) {
380 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
381 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
384 if(foot->Magic != MAGIC_FOOT) {
385 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)",
386 head, &foot->Magic, foot->Magic);
387 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
392 Log_Debug("Heap", "free: %p freed by %s:%i (%i old)",
393 Ptr, File, Line, now()-head->AllocateTime);
397 Mutex_Acquire( &glHeap );
400 head->Magic = MAGIC_FREE;
408 Mutex_Release( &glHeap );
412 * \brief Increase/Decrease the size of an allocation
413 * \param File Calling File
414 * \param Line Calling Line
415 * \param __ptr Old memory
416 * \param __size New Size
418 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
420 tHeapHead *head = (void*)( (Uint)__ptr-sizeof(tHeapHead) );
423 Uint newSize = (__size + sizeof(tHeapFoot)+sizeof(tHeapHead)+MIN_SIZE-1)&~(MIN_SIZE-1);
425 // Check for reallocating NULL
426 if(__ptr == NULL) return Heap_Allocate(File, Line, __size);
428 // Check if resize is needed
429 if(newSize <= head->Size) return __ptr;
431 // Check if next block is free
432 nextHead = Heap_NextHead(head);
434 // Extend into next block
435 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
437 Uint size = nextHead->Size + head->Size;
438 foot = Heap_ThisFoot(nextHead);
440 if(size == newSize) {
441 head->Size = newSize;
442 head->ValidSize = __size;
450 // Create a new heap block
451 // - Update old with new information
452 head->Size = newSize;
453 head->File = File; // Kinda counts as a new allocation
455 head->ValidSize = __size;
456 // - Create new footer
457 foot = Heap_ThisFoot(head);
459 foot->Magic = MAGIC_FOOT;
460 // - Create new header
461 nextHead = Heap_NextHead(head);
462 nextHead->Size = size - newSize;
463 nextHead->Magic = MAGIC_FREE;
464 // - Update old footer
465 foot->Head = nextHead;
470 foot = Heap_PrevFoot(head);
471 nextHead = foot->Head;
472 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
474 Uint size = nextHead->Size + head->Size;
475 // Inexact fit, split things up
478 // TODO: Handle splitting of downwards blocks
479 Warning("[Heap ] TODO: Space efficient realloc when new size is smaller");
484 // Set 1st (new/lower) header
485 nextHead->Magic = MAGIC_USED;
486 nextHead->Size = newSize;
487 nextHead->File = File;
488 nextHead->Line = Line;
489 nextHead->ValidSize = __size;
490 // Get 2nd (old) footer
491 foot = Heap_ThisFoot(nextHead);
492 foot->Head = nextHead;
493 // Save old data size
494 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
499 memmove(nextHead->Data, __ptr, oldDataSize);
501 return nextHead->Data;
505 nextHead = Heap_Allocate( File, Line, __size );
506 if(!nextHead) return NULL;
508 nextHead->File = File;
509 nextHead->Line = Line;
510 nextHead->ValidSize = __size;
515 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
520 return nextHead->Data;
524 * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
525 * \brief Allocate and Zero a buffer in memory
526 * \param File Allocating file
527 * \param Line Line of allocation
528 * \param Bytes Size of the allocation
530 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
532 void *ret = Heap_Allocate(File, Line, Bytes);
533 if(ret == NULL) return NULL;
535 memset( ret, 0, Bytes );
541 * \fn int Heap_IsHeapAddr(void *Ptr)
542 * \brief Checks if an address is a heap pointer
544 int Heap_IsHeapAddr(void *Ptr)
547 if((Uint)Ptr < (Uint)gHeapStart) return 0;
548 if((Uint)Ptr > (Uint)gHeapEnd) return 0;
549 if((Uint)Ptr & (sizeof(Uint)-1)) return 0;
551 head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
552 if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
560 void Heap_Validate(void)
562 // Call dump non-verbosely.
563 // - If a heap error is detected, it'll print
564 Heap_ValidateDump(0);
569 Heap_ValidateDump(1);
572 void Heap_ValidateDump(int bVerbose)
574 tHeapHead *head, *badHead;
575 tHeapFoot *foot = NULL;
576 static int in_heap_dump;
578 if( in_heap_dump ) return;
583 while( (Uint)head < (Uint)gHeapEnd )
585 foot = Heap_ThisFoot(head);
589 Log_Log("Heap", "%p (%P): 0x%08x (%i) %4C",
590 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
591 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
593 Log_Log("Heap", "%sowned by %s:%i",
594 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
598 // Sanity Check Header
599 if(head->Size == 0) {
600 Log_Warning("Heap", "HALTED - Size is zero");
603 if(head->Size & (MIN_SIZE-1)) {
604 Log_Warning("Heap", "HALTED - Size is malaligned");
607 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
608 Log_Warning("Heap", "HALTED - Head Magic is Bad");
613 if(foot->Magic != MAGIC_FOOT) {
614 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
617 if(head != foot->Head) {
618 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
627 // All OK? Go to next
628 head = foot->NextHead;
631 // If the heap is valid, ok!
632 if( (tVAddr)head == (tVAddr)gHeapEnd ) {
637 // Check for a bad return
638 if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
643 // If not verbose, we need to dump the failing block
646 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
647 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
649 Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
651 Log_Log("Heap", "%sowned by %s:%i",
652 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
660 foot = Heap_PrevFoot(gHeapEnd);
661 Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
663 while( (tVAddr)head >= (tVAddr)badHead )
665 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
666 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
667 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
669 Log_Log("Heap", "%sowned by %s:%i",
670 (head->Magic!=MAGIC_USED?"was ":""),
671 head->File, head->Line);
674 // Sanity Check Header
675 if(head->Size == 0) {
676 Log_Warning("Heap", "HALTED - Size is zero");
679 if(head->Size & (MIN_SIZE-1)) {
680 Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
683 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
684 Log_Warning("Heap", "HALTED - Head Magic is Bad");
689 if(foot->Magic != MAGIC_FOOT) {
690 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
693 if(head != foot->Head) {
694 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
698 if(head == badHead) break;
700 foot = Heap_PrevFoot(head);
702 Log_Debug("Heap", "head=%p", head);
705 Panic("Heap_Dump - Heap is corrupted, kernel panic! (%p)", badHead);
708 void Heap_Stats(void)
714 int maxAlloc=0, minAlloc=-1;
715 int avgAlloc, frag, overhead;
717 for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
720 totalBytes += head->Size;
721 if( head->Magic == MAGIC_FREE )
724 freeBytes += head->Size;
726 else if( head->Magic == MAGIC_USED) {
727 if(maxAlloc < head->Size) maxAlloc = head->Size;
728 if(minAlloc == -1 || minAlloc > head->Size)
729 minAlloc = head->Size;
732 Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
736 // Print the block info?
738 if( head->Magic == MAGIC_FREE )
739 Log_Debug("Heap", "%p (%P) - 0x%x free",
740 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
742 Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
743 head->Data, MM_GetPhysAddr(&head->Data), head->Size,
744 head->ValidSize, head->File, head->Line,
745 now() - head->AllocateTime
750 Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
751 Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
753 frag = (nFree-1)*10000/nBlocks;
756 Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
760 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
762 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
765 Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
766 avgAlloc, overhead/100, overhead%100
768 Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
771 // Scan and get distribution
778 } sizeCounts[nBlocks];
781 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
783 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
785 for( i = 0; i < nBlocks; i ++ ) {
786 if( sizeCounts[i].Size == 0 )
788 if( sizeCounts[i].Size == head->Size )
791 // Should never reach this part (in a non-concurrent case)
792 if( i == nBlocks ) continue;
793 sizeCounts[i].Size = head->Size;
794 sizeCounts[i].Count ++;
796 Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
797 head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
799 Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
800 sizeCounts[i].Size, sizeCounts[i].Count);
804 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
806 Log("Heap_Stats: 0x%x - %i blocks",
807 sizeCounts[i].Size, sizeCounts[i].Count
815 EXPORT(Heap_Allocate);
816 EXPORT(Heap_AllocateZero);
817 EXPORT(Heap_Reallocate);
818 EXPORT(Heap_Deallocate);
819 EXPORT(Heap_IsHeapAddr);