3 * - By John Hodge (thePowersGang)
6 * - Dynamic memory allocation
13 #define WARNINGS 1 // Warn and dump on heap errors
14 #define DEBUG_TRACE 0 // Enable tracing of allocations
15 #define VERBOSE_DUMP 0 // Set to 1 to enable a verbose dump when heap errors are encountered
16 #define VALIDATE_ON_ALLOC 1 // Set to 1 to enable validation of the heap on all malloc() calls
19 #define HEAP_INIT_SIZE 0x8000 // 32 KiB
20 #define MIN_SIZE (sizeof(void*))*8 // 8 Machine Words
22 #define COMPACT_HEAP 0 // Use 4 byte header?
25 //#define MAGIC_FOOT 0x2ACE5505
26 //#define MAGIC_FREE 0xACE55000
27 //#define MAGIC_USED 0x862B0505 // MAGIC_FOOT ^ MAGIC_FREE
28 #define MAGIC_FOOT 0x544F4F46 // 'FOOT'
29 #define MAGIC_FREE 0x45455246 // 'FREE'
30 #define MAGIC_USED 0x44455355 // 'USED'
33 void Heap_Install(void);
34 void *Heap_Extend(size_t Bytes);
35 void *Heap_Merge(tHeapHead *Head);
36 //void *Heap_Allocate(const char *File, int Line, size_t Bytes);
37 //void *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
38 //void *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
39 //void Heap_Deallocate(const char *File, int Line, void *Ptr);
40 void Heap_Dump(int bVerbose);
41 void Heap_Stats(void);
45 tHeapHead *gHeapStart;
49 void Heap_Install(void)
51 gHeapStart = (void*)MM_KHEAP_BASE;
52 gHeapEnd = gHeapStart;
53 Heap_Extend(HEAP_INIT_SIZE);
56 static inline tHeapHead *Heap_NextHead(tHeapHead *Head) {
57 return (void*)( (char*)Head + Head->Size );
59 static inline tHeapFoot *Heap_ThisFoot(tHeapHead *Head) {
60 return (void*)( (char*)Head + Head->Size - sizeof(tHeapFoot) );
62 static inline tHeapFoot *Heap_PrevFoot(tHeapHead *Head) {
63 //ASSERT(Head != gHeapStart);
64 return (void*)( (char*)Head - sizeof(tHeapFoot) );
68 * \brief Extend the size of the heap
70 void *Heap_Extend(size_t Bytes)
72 Debug("Heap_Extend(0x%x)", Bytes);
75 if( gHeapEnd == (tHeapHead*)MM_KHEAP_MAX ) {
76 Log_Error("Heap", "Heap limit reached (%p)", (void*)MM_KHEAP_MAX);
81 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
85 const Uint pages = (Bytes + 0xFFF) >> 12;
86 tHeapHead *new_end = (void*)( (tVAddr)gHeapEnd + (pages << 12) );
88 if( new_end > (tHeapHead*)MM_KHEAP_MAX )
90 Log_Error("Heap", "Heap limit exceeded (%p)", (void*)new_end);
91 // TODO: Clip allocation to available space, and have caller check returned block
95 // Heap expands in pages
96 for( Uint i = 0; i < pages; i ++ )
98 if( !MM_Allocate( (tVAddr)gHeapEnd+(i<<12) ) )
100 Warning("OOM - Heap_Extend (%i bytes)");
107 tHeapHead *head = gHeapEnd;
111 head->Size = (Bytes+0xFFF)&~0xFFF;
112 head->Magic = MAGIC_FREE;
113 tHeapFoot *foot = Heap_ThisFoot(head);
115 foot->Magic = MAGIC_FOOT;
117 return Heap_Merge(head); // Merge with previous block
121 * \brief Merges two adjacent heap blocks
123 void *Heap_Merge(tHeapHead *Head)
129 //Log("Heap_Merge: (Head=%p)", Head);
131 thisFoot = Heap_ThisFoot(Head);
132 if( (void*)thisFoot > (void*)gHeapEnd )
136 foot = Heap_PrevFoot(Head);
137 if( ((Uint)foot < (Uint)gHeapEnd && (Uint)foot > (Uint)gHeapStart)
138 && foot->Head->Magic == MAGIC_FREE) {
139 foot->Head->Size += Head->Size; // Increase size
140 thisFoot->Head = foot->Head; // Change backlink
141 Head->Magic = 0; // Clear old head
143 Head = foot->Head; // Save new head address
144 foot->Head = NULL; // Clear central footer
148 // Merge Right (Upwards)
149 head = Heap_NextHead(Head);
150 if((Uint)head < (Uint)gHeapEnd && head->Magic == MAGIC_FREE)
152 Head->Size += head->Size;
153 foot = Heap_ThisFoot(Head);
154 foot->Head = Head; // Update Backlink
155 thisFoot->Head = NULL; // Clear old footer
157 head->Size = 0; // Clear old header
161 // Return new address
166 * \param File Allocating source file
167 * \param Line Source line
168 * \param __Bytes Size of region to allocate
170 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
173 tHeapFoot *foot, *newfoot;
174 tHeapHead *best = NULL;
175 Uint bestSize = UINT_MAX; // Speed hack
179 return NULL; // TODO: Return a known un-mapped range.
183 #if VALIDATE_ON_ALLOC
189 Bytes = __Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot);
190 Bytes = 1UUL << LOG2(__Bytes);
192 Bytes = (__Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
196 Mutex_Acquire(&glHeap);
199 for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
203 if( head->Size != 1UUL << LOG2(head->Size) ) {
205 if( head->Size & (MIN_SIZE-1) ) {
207 Mutex_Release(&glHeap); // Release spinlock
209 Log_Warning("Heap", "Size of heap address %p is invalid"
210 " - not aligned (0x%x) [at paddr 0x%x]",
211 head, head->Size, MM_GetPhysAddr(&head->Size));
212 Heap_Dump(VERBOSE_DUMP);
216 if( head->Size < MIN_SIZE ) {
217 Mutex_Release(&glHeap);
218 Log_Warning("Heap", "Size of heap address %p is invalid"
219 " - Too small (0x%x) [at paddr 0x%x]",
220 head, head->Size, MM_GetPhysAddr(&head->Size));
221 Heap_Dump(VERBOSE_DUMP);
224 if( head->Size > (2<<30) ) {
225 Mutex_Release(&glHeap);
226 Log_Warning("Heap", "Size of heap address %p is invalid"
227 " - Over 2GiB (0x%x) [at paddr 0x%x]",
228 head, head->Size, MM_GetPhysAddr(&head->Size));
229 Heap_Dump(VERBOSE_DUMP);
233 // Check if allocated
234 if(head->Magic == MAGIC_USED) continue;
236 if(head->Magic != MAGIC_FREE) {
237 Mutex_Release(&glHeap); // Release spinlock
239 Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
240 head, &head->Magic, head->Magic);
241 Heap_Dump(VERBOSE_DUMP);
247 if(head->Size < Bytes) continue;
250 if(head->Size == Bytes) {
251 head->Magic = MAGIC_USED;
254 head->ValidSize = __Bytes;
255 head->AllocateTime = now();
256 Mutex_Release(&glHeap); // Release spinlock
258 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
259 head->Data, head->Size, File, Line);
267 bestSize = head->Size;
270 // or check if the block is the best size
271 if(bestSize > head->Size) {
273 bestSize = head->Size;
278 // If no block large enough is found, create one
281 best = Heap_Extend( Bytes );
284 Warning("OOM when allocating 0x%x bytes", Bytes);
285 Mutex_Release(&glHeap); // Release spinlock
289 if(best->Size == Bytes) {
290 best->Magic = MAGIC_USED; // Mark block as used
293 best->ValidSize = __Bytes;
294 best->AllocateTime = now();
295 Mutex_Release(&glHeap); // Release spinlock
297 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
298 best->Data, best->Size, File, Line);
305 // - Save size for new block
306 size_t newsize = best->Size - Bytes;
307 // - Allocate beginning of old block
308 best->Size = Bytes; // Update size in old header
309 best->ValidSize = __Bytes;
310 best->Magic = MAGIC_USED; // Mark block as used
313 best->AllocateTime = now();
314 // - Create a new foot on old block
315 newfoot = Heap_ThisFoot(best);
316 newfoot->Head = best; // Create new footer
317 newfoot->Magic = MAGIC_FOOT;
318 // - Create a new header after resized old
319 newhead = Heap_NextHead(best);
320 newhead->Size = newsize;
321 newhead->Magic = MAGIC_FREE;
322 newhead->ValidSize = 0;
323 newhead->File = NULL;
326 foot = Heap_ThisFoot(newhead);
327 foot->Head = newhead;
329 Mutex_Release(&glHeap); // Release spinlock
331 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
332 best->Data, best->Size, File, Line);
338 * \brief Free an allocated memory block
340 void Heap_Deallocate(const char *File, int Line, void *Ptr)
342 // INVLPTR is returned from Heap_Allocate when the allocation
344 if( Ptr == INVLPTR ) return;
347 if( (tVAddr)Ptr % sizeof(void*) != 0 ) {
348 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
353 if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
355 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
356 __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
360 // Check memory block - Header
361 tHeapHead *head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
362 if(head->Magic == MAGIC_FREE) {
363 Log_Warning("Heap", "free - Passed a freed block (%p) by %s:%i (was freed by %s:%i)",
365 head->File, head->Line);
366 Proc_PrintBacktrace();
369 if(head->Magic != MAGIC_USED) {
370 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
371 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
375 // Check memory block - Footer
376 tHeapFoot *foot = Heap_ThisFoot(head);
377 if(foot->Head != head) {
378 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
379 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
382 if(foot->Magic != MAGIC_FOOT) {
383 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)",
384 head, &foot->Magic, foot->Magic);
385 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
390 Log_Debug("Heap", "free: %p freed by %s:%i (%i old)",
391 Ptr, File, Line, now()-head->AllocateTime);
395 Mutex_Acquire( &glHeap );
398 head->Magic = MAGIC_FREE;
406 Mutex_Release( &glHeap );
410 * \brief Increase/Decrease the size of an allocation
411 * \param File Calling File
412 * \param Line Calling Line
413 * \param __ptr Old memory
414 * \param __size New Size
416 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
418 tHeapHead *head = (void*)( (Uint)__ptr-sizeof(tHeapHead) );
421 Uint newSize = (__size + sizeof(tHeapFoot)+sizeof(tHeapHead)+MIN_SIZE-1)&~(MIN_SIZE-1);
423 // Check for reallocating NULL
424 if(__ptr == NULL) return Heap_Allocate(File, Line, __size);
426 // Check if resize is needed
427 if(newSize <= head->Size) return __ptr;
429 // Check if next block is free
430 nextHead = Heap_NextHead(head);
432 // Extend into next block
433 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
435 Uint size = nextHead->Size + head->Size;
436 foot = Heap_ThisFoot(nextHead);
438 if(size == newSize) {
439 head->Size = newSize;
440 head->ValidSize = __size;
448 // Create a new heap block
449 // - Update old with new information
450 head->Size = newSize;
451 head->File = File; // Kinda counts as a new allocation
453 head->ValidSize = __size;
454 // - Create new footer
455 foot = Heap_ThisFoot(head);
457 foot->Magic = MAGIC_FOOT;
458 // - Create new header
459 nextHead = Heap_NextHead(head);
460 nextHead->Size = size - newSize;
461 nextHead->Magic = MAGIC_FREE;
462 // - Update old footer
463 foot->Head = nextHead;
468 foot = Heap_PrevFoot(head);
469 nextHead = foot->Head;
470 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
472 Uint size = nextHead->Size + head->Size;
473 // Inexact fit, split things up
476 // TODO: Handle splitting of downwards blocks
477 Warning("[Heap ] TODO: Space efficient realloc when new size is smaller");
482 // Set 1st (new/lower) header
483 nextHead->Magic = MAGIC_USED;
484 nextHead->Size = newSize;
485 nextHead->File = File;
486 nextHead->Line = Line;
487 nextHead->ValidSize = __size;
488 // Get 2nd (old) footer
489 foot = Heap_ThisFoot(nextHead);
490 foot->Head = nextHead;
491 // Save old data size
492 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
497 memmove(nextHead->Data, __ptr, oldDataSize);
499 return nextHead->Data;
503 nextHead = Heap_Allocate( File, Line, __size );
504 if(!nextHead) return NULL;
506 nextHead->File = File;
507 nextHead->Line = Line;
508 nextHead->ValidSize = __size;
513 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
518 return nextHead->Data;
522 * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
523 * \brief Allocate and Zero a buffer in memory
524 * \param File Allocating file
525 * \param Line Line of allocation
526 * \param Bytes Size of the allocation
528 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
530 void *ret = Heap_Allocate(File, Line, Bytes);
531 if(ret == NULL) return NULL;
533 memset( ret, 0, Bytes );
539 * \fn int Heap_IsHeapAddr(void *Ptr)
540 * \brief Checks if an address is a heap pointer
542 int Heap_IsHeapAddr(void *Ptr)
545 if((Uint)Ptr < (Uint)gHeapStart) return 0;
546 if((Uint)Ptr > (Uint)gHeapEnd) return 0;
547 if((Uint)Ptr & (sizeof(Uint)-1)) return 0;
549 head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
550 if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
558 void Heap_Validate(void)
560 // Call dump non-verbosely.
561 // - If a heap error is detected, it'll print
565 void Heap_Dump(int bVerbose)
567 tHeapHead *head, *badHead;
568 tHeapFoot *foot = NULL;
569 static int in_heap_dump;
571 if( in_heap_dump ) return;
576 while( (Uint)head < (Uint)gHeapEnd )
578 foot = Heap_ThisFoot(head);
582 Log_Log("Heap", "%p (%P): 0x%08x (%i) %4C",
583 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
584 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
586 Log_Log("Heap", "%sowned by %s:%i",
587 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
591 // Sanity Check Header
592 if(head->Size == 0) {
593 Log_Warning("Heap", "HALTED - Size is zero");
596 if(head->Size & (MIN_SIZE-1)) {
597 Log_Warning("Heap", "HALTED - Size is malaligned");
600 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
601 Log_Warning("Heap", "HALTED - Head Magic is Bad");
606 if(foot->Magic != MAGIC_FOOT) {
607 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
610 if(head != foot->Head) {
611 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
620 // All OK? Go to next
621 head = foot->NextHead;
624 // If the heap is valid, ok!
625 if( (tVAddr)head == (tVAddr)gHeapEnd ) {
630 // Check for a bad return
631 if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
636 // If not verbose, we need to dump the failing block
639 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
640 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
642 Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
644 Log_Log("Heap", "%sowned by %s:%i",
645 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
653 foot = Heap_PrevFoot(gHeapEnd);
654 Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
656 while( (tVAddr)head >= (tVAddr)badHead )
658 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
659 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
660 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
662 Log_Log("Heap", "%sowned by %s:%i",
663 (head->Magic!=MAGIC_USED?"was ":""),
664 head->File, head->Line);
667 // Sanity Check Header
668 if(head->Size == 0) {
669 Log_Warning("Heap", "HALTED - Size is zero");
672 if(head->Size & (MIN_SIZE-1)) {
673 Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
676 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
677 Log_Warning("Heap", "HALTED - Head Magic is Bad");
682 if(foot->Magic != MAGIC_FOOT) {
683 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
686 if(head != foot->Head) {
687 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
691 if(head == badHead) break;
693 foot = Heap_PrevFoot(head);
695 Log_Debug("Heap", "head=%p", head);
698 Panic("Heap_Dump - Heap is corrupted, kernel panic! (%p)", badHead);
701 void Heap_Stats(void)
707 int maxAlloc=0, minAlloc=-1;
708 int avgAlloc, frag, overhead;
710 for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
713 totalBytes += head->Size;
714 if( head->Magic == MAGIC_FREE )
717 freeBytes += head->Size;
719 else if( head->Magic == MAGIC_USED) {
720 if(maxAlloc < head->Size) maxAlloc = head->Size;
721 if(minAlloc == -1 || minAlloc > head->Size)
722 minAlloc = head->Size;
725 Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
729 // Print the block info?
731 if( head->Magic == MAGIC_FREE )
732 Log_Debug("Heap", "%p (%P) - 0x%x free",
733 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
735 Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
736 head->Data, MM_GetPhysAddr(&head->Data), head->Size,
737 head->ValidSize, head->File, head->Line,
738 now() - head->AllocateTime
743 Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
744 Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
746 frag = (nFree-1)*10000/nBlocks;
749 Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
753 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
755 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
758 Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
759 avgAlloc, overhead/100, overhead%100
761 Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
764 // Scan and get distribution
771 } sizeCounts[nBlocks];
774 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
776 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
778 for( i = 0; i < nBlocks; i ++ ) {
779 if( sizeCounts[i].Size == 0 )
781 if( sizeCounts[i].Size == head->Size )
784 // Should never reach this part (in a non-concurrent case)
785 if( i == nBlocks ) continue;
786 sizeCounts[i].Size = head->Size;
787 sizeCounts[i].Count ++;
789 Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
790 head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
792 Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
793 sizeCounts[i].Size, sizeCounts[i].Count);
797 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
799 Log("Heap_Stats: 0x%x - %i blocks",
800 sizeCounts[i].Size, sizeCounts[i].Count
808 EXPORT(Heap_Allocate);
809 EXPORT(Heap_AllocateZero);
810 EXPORT(Heap_Reallocate);
811 EXPORT(Heap_Deallocate);
812 EXPORT(Heap_IsHeapAddr);