3 * - By John Hodge (thePowersGang)
6 * - Dynamic memory allocation
13 #define WARNINGS 1 // Warn and dump on heap errors
14 #define DEBUG_TRACE 0 // Enable tracing of allocations
15 #define VERBOSE_DUMP 0 // Set to 1 to enable a verbose dump when heap errors are encountered
18 #define HEAP_INIT_SIZE 0x8000 // 32 KiB
19 #define MIN_SIZE (sizeof(void*))*8 // 8 Machine Words
21 #define COMPACT_HEAP 0 // Use 4 byte header?
24 //#define MAGIC_FOOT 0x2ACE5505
25 //#define MAGIC_FREE 0xACE55000
26 //#define MAGIC_USED 0x862B0505 // MAGIC_FOOT ^ MAGIC_FREE
27 #define MAGIC_FOOT 0x544F4F46 // 'FOOT'
28 #define MAGIC_FREE 0x45455246 // 'FREE'
29 #define MAGIC_USED 0x44455355 // 'USED'
32 void Heap_Install(void);
33 void *Heap_Extend(size_t Bytes);
34 void *Heap_Merge(tHeapHead *Head);
35 //void *Heap_Allocate(const char *File, int Line, size_t Bytes);
36 //void *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
37 //void *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
38 //void Heap_Deallocate(const char *File, int Line, void *Ptr);
39 void Heap_Dump(int bVerbose);
40 void Heap_Stats(void);
44 tHeapHead *gHeapStart;
48 void Heap_Install(void)
50 gHeapStart = (void*)MM_KHEAP_BASE;
51 gHeapEnd = gHeapStart;
52 Heap_Extend(HEAP_INIT_SIZE);
55 static inline tHeapHead *Heap_NextHead(tHeapHead *Head) {
56 return (void*)( (char*)Head + Head->Size );
58 static inline tHeapFoot *Heap_ThisFoot(tHeapHead *Head) {
59 return (void*)( (char*)Head + Head->Size - sizeof(tHeapFoot) );
61 static inline tHeapFoot *Heap_PrevFoot(tHeapHead *Head) {
62 //ASSERT(Head != gHeapStart);
63 return (void*)( (char*)Head - sizeof(tHeapFoot) );
67 * \brief Extend the size of the heap
69 void *Heap_Extend(size_t Bytes)
72 Debug("Heap_Extend(0x%x)", Bytes);
75 if( gHeapEnd == (tHeapHead*)MM_KHEAP_MAX ) {
76 Log_Error("Heap", "Heap limit reached (%p)", (void*)MM_KHEAP_MAX);
81 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
85 const Uint pages = (Bytes + 0xFFF) >> 12;
86 tHeapHead *new_end = (void*)( (tVAddr)gHeapEnd + (pages << 12) );
88 if( new_end > (tHeapHead*)MM_KHEAP_MAX )
90 Log_Error("Heap", "Heap limit exceeded (%p)", (void*)new_end);
91 // TODO: Clip allocation to avaliable space, and have caller check returned block
95 // Heap expands in pages
96 for( Uint i = 0; i < pages; i ++ )
98 if( !MM_Allocate( (tVAddr)gHeapEnd+(i<<12) ) )
100 Warning("OOM - Heap_Extend (%i bytes)");
107 tHeapHead *head = gHeapEnd;
111 head->Size = (Bytes+0xFFF)&~0xFFF;
112 head->Magic = MAGIC_FREE;
113 tHeapFoot *foot = Heap_ThisFoot(head);
115 foot->Magic = MAGIC_FOOT;
117 return Heap_Merge(head); // Merge with previous block
121 * \brief Merges two ajacent heap blocks
123 void *Heap_Merge(tHeapHead *Head)
129 //Log("Heap_Merge: (Head=%p)", Head);
131 thisFoot = Heap_ThisFoot(Head);
132 if( (void*)thisFoot > (void*)gHeapEnd )
136 foot = Heap_PrevFoot(Head);
137 if( ((Uint)foot < (Uint)gHeapEnd && (Uint)foot > (Uint)gHeapStart)
138 && foot->Head->Magic == MAGIC_FREE) {
139 foot->Head->Size += Head->Size; // Increase size
140 thisFoot->Head = foot->Head; // Change backlink
141 Head->Magic = 0; // Clear old head
143 Head = foot->Head; // Save new head address
144 foot->Head = NULL; // Clear central footer
148 // Merge Right (Upwards)
149 head = Heap_NextHead(Head);
150 if((Uint)head < (Uint)gHeapEnd && head->Magic == MAGIC_FREE)
152 Head->Size += head->Size;
153 foot = Heap_ThisFoot(Head);
154 foot->Head = Head; // Update Backlink
155 thisFoot->Head = NULL; // Clear old footer
157 head->Size = 0; // Clear old header
161 // Return new address
166 * \param File Allocating source file
167 * \param Line Source line
168 * \param __Bytes Size of region to allocate
170 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
173 tHeapFoot *foot, *newfoot;
174 tHeapHead *best = NULL;
175 Uint bestSize = UINT_MAX; // Speed hack
179 return NULL; // TODO: Return a known un-mapped range.
185 Bytes = __Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot);
186 Bytes = 1UUL << LOG2(__Bytes);
188 Bytes = (__Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
192 Mutex_Acquire(&glHeap);
195 for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
199 if( head->Size != 1UUL << LOG2(head->Size) ) {
201 if( head->Size & (MIN_SIZE-1) ) {
203 Mutex_Release(&glHeap); // Release spinlock
205 Log_Warning("Heap", "Size of heap address %p is invalid"
206 " - not aligned (0x%x) [at paddr 0x%x]",
207 head, head->Size, MM_GetPhysAddr(&head->Size));
208 Heap_Dump(VERBOSE_DUMP);
212 if( head->Size < MIN_SIZE ) {
213 Mutex_Release(&glHeap);
214 Log_Warning("Heap", "Size of heap address %p is invalid"
215 " - Too small (0x%x) [at paddr 0x%x]",
216 head, head->Size, MM_GetPhysAddr(&head->Size));
217 Heap_Dump(VERBOSE_DUMP);
220 if( head->Size > (2<<30) ) {
221 Mutex_Release(&glHeap);
222 Log_Warning("Heap", "Size of heap address %p is invalid"
223 " - Over 2GiB (0x%x) [at paddr 0x%x]",
224 head, head->Size, MM_GetPhysAddr(&head->Size));
225 Heap_Dump(VERBOSE_DUMP);
229 // Check if allocated
230 if(head->Magic == MAGIC_USED) continue;
232 if(head->Magic != MAGIC_FREE) {
233 Mutex_Release(&glHeap); // Release spinlock
235 Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
236 head, &head->Magic, head->Magic);
237 Heap_Dump(VERBOSE_DUMP);
243 if(head->Size < Bytes) continue;
246 if(head->Size == Bytes) {
247 head->Magic = MAGIC_USED;
250 head->ValidSize = __Bytes;
251 head->AllocateTime = now();
252 Mutex_Release(&glHeap); // Release spinlock
254 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
255 head->Data, head->Size, File, Line);
263 bestSize = head->Size;
266 // or check if the block is the best size
267 if(bestSize > head->Size) {
269 bestSize = head->Size;
274 // If no block large enough is found, create one
277 best = Heap_Extend( Bytes );
280 Warning("OOM when allocating 0x%x bytes", Bytes);
281 Mutex_Release(&glHeap); // Release spinlock
285 if(best->Size == Bytes) {
286 best->Magic = MAGIC_USED; // Mark block as used
289 best->ValidSize = __Bytes;
290 best->AllocateTime = now();
291 Mutex_Release(&glHeap); // Release spinlock
293 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
294 best->Data, best->Size, File, Line);
301 // - Save size for new block
302 size_t newsize = best->Size - Bytes;
303 // - Allocate beginning of old block
304 best->Size = Bytes; // Update size in old header
305 best->ValidSize = __Bytes;
306 best->Magic = MAGIC_USED; // Mark block as used
309 best->AllocateTime = now();
310 // - Create a new foot on old block
311 newfoot = Heap_ThisFoot(best);
312 newfoot->Head = best; // Create new footer
313 newfoot->Magic = MAGIC_FOOT;
314 // - Create a new header after resized old
315 newhead = Heap_NextHead(best);
316 newhead->Size = newsize;
317 newhead->Magic = MAGIC_FREE;
318 newhead->ValidSize = 0;
319 newhead->File = NULL;
322 foot = Heap_ThisFoot(newhead);
323 foot->Head = newhead;
325 Mutex_Release(&glHeap); // Release spinlock
327 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
328 best->Data, best->Size, File, Line);
334 * \brief Free an allocated memory block
336 void Heap_Deallocate(const char *File, int Line, void *Ptr)
338 // INVLPTR is returned from Heap_Allocate when the allocation
340 if( Ptr == INVLPTR ) return;
343 if( (tVAddr)Ptr % sizeof(void*) != 0 ) {
344 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
349 if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
351 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
352 __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
356 // Check memory block - Header
357 tHeapHead *head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
358 if(head->Magic == MAGIC_FREE) {
359 Log_Warning("Heap", "free - Passed a freed block (%p) by %s:%i (was freed by %s:%i)",
361 head->File, head->Line);
364 if(head->Magic != MAGIC_USED) {
365 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
366 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
370 // Check memory block - Footer
371 tHeapFoot *foot = Heap_ThisFoot(head);
372 if(foot->Head != head) {
373 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
374 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
377 if(foot->Magic != MAGIC_FOOT) {
378 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)",
379 head, &foot->Magic, foot->Magic);
380 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
385 Log_Debug("Heap", "free: %p freed by %s:%i (%i old)",
386 Ptr, File, Line, now()-head->AllocateTime);
390 Mutex_Acquire( &glHeap );
393 head->Magic = MAGIC_FREE;
401 Mutex_Release( &glHeap );
405 * \brief Increase/Decrease the size of an allocation
406 * \param File Calling File
407 * \param Line Calling Line
408 * \param __ptr Old memory
409 * \param __size New Size
411 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
413 tHeapHead *head = (void*)( (Uint)__ptr-sizeof(tHeapHead) );
416 Uint newSize = (__size + sizeof(tHeapFoot)+sizeof(tHeapHead)+MIN_SIZE-1)&~(MIN_SIZE-1);
418 // Check for reallocating NULL
419 if(__ptr == NULL) return Heap_Allocate(File, Line, __size);
421 // Check if resize is needed
422 if(newSize <= head->Size) return __ptr;
424 // Check if next block is free
425 nextHead = Heap_NextHead(head);
427 // Extend into next block
428 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
430 Uint size = nextHead->Size + head->Size;
431 foot = Heap_ThisFoot(nextHead);
433 if(size == newSize) {
434 head->Size = newSize;
435 head->ValidSize = __size;
443 // Create a new heap block
444 // - Update old with new information
445 head->Size = newSize;
446 head->File = File; // Kinda counts as a new allocation
448 head->ValidSize = __size;
449 // - Create new footer
450 foot = Heap_ThisFoot(head);
452 foot->Magic = MAGIC_FOOT;
453 // - Create new header
454 nextHead = Heap_NextHead(head);
455 nextHead->Size = size - newSize;
456 nextHead->Magic = MAGIC_FREE;
457 // - Update old footer
458 foot->Head = nextHead;
463 foot = Heap_PrevFoot(head);
464 nextHead = foot->Head;
465 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
467 Uint size = nextHead->Size + head->Size;
468 // Inexact fit, split things up
471 // TODO: Handle splitting of downwards blocks
472 Warning("[Heap ] TODO: Space efficient realloc when new size is smaller");
477 // Set 1st (new/lower) header
478 nextHead->Magic = MAGIC_USED;
479 nextHead->Size = newSize;
480 nextHead->File = File;
481 nextHead->Line = Line;
482 nextHead->ValidSize = __size;
483 // Get 2nd (old) footer
484 foot = Heap_ThisFoot(nextHead);
485 foot->Head = nextHead;
486 // Save old data size
487 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
492 memmove(nextHead->Data, __ptr, oldDataSize);
494 return nextHead->Data;
498 nextHead = Heap_Allocate( File, Line, __size );
499 if(!nextHead) return NULL;
501 nextHead->File = File;
502 nextHead->Line = Line;
503 nextHead->ValidSize = __size;
508 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
513 return nextHead->Data;
517 * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
518 * \brief Allocate and Zero a buffer in memory
519 * \param File Allocating file
520 * \param Line Line of allocation
521 * \param Bytes Size of the allocation
523 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
525 void *ret = Heap_Allocate(File, Line, Bytes);
526 if(ret == NULL) return NULL;
528 memset( ret, 0, Bytes );
534 * \fn int Heap_IsHeapAddr(void *Ptr)
535 * \brief Checks if an address is a heap pointer
537 int Heap_IsHeapAddr(void *Ptr)
540 if((Uint)Ptr < (Uint)gHeapStart) return 0;
541 if((Uint)Ptr > (Uint)gHeapEnd) return 0;
542 if((Uint)Ptr & (sizeof(Uint)-1)) return 0;
544 head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
545 if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
553 void Heap_Validate(void)
555 // Call dump non-verbosely.
556 // - If a heap error is detected, it'll print
560 void Heap_Dump(int bVerbose)
562 tHeapHead *head, *badHead;
563 tHeapFoot *foot = NULL;
564 static int in_heap_dump;
566 if( in_heap_dump ) return;
571 while( (Uint)head < (Uint)gHeapEnd )
573 foot = Heap_ThisFoot(head);
577 Log_Log("Heap", "%p (%P): 0x%08x (%i) %4C",
578 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
579 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
581 Log_Log("Heap", "%sowned by %s:%i",
582 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
586 // Sanity Check Header
587 if(head->Size == 0) {
588 Log_Warning("Heap", "HALTED - Size is zero");
591 if(head->Size & (MIN_SIZE-1)) {
592 Log_Warning("Heap", "HALTED - Size is malaligned");
595 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
596 Log_Warning("Heap", "HALTED - Head Magic is Bad");
601 if(foot->Magic != MAGIC_FOOT) {
602 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
605 if(head != foot->Head) {
606 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
615 // All OK? Go to next
616 head = foot->NextHead;
619 // If the heap is valid, ok!
620 if( (tVAddr)head == (tVAddr)gHeapEnd ) {
625 // Check for a bad return
626 if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
631 // If not verbose, we need to dump the failing block
634 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
635 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
637 Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
639 Log_Log("Heap", "%sowned by %s:%i",
640 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
649 foot = Heap_PrevFoot(gHeapEnd);
650 Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
652 while( (tVAddr)head >= (tVAddr)badHead )
654 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
655 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
656 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
658 Log_Log("Heap", "%sowned by %s:%i",
659 (head->Magic!=MAGIC_USED?"was ":""),
660 head->File, head->Line);
663 // Sanity Check Header
664 if(head->Size == 0) {
665 Log_Warning("Heap", "HALTED - Size is zero");
668 if(head->Size & (MIN_SIZE-1)) {
669 Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
672 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
673 Log_Warning("Heap", "HALTED - Head Magic is Bad");
678 if(foot->Magic != MAGIC_FOOT) {
679 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
682 if(head != foot->Head) {
683 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
687 if(head == badHead) break;
689 foot = Heap_PrevFoot(head);
691 Log_Debug("Heap", "head=%p", head);
694 Panic("Heap_Dump - Heap is corrupted, kernel panic!");
697 void Heap_Stats(void)
703 int maxAlloc=0, minAlloc=-1;
704 int avgAlloc, frag, overhead;
706 for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
709 totalBytes += head->Size;
710 if( head->Magic == MAGIC_FREE )
713 freeBytes += head->Size;
715 else if( head->Magic == MAGIC_USED) {
716 if(maxAlloc < head->Size) maxAlloc = head->Size;
717 if(minAlloc == -1 || minAlloc > head->Size)
718 minAlloc = head->Size;
721 Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
725 // Print the block info?
727 if( head->Magic == MAGIC_FREE )
728 Log_Debug("Heap", "%p (%P) - 0x%x free",
729 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
731 Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
732 head->Data, MM_GetPhysAddr(&head->Data), head->Size,
733 head->ValidSize, head->File, head->Line,
734 now() - head->AllocateTime
739 Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
740 Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
742 frag = (nFree-1)*10000/nBlocks;
745 Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
749 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
751 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
754 Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
755 avgAlloc, overhead/100, overhead%100
757 Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
760 // Scan and get distribution
767 } sizeCounts[nBlocks];
770 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
772 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
774 for( i = 0; i < nBlocks; i ++ ) {
775 if( sizeCounts[i].Size == 0 )
777 if( sizeCounts[i].Size == head->Size )
780 // Should never reach this part (in a non-concurrent case)
781 if( i == nBlocks ) continue;
782 sizeCounts[i].Size = head->Size;
783 sizeCounts[i].Count ++;
785 Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
786 head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
788 Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
789 sizeCounts[i].Size, sizeCounts[i].Count);
793 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
795 Log("Heap_Stats: 0x%x - %i blocks",
796 sizeCounts[i].Size, sizeCounts[i].Count
804 EXPORT(Heap_Allocate);
805 EXPORT(Heap_AllocateZero);
806 EXPORT(Heap_Reallocate);
807 EXPORT(Heap_Deallocate);
808 EXPORT(Heap_IsHeapAddr);