3 * - By John Hodge (thePowersGang)
6 * - Dynamic memory allocation
12 #define WARNINGS 1 // Warn and dump on heap errors
13 #define DEBUG_TRACE 0 // Enable tracing of allocations
14 #define VERBOSE_DUMP 0 // Set to 1 to enable a verbose dump when heap errors are encountered
17 #define HEAP_INIT_SIZE 0x8000 // 32 KiB
18 #define MIN_SIZE (sizeof(void*))*8 // 8 Machine Words
20 #define COMPACT_HEAP 0 // Use 4 byte header?
23 //#define MAGIC_FOOT 0x2ACE5505
24 //#define MAGIC_FREE 0xACE55000
25 //#define MAGIC_USED 0x862B0505 // MAGIC_FOOT ^ MAGIC_FREE
26 #define MAGIC_FOOT 0x544F4F46 // 'FOOT'
27 #define MAGIC_FREE 0x45455246 // 'FREE'
28 #define MAGIC_USED 0x44455355 // 'USED'
31 void Heap_Install(void);
32 void *Heap_Extend(size_t Bytes);
33 void *Heap_Merge(tHeapHead *Head);
34 //void *Heap_Allocate(const char *File, int Line, size_t Bytes);
35 //void *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
36 //void *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
37 //void Heap_Deallocate(const char *File, int Line, void *Ptr);
38 void Heap_Dump(int bVerbose);
39 void Heap_Stats(void);
43 tHeapHead *gHeapStart;
47 void Heap_Install(void)
49 gHeapStart = (void*)MM_KHEAP_BASE;
50 gHeapEnd = gHeapStart;
51 Heap_Extend(HEAP_INIT_SIZE);
55 * \brief Extend the size of the heap
57 void *Heap_Extend(size_t Bytes)
59 tHeapHead *head = gHeapEnd;
63 if( gHeapEnd == (tHeapHead*)MM_KHEAP_MAX ) {
64 Log_Error("Heap", "Heap limit reached (%p)", (void*)MM_KHEAP_MAX);
69 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
73 const Uint pages = (Bytes + 0xFFF) >> 12;
74 tHeapHead *new_end = (void*)( (tVAddr)gHeapEnd + (pages << 12) );
76 if( new_end > (tHeapHead*)MM_KHEAP_MAX )
78 Log_Error("Heap", "Heap limit exceeded (%p)", (void*)new_end);
79 // TODO: Clip allocation to avaliable space, and have caller check returned block
83 // Heap expands in pages
84 for( Uint i = 0; i < pages; i ++ )
86 if( !MM_Allocate( (tVAddr)gHeapEnd+(i<<12) ) )
88 Warning("OOM - Heap_Extend");
98 head->Size = (Bytes+0xFFF)&~0xFFF;
99 head->Magic = MAGIC_FREE;
100 foot = (void*)( (Uint)gHeapEnd - sizeof(tHeapFoot) );
102 foot->Magic = MAGIC_FOOT;
104 return Heap_Merge(head); // Merge with previous block
108 * \brief Merges two ajacent heap blocks
110 void *Heap_Merge(tHeapHead *Head)
116 //Log("Heap_Merge: (Head=%p)", Head);
118 thisFoot = (void*)( (Uint)Head + Head->Size - sizeof(tHeapFoot) );
119 if((Uint)thisFoot > (Uint)gHeapEnd) return NULL;
122 foot = (void*)( (Uint)Head - sizeof(tHeapFoot) );
123 if( ((Uint)foot < (Uint)gHeapEnd && (Uint)foot > (Uint)gHeapStart)
124 && foot->Head->Magic == MAGIC_FREE) {
125 foot->Head->Size += Head->Size; // Increase size
126 thisFoot->Head = foot->Head; // Change backlink
127 Head->Magic = 0; // Clear old head
129 Head = foot->Head; // Save new head address
130 foot->Head = NULL; // Clear central footer
134 // Merge Right (Upwards)
135 head = (void*)( (Uint)Head + Head->Size );
136 if((Uint)head < (Uint)gHeapEnd && head->Magic == MAGIC_FREE)
138 Head->Size += head->Size;
139 foot = (void*)( (Uint)Head + Head->Size - sizeof(tHeapFoot) );
140 foot->Head = Head; // Update Backlink
141 thisFoot->Head = NULL; // Clear old footer
143 head->Size = 0; // Clear old header
147 // Return new address
152 * \param File Allocating source file
153 * \param Line Source line
154 * \param __Bytes Size of region to allocate
156 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
158 tHeapHead *head, *newhead;
159 tHeapFoot *foot, *newfoot;
160 tHeapHead *best = NULL;
161 Uint bestSize = 0; // Speed hack
165 return NULL; // TODO: Return a known un-mapped range.
171 Bytes = __Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot);
172 Bytes = 1UUL << LOG2(__Bytes);
174 Bytes = (__Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
178 Mutex_Acquire(&glHeap);
181 for( head = gHeapStart; head < gHeapEnd; head = (void*)((Uint)head + head->Size) )
185 if( head->Size != 1UUL << LOG2(head->Size) ) {
187 if( head->Size & (MIN_SIZE-1) ) {
189 Mutex_Release(&glHeap); // Release spinlock
191 Log_Warning("Heap", "Size of heap address %p is invalid - not aligned (0x%x) [at paddr 0x%x]",
192 head, head->Size, MM_GetPhysAddr(&head->Size));
193 Heap_Dump(VERBOSE_DUMP);
197 if( head->Size < MIN_SIZE ) {
198 Mutex_Release(&glHeap);
199 Log_Warning("Heap", "Size of heap address %p is invalid - Too small (0x%x) [at paddr 0x%x]",
200 head, head->Size, MM_GetPhysAddr(&head->Size));
201 Heap_Dump(VERBOSE_DUMP);
204 if( head->Size > (2<<30) ) {
205 Mutex_Release(&glHeap);
206 Log_Warning("Heap", "Size of heap address %p is invalid - Over 2GiB (0x%x) [at paddr 0x%x]",
207 head, head->Size, MM_GetPhysAddr(&head->Size));
208 Heap_Dump(VERBOSE_DUMP);
212 // Check if allocated
213 if(head->Magic == MAGIC_USED) continue;
215 if(head->Magic != MAGIC_FREE) {
216 Mutex_Release(&glHeap); // Release spinlock
218 Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
219 head, &head->Magic, head->Magic);
220 Heap_Dump(VERBOSE_DUMP);
226 if(head->Size < Bytes) continue;
229 if(head->Size == Bytes) {
230 head->Magic = MAGIC_USED;
233 head->ValidSize = __Bytes;
234 head->AllocateTime = now();
235 Mutex_Release(&glHeap); // Release spinlock
237 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
238 head->Data, head->Size, File, Line);
246 bestSize = head->Size;
249 // or check if the block is the best size
250 if(bestSize > head->Size) {
252 bestSize = head->Size;
257 // If no block large enough is found, create one
260 best = Heap_Extend( Bytes );
263 Mutex_Release(&glHeap); // Release spinlock
267 if(best->Size == Bytes) {
268 best->Magic = MAGIC_USED; // Mark block as used
271 best->ValidSize = __Bytes;
272 best->AllocateTime = now();
273 Mutex_Release(&glHeap); // Release spinlock
275 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
276 best->Data, best->Size, File, Line);
283 newhead = (void*)( (Uint)best + Bytes );
284 newfoot = (void*)( (Uint)best + Bytes - sizeof(tHeapFoot) );
285 foot = (void*)( (Uint)best + best->Size - sizeof(tHeapFoot) );
287 newfoot->Head = best; // Create new footer
288 newfoot->Magic = MAGIC_FOOT;
289 newhead->Size = best->Size - Bytes; // Create new header
290 newhead->Magic = MAGIC_FREE;
291 foot->Head = newhead; // Update backlink in old footer
292 best->Size = Bytes; // Update size in old header
293 best->ValidSize = __Bytes;
294 best->Magic = MAGIC_USED; // Mark block as used
297 best->AllocateTime = now();
299 Mutex_Release(&glHeap); // Release spinlock
301 Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
302 best->Data, best->Size, File, Line);
308 * \brief Free an allocated memory block
310 void Heap_Deallocate(const char *File, int Line, void *Ptr)
312 tHeapHead *head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
315 // INVLPTR is returned from Heap_Allocate when the allocation
317 if( Ptr == INVLPTR ) return;
320 if( (Uint)Ptr & (sizeof(Uint)-1) ) {
321 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
326 if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
328 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
329 __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
333 // Check memory block - Header
334 head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
335 if(head->Magic == MAGIC_FREE) {
336 Log_Warning("Heap", "free - Passed a freed block (%p) by %s:%i (was freed by %s:%i)",
338 head->File, head->Line);
341 if(head->Magic != MAGIC_USED) {
342 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
343 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
347 // Check memory block - Footer
348 foot = (void*)( (Uint)head + head->Size - sizeof(tHeapFoot) );
349 if(foot->Head != head) {
350 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
351 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
354 if(foot->Magic != MAGIC_FOOT) {
355 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)", head, &foot->Magic, foot->Magic);
356 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
361 Log_Debug("Heap", "free: %p freed by %s:%i (%i old)",
362 Ptr, File, Line, now()-head->AllocateTime);
366 Mutex_Acquire( &glHeap );
369 head->Magic = MAGIC_FREE;
377 Mutex_Release( &glHeap );
381 * \brief Increase/Decrease the size of an allocation
382 * \param File Calling File
383 * \param Line Calling Line
384 * \param __ptr Old memory
385 * \param __size New Size
387 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
389 tHeapHead *head = (void*)( (Uint)__ptr-sizeof(tHeapHead) );
392 Uint newSize = (__size + sizeof(tHeapFoot)+sizeof(tHeapHead)+MIN_SIZE-1)&~(MIN_SIZE-1);
394 // Check for reallocating NULL
395 if(__ptr == NULL) return Heap_Allocate(File, Line, __size);
397 // Check if resize is needed
398 if(newSize <= head->Size) return __ptr;
400 // Check if next block is free
401 nextHead = (void*)( (Uint)head + head->Size );
403 // Extend into next block
404 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
406 Uint size = nextHead->Size + head->Size;
407 foot = (void*)( (Uint)nextHead + nextHead->Size - sizeof(tHeapFoot) );
409 if(size == newSize) {
410 head->Size = newSize;
411 head->ValidSize = __size;
419 // Create a new heap block
420 nextHead = (void*)( (Uint)head + newSize );
421 nextHead->Size = size - newSize;
422 nextHead->Magic = MAGIC_FREE;
423 foot->Head = nextHead; // Edit 2nd footer
424 head->Size = newSize; // Edit first header
427 head->ValidSize = __size;
429 foot = (void*)( (Uint)head + newSize - sizeof(tHeapFoot) );
431 foot->Magic = MAGIC_FOOT;
436 foot = (void*)( (Uint)head - sizeof(tHeapFoot) );
437 nextHead = foot->Head;
438 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
440 Uint size = nextHead->Size + head->Size;
441 // Inexact fit, split things up
445 Warning("[Heap ] TODO: Space efficient realloc when new size is smaller");
452 // Set 1st (new/lower) header
453 nextHead->Magic = MAGIC_USED;
454 nextHead->Size = newSize;
455 nextHead->File = File;
456 nextHead->Line = Line;
457 nextHead->ValidSize = __size;
458 // Get 2nd (old) footer
459 foot = (void*)( (Uint)nextHead + newSize );
460 foot->Head = nextHead;
461 // Save old data size
462 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
467 memcpy(nextHead->Data, __ptr, oldDataSize);
469 return nextHead->Data;
471 // On to the expensive then
475 nextHead = Heap_Allocate( File, Line, __size );
476 if(!nextHead) return NULL;
478 nextHead->File = File;
479 nextHead->Line = Line;
480 nextHead->ValidSize = __size;
485 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
490 return nextHead->Data;
494 * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
495 * \brief Allocate and Zero a buffer in memory
496 * \param File Allocating file
497 * \param Line Line of allocation
498 * \param Bytes Size of the allocation
500 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
502 void *ret = Heap_Allocate(File, Line, Bytes);
503 if(ret == NULL) return NULL;
505 memset( ret, 0, Bytes );
511 * \fn int Heap_IsHeapAddr(void *Ptr)
512 * \brief Checks if an address is a heap pointer
514 int Heap_IsHeapAddr(void *Ptr)
517 if((Uint)Ptr < (Uint)gHeapStart) return 0;
518 if((Uint)Ptr > (Uint)gHeapEnd) return 0;
519 if((Uint)Ptr & (sizeof(Uint)-1)) return 0;
521 head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
522 if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
530 void Heap_Validate(void)
532 // Call dump non-verbosely.
533 // - If a heap error is detected, it'll print
537 void Heap_Dump(int bVerbose)
539 tHeapHead *head, *badHead;
540 tHeapFoot *foot = NULL;
541 static int in_heap_dump;
543 if( in_heap_dump ) return;
548 while( (Uint)head < (Uint)gHeapEnd )
550 foot = (void*)( (Uint)head + head->Size - sizeof(tHeapFoot) );
554 Log_Log("Heap", "%p (0x%P): 0x%08x (%i) %4C",
555 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
556 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
558 Log_Log("Heap", "%sowned by %s:%i",
559 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
563 // Sanity Check Header
564 if(head->Size == 0) {
565 Log_Warning("Heap", "HALTED - Size is zero");
568 if(head->Size & (MIN_SIZE-1)) {
569 Log_Warning("Heap", "HALTED - Size is malaligned");
572 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
573 Log_Warning("Heap", "HALTED - Head Magic is Bad");
578 if(foot->Magic != MAGIC_FOOT) {
579 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
582 if(head != foot->Head) {
583 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
592 // All OK? Go to next
593 head = foot->NextHead;
596 // If the heap is valid, ok!
597 if( (tVAddr)head == (tVAddr)gHeapEnd ) {
602 // Check for a bad return
603 if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
608 // If not verbose, we need to dump the failing block
611 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
612 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
614 Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
616 Log_Log("Heap", "%sowned by %s:%i",
617 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
626 foot = (void*)( (tVAddr)gHeapEnd - sizeof(tHeapFoot) );
627 Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
629 while( (tVAddr)head >= (tVAddr)badHead )
631 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
632 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
633 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
635 Log_Log("Heap", "%sowned by %s:%i",
636 (head->Magic!=MAGIC_USED?"was ":""),
637 head->File, head->Line);
640 // Sanity Check Header
641 if(head->Size == 0) {
642 Log_Warning("Heap", "HALTED - Size is zero");
645 if(head->Size & (MIN_SIZE-1)) {
646 Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
649 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
650 Log_Warning("Heap", "HALTED - Head Magic is Bad");
655 if(foot->Magic != MAGIC_FOOT) {
656 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
659 if(head != foot->Head) {
660 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
664 if(head == badHead) break;
666 foot = (void*)( (tVAddr)head - sizeof(tHeapFoot) );
668 Log_Debug("Heap", "head=%p", head);
671 Panic("Heap_Dump - Heap is corrupted, kernel panic!");
674 void Heap_Stats(void)
680 int maxAlloc=0, minAlloc=-1;
681 int avgAlloc, frag, overhead;
683 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
686 totalBytes += head->Size;
687 if( head->Magic == MAGIC_FREE )
690 freeBytes += head->Size;
692 else if( head->Magic == MAGIC_USED) {
693 if(maxAlloc < head->Size) maxAlloc = head->Size;
694 if(minAlloc == -1 || minAlloc > head->Size)
695 minAlloc = head->Size;
698 Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
702 // Print the block info?
704 if( head->Magic == MAGIC_FREE )
705 Log_Debug("Heap", "%p (%P) - 0x%x free",
706 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
708 Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
709 head->Data, MM_GetPhysAddr(&head->Data), head->Size,
710 head->ValidSize, head->File, head->Line,
711 now() - head->AllocateTime
716 Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
717 Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
719 frag = (nFree-1)*10000/nBlocks;
722 Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
726 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
728 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
731 Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
732 avgAlloc, overhead/100, overhead%100
734 Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
737 // Scan and get distribution
744 } sizeCounts[nBlocks];
747 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
749 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
751 for( i = 0; i < nBlocks; i ++ ) {
752 if( sizeCounts[i].Size == 0 )
754 if( sizeCounts[i].Size == head->Size )
757 // Should never reach this part (in a non-concurrent case)
758 if( i == nBlocks ) continue;
759 sizeCounts[i].Size = head->Size;
760 sizeCounts[i].Count ++;
762 Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
763 head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
765 Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
766 sizeCounts[i].Size, sizeCounts[i].Count);
770 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
772 Log("Heap_Stats: 0x%x - %i blocks",
773 sizeCounts[i].Size, sizeCounts[i].Count
781 EXPORT(Heap_Allocate);
782 EXPORT(Heap_AllocateZero);
783 EXPORT(Heap_Reallocate);
784 EXPORT(Heap_Deallocate);
785 EXPORT(Heap_IsHeapAddr);