2 * AcessOS Microkernel Version
11 #define VERBOSE_DUMP 0
14 #define HEAP_INIT_SIZE 0x8000 // 32 KiB
15 #define MIN_SIZE (sizeof(void*))*8 // 8 Machine Words
17 #define COMPACT_HEAP 0 // Use 4 byte header?
20 //#define MAGIC_FOOT 0x2ACE5505
21 //#define MAGIC_FREE 0xACE55000
22 //#define MAGIC_USED 0x862B0505 // MAGIC_FOOT ^ MAGIC_FREE
23 #define MAGIC_FOOT 0x544F4F46 // 'FOOT'
24 #define MAGIC_FREE 0x45455246 // 'FREE'
25 #define MAGIC_USED 0x44455355 // 'USED'
28 void Heap_Install(void);
29 void *Heap_Extend(int Bytes);
30 void *Heap_Merge(tHeapHead *Head);
31 //void *Heap_Allocate(const char *File, int Line, size_t Bytes);
32 //void *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
33 //void *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
34 //void Heap_Deallocate(void *Ptr);
36 void Heap_Stats(void);
44 void Heap_Install(void)
46 gHeapStart = (void*)MM_KHEAP_BASE;
47 gHeapEnd = (void*)MM_KHEAP_BASE;
48 Heap_Extend(HEAP_INIT_SIZE);
52 * \brief Extend the size of the heap
54 void *Heap_Extend(int Bytes)
57 tHeapHead *head = gHeapEnd;
61 if( (tVAddr)gHeapEnd == MM_KHEAP_MAX )
65 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
70 if( (tVAddr)gHeapEnd + ((Bytes+0xFFF)&~0xFFF) > MM_KHEAP_MAX ) {
71 // Bytes = MM_KHEAP_MAX - (tVAddr)gHeapEnd;
75 // Heap expands in pages
76 for( i = 0; i < (Bytes+0xFFF) >> 12; i ++ )
78 if( !MM_Allocate( (tVAddr)gHeapEnd+(i<<12) ) )
80 Warning("OOM - Heap_Extend");
86 gHeapEnd = (Uint8*)gHeapEnd + (i << 12);
89 head->Size = (Bytes+0xFFF)&~0xFFF;
90 head->Magic = MAGIC_FREE;
91 foot = (void*)( (Uint)gHeapEnd - sizeof(tHeapFoot) );
93 foot->Magic = MAGIC_FOOT;
95 return Heap_Merge(head); // Merge with previous block
99 * \brief Merges two ajacent heap blocks
101 void *Heap_Merge(tHeapHead *Head)
107 //Log("Heap_Merge: (Head=%p)", Head);
109 thisFoot = (void*)( (Uint)Head + Head->Size - sizeof(tHeapFoot) );
110 if((Uint)thisFoot > (Uint)gHeapEnd) return NULL;
113 foot = (void*)( (Uint)Head - sizeof(tHeapFoot) );
114 if( ((Uint)foot < (Uint)gHeapEnd && (Uint)foot > (Uint)gHeapStart)
115 && foot->Head->Magic == MAGIC_FREE) {
116 foot->Head->Size += Head->Size; // Increase size
117 thisFoot->Head = foot->Head; // Change backlink
118 Head->Magic = 0; // Clear old head
120 Head = foot->Head; // Save new head address
121 foot->Head = NULL; // Clear central footer
125 // Merge Right (Upwards)
126 head = (void*)( (Uint)Head + Head->Size );
127 if((Uint)head < (Uint)gHeapEnd && head->Magic == MAGIC_FREE)
129 Head->Size += head->Size;
130 foot = (void*)( (Uint)Head + Head->Size - sizeof(tHeapFoot) );
131 foot->Head = Head; // Update Backlink
132 thisFoot->Head = NULL; // Clear old footer
134 head->Size = 0; // Clear old header
138 // Return new address
143 * \param File Allocating source file
144 * \param Line Source line
145 * \param __Bytes Size of region to allocate
147 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
149 tHeapHead *head, *newhead;
150 tHeapFoot *foot, *newfoot;
151 tHeapHead *best = NULL;
152 Uint bestSize = 0; // Speed hack
156 return NULL; // TODO: Return a known un-mapped range.
162 Bytes = __Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot);
163 Bytes = 1UUL << LOG2(__Bytes);
165 Bytes = (__Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
169 Mutex_Acquire(&glHeap);
172 for( head = gHeapStart;
173 (Uint)head < (Uint)gHeapEnd;
174 head = (void*)((Uint)head + head->Size)
179 if( head->Size != 1UUL << LOG2(head->Size) ) {
181 if( head->Size & (MIN_SIZE-1) ) {
183 Mutex_Release(&glHeap); // Release spinlock
185 Log_Warning("Heap", "Size of heap address %p is invalid - not aligned (0x%x) [at paddr 0x%x]",
186 head, head->Size, MM_GetPhysAddr(&head->Size));
191 if( head->Size < MIN_SIZE ) {
192 Mutex_Release(&glHeap);
193 Log_Warning("Heap", "Size of heap address %p is invalid - Too small (0x%x) [at paddr 0x%x]",
194 head, head->Size, MM_GetPhysAddr(&head->Size));
198 if( head->Size > (2<<30) ) {
199 Mutex_Release(&glHeap);
200 Log_Warning("Heap", "Size of heap address %p is invalid - Over 2GiB (0x%x) [at paddr 0x%x]",
201 head, head->Size, MM_GetPhysAddr(&head->Size));
206 // Check if allocated
207 if(head->Magic == MAGIC_USED) continue;
209 if(head->Magic != MAGIC_FREE) {
210 Mutex_Release(&glHeap); // Release spinlock
212 Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
213 head, &head->Magic, head->Magic);
220 if(head->Size < Bytes) continue;
223 if(head->Size == Bytes) {
224 head->Magic = MAGIC_USED;
227 head->ValidSize = __Bytes;
228 head->AllocateTime = now();
229 Mutex_Release(&glHeap); // Release spinlock
231 Debug("[Heap ] Malloc'd %p (%i bytes), returning to %p",
232 head->Data, head->Size, __builtin_return_address(0));
240 bestSize = head->Size;
243 // or check if the block is the best size
244 if(bestSize > head->Size) {
246 bestSize = head->Size;
251 // If no block large enough is found, create one
254 best = Heap_Extend( Bytes );
257 Mutex_Release(&glHeap); // Release spinlock
261 if(best->Size == Bytes) {
262 best->Magic = MAGIC_USED; // Mark block as used
265 best->ValidSize = __Bytes;
266 best->AllocateTime = now();
267 Mutex_Release(&glHeap); // Release spinlock
269 Debug("[Heap ] Malloc'd %p (%i bytes), returning to %s:%i", best->Data, best->Size, File, Line);
276 newhead = (void*)( (Uint)best + Bytes );
277 newfoot = (void*)( (Uint)best + Bytes - sizeof(tHeapFoot) );
278 foot = (void*)( (Uint)best + best->Size - sizeof(tHeapFoot) );
280 newfoot->Head = best; // Create new footer
281 newfoot->Magic = MAGIC_FOOT;
282 newhead->Size = best->Size - Bytes; // Create new header
283 newhead->Magic = MAGIC_FREE;
284 foot->Head = newhead; // Update backlink in old footer
285 best->Size = Bytes; // Update size in old header
286 best->ValidSize = __Bytes;
287 best->Magic = MAGIC_USED; // Mark block as used
290 best->AllocateTime = now();
292 Mutex_Release(&glHeap); // Release spinlock
294 Debug("[Heap ] Malloc'd %p (0x%x bytes), returning to %s:%i",
295 best->Data, best->Size, File, Line);
301 * \brief Free an allocated memory block
303 void Heap_Deallocate(void *Ptr)
305 tHeapHead *head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
308 // INVLPTR is returned from Heap_Allocate when the allocation
310 if( Ptr == INVLPTR ) return;
313 Debug("[Heap ] free: %p freed by %p (%i old)", Ptr, __builtin_return_address(0), now()-head->AllocateTime);
317 if( (Uint)Ptr & (sizeof(Uint)-1) ) {
318 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
323 if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
325 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
326 __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
330 // Check memory block - Header
331 head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
332 if(head->Magic == MAGIC_FREE) {
333 Log_Warning("Heap", "free - Passed a freed block (%p) by %p", head, __builtin_return_address(0));
336 if(head->Magic != MAGIC_USED) {
337 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
338 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
342 // Check memory block - Footer
343 foot = (void*)( (Uint)head + head->Size - sizeof(tHeapFoot) );
344 if(foot->Head != head) {
345 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
346 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
349 if(foot->Magic != MAGIC_FOOT) {
350 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)", head, &foot->Magic, foot->Magic);
351 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
356 Mutex_Acquire( &glHeap );
359 head->Magic = MAGIC_FREE;
367 Mutex_Release( &glHeap );
371 * \brief Increase/Decrease the size of an allocation
372 * \param File Calling File
373 * \param Line Calling Line
374 * \param __ptr Old memory
375 * \param __size New Size
377 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
379 tHeapHead *head = (void*)( (Uint)__ptr-sizeof(tHeapHead) );
382 Uint newSize = (__size + sizeof(tHeapFoot)+sizeof(tHeapHead)+MIN_SIZE-1)&~(MIN_SIZE-1);
384 // Check for reallocating NULL
385 if(__ptr == NULL) return Heap_Allocate(File, Line, __size);
387 // Check if resize is needed
388 if(newSize <= head->Size) return __ptr;
390 // Check if next block is free
391 nextHead = (void*)( (Uint)head + head->Size );
393 // Extend into next block
394 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
396 Uint size = nextHead->Size + head->Size;
397 foot = (void*)( (Uint)nextHead + nextHead->Size - sizeof(tHeapFoot) );
399 if(size == newSize) {
400 head->Size = newSize;
401 head->ValidSize = __size;
409 // Create a new heap block
410 nextHead = (void*)( (Uint)head + newSize );
411 nextHead->Size = size - newSize;
412 nextHead->Magic = MAGIC_FREE;
413 foot->Head = nextHead; // Edit 2nd footer
414 head->Size = newSize; // Edit first header
417 head->ValidSize = __size;
419 foot = (void*)( (Uint)head + newSize - sizeof(tHeapFoot) );
421 foot->Magic = MAGIC_FOOT;
426 foot = (void*)( (Uint)head - sizeof(tHeapFoot) );
427 nextHead = foot->Head;
428 if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
430 Uint size = nextHead->Size + head->Size;
431 // Inexact fit, split things up
435 Warning("[Heap ] TODO: Space efficient realloc when new size is smaller");
442 // Set 1st (new/lower) header
443 nextHead->Magic = MAGIC_USED;
444 nextHead->Size = newSize;
445 nextHead->File = File;
446 nextHead->Line = Line;
447 nextHead->ValidSize = __size;
448 // Get 2nd (old) footer
449 foot = (void*)( (Uint)nextHead + newSize );
450 foot->Head = nextHead;
451 // Save old data size
452 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
457 memcpy(nextHead->Data, __ptr, oldDataSize);
459 return nextHead->Data;
461 // On to the expensive then
465 nextHead = Heap_Allocate( File, Line, __size );
467 nextHead->File = File;
468 nextHead->Line = Line;
469 nextHead->ValidSize = __size;
474 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
479 return nextHead->Data;
483 * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
484 * \brief Allocate and Zero a buffer in memory
485 * \param File Allocating file
486 * \param Line Line of allocation
487 * \param Bytes Size of the allocation
489 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
491 void *ret = Heap_Allocate(File, Line, Bytes);
492 if(ret == NULL) return NULL;
494 memset( ret, 0, Bytes );
500 * \fn int Heap_IsHeapAddr(void *Ptr)
501 * \brief Checks if an address is a heap pointer
503 int Heap_IsHeapAddr(void *Ptr)
506 if((Uint)Ptr < (Uint)gHeapStart) return 0;
507 if((Uint)Ptr > (Uint)gHeapEnd) return 0;
508 if((Uint)Ptr & (sizeof(Uint)-1)) return 0;
510 head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
511 if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
519 void Heap_Validate(void)
527 tHeapHead *head, *badHead;
528 tHeapFoot *foot = NULL;
529 static int in_heap_dump;
531 if( in_heap_dump ) return;
536 while( (Uint)head < (Uint)gHeapEnd )
538 foot = (void*)( (Uint)head + head->Size - sizeof(tHeapFoot) );
540 Log_Log("Heap", "%p (0x%P): 0x%08x (%i) %4C",
541 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
542 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
544 Log_Log("Heap", "%sowned by %s:%i",
545 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
549 // Sanity Check Header
550 if(head->Size == 0) {
551 Log_Warning("Heap", "HALTED - Size is zero");
554 if(head->Size & (MIN_SIZE-1)) {
555 Log_Warning("Heap", "HALTED - Size is malaligned");
558 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
559 Log_Warning("Heap", "HALTED - Head Magic is Bad");
564 if(foot->Magic != MAGIC_FOOT) {
565 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
568 if(head != foot->Head) {
569 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
577 // All OK? Go to next
578 head = foot->NextHead;
581 // If the heap is valid, ok!
582 if( (tVAddr)head == (tVAddr)gHeapEnd ) {
587 // Check for a bad return
588 if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
594 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
595 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
597 Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
599 Log_Log("Heap", "%sowned by %s:%i",
600 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
609 foot = (void*)( (tVAddr)gHeapEnd - sizeof(tHeapFoot) );
610 Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
612 while( (tVAddr)head >= (tVAddr)badHead )
614 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
615 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
616 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
618 Log_Log("Heap", "%sowned by %s:%i",
619 (head->Magic!=MAGIC_USED?"was ":""),
620 head->File, head->Line);
623 // Sanity Check Header
624 if(head->Size == 0) {
625 Log_Warning("Heap", "HALTED - Size is zero");
628 if(head->Size & (MIN_SIZE-1)) {
629 Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
632 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
633 Log_Warning("Heap", "HALTED - Head Magic is Bad");
638 if(foot->Magic != MAGIC_FOOT) {
639 Log_Warning("Heap", "HALTED - Foot Magic is Bad");
642 if(head != foot->Head) {
643 Log_Warning("Heap", "HALTED - Footer backlink is invalid");
647 if(head == badHead) break;
649 foot = (void*)( (tVAddr)head - sizeof(tHeapFoot) );
651 Log_Debug("Heap", "head=%p", head);
654 Panic("Heap_Dump - Heap is corrupted, kernel panic!");
659 void Heap_Stats(void)
666 int maxAlloc=0, minAlloc=-1;
667 int avgAlloc, frag, overhead;
669 for(head = gHeapStart;
670 (Uint)head < (Uint)gHeapEnd;
671 head = (void*)( (Uint)head + head->Size )
675 totalBytes += head->Size;
676 if( head->Magic == MAGIC_FREE )
679 freeBytes += head->Size;
681 else if( head->Magic == MAGIC_USED) {
682 if(maxAlloc < head->Size) maxAlloc = head->Size;
683 if(minAlloc == -1 || minAlloc > head->Size)
684 minAlloc = head->Size;
687 Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
691 // Print the block info?
693 if( head->Magic == MAGIC_FREE )
694 Log_Debug("Heap", "%p (%P) - 0x%x free",
695 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
697 Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
698 head->Data, MM_GetPhysAddr(&head->Data), head->Size, head->ValidSize, head->File, head->Line,
699 now() - head->AllocateTime
704 Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
705 Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
707 frag = (nFree-1)*10000/nBlocks;
710 Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
714 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
716 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
719 Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
720 avgAlloc, overhead/100, overhead%100
722 Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
725 // Scan and get distribution
732 } sizeCounts[nBlocks];
735 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
737 for(head = gHeapStart;
738 (Uint)head < (Uint)gHeapEnd;
739 head = (void*)( (Uint)head + head->Size )
742 for( i = 0; i < nBlocks; i ++ ) {
743 if( sizeCounts[i].Size == 0 )
745 if( sizeCounts[i].Size == head->Size )
748 // Should never reach this part (in a non-concurrent case)
749 if( i == nBlocks ) continue;
750 sizeCounts[i].Size = head->Size;
751 sizeCounts[i].Count ++;
753 //Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
754 // head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
756 //Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
757 // sizeCounts[i].Size, sizeCounts[i].Count);
761 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
763 Log("Heap_Stats: 0x%x - %i blocks",
764 sizeCounts[i].Size, sizeCounts[i].Count
773 EXPORT(Heap_Allocate);
774 EXPORT(Heap_AllocateZero);
775 EXPORT(Heap_Reallocate);
776 EXPORT(Heap_Deallocate);
777 EXPORT(Heap_IsHeapAddr);