KernelLand/Kernel/heap.c

   1 /*
   2  * Acess2 Kernel
   3  * - By John Hodge (thePowersGang)
   4  *
   5  * heap.c
   6  * - Dynamic memory allocation
   7  */
   8 #include <acess.h>
   9 #include <mm_virt.h>
  10 #include <heap_int.h>
  11 #include <limits.h>
  12
  13 #define WARNINGS        1       // Warn and dump on heap errors
  14 #define DEBUG_TRACE     0       // Enable tracing of allocations
  15 #define VERBOSE_DUMP    0       // Set to 1 to enable a verbose dump when heap errors are encountered
  16 #define VALIDATE_ON_ALLOC       1       // Set to 1 to enable validation of the heap on all malloc() calls
  17
  18 // === CONSTANTS ===
  19 #define HEAP_INIT_SIZE  0x8000  // 32 KiB
  20 #define MIN_SIZE        (sizeof(void*))*8       // 8 Machine Words
  21 #define POW2_SIZES      0
  22 #define COMPACT_HEAP    0       // Use 4 byte header?
  23 #define FIRST_FIT       0
  24
  25 //#define       MAGIC_FOOT      0x2ACE5505
  26 //#define       MAGIC_FREE      0xACE55000
  27 //#define       MAGIC_USED      0x862B0505      // MAGIC_FOOT ^ MAGIC_FREE
  28 #define MAGIC_FOOT      0x544F4F46      // 'FOOT'
  29 #define MAGIC_FREE      0x45455246      // 'FREE'
  30 #define MAGIC_USED      0x44455355      // 'USED'
  31
  32 // === PROTOTYPES ===
  33 void    Heap_Install(void);
  34 void    *Heap_Extend(size_t Bytes);
  35 void    *Heap_Merge(tHeapHead *Head);
  36 //void  *Heap_Allocate(const char *File, int Line, size_t Bytes);
  37 //void  *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
  38 //void  *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
  39 //void  Heap_Deallocate(const char *File, int Line, void *Ptr);
  40 void    Heap_Dump(int bVerbose);
  41 void    Heap_Stats(void);
  42
  43 // === GLOBALS ===
  44 tMutex  glHeap;
  45 tHeapHead       *gHeapStart;
  46 tHeapHead       *gHeapEnd;
  47
  48 // === CODE ===
  49 void Heap_Install(void)
  50 {
  51         gHeapStart = (void*)MM_KHEAP_BASE;
  52         gHeapEnd   = gHeapStart;
  53         Heap_Extend(HEAP_INIT_SIZE);
  54 }
  55
  56 static inline tHeapHead *Heap_NextHead(tHeapHead *Head) {
  57         return (void*)( (char*)Head + Head->Size );
  58 }
  59 static inline tHeapFoot *Heap_ThisFoot(tHeapHead *Head) {
  60         return (void*)( (char*)Head + Head->Size - sizeof(tHeapFoot) );
  61 }
  62 static inline tHeapFoot *Heap_PrevFoot(tHeapHead *Head) {
  63         //ASSERT(Head != gHeapStart);
  64         return (void*)( (char*)Head - sizeof(tHeapFoot) );
  65 }
  66
  67 /**
  68  * \brief Extend the size of the heap
  69  */
  70 void *Heap_Extend(size_t Bytes)
  71 {
  72         Debug("Heap_Extend(0x%x)", Bytes);
  73
  74         // Bounds Check
  75         if( gHeapEnd == (tHeapHead*)MM_KHEAP_MAX ) {
  76                 Log_Error("Heap", "Heap limit reached (%p)", (void*)MM_KHEAP_MAX);
  77                 return NULL;
  78         }
  79
  80         if( Bytes == 0 ) {
  81                 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
  82                 return NULL;
  83         }
  84
  85         const Uint      pages = (Bytes + 0xFFF) >> 12;
  86         tHeapHead       *new_end = (void*)( (tVAddr)gHeapEnd + (pages << 12) );
  87         // Bounds Check
  88         if( new_end > (tHeapHead*)MM_KHEAP_MAX )
  89         {
  90                 Log_Error("Heap", "Heap limit exceeded (%p)", (void*)new_end);
  91                 // TODO: Clip allocation to available space, and have caller check returned block
  92                 return NULL;
  93         }
  94
  95         // Heap expands in pages
  96         for( Uint i = 0; i < pages; i ++ )
  97         {
  98                 if( !MM_Allocate( (tVAddr)gHeapEnd+(i<<12) ) )
  99                 {
 100                         Warning("OOM - Heap_Extend (%i bytes)");
 101                         Heap_Dump(1);
 102                         return NULL;
 103                 }
 104         }
 105
 106         // Increase heap end
 107         tHeapHead       *head = gHeapEnd;
 108         gHeapEnd = new_end;
 109
 110         // Create Block
 111         head->Size = (Bytes+0xFFF)&~0xFFF;
 112         head->Magic = MAGIC_FREE;
 113         tHeapFoot *foot = Heap_ThisFoot(head);
 114         foot->Head = head;
 115         foot->Magic = MAGIC_FOOT;
 116
 117         return Heap_Merge(head);        // Merge with previous block
 118 }
 119
 120 /**
 121  * \brief Merges two adjacent heap blocks
 122  */
 123 void *Heap_Merge(tHeapHead *Head)
 124 {
 125         tHeapFoot       *foot;
 126         tHeapFoot       *thisFoot;
 127         tHeapHead       *head;
 128
 129         //Log("Heap_Merge: (Head=%p)", Head);
 130
 131         thisFoot = Heap_ThisFoot(Head);
 132         if( (void*)thisFoot > (void*)gHeapEnd )
 133                 return NULL;
 134
 135         // Merge Left (Down)
 136         foot = Heap_PrevFoot(Head);
 137         if( ((Uint)foot < (Uint)gHeapEnd && (Uint)foot > (Uint)gHeapStart)
 138         && foot->Head->Magic == MAGIC_FREE) {
 139                 foot->Head->Size += Head->Size; // Increase size
 140                 thisFoot->Head = foot->Head;    // Change backlink
 141                 Head->Magic = 0;        // Clear old head
 142                 Head->Size = 0;
 143                 Head = foot->Head;      // Save new head address
 144                 foot->Head = NULL;      // Clear central footer
 145                 foot->Magic = 0;
 146         }
 147
 148         // Merge Right (Upwards)
 149         head = Heap_NextHead(Head);
 150         if((Uint)head < (Uint)gHeapEnd && head->Magic == MAGIC_FREE)
 151         {
 152                 Head->Size += head->Size;
 153                 foot = Heap_ThisFoot(Head);
 154                 foot->Head = Head;      // Update Backlink
 155                 thisFoot->Head = NULL;  // Clear old footer
 156                 thisFoot->Magic = 0;
 157                 head->Size = 0;         // Clear old header
 158                 head->Magic = 0;
 159         }
 160
 161         // Return new address
 162         return Head;
 163 }
 164
 165 /**
 166  * \param File  Allocating source file
 167  * \param Line  Source line
 168  * \param __Bytes       Size of region to allocate
 169  */
 170 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
 171 {
 172         tHeapHead       *newhead;
 173         tHeapFoot       *foot, *newfoot;
 174         tHeapHead       *best = NULL;
 175         Uint    bestSize = UINT_MAX;    // Speed hack
 176         size_t  Bytes;
 177
 178         if( __Bytes == 0 ) {
 179                 return NULL;    // TODO: Return a known un-mapped range.
 180 //              return INVLPTR;
 181         }
 182
 183         #if VALIDATE_ON_ALLOC
 184         Heap_Dump(0);
 185         #endif
 186
 187         // Get required size
 188         #if POW2_SIZES
 189         Bytes = __Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot);
 190         Bytes = 1UUL << LOG2(__Bytes);
 191         #else
 192         Bytes = (__Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
 193         #endif
 194
 195         // Lock Heap
 196         Mutex_Acquire(&glHeap);
 197
 198         // Traverse Heap
 199         for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
 200         {
 201                 // Alignment Check
 202                 #if POW2_SIZES
 203                 if( head->Size != 1UUL << LOG2(head->Size) ) {
 204                 #else
 205                 if( head->Size & (MIN_SIZE-1) ) {
 206                 #endif
 207                         Mutex_Release(&glHeap); // Release spinlock
 208                         #if WARNINGS
 209                         Log_Warning("Heap", "Size of heap address %p is invalid"
 210                                 " - not aligned (0x%x) [at paddr 0x%x]",
 211                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 212                         Heap_Dump(VERBOSE_DUMP);
 213                         #endif
 214                         return NULL;
 215                 }
 216                 if( head->Size < MIN_SIZE ) {
 217                         Mutex_Release(&glHeap);
 218                         Log_Warning("Heap", "Size of heap address %p is invalid"
 219                                 " - Too small (0x%x) [at paddr 0x%x]",
 220                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 221                         Heap_Dump(VERBOSE_DUMP);
 222                         return NULL;
 223                 }
 224                 if( head->Size > (2<<30) ) {
 225                         Mutex_Release(&glHeap);
 226                         Log_Warning("Heap", "Size of heap address %p is invalid"
 227                                 " - Over 2GiB (0x%x) [at paddr 0x%x]",
 228                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 229                         Heap_Dump(VERBOSE_DUMP);
 230                         return NULL;
 231                 }
 232
 233                 // Check if allocated
 234                 if(head->Magic == MAGIC_USED)   continue;
 235                 // Error check
 236                 if(head->Magic != MAGIC_FREE)   {
 237                         Mutex_Release(&glHeap); // Release spinlock
 238                         #if WARNINGS
 239                         Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
 240                                 head, &head->Magic, head->Magic);
 241                         Heap_Dump(VERBOSE_DUMP);
 242                         #endif
 243                         return NULL;
 244                 }
 245
 246                 // Size check
 247                 if(head->Size < Bytes)  continue;
 248
 249                 // Perfect fit
 250                 if(head->Size == Bytes) {
 251                         head->Magic = MAGIC_USED;
 252                         head->File = File;
 253                         head->Line = Line;
 254                         head->ValidSize = __Bytes;
 255                         head->AllocateTime = now();
 256                         Mutex_Release(&glHeap); // Release spinlock
 257                         #if DEBUG_TRACE
 258                         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 259                                 head->Data, head->Size, File, Line);
 260                         #endif
 261                         return head->Data;
 262                 }
 263
 264                 // Break out of loop
 265                 #if FIRST_FIT
 266                 best = head;
 267                 bestSize = head->Size;
 268                 break;
 269                 #else
 270                 // or check if the block is the best size
 271                 if(bestSize > head->Size) {
 272                         best = head;
 273                         bestSize = head->Size;
 274                 }
 275                 #endif
 276         }
 277
 278         // If no block large enough is found, create one
 279         if(!best)
 280         {
 281                 best = Heap_Extend( Bytes );
 282                 // Check for errors
 283                 if(!best) {
 284                         Warning("OOM when allocating 0x%x bytes", Bytes);
 285                         Mutex_Release(&glHeap); // Release spinlock
 286                         return NULL;
 287                 }
 288                 // Check size
 289                 if(best->Size == Bytes) {
 290                         best->Magic = MAGIC_USED;       // Mark block as used
 291                         best->File = File;
 292                         best->Line = Line;
 293                         best->ValidSize = __Bytes;
 294                         best->AllocateTime = now();
 295                         Mutex_Release(&glHeap); // Release spinlock
 296                         #if DEBUG_TRACE
 297                         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 298                                 best->Data, best->Size, File, Line);
 299                         #endif
 300                         return best->Data;
 301                 }
 302         }
 303
 304         // Split Block
 305         // - Save size for new block
 306         size_t  newsize = best->Size - Bytes;
 307         // - Allocate beginning of old block
 308         best->Size = Bytes;             // Update size in old header
 309         best->ValidSize = __Bytes;
 310         best->Magic = MAGIC_USED;       // Mark block as used
 311         best->File = File;
 312         best->Line = Line;
 313         best->AllocateTime = now();
 314         // - Create a new foot on old block
 315         newfoot = Heap_ThisFoot(best);
 316         newfoot->Head = best;   // Create new footer
 317         newfoot->Magic = MAGIC_FOOT;
 318         // - Create a new header after resized old
 319         newhead = Heap_NextHead(best);
 320         newhead->Size = newsize;
 321         newhead->Magic = MAGIC_FREE;
 322         newhead->ValidSize = 0;
 323         newhead->File = NULL;
 324         newhead->Line = 0;
 325         // - Update footer
 326         foot = Heap_ThisFoot(newhead);
 327         foot->Head = newhead;
 328
 329         Mutex_Release(&glHeap); // Release spinlock
 330         #if DEBUG_TRACE
 331         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 332                 best->Data, best->Size, File, Line);
 333         #endif
 334         return best->Data;
 335 }
 336
 337 /**
 338  * \brief Free an allocated memory block
 339  */
 340 void Heap_Deallocate(const char *File, int Line, void *Ptr)
 341 {
 342         // INVLPTR is returned from Heap_Allocate when the allocation
 343         // size is zero.
 344         if( Ptr == INVLPTR )    return;
 345
 346         // Alignment Check
 347         if( (tVAddr)Ptr % sizeof(void*) != 0 ) {
 348                 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
 349                 return;
 350         }
 351
 352         // Sanity check
 353         if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
 354         {
 355                 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
 356                         __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
 357                 return;
 358         }
 359
 360         // Check memory block - Header
 361         tHeapHead *head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
 362         if(head->Magic == MAGIC_FREE) {
 363                 Log_Warning("Heap", "free - Passed a freed block (%p) by %s:%i (was freed by %s:%i)",
 364                         head, File, Line,
 365                         head->File, head->Line);
 366                 Proc_PrintBacktrace();
 367                 return;
 368         }
 369         if(head->Magic != MAGIC_USED) {
 370                 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
 371                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 372                 return;
 373         }
 374
 375         // Check memory block - Footer
 376         tHeapFoot *foot = Heap_ThisFoot(head);
 377         if(foot->Head != head) {
 378                 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
 379                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 380                 return;
 381         }
 382         if(foot->Magic != MAGIC_FOOT) {
 383                 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)",
 384                         head, &foot->Magic, foot->Magic);
 385                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 386                 return;
 387         }
 388
 389         #if DEBUG_TRACE
 390         Log_Debug("Heap", "free: %p freed by %s:%i (%i old)",
 391                 Ptr, File, Line, now()-head->AllocateTime);
 392         #endif
 393
 394         // Lock
 395         Mutex_Acquire( &glHeap );
 396
 397         // Mark as free
 398         head->Magic = MAGIC_FREE;
 399         head->File = File;
 400         head->Line = Line;
 401         head->ValidSize = 0;
 402         // Merge blocks
 403         Heap_Merge( head );
 404
 405         // Release
 406         Mutex_Release( &glHeap );
 407 }
 408
 409 /**
 410  * \brief Increase/Decrease the size of an allocation
 411  * \param File  Calling File
 412  * \param Line  Calling Line
 413  * \param __ptr Old memory
 414  * \param __size        New Size
 415  */
 416 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
 417 {
 418         tHeapHead       *head = (void*)( (Uint)__ptr-sizeof(tHeapHead) );
 419         tHeapHead       *nextHead;
 420         tHeapFoot       *foot;
 421         Uint    newSize = (__size + sizeof(tHeapFoot)+sizeof(tHeapHead)+MIN_SIZE-1)&~(MIN_SIZE-1);
 422
 423         // Check for reallocating NULL
 424         if(__ptr == NULL)       return Heap_Allocate(File, Line, __size);
 425
 426         // Check if resize is needed
 427         if(newSize <= head->Size)       return __ptr;
 428
 429         // Check if next block is free
 430         nextHead = Heap_NextHead(head);
 431
 432         // Extend into next block
 433         if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
 434         {
 435                 Uint    size = nextHead->Size + head->Size;
 436                 foot = Heap_ThisFoot(nextHead);
 437                 // Exact Fit
 438                 if(size == newSize) {
 439                         head->Size = newSize;
 440                         head->ValidSize = __size;
 441                         head->File = File;
 442                         head->Line = Line;
 443                         foot->Head = head;
 444                         nextHead->Magic = 0;
 445                         nextHead->Size = 0;
 446                         return __ptr;
 447                 }
 448                 // Create a new heap block
 449                 // - Update old with new information
 450                 head->Size = newSize;
 451                 head->File = File;      // Kinda counts as a new allocation
 452                 head->Line = Line;
 453                 head->ValidSize = __size;
 454                 // - Create new footer
 455                 foot = Heap_ThisFoot(head);
 456                 foot->Head = head;
 457                 foot->Magic = MAGIC_FOOT;
 458                 // - Create new header
 459                 nextHead = Heap_NextHead(head);
 460                 nextHead->Size = size - newSize;
 461                 nextHead->Magic = MAGIC_FREE;
 462                 // - Update old footer
 463                 foot->Head = nextHead;
 464                 return __ptr;
 465         }
 466
 467         // Extend downwards?
 468         foot = Heap_PrevFoot(head);
 469         nextHead = foot->Head;
 470         if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
 471         {
 472                 Uint    size = nextHead->Size + head->Size;
 473                 // Inexact fit, split things up
 474                 if(size > newSize)
 475                 {
 476                         // TODO: Handle splitting of downwards blocks
 477                         Warning("[Heap   ] TODO: Space efficient realloc when new size is smaller");
 478                 }
 479
 480                 // Exact fit
 481                 Uint    oldDataSize;
 482                 // Set 1st (new/lower) header
 483                 nextHead->Magic = MAGIC_USED;
 484                 nextHead->Size = newSize;
 485                 nextHead->File = File;
 486                 nextHead->Line = Line;
 487                 nextHead->ValidSize = __size;
 488                 // Get 2nd (old) footer
 489                 foot = Heap_ThisFoot(nextHead);
 490                 foot->Head = nextHead;
 491                 // Save old data size
 492                 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
 493                 // Clear old header
 494                 head->Size = 0;
 495                 head->Magic = 0;
 496                 // Copy data
 497                 memmove(nextHead->Data, __ptr, oldDataSize);
 498                 // Return
 499                 return nextHead->Data;
 500         }
 501
 502         // Well, darn
 503         nextHead = Heap_Allocate( File, Line, __size );
 504         if(!nextHead)   return NULL;
 505         nextHead -= 1;
 506         nextHead->File = File;
 507         nextHead->Line = Line;
 508         nextHead->ValidSize = __size;
 509
 510         memcpy(
 511                 nextHead->Data,
 512                 __ptr,
 513                 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
 514                 );
 515
 516         free(__ptr);
 517
 518         return nextHead->Data;
 519 }
 520
 521 /**
 522  * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
 523  * \brief Allocate and Zero a buffer in memory
 524  * \param File  Allocating file
 525  * \param Line  Line of allocation
 526  * \param Bytes Size of the allocation
 527  */
 528 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
 529 {
 530         void    *ret = Heap_Allocate(File, Line, Bytes);
 531         if(ret == NULL) return NULL;
 532
 533         memset( ret, 0, Bytes );
 534
 535         return ret;
 536 }
 537
 538 /**
 539  * \fn int Heap_IsHeapAddr(void *Ptr)
 540  * \brief Checks if an address is a heap pointer
 541  */
 542 int Heap_IsHeapAddr(void *Ptr)
 543 {
 544         tHeapHead       *head;
 545         if((Uint)Ptr < (Uint)gHeapStart)        return 0;
 546         if((Uint)Ptr > (Uint)gHeapEnd)  return 0;
 547         if((Uint)Ptr & (sizeof(Uint)-1))        return 0;
 548
 549         head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
 550         if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
 551                 return 0;
 552
 553         return 1;
 554 }
 555
 556 /**
 557  */
 558 void Heap_Validate(void)
 559 {
 560         // Call dump non-verbosely.
 561         // - If a heap error is detected, it'll print
 562         Heap_Dump(0);
 563 }
 564
 565 void Heap_Dump(int bVerbose)
 566 {
 567         tHeapHead       *head, *badHead;
 568         tHeapFoot       *foot = NULL;
 569         static int      in_heap_dump;
 570
 571         if( in_heap_dump )      return;
 572
 573         in_heap_dump = 1;
 574
 575         head = gHeapStart;
 576         while( (Uint)head < (Uint)gHeapEnd )
 577         {
 578                 foot = Heap_ThisFoot(head);
 579
 580                 if( bVerbose )
 581                 {
 582                         Log_Log("Heap", "%p (%P): 0x%08x (%i) %4C",
 583                                 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 584                         Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
 585                         if(head->File) {
 586                                 Log_Log("Heap", "%sowned by %s:%i",
 587                                         (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
 588                         }
 589                 }
 590
 591                 // Sanity Check Header
 592                 if(head->Size == 0) {
 593                         Log_Warning("Heap", "HALTED - Size is zero");
 594                         break;
 595                 }
 596                 if(head->Size & (MIN_SIZE-1)) {
 597                         Log_Warning("Heap", "HALTED - Size is malaligned");
 598                         break;
 599                 }
 600                 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
 601                         Log_Warning("Heap", "HALTED - Head Magic is Bad");
 602                         break;
 603                 }
 604
 605                 // Check footer
 606                 if(foot->Magic != MAGIC_FOOT) {
 607                         Log_Warning("Heap", "HALTED - Foot Magic is Bad");
 608                         break;
 609                 }
 610                 if(head != foot->Head) {
 611                         Log_Warning("Heap", "HALTED - Footer backlink is invalid");
 612                         break;
 613                 }
 614
 615                 if( bVerbose )
 616                 {
 617                         Log_Log("Heap", "");
 618                 }
 619
 620                 // All OK? Go to next
 621                 head = foot->NextHead;
 622         }
 623
 624         // If the heap is valid, ok!
 625         if( (tVAddr)head == (tVAddr)gHeapEnd ) {
 626                 in_heap_dump = 0;
 627                 return ;
 628         }
 629
 630         // Check for a bad return
 631         if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
 632                 in_heap_dump = 0;
 633                 return ;
 634         }
 635
 636         // If not verbose, we need to dump the failing block
 637         if( !bVerbose )
 638         {
 639                 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
 640                         head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 641                 if(foot)
 642                         Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
 643                 if(head->File) {
 644                         Log_Log("Heap", "%sowned by %s:%i",
 645                                 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
 646                 }
 647                 Log_Log("Heap", "");
 648         }
 649
 650         badHead = head;
 651
 652         // Work backwards
 653         foot = Heap_PrevFoot(gHeapEnd);
 654         Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
 655         head = foot->Head;
 656         while( (tVAddr)head >= (tVAddr)badHead )
 657         {
 658                 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
 659                         head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 660                 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
 661                 if(head->File)
 662                         Log_Log("Heap", "%sowned by %s:%i",
 663                                 (head->Magic!=MAGIC_USED?"was ":""),
 664                                 head->File, head->Line);
 665                 Log_Log("Heap", "");
 666
 667                 // Sanity Check Header
 668                 if(head->Size == 0) {
 669                         Log_Warning("Heap", "HALTED - Size is zero");
 670                         break;
 671                 }
 672                 if(head->Size & (MIN_SIZE-1)) {
 673                         Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
 674                         break ;
 675                 }
 676                 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
 677                         Log_Warning("Heap", "HALTED - Head Magic is Bad");
 678                         break;
 679                 }
 680
 681                 // Check footer
 682                 if(foot->Magic != MAGIC_FOOT) {
 683                         Log_Warning("Heap", "HALTED - Foot Magic is Bad");
 684                         break;
 685                 }
 686                 if(head != foot->Head) {
 687                         Log_Warning("Heap", "HALTED - Footer backlink is invalid");
 688                         break;
 689                 }
 690
 691                 if(head == badHead)     break;
 692
 693                 foot = Heap_PrevFoot(head);
 694                 head = foot->Head;
 695                 Log_Debug("Heap", "head=%p", head);
 696         }
 697
 698         Panic("Heap_Dump - Heap is corrupted, kernel panic! (%p)", badHead);
 699 }
 700
 701 void Heap_Stats(void)
 702 {
 703          int    nBlocks = 0;
 704          int    nFree = 0;
 705          int    totalBytes = 0;
 706          int    freeBytes = 0;
 707          int    maxAlloc=0, minAlloc=-1;
 708          int    avgAlloc, frag, overhead;
 709
 710         for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
 711         {
 712                 nBlocks ++;
 713                 totalBytes += head->Size;
 714                 if( head->Magic == MAGIC_FREE )
 715                 {
 716                         nFree ++;
 717                         freeBytes += head->Size;
 718                 }
 719                 else if( head->Magic == MAGIC_USED) {
 720                         if(maxAlloc < head->Size)       maxAlloc = head->Size;
 721                         if(minAlloc == -1 || minAlloc > head->Size)
 722                                 minAlloc = head->Size;
 723                 }
 724                 else {
 725                         Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
 726                         break;
 727                 }
 728
 729                 // Print the block info?
 730                 #if 1
 731                 if( head->Magic == MAGIC_FREE )
 732                         Log_Debug("Heap", "%p (%P) - 0x%x free",
 733                                 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
 734                 else
 735                         Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
 736                                 head->Data, MM_GetPhysAddr(&head->Data), head->Size,
 737                                 head->ValidSize, head->File, head->Line,
 738                                 now() - head->AllocateTime
 739                                 );
 740                 #endif
 741         }
 742
 743         Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
 744         Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
 745         if(nBlocks != 0)
 746                 frag = (nFree-1)*10000/nBlocks;
 747         else
 748                 frag = 0;
 749         Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
 750         if(nBlocks <= nFree)
 751                 avgAlloc = 0;
 752         else
 753                 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
 754         if(avgAlloc != 0)
 755                 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
 756         else
 757                 overhead = 0;
 758         Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
 759                 avgAlloc, overhead/100, overhead%100
 760                 );
 761         Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
 762                 minAlloc, maxAlloc);
 763
 764         // Scan and get distribution
 765         #if 1
 766         if(nBlocks > 0)
 767         {
 768                 struct {
 769                         Uint    Size;
 770                         Uint    Count;
 771                 }       sizeCounts[nBlocks];
 772                  int    i;
 773
 774                 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
 775
 776                 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
 777                 {
 778                         for( i = 0; i < nBlocks; i ++ ) {
 779                                 if( sizeCounts[i].Size == 0 )
 780                                         break;
 781                                 if( sizeCounts[i].Size == head->Size )
 782                                         break;
 783                         }
 784                         // Should never reach this part (in a non-concurrent case)
 785                         if( i == nBlocks )      continue;
 786                         sizeCounts[i].Size = head->Size;
 787                         sizeCounts[i].Count ++;
 788                         #if 0
 789                         Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
 790                                 head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
 791                                 );
 792                         Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
 793                                 sizeCounts[i].Size, sizeCounts[i].Count);
 794                         #endif
 795                 }
 796
 797                 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
 798                 {
 799                         Log("Heap_Stats: 0x%x - %i blocks",
 800                                 sizeCounts[i].Size, sizeCounts[i].Count
 801                                 );
 802                 }
 803         }
 804         #endif
 805 }
 806
 807 // === EXPORTS ===
 808 EXPORT(Heap_Allocate);
 809 EXPORT(Heap_AllocateZero);
 810 EXPORT(Heap_Reallocate);
 811 EXPORT(Heap_Deallocate);
 812 EXPORT(Heap_IsHeapAddr);