KernelLand/Kernel/heap.c

   1 /*
   2  * Acess2 Kernel
   3  * - By John Hodge (thePowersGang)
   4  *
   5  * heap.c
   6  * - Dynamic memory allocation
   7  */
   8 #include <acess.h>
   9 #include <mm_virt.h>
  10 #include <heap_int.h>
  11 #include <limits.h>
  12 #include <debug_hooks.h>
  13
  14 #define WARNINGS        1       // Warn and dump on heap errors
  15 #define DEBUG_TRACE     0       // Enable tracing of allocations
  16 #define VERBOSE_DUMP    0       // Set to 1 to enable a verbose dump when heap errors are encountered
  17 #define VALIDATE_ON_ALLOC       1       // Set to 1 to enable validation of the heap on all malloc() calls
  18
  19 // === CONSTANTS ===
  20 #define HEAP_INIT_SIZE  0x8000  // 32 KiB
  21 #define MIN_SIZE        (sizeof(void*))*8       // 8 Machine Words
  22 #define POW2_SIZES      0
  23 #define COMPACT_HEAP    0       // Use 4 byte header?
  24 #define FIRST_FIT       0
  25
  26 //#define       MAGIC_FOOT      0x2ACE5505
  27 //#define       MAGIC_FREE      0xACE55000
  28 //#define       MAGIC_USED      0x862B0505      // MAGIC_FOOT ^ MAGIC_FREE
  29 #define MAGIC_FOOT      0x544F4F46      // 'FOOT'
  30 #define MAGIC_FREE      0x45455246      // 'FREE'
  31 #define MAGIC_USED      0x44455355      // 'USED'
  32
  33 // === PROTOTYPES ===
  34 void    Heap_Install(void);
  35 void    *Heap_Extend(size_t Bytes);
  36 void    *Heap_Merge(tHeapHead *Head);
  37 //void  *Heap_Allocate(const char *File, int Line, size_t Bytes);
  38 //void  *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
  39 //void  *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
  40 //void  Heap_Deallocate(const char *File, int Line, void *Ptr);
  41 //void  Heap_Dump(void);
  42 void    Heap_ValidateDump(int bVerbose);
  43 //void  Heap_Stats(void);
  44
  45 // === GLOBALS ===
  46 tMutex  glHeap;
  47 tHeapHead       *gHeapStart;
  48 tHeapHead       *gHeapEnd;
  49
  50 // === CODE ===
  51 void Heap_Install(void)
  52 {
  53         gHeapStart = (void*)MM_KHEAP_BASE;
  54         gHeapEnd   = gHeapStart;
  55         Heap_Extend(HEAP_INIT_SIZE);
  56 }
  57
  58 static inline tHeapHead *Heap_NextHead(tHeapHead *Head) {
  59         return (void*)( (char*)Head + Head->Size );
  60 }
  61 static inline tHeapFoot *Heap_ThisFoot(tHeapHead *Head) {
  62         return (void*)( (char*)Head + Head->Size - sizeof(tHeapFoot) );
  63 }
  64 static inline tHeapFoot *Heap_PrevFoot(tHeapHead *Head) {
  65         //ASSERT(Head != gHeapStart);
  66         return (void*)( (char*)Head - sizeof(tHeapFoot) );
  67 }
  68
  69 /**
  70  * \brief Extend the size of the heap
  71  */
  72 void *Heap_Extend(size_t Bytes)
  73 {
  74         //Debug("Heap_Extend(0x%x)", Bytes);
  75
  76         // Bounds Check
  77         if( gHeapEnd == (tHeapHead*)MM_KHEAP_MAX ) {
  78                 Log_Error("Heap", "Heap limit reached (%p)", (void*)MM_KHEAP_MAX);
  79                 return NULL;
  80         }
  81
  82         if( Bytes == 0 ) {
  83                 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
  84                 return NULL;
  85         }
  86
  87         const Uint      pages = (Bytes + 0xFFF) >> 12;
  88         tHeapHead       *new_end = (void*)( (tVAddr)gHeapEnd + (pages << 12) );
  89         // Bounds Check
  90         if( new_end > (tHeapHead*)MM_KHEAP_MAX )
  91         {
  92                 Log_Error("Heap", "Heap limit exceeded (%p)", (void*)new_end);
  93                 // TODO: Clip allocation to available space, and have caller check returned block
  94                 return NULL;
  95         }
  96
  97         // Heap expands in pages
  98         for( Uint i = 0; i < pages; i ++ )
  99         {
 100                 if( !MM_Allocate( (tPage*)gHeapEnd + i ) )
 101                 {
 102                         Warning("OOM - Heap_Extend (%i bytes)");
 103                         Heap_Dump();
 104                         return NULL;
 105                 }
 106         }
 107
 108         // Increase heap end
 109         tHeapHead       *head = gHeapEnd;
 110         gHeapEnd = new_end;
 111
 112         // Create Block
 113         head->Size = (Bytes+0xFFF)&~0xFFF;
 114         head->Magic = MAGIC_FREE;
 115         tHeapFoot *foot = Heap_ThisFoot(head);
 116         foot->Head = head;
 117         foot->Magic = MAGIC_FOOT;
 118
 119         return Heap_Merge(head);        // Merge with previous block
 120 }
 121
 122 /**
 123  * \brief Merges two adjacent heap blocks
 124  */
 125 void *Heap_Merge(tHeapHead *Head)
 126 {
 127         tHeapFoot       *foot;
 128         tHeapFoot       *thisFoot;
 129         tHeapHead       *head;
 130
 131         //Log("Heap_Merge: (Head=%p)", Head);
 132
 133         thisFoot = Heap_ThisFoot(Head);
 134         if( (void*)thisFoot > (void*)gHeapEnd )
 135                 return NULL;
 136
 137         // Merge Left (Down)
 138         foot = Heap_PrevFoot(Head);
 139         if( ((Uint)foot < (Uint)gHeapEnd && (Uint)foot > (Uint)gHeapStart)
 140         && foot->Head->Magic == MAGIC_FREE) {
 141                 foot->Head->Size += Head->Size; // Increase size
 142                 thisFoot->Head = foot->Head;    // Change backlink
 143                 Head->Magic = 0;        // Clear old head
 144                 Head->Size = 0;
 145                 Head = foot->Head;      // Save new head address
 146                 foot->Head = NULL;      // Clear central footer
 147                 foot->Magic = 0;
 148         }
 149
 150         // Merge Right (Upwards)
 151         head = Heap_NextHead(Head);
 152         if((Uint)head < (Uint)gHeapEnd && head->Magic == MAGIC_FREE)
 153         {
 154                 Head->Size += head->Size;
 155                 foot = Heap_ThisFoot(Head);
 156                 foot->Head = Head;      // Update Backlink
 157                 thisFoot->Head = NULL;  // Clear old footer
 158                 thisFoot->Magic = 0;
 159                 head->Size = 0;         // Clear old header
 160                 head->Magic = 0;
 161         }
 162
 163         // Return new address
 164         return Head;
 165 }
 166
 167 /**
 168  * \param File  Allocating source file
 169  * \param Line  Source line
 170  * \param __Bytes       Size of region to allocate
 171  */
 172 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
 173 {
 174         tHeapHead       *newhead;
 175         tHeapFoot       *foot, *newfoot;
 176         tHeapHead       *best = NULL;
 177         Uint    bestSize = UINT_MAX;    // Speed hack
 178         size_t  Bytes;
 179
 180         if( __Bytes == 0 ) {
 181                 return NULL;    // TODO: Return a known un-mapped range.
 182 //              return INVLPTR;
 183         }
 184
 185         #if VALIDATE_ON_ALLOC
 186         Heap_Validate();
 187         #endif
 188
 189         // Get required size
 190         #if POW2_SIZES
 191         Bytes = __Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot);
 192         Bytes = 1UUL << LOG2(__Bytes);
 193         #else
 194         Bytes = (__Bytes + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
 195         #endif
 196
 197         // Lock Heap
 198         Mutex_Acquire(&glHeap);
 199
 200         // Traverse Heap
 201         for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
 202         {
 203                 // Alignment Check
 204                 #if POW2_SIZES
 205                 if( head->Size != 1UUL << LOG2(head->Size) ) {
 206                 #else
 207                 if( head->Size & (MIN_SIZE-1) ) {
 208                 #endif
 209                         Mutex_Release(&glHeap); // Release spinlock
 210                         #if WARNINGS
 211                         Log_Warning("Heap", "Size of heap address %p is invalid"
 212                                 " - not aligned (0x%x) [at paddr 0x%x]",
 213                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 214                         Heap_ValidateDump(VERBOSE_DUMP);
 215                         #endif
 216                         return NULL;
 217                 }
 218                 if( head->Size < MIN_SIZE ) {
 219                         Mutex_Release(&glHeap);
 220                         Log_Warning("Heap", "Size of heap address %p is invalid"
 221                                 " - Too small (0x%x) [at paddr 0x%x]",
 222                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 223                         Heap_ValidateDump(VERBOSE_DUMP);
 224                         return NULL;
 225                 }
 226                 if( head->Size > (2<<30) ) {
 227                         Mutex_Release(&glHeap);
 228                         Log_Warning("Heap", "Size of heap address %p is invalid"
 229                                 " - Over 2GiB (0x%x) [at paddr 0x%x]",
 230                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 231                         Heap_ValidateDump(VERBOSE_DUMP);
 232                         return NULL;
 233                 }
 234
 235                 // Check if allocated
 236                 if(head->Magic == MAGIC_USED)   continue;
 237                 // Error check
 238                 if(head->Magic != MAGIC_FREE)   {
 239                         Mutex_Release(&glHeap); // Release spinlock
 240                         #if WARNINGS
 241                         Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
 242                                 head, &head->Magic, head->Magic);
 243                         Heap_ValidateDump(VERBOSE_DUMP);
 244                         #endif
 245                         return NULL;
 246                 }
 247
 248                 // Size check
 249                 if(head->Size < Bytes)  continue;
 250
 251                 // Perfect fit
 252                 if(head->Size == Bytes) {
 253                         head->Magic = MAGIC_USED;
 254                         head->File = File;
 255                         head->Line = Line;
 256                         head->ValidSize = __Bytes;
 257                         head->AllocateTime = now();
 258                         Mutex_Release(&glHeap); // Release spinlock
 259                         #if DEBUG_TRACE
 260                         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 261                                 head->Data, head->Size, File, Line);
 262                         #endif
 263                         return head->Data;
 264                 }
 265
 266                 // Break out of loop
 267                 #if FIRST_FIT
 268                 best = head;
 269                 bestSize = head->Size;
 270                 break;
 271                 #else
 272                 // or check if the block is the best size
 273                 if(bestSize > head->Size) {
 274                         best = head;
 275                         bestSize = head->Size;
 276                 }
 277                 #endif
 278         }
 279
 280         // If no block large enough is found, create one
 281         if(!best)
 282         {
 283                 best = Heap_Extend( Bytes );
 284                 // Check for errors
 285                 if(!best) {
 286                         Warning("OOM when allocating 0x%x bytes", Bytes);
 287                         Mutex_Release(&glHeap); // Release spinlock
 288                         return NULL;
 289                 }
 290                 // Check size
 291                 if(best->Size == Bytes) {
 292                         best->Magic = MAGIC_USED;       // Mark block as used
 293                         best->File = File;
 294                         best->Line = Line;
 295                         best->ValidSize = __Bytes;
 296                         best->AllocateTime = now();
 297                         Mutex_Release(&glHeap); // Release spinlock
 298                         #if DEBUG_TRACE
 299                         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 300                                 best->Data, best->Size, File, Line);
 301                         #endif
 302                         return best->Data;
 303                 }
 304         }
 305
 306         // Split Block
 307         // - Save size for new block
 308         size_t  newsize = best->Size - Bytes;
 309         // - Allocate beginning of old block
 310         best->Size = Bytes;             // Update size in old header
 311         best->ValidSize = __Bytes;
 312         best->Magic = MAGIC_USED;       // Mark block as used
 313         best->File = File;
 314         best->Line = Line;
 315         best->AllocateTime = now();
 316         // - Create a new foot on old block
 317         newfoot = Heap_ThisFoot(best);
 318         newfoot->Head = best;   // Create new footer
 319         newfoot->Magic = MAGIC_FOOT;
 320         // - Create a new header after resized old
 321         newhead = Heap_NextHead(best);
 322         newhead->Size = newsize;
 323         newhead->Magic = MAGIC_FREE;
 324         newhead->ValidSize = 0;
 325         newhead->File = NULL;
 326         newhead->Line = 0;
 327         // - Update footer
 328         foot = Heap_ThisFoot(newhead);
 329         foot->Head = newhead;
 330
 331         Mutex_Release(&glHeap); // Release spinlock
 332         #if DEBUG_TRACE
 333         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 334                 best->Data, best->Size, File, Line);
 335         #endif
 336         return best->Data;
 337 }
 338
 339 /**
 340  * \brief Free an allocated memory block
 341  */
 342 void Heap_Deallocate(const char *File, int Line, void *Ptr)
 343 {
 344         // INVLPTR is returned from Heap_Allocate when the allocation
 345         // size is zero.
 346         if( Ptr == INVLPTR )    return;
 347
 348         // Alignment Check
 349         if( (tVAddr)Ptr % sizeof(void*) != 0 ) {
 350                 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
 351                 return;
 352         }
 353
 354         // Sanity check
 355         if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
 356         {
 357                 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
 358                         __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
 359                 return;
 360         }
 361
 362         // Check memory block - Header
 363         tHeapHead *head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
 364         if(head->Magic == MAGIC_FREE) {
 365                 Log_Warning("Heap", "free - Passed a freed block (%p) by %s:%i (was freed by %s:%i)",
 366                         head, File, Line,
 367                         head->File, head->Line);
 368                 Proc_PrintBacktrace();
 369                 return;
 370         }
 371         if(head->Magic != MAGIC_USED) {
 372                 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
 373                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 374                 return;
 375         }
 376
 377         // Check memory block - Footer
 378         tHeapFoot *foot = Heap_ThisFoot(head);
 379         if(foot->Head != head) {
 380                 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
 381                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 382                 return;
 383         }
 384         if(foot->Magic != MAGIC_FOOT) {
 385                 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)",
 386                         head, &foot->Magic, foot->Magic);
 387                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 388                 return;
 389         }
 390
 391         #if DEBUG_TRACE
 392         Log_Debug("Heap", "free: %p freed by %s:%i (%i old)",
 393                 Ptr, File, Line, now()-head->AllocateTime);
 394         #endif
 395
 396         // Lock
 397         Mutex_Acquire( &glHeap );
 398
 399         // Mark as free
 400         head->Magic = MAGIC_FREE;
 401         head->File = File;
 402         head->Line = Line;
 403         head->ValidSize = 0;
 404         // Merge blocks
 405         Heap_Merge( head );
 406
 407         // Release
 408         Mutex_Release( &glHeap );
 409 }
 410
 411 /**
 412  * \brief Increase/Decrease the size of an allocation
 413  * \param File  Calling File
 414  * \param Line  Calling Line
 415  * \param __ptr Old memory
 416  * \param __size        New Size
 417  */
 418 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
 419 {
 420         tHeapHead       *head = (void*)( (Uint)__ptr-sizeof(tHeapHead) );
 421         tHeapHead       *nextHead;
 422         tHeapFoot       *foot;
 423         Uint    newSize = (__size + sizeof(tHeapFoot)+sizeof(tHeapHead)+MIN_SIZE-1)&~(MIN_SIZE-1);
 424
 425         // Check for reallocating NULL
 426         if(__ptr == NULL)       return Heap_Allocate(File, Line, __size);
 427
 428         // Check if resize is needed
 429         if(newSize <= head->Size)       return __ptr;
 430
 431         // Check if next block is free
 432         nextHead = Heap_NextHead(head);
 433
 434         // Extend into next block
 435         if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
 436         {
 437                 Uint    size = nextHead->Size + head->Size;
 438                 foot = Heap_ThisFoot(nextHead);
 439                 // Exact Fit
 440                 if(size == newSize) {
 441                         head->Size = newSize;
 442                         head->ValidSize = __size;
 443                         head->File = File;
 444                         head->Line = Line;
 445                         foot->Head = head;
 446                         nextHead->Magic = 0;
 447                         nextHead->Size = 0;
 448                         return __ptr;
 449                 }
 450                 // Create a new heap block
 451                 // - Update old with new information
 452                 head->Size = newSize;
 453                 head->File = File;      // Kinda counts as a new allocation
 454                 head->Line = Line;
 455                 head->ValidSize = __size;
 456                 // - Create new footer
 457                 foot = Heap_ThisFoot(head);
 458                 foot->Head = head;
 459                 foot->Magic = MAGIC_FOOT;
 460                 // - Create new header
 461                 nextHead = Heap_NextHead(head);
 462                 nextHead->Size = size - newSize;
 463                 nextHead->Magic = MAGIC_FREE;
 464                 // - Update old footer
 465                 foot->Head = nextHead;
 466                 return __ptr;
 467         }
 468
 469         // Extend downwards?
 470         foot = Heap_PrevFoot(head);
 471         nextHead = foot->Head;
 472         if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
 473         {
 474                 Uint    size = nextHead->Size + head->Size;
 475                 // Inexact fit, split things up
 476                 if(size > newSize)
 477                 {
 478                         // TODO: Handle splitting of downwards blocks
 479                         Warning("[Heap   ] TODO: Space efficient realloc when new size is smaller");
 480                 }
 481
 482                 // Exact fit
 483                 Uint    oldDataSize;
 484                 // Set 1st (new/lower) header
 485                 nextHead->Magic = MAGIC_USED;
 486                 nextHead->Size = newSize;
 487                 nextHead->File = File;
 488                 nextHead->Line = Line;
 489                 nextHead->ValidSize = __size;
 490                 // Get 2nd (old) footer
 491                 foot = Heap_ThisFoot(nextHead);
 492                 foot->Head = nextHead;
 493                 // Save old data size
 494                 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
 495                 // Clear old header
 496                 head->Size = 0;
 497                 head->Magic = 0;
 498                 // Copy data
 499                 memmove(nextHead->Data, __ptr, oldDataSize);
 500                 // Return
 501                 return nextHead->Data;
 502         }
 503
 504         // Well, darn
 505         nextHead = Heap_Allocate( File, Line, __size );
 506         if(!nextHead)   return NULL;
 507         nextHead -= 1;
 508         nextHead->File = File;
 509         nextHead->Line = Line;
 510         nextHead->ValidSize = __size;
 511
 512         memcpy(
 513                 nextHead->Data,
 514                 __ptr,
 515                 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
 516                 );
 517
 518         free(__ptr);
 519
 520         return nextHead->Data;
 521 }
 522
 523 /**
 524  * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
 525  * \brief Allocate and Zero a buffer in memory
 526  * \param File  Allocating file
 527  * \param Line  Line of allocation
 528  * \param Bytes Size of the allocation
 529  */
 530 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
 531 {
 532         void    *ret = Heap_Allocate(File, Line, Bytes);
 533         if(ret == NULL) return NULL;
 534
 535         memset( ret, 0, Bytes );
 536
 537         return ret;
 538 }
 539
 540 /**
 541  * \fn int Heap_IsHeapAddr(void *Ptr)
 542  * \brief Checks if an address is a heap pointer
 543  */
 544 int Heap_IsHeapAddr(void *Ptr)
 545 {
 546         tHeapHead       *head;
 547         if((Uint)Ptr < (Uint)gHeapStart)        return 0;
 548         if((Uint)Ptr > (Uint)gHeapEnd)  return 0;
 549         if((Uint)Ptr & (sizeof(Uint)-1))        return 0;
 550
 551         head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
 552         if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
 553                 return 0;
 554
 555         return 1;
 556 }
 557
 558 /**
 559  */
 560 void Heap_Validate(void)
 561 {
 562         // Call dump non-verbosely.
 563         // - If a heap error is detected, it'll print
 564         Heap_ValidateDump(0);
 565 }
 566
 567 void Heap_Dump(void)
 568 {
 569         Heap_ValidateDump(1);
 570 }
 571
 572 void Heap_ValidateDump(int bVerbose)
 573 {
 574         tHeapHead       *head, *badHead;
 575         tHeapFoot       *foot = NULL;
 576         static int      in_heap_dump;
 577
 578         if( in_heap_dump )      return;
 579
 580         in_heap_dump = 1;
 581
 582         head = gHeapStart;
 583         while( (Uint)head < (Uint)gHeapEnd )
 584         {
 585                 foot = Heap_ThisFoot(head);
 586
 587                 if( bVerbose )
 588                 {
 589                         Log_Log("Heap", "%p (%P): 0x%08x (%i) %4C",
 590                                 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 591                         Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
 592                         if(head->File) {
 593                                 Log_Log("Heap", "%sowned by %s:%i",
 594                                         (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
 595                         }
 596                 }
 597
 598                 // Sanity Check Header
 599                 if(head->Size == 0) {
 600                         Log_Warning("Heap", "HALTED - Size is zero");
 601                         break;
 602                 }
 603                 if(head->Size & (MIN_SIZE-1)) {
 604                         Log_Warning("Heap", "HALTED - Size is malaligned");
 605                         break;
 606                 }
 607                 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
 608                         Log_Warning("Heap", "HALTED - Head Magic is Bad");
 609                         break;
 610                 }
 611
 612                 // Check footer
 613                 if(foot->Magic != MAGIC_FOOT) {
 614                         Log_Warning("Heap", "HALTED - Foot Magic is Bad");
 615                         break;
 616                 }
 617                 if(head != foot->Head) {
 618                         Log_Warning("Heap", "HALTED - Footer backlink is invalid");
 619                         break;
 620                 }
 621
 622                 if( bVerbose )
 623                 {
 624                         Log_Log("Heap", "");
 625                 }
 626
 627                 // All OK? Go to next
 628                 head = foot->NextHead;
 629         }
 630
 631         // If the heap is valid, ok!
 632         if( (tVAddr)head == (tVAddr)gHeapEnd ) {
 633                 in_heap_dump = 0;
 634                 return ;
 635         }
 636
 637         // Check for a bad return
 638         if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
 639                 in_heap_dump = 0;
 640                 return ;
 641         }
 642
 643         // If not verbose, we need to dump the failing block
 644         if( !bVerbose )
 645         {
 646                 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
 647                         head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 648                 if(foot)
 649                         Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
 650                 if(head->File) {
 651                         Log_Log("Heap", "%sowned by %s:%i",
 652                                 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
 653                 }
 654                 Log_Log("Heap", "");
 655         }
 656
 657         badHead = head;
 658
 659         // Work backwards
 660         foot = Heap_PrevFoot(gHeapEnd);
 661         Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
 662         head = foot->Head;
 663         while( (tVAddr)head >= (tVAddr)badHead )
 664         {
 665                 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
 666                         head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 667                 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
 668                 if(head->File)
 669                         Log_Log("Heap", "%sowned by %s:%i",
 670                                 (head->Magic!=MAGIC_USED?"was ":""),
 671                                 head->File, head->Line);
 672                 Log_Log("Heap", "");
 673
 674                 // Sanity Check Header
 675                 if(head->Size == 0) {
 676                         Log_Warning("Heap", "HALTED - Size is zero");
 677                         break;
 678                 }
 679                 if(head->Size & (MIN_SIZE-1)) {
 680                         Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
 681                         break ;
 682                 }
 683                 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
 684                         Log_Warning("Heap", "HALTED - Head Magic is Bad");
 685                         break;
 686                 }
 687
 688                 // Check footer
 689                 if(foot->Magic != MAGIC_FOOT) {
 690                         Log_Warning("Heap", "HALTED - Foot Magic is Bad");
 691                         break;
 692                 }
 693                 if(head != foot->Head) {
 694                         Log_Warning("Heap", "HALTED - Footer backlink is invalid");
 695                         break;
 696                 }
 697
 698                 if(head == badHead)     break;
 699
 700                 foot = Heap_PrevFoot(head);
 701                 head = foot->Head;
 702                 Log_Debug("Heap", "head=%p", head);
 703         }
 704
 705         Panic("Heap_Dump - Heap is corrupted, kernel panic! (%p)", badHead);
 706 }
 707
 708 void Heap_Stats(void)
 709 {
 710          int    nBlocks = 0;
 711          int    nFree = 0;
 712          int    totalBytes = 0;
 713          int    freeBytes = 0;
 714          int    maxAlloc=0, minAlloc=-1;
 715          int    avgAlloc, frag, overhead;
 716
 717         for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
 718         {
 719                 nBlocks ++;
 720                 totalBytes += head->Size;
 721                 if( head->Magic == MAGIC_FREE )
 722                 {
 723                         nFree ++;
 724                         freeBytes += head->Size;
 725                 }
 726                 else if( head->Magic == MAGIC_USED) {
 727                         if(maxAlloc < head->Size)       maxAlloc = head->Size;
 728                         if(minAlloc == -1 || minAlloc > head->Size)
 729                                 minAlloc = head->Size;
 730                 }
 731                 else {
 732                         Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
 733                         break;
 734                 }
 735
 736                 // Print the block info?
 737                 #if 1
 738                 if( head->Magic == MAGIC_FREE )
 739                         Log_Debug("Heap", "%p (%P) - 0x%x free",
 740                                 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
 741                 else
 742                         Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
 743                                 head->Data, MM_GetPhysAddr(&head->Data), head->Size,
 744                                 head->ValidSize, head->File, head->Line,
 745                                 now() - head->AllocateTime
 746                                 );
 747                 #endif
 748         }
 749
 750         Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
 751         Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
 752         if(nBlocks != 0)
 753                 frag = (nFree-1)*10000/nBlocks;
 754         else
 755                 frag = 0;
 756         Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
 757         if(nBlocks <= nFree)
 758                 avgAlloc = 0;
 759         else
 760                 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
 761         if(avgAlloc != 0)
 762                 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
 763         else
 764                 overhead = 0;
 765         Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
 766                 avgAlloc, overhead/100, overhead%100
 767                 );
 768         Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
 769                 minAlloc, maxAlloc);
 770
 771         // Scan and get distribution
 772         #if 1
 773         if(nBlocks > 0)
 774         {
 775                 struct {
 776                         Uint    Size;
 777                         Uint    Count;
 778                 }       sizeCounts[nBlocks];
 779                  int    i;
 780
 781                 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
 782
 783                 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
 784                 {
 785                         for( i = 0; i < nBlocks; i ++ ) {
 786                                 if( sizeCounts[i].Size == 0 )
 787                                         break;
 788                                 if( sizeCounts[i].Size == head->Size )
 789                                         break;
 790                         }
 791                         // Should never reach this part (in a non-concurrent case)
 792                         if( i == nBlocks )      continue;
 793                         sizeCounts[i].Size = head->Size;
 794                         sizeCounts[i].Count ++;
 795                         #if 0
 796                         Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
 797                                 head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
 798                                 );
 799                         Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
 800                                 sizeCounts[i].Size, sizeCounts[i].Count);
 801                         #endif
 802                 }
 803
 804                 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
 805                 {
 806                         Log("Heap_Stats: 0x%x - %i blocks",
 807                                 sizeCounts[i].Size, sizeCounts[i].Count
 808                                 );
 809                 }
 810         }
 811         #endif
 812 }
 813
 814 // === EXPORTS ===
 815 EXPORT(Heap_Allocate);
 816 EXPORT(Heap_AllocateZero);
 817 EXPORT(Heap_Reallocate);
 818 EXPORT(Heap_Deallocate);
 819 EXPORT(Heap_IsHeapAddr);