KernelLand/Kernel/heap.c

   1 /*
   2  * Acess2 Kernel
   3  * - By John Hodge (thePowersGang)
   4  *
   5  * heap.c
   6  * - Dynamic memory allocation
   7  */
   8 #include <acess.h>
   9 #include <mm_virt.h>
  10 #include <heap_int.h>
  11 #include <limits.h>
  12 #include <debug_hooks.h>
  13
  14 #define WARNINGS        1       // Warn and dump on heap errors
  15 #define DEBUG_TRACE     0       // Enable tracing of allocations
  16 #define VERBOSE_DUMP    0       // Set to 1 to enable a verbose dump when heap errors are encountered
  17 #define VALIDATE_ON_ALLOC       1       // Set to 1 to enable validation of the heap on all malloc() calls
  18
  19 // === CONSTANTS ===
  20 #define HEAP_INIT_SIZE  0x8000  // 32 KiB
  21 #define MIN_SIZE        (sizeof(void*))*8       // 8 Machine Words
  22 #define POW2_SIZES      0
  23 #define COMPACT_HEAP    0       // Use 4 byte header?
  24 #define FIRST_FIT       0
  25
  26 //#define       MAGIC_FOOT      0x2ACE5505
  27 //#define       MAGIC_FREE      0xACE55000
  28 //#define       MAGIC_USED      0x862B0505      // MAGIC_FOOT ^ MAGIC_FREE
  29 #define MAGIC_FOOT      0x544F4F46      // 'FOOT'
  30 #define MAGIC_FREE      0x45455246      // 'FREE'
  31 #define MAGIC_USED      0x44455355      // 'USED'
  32
  33 // === PROTOTYPES ===
  34 void    Heap_Install(void);
  35 void    *Heap_Extend(size_t Bytes);
  36 void    *Heap_Merge(tHeapHead *Head);
  37 static const size_t     Heap_int_GetBlockSize(size_t AllocSize);
  38 //void  *Heap_Allocate(const char *File, int Line, size_t Bytes);
  39 //void  *Heap_AllocateZero(const char *File, int Line, size_t Bytes);
  40 //void  *Heap_Reallocate(const char *File, int Line, void *Ptr, size_t Bytes);
  41 //void  Heap_Deallocate(const char *File, int Line, void *Ptr);
  42  int    Heap_int_ApplyWatchpont(void *Addr, bool Enabled);
  43 void    Heap_int_WatchBlock(tHeapHead *Head, bool Enabled);
  44 //void  Heap_Dump(void);
  45 void    Heap_ValidateDump(int bVerbose);
  46 //void  Heap_Stats(void);
  47
  48 // === GLOBALS ===
  49 tMutex  glHeap;
  50 tHeapHead       *gHeapStart;
  51 tHeapHead       *gHeapEnd;
  52
  53 // === CODE ===
  54 void Heap_Install(void)
  55 {
  56         gHeapStart = (void*)MM_KHEAP_BASE;
  57         gHeapEnd   = gHeapStart;
  58         Heap_Extend(HEAP_INIT_SIZE);
  59 }
  60
  61 static inline tHeapHead *Heap_NextHead(tHeapHead *Head) {
  62         return (void*)( (char*)Head + Head->Size );
  63 }
  64 static inline tHeapFoot *Heap_ThisFoot(tHeapHead *Head) {
  65         return (void*)( (char*)Head + Head->Size - sizeof(tHeapFoot) );
  66 }
  67 static inline tHeapFoot *Heap_PrevFoot(tHeapHead *Head) {
  68         //ASSERT(Head != gHeapStart);
  69         return (void*)( (char*)Head - sizeof(tHeapFoot) );
  70 }
  71
  72 /**
  73  * \brief Extend the size of the heap
  74  */
  75 void *Heap_Extend(size_t Bytes)
  76 {
  77         //Debug("Heap_Extend(0x%x)", Bytes);
  78
  79         // Bounds Check
  80         if( gHeapEnd == (tHeapHead*)MM_KHEAP_MAX ) {
  81                 Log_Error("Heap", "Heap limit reached (%p)", (void*)MM_KHEAP_MAX);
  82                 return NULL;
  83         }
  84
  85         if( Bytes == 0 ) {
  86                 Log_Warning("Heap", "Heap_Extend called with Bytes=%i", Bytes);
  87                 return NULL;
  88         }
  89
  90         const Uint      pages = (Bytes + 0xFFF) >> 12;
  91         tHeapHead       *new_end = (void*)( (tVAddr)gHeapEnd + (pages << 12) );
  92         // Bounds Check
  93         if( new_end > (tHeapHead*)MM_KHEAP_MAX )
  94         {
  95                 Log_Error("Heap", "Heap limit exceeded (%p)", (void*)new_end);
  96                 // TODO: Clip allocation to available space, and have caller check returned block
  97                 return NULL;
  98         }
  99
 100         // Heap expands in pages
 101         for( Uint i = 0; i < pages; i ++ )
 102         {
 103                 if( !MM_Allocate( (tPage*)gHeapEnd + i ) )
 104                 {
 105                         Warning("OOM - Heap_Extend (%i bytes)");
 106                         Heap_Dump();
 107                         return NULL;
 108                 }
 109         }
 110
 111         // Increase heap end
 112         tHeapHead       *head = gHeapEnd;
 113         gHeapEnd = new_end;
 114
 115         // Create Block
 116         head->Size = (Bytes+0xFFF)&~0xFFF;
 117         head->Magic = MAGIC_FREE;
 118         tHeapFoot *foot = Heap_ThisFoot(head);
 119         foot->Head = head;
 120         foot->Magic = MAGIC_FOOT;
 121
 122         return Heap_Merge(head);        // Merge with previous block
 123 }
 124
 125 /**
 126  * \brief Merges two adjacent heap blocks
 127  */
 128 void *Heap_Merge(tHeapHead *Head)
 129 {
 130         //Log("Heap_Merge: (Head=%p)", Head);
 131         tHeapFoot *thisFoot = Heap_ThisFoot(Head);
 132
 133         ASSERT( Heap_NextHead(Head) <= gHeapEnd );
 134
 135         // Merge Left (Down)
 136         tHeapFoot *foot = Heap_PrevFoot(Head);
 137         if( Head > gHeapStart && foot->Head->Magic == MAGIC_FREE)
 138         {
 139                 foot->Head->Size += Head->Size; // Increase size
 140                 thisFoot->Head = foot->Head;    // Change backlink
 141                 Head->Magic = 0;        // Clear old head
 142                 Head->Size = 0;
 143                 Head = foot->Head;      // Save new head address
 144                 foot->Head = NULL;      // Clear central footer
 145                 foot->Magic = 0;
 146         }
 147
 148         // Merge Right (Upwards)
 149         tHeapHead *nexthead = Heap_NextHead(Head);
 150         if(nexthead < gHeapEnd && nexthead->Magic == MAGIC_FREE)
 151         {
 152                 Head->Size += nexthead->Size;
 153                 foot = Heap_ThisFoot(Head);
 154                 foot->Head = Head;      // Update Backlink
 155                 thisFoot->Head = NULL;  // Clear old footer
 156                 thisFoot->Magic = 0;
 157                 nexthead->Size = 0;             // Clear old header
 158                 nexthead->Magic = 0;
 159         }
 160
 161         // Return new address
 162         return Head;
 163 }
 164
 165 static const size_t Heap_int_GetBlockSize(size_t AllocSize)
 166 {
 167         size_t Bytes;
 168         #if POW2_SIZES
 169         Bytes = AllocSize + sizeof(tHeapHead) + sizeof(tHeapFoot);
 170         Bytes = 1UUL << LOG2(Bytes);
 171         #else
 172         Bytes = (AllocSize + sizeof(tHeapHead) + sizeof(tHeapFoot) + MIN_SIZE-1) & ~(MIN_SIZE-1);
 173         #endif
 174         return Bytes;
 175 }
 176
 177 /**
 178  * \param File  Allocating source file
 179  * \param Line  Source line
 180  * \param __Bytes       Size of region to allocate
 181  */
 182 void *Heap_Allocate(const char *File, int Line, size_t __Bytes)
 183 {
 184         tHeapHead       *newhead;
 185         tHeapFoot       *foot, *newfoot;
 186         tHeapHead       *best = NULL;
 187         Uint    bestSize = UINT_MAX;    // Speed hack
 188         size_t  Bytes;
 189
 190         if( __Bytes == 0 ) {
 191                 return NULL;    // TODO: Return a known un-mapped range.
 192 //              return INVLPTR;
 193         }
 194
 195         #if VALIDATE_ON_ALLOC
 196         Heap_Validate();
 197         #endif
 198
 199         // Get required size
 200         Bytes = Heap_int_GetBlockSize(__Bytes);
 201
 202         // Lock Heap
 203         Mutex_Acquire(&glHeap);
 204
 205         // Traverse Heap
 206         for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
 207         {
 208                 // Alignment Check
 209                 #if POW2_SIZES
 210                 if( head->Size != 1UUL << LOG2(head->Size) ) {
 211                 #else
 212                 if( head->Size & (MIN_SIZE-1) ) {
 213                 #endif
 214                         Mutex_Release(&glHeap); // Release spinlock
 215                         #if WARNINGS
 216                         Log_Warning("Heap", "Size of heap address %p is invalid"
 217                                 " - not aligned (0x%x) [at paddr 0x%x]",
 218                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 219                         Heap_ValidateDump(VERBOSE_DUMP);
 220                         #endif
 221                         return NULL;
 222                 }
 223                 if( head->Size < MIN_SIZE ) {
 224                         Mutex_Release(&glHeap);
 225                         #if WARNINGS
 226                         Log_Warning("Heap", "Size of heap address %p is invalid"
 227                                 " - Too small (0x%x) [at paddr 0x%x]",
 228                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 229                         Heap_ValidateDump(VERBOSE_DUMP);
 230                         #endif
 231                         return NULL;
 232                 }
 233                 if( head->Size > (2<<30) ) {
 234                         Mutex_Release(&glHeap);
 235                         #if WARNINGS
 236                         Log_Warning("Heap", "Size of heap address %p is invalid"
 237                                 " - Over 2GiB (0x%x) [at paddr 0x%x]",
 238                                 head, head->Size, MM_GetPhysAddr(&head->Size));
 239                         Heap_ValidateDump(VERBOSE_DUMP);
 240                         #endif
 241                         return NULL;
 242                 }
 243
 244                 // Check if allocated
 245                 if(head->Magic == MAGIC_USED)   continue;
 246                 // Error check
 247                 if(head->Magic != MAGIC_FREE)   {
 248                         Mutex_Release(&glHeap); // Release spinlock
 249                         #if WARNINGS
 250                         Log_Warning("Heap", "Magic of heap address %p is invalid (%p = 0x%x)",
 251                                 head, &head->Magic, head->Magic);
 252                         Heap_ValidateDump(VERBOSE_DUMP);
 253                         #endif
 254                         return NULL;
 255                 }
 256
 257                 // Size check
 258                 if(head->Size < Bytes)  continue;
 259
 260                 // Perfect fit
 261                 if(head->Size == Bytes) {
 262                         head->Magic = MAGIC_USED;
 263                         head->File = File;
 264                         head->Line = Line;
 265                         head->ValidSize = __Bytes;
 266                         head->AllocateTime = now();
 267                         Mutex_Release(&glHeap); // Release spinlock
 268                         #if DEBUG_TRACE
 269                         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 270                                 head->Data, head->Size, File, Line);
 271                         #endif
 272                         return head->Data;
 273                 }
 274
 275                 // Break out of loop
 276                 #if FIRST_FIT
 277                 best = head;
 278                 bestSize = head->Size;
 279                 break;
 280                 #else
 281                 // or check if the block is the best size
 282                 if(bestSize > head->Size) {
 283                         best = head;
 284                         bestSize = head->Size;
 285                 }
 286                 #endif
 287         }
 288
 289         // If no block large enough is found, create one
 290         if(!best)
 291         {
 292                 best = Heap_Extend( Bytes );
 293                 // Check for errors
 294                 if(!best) {
 295                         Warning("OOM when allocating 0x%x bytes", Bytes);
 296                         Mutex_Release(&glHeap); // Release spinlock
 297                         return NULL;
 298                 }
 299                 // Check size
 300                 if(best->Size == Bytes) {
 301                         best->Magic = MAGIC_USED;       // Mark block as used
 302                         best->File = File;
 303                         best->Line = Line;
 304                         best->ValidSize = __Bytes;
 305                         best->AllocateTime = now();
 306                         Mutex_Release(&glHeap); // Release spinlock
 307                         #if DEBUG_TRACE
 308                         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 309                                 best->Data, best->Size, File, Line);
 310                         #endif
 311                         return best->Data;
 312                 }
 313         }
 314
 315         // Split Block
 316         // - Save size for new block
 317         size_t  newsize = best->Size - Bytes;
 318         // - Allocate beginning of old block
 319         best->Size = Bytes;             // Update size in old header
 320         best->ValidSize = __Bytes;
 321         best->Magic = MAGIC_USED;       // Mark block as used
 322         best->File = File;
 323         best->Line = Line;
 324         best->AllocateTime = now();
 325         // - Create a new foot on old block
 326         newfoot = Heap_ThisFoot(best);
 327         newfoot->Head = best;   // Create new footer
 328         newfoot->Magic = MAGIC_FOOT;
 329         // - Create a new header after resized old
 330         newhead = Heap_NextHead(best);
 331         newhead->Size = newsize;
 332         newhead->Magic = MAGIC_FREE;
 333         newhead->ValidSize = 0;
 334         newhead->File = NULL;
 335         newhead->Line = 0;
 336         // - Update footer
 337         foot = Heap_ThisFoot(newhead);
 338         foot->Head = newhead;
 339
 340         Mutex_Release(&glHeap); // Release spinlock
 341         #if DEBUG_TRACE
 342         Log_Debug("Heap", "Malloc'd %p (0x%x bytes), returning to %s:%i",
 343                 best->Data, best->Size, File, Line);
 344         #endif
 345         return best->Data;
 346 }
 347
 348 /**
 349  * \brief Free an allocated memory block
 350  */
 351 void Heap_Deallocate(const char *File, int Line, void *Ptr)
 352 {
 353         // INVLPTR is returned from Heap_Allocate when the allocation
 354         // size is zero.
 355         if( Ptr == INVLPTR )    return;
 356         // free(NULL) is a no-op
 357         if( Ptr == NULL )       return;
 358
 359         // Alignment Check
 360         if( (tVAddr)Ptr % sizeof(void*) != 0 ) {
 361                 Log_Warning("Heap", "free - Passed a non-aligned address (%p)", Ptr);
 362                 return;
 363         }
 364
 365         // Sanity check
 366         if((Uint)Ptr < (Uint)gHeapStart || (Uint)Ptr > (Uint)gHeapEnd)
 367         {
 368                 Log_Warning("Heap", "free - Passed a non-heap address by %p (%p < %p < %p)",
 369                         __builtin_return_address(0), gHeapStart, Ptr, gHeapEnd);
 370                 return;
 371         }
 372
 373         // Check memory block - Header
 374         tHeapHead *head = (tHeapHead*)Ptr - 1;
 375         if(head->Magic == MAGIC_FREE) {
 376                 Log_Warning("Heap", "free - Passed a freed block (%p) by %s:%i (was freed by %s:%i)",
 377                         head, File, Line,
 378                         head->File, head->Line);
 379                 Proc_PrintBacktrace();
 380                 return;
 381         }
 382         if(head->Magic != MAGIC_USED) {
 383                 Log_Warning("Heap", "free - Magic value is invalid (%p, 0x%x)", head, head->Magic);
 384                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 385                 return;
 386         }
 387
 388         // Check memory block - Footer
 389         tHeapFoot *foot = Heap_ThisFoot(head);
 390         if(foot->Head != head) {
 391                 Log_Warning("Heap", "free - Footer backlink is incorrect (%p, 0x%x)", head, foot->Head);
 392                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 393                 return;
 394         }
 395         if(foot->Magic != MAGIC_FOOT) {
 396                 Log_Warning("Heap", "free - Footer magic is invalid (%p, %p = 0x%x)",
 397                         head, &foot->Magic, foot->Magic);
 398                 Log_Notice("Heap", "Allocated by %s:%i", head->File, head->Line);
 399                 return;
 400         }
 401
 402         #if DEBUG_TRACE
 403         Log_Debug("Heap", "free: %p freed by %s:%i (%i old)",
 404                 Ptr, File, Line, now()-head->AllocateTime);
 405         #endif
 406
 407         // Lock
 408         Mutex_Acquire( &glHeap );
 409
 410         Heap_int_WatchBlock(head, false);
 411
 412         // Mark as free
 413         head->Magic = MAGIC_FREE;
 414         head->File = File;
 415         head->Line = Line;
 416         head->ValidSize = 0;
 417         // Merge blocks
 418         Heap_Merge( head );
 419
 420         // Release
 421         Mutex_Release( &glHeap );
 422 }
 423
 424 /**
 425  * \brief Increase/Decrease the size of an allocation
 426  * \param File  Calling File
 427  * \param Line  Calling Line
 428  * \param __ptr Old memory
 429  * \param __size        New Size
 430  */
 431 void *Heap_Reallocate(const char *File, int Line, void *__ptr, size_t __size)
 432 {
 433         tHeapHead       *head = (tHeapHead*)__ptr - 1;
 434         tHeapHead       *nextHead;
 435         tHeapFoot       *foot;
 436         Uint    newSize = Heap_int_GetBlockSize(__size);
 437
 438         // Check for reallocating NULL
 439         if(__ptr == NULL)
 440                 return Heap_Allocate(File, Line, __size);
 441
 442         if( !Heap_IsHeapAddr(__ptr)) {
 443                 Log_Error("Heap", "%s:%i passed non-heap address %p when reallocating to %zi",
 444                         File, Line, __ptr, __size);
 445                 return NULL;
 446         }
 447
 448         // Check if resize is needed
 449         // TODO: Reduce size of block if needed
 450         if(newSize <= head->Size) {
 451                 #if DEBUG_TRACE
 452                 Log_Debug("Heap", "realloc maintain %p (0x%x >= 0x%x), returning to %s:%i",
 453                         head->Data, head->Size, newSize, File, Line);
 454                 #endif
 455                 return __ptr;
 456         }
 457
 458         #if VALIDATE_ON_ALLOC
 459         Heap_Validate();
 460         #endif
 461
 462         Heap_int_WatchBlock(head, false);
 463
 464         // Check if next block is free
 465         nextHead = Heap_NextHead(head);
 466
 467         // Extend into next block
 468         if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
 469         {
 470                 #if DEBUG_TRACE
 471                 Log_Debug("Heap", "realloc expand %p (0x%x to 0x%x), returning to %s:%i",
 472                         head->Data, head->Size, newSize, File, Line);
 473                 #endif
 474                 Uint    size = nextHead->Size + head->Size;
 475                 foot = Heap_ThisFoot(nextHead);
 476                 // Exact Fit
 477                 if(size == newSize) {
 478                         head->Size = newSize;
 479                         head->ValidSize = __size;
 480                         head->File = File;
 481                         head->Line = Line;
 482                         foot->Head = head;
 483                         nextHead->Magic = 0;
 484                         nextHead->Size = 0;
 485                         return __ptr;
 486                 }
 487                 // Create a new heap block
 488                 // - Update old with new information
 489                 head->Size = newSize;
 490                 head->File = File;      // Kinda counts as a new allocation
 491                 head->Line = Line;
 492                 head->ValidSize = __size;
 493                 // - Create new footer
 494                 foot = Heap_ThisFoot(head);
 495                 foot->Head = head;
 496                 foot->Magic = MAGIC_FOOT;
 497                 // - Create new header
 498                 nextHead = Heap_NextHead(head);
 499                 nextHead->Size = size - newSize;
 500                 nextHead->Magic = MAGIC_FREE;
 501                 // - Update old footer
 502                 Heap_ThisFoot(nextHead)->Head = nextHead;
 503                 return __ptr;
 504         }
 505
 506         // Extend downwards?
 507         foot = Heap_PrevFoot(head);
 508         nextHead = foot->Head;
 509         if(nextHead->Magic == MAGIC_FREE && nextHead->Size+head->Size >= newSize)
 510         {
 511                 Uint    size = nextHead->Size + head->Size;
 512                 // Inexact fit, split things up
 513                 if(size > newSize)
 514                 {
 515                         // TODO: Handle splitting of downwards blocks
 516                         Warning("[Heap   ] TODO: Space efficient realloc when new size is smaller");
 517                 }
 518
 519                 #if DEBUG_TRACE
 520                 Log_Debug("Heap", "realloc expand down %p (0x%x to 0x%x), returning to %s:%i",
 521                         head->Data, head->Size, newSize, File, Line);
 522                 #endif
 523
 524                 // Exact fit
 525                 Uint    oldDataSize;
 526                 // Set 1st (new/lower) header
 527                 nextHead->Magic = MAGIC_USED;
 528                 nextHead->Size = newSize;
 529                 nextHead->File = File;
 530                 nextHead->Line = Line;
 531                 nextHead->ValidSize = __size;
 532                 // Get 2nd (old) footer
 533                 foot = Heap_ThisFoot(nextHead);
 534                 foot->Head = nextHead;
 535                 // Save old data size
 536                 oldDataSize = head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead);
 537                 // Clear old header
 538                 head->Size = 0;
 539                 head->Magic = 0;
 540                 // Copy data
 541                 memmove(nextHead->Data, __ptr, oldDataSize);
 542                 // Return
 543                 return nextHead->Data;
 544         }
 545
 546         // Well, darn
 547         nextHead = Heap_Allocate( File, Line, __size );
 548         if(!nextHead)   return NULL;
 549         nextHead -= 1;
 550         nextHead->File = File;
 551         nextHead->Line = Line;
 552         nextHead->ValidSize = __size;
 553
 554         ASSERTC(head->Size, <, nextHead->Size);
 555         ASSERTC(__ptr, ==, head->Data);
 556         memcpy(
 557                 nextHead->Data,
 558                 __ptr,
 559                 head->Size - sizeof(tHeapFoot) - sizeof(tHeapHead)
 560                 );
 561
 562         free(__ptr);
 563         Heap_Validate();
 564
 565         return nextHead->Data;
 566 }
 567
 568 /**
 569  * \fn void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
 570  * \brief Allocate and Zero a buffer in memory
 571  * \param File  Allocating file
 572  * \param Line  Line of allocation
 573  * \param Bytes Size of the allocation
 574  */
 575 void *Heap_AllocateZero(const char *File, int Line, size_t Bytes)
 576 {
 577         void    *ret = Heap_Allocate(File, Line, Bytes);
 578         if(ret == NULL) return NULL;
 579
 580         memset( ret, 0, Bytes );
 581
 582         return ret;
 583 }
 584
 585 /**
 586  * \fn int Heap_IsHeapAddr(void *Ptr)
 587  * \brief Checks if an address is a heap pointer
 588  */
 589 int Heap_IsHeapAddr(void *Ptr)
 590 {
 591         tHeapHead       *head;
 592         if((Uint)Ptr < (Uint)gHeapStart)        return 0;
 593         if((Uint)Ptr > (Uint)gHeapEnd)  return 0;
 594         if((Uint)Ptr & (sizeof(Uint)-1))        return 0;
 595
 596         head = (void*)( (Uint)Ptr - sizeof(tHeapHead) );
 597         if(head->Magic != MAGIC_USED && head->Magic != MAGIC_FREE)
 598                 return 0;
 599
 600         return 1;
 601 }
 602
 603 int Heap_int_ApplyWatchpont(void *Word, bool enabled)
 604 {
 605         #if ARCHDIR_IS_x86
 606         static void     *active_wps[4];
 607         unsigned int    dr;
 608         for( dr = 2; dr < 4; dr ++ )
 609         {
 610                 if( (enabled && active_wps[dr] == NULL) || active_wps[dr] == Word)
 611                         break;
 612         }
 613         if(dr == 4) {
 614                 return 1;
 615         }
 616         if( enabled )
 617         {
 618                 active_wps[dr] = Word;
 619                 switch(dr)
 620                 {
 621                 //case 0:       ASM("mov %0, %%dr0" : : "r" (Word));    break;
 622                 //case 1:       ASM("mov %0, %%dr1" : : "r" (Word));    break;
 623                 case 2: ASM("mov %0, %%dr2" : : "r" (Word));    break;
 624                 case 3: ASM("mov %0, %%dr3" : : "r" (Word));    break;
 625                 default:        ASSERTC(dr,<,4);        return 1;
 626                 }
 627         }
 628         else
 629         {
 630                 active_wps[dr] = NULL;
 631         }
 632         Uint32  dr7flag;
 633         ASM("MOV %%dr7, %0" : "=r" (dr7flag));
 634         dr7flag &= ~(0x2 << (dr*2));
 635         dr7flag &= ~(0xF000 << (dr*4));
 636         if( enabled ) {
 637                 dr7flag |= 0x2 << (dr*2);
 638                 dr7flag |= 0xD000 << (dr*4);    // 4 bytes, write
 639                 Debug("Heap_int_ApplyWatchpont: Watchpoint #%i %p ENABLED", dr, Word);
 640         }
 641         else {
 642                 Debug("Heap_int_ApplyWatchpont: Watchpoint #%i %p disabled", dr, Word);
 643         }
 644         ASM("MOV %0, %%dr7" : : "r" (dr7flag));
 645         return 0;
 646         #else
 647         return 1;
 648         #endif
 649 }
 650
 651 void Heap_int_WatchBlock(tHeapHead *Head, bool Enabled)
 652 {
 653         int rv;
 654         rv = Heap_int_ApplyWatchpont( &Head->Size, Enabled );
 655         //rv = Heap_int_ApplyWatchpont( &Head->Magic, Enabled );
 656         rv = rv + Heap_int_ApplyWatchpont( &Heap_ThisFoot(Head)->Head, Enabled );
 657         if(rv && Enabled) {
 658                 Warning("Can't apply watch on %p", Head);
 659         }
 660 }
 661
 662 int Heap_WatchBlock(void *Ptr)
 663 {
 664         //Heap_int_ApplyWatchpont();
 665         tHeapHead       *head;
 666         if((Uint)Ptr < (Uint)gHeapStart)        return 0;
 667         if((Uint)Ptr > (Uint)gHeapEnd)  return 0;
 668         if((Uint)Ptr & (sizeof(Uint)-1))        return 0;
 669
 670         head = (tHeapHead*)Ptr - 1;
 671
 672         Heap_int_WatchBlock( head, true );
 673
 674         return 0;
 675 }
 676
 677 /**
 678  */
 679 void Heap_Validate(void)
 680 {
 681         // Call dump non-verbosely.
 682         // - If a heap error is detected, it'll print
 683         Heap_ValidateDump(0);
 684 }
 685
 686 void Heap_Dump(void)
 687 {
 688         Heap_ValidateDump(1);
 689 }
 690
 691 void Heap_ValidateDump(int bVerbose)
 692 {
 693         tHeapHead       *head, *badHead;
 694         tHeapFoot       *foot = NULL;
 695         static int      in_heap_dump;
 696
 697         if( in_heap_dump )      return;
 698
 699         in_heap_dump = 1;
 700
 701         head = gHeapStart;
 702         while( (Uint)head < (Uint)gHeapEnd )
 703         {
 704                 foot = Heap_ThisFoot(head);
 705
 706                 if( bVerbose )
 707                 {
 708                         Log_Log("Heap", "%p (%P): 0x%08x (%i) %4C",
 709                                 head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 710                         Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
 711                         if(head->File) {
 712                                 Log_Log("Heap", "%sowned by %s:%i",
 713                                         (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
 714                         }
 715                 }
 716
 717                 // Sanity Check Header
 718                 if(head->Size == 0) {
 719                         Log_Warning("Heap", "HALTED - Size is zero");
 720                         break;
 721                 }
 722                 if(head->Size & (MIN_SIZE-1)) {
 723                         Log_Warning("Heap", "HALTED - Size is malaligned");
 724                         break;
 725                 }
 726                 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
 727                         Log_Warning("Heap", "HALTED - Head Magic is Bad");
 728                         break;
 729                 }
 730
 731                 // Check footer
 732                 if(foot->Magic != MAGIC_FOOT) {
 733                         Log_Warning("Heap", "HALTED - Foot Magic is Bad");
 734                         break;
 735                 }
 736                 if(head != foot->Head) {
 737                         Log_Warning("Heap", "HALTED - Footer backlink is invalid");
 738                         break;
 739                 }
 740
 741                 if( bVerbose )
 742                 {
 743                         Log_Log("Heap", "");
 744                 }
 745
 746                 // All OK? Go to next
 747                 head = foot->NextHead;
 748         }
 749
 750         // If the heap is valid, ok!
 751         if( (tVAddr)head == (tVAddr)gHeapEnd ) {
 752                 in_heap_dump = 0;
 753                 return ;
 754         }
 755
 756         // Check for a bad return
 757         if( (tVAddr)head >= (tVAddr)gHeapEnd ) {
 758                 in_heap_dump = 0;
 759                 return ;
 760         }
 761
 762         // If not verbose, we need to dump the failing block
 763         if( !bVerbose )
 764         {
 765                 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
 766                         head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 767                 if(foot)
 768                         Log_Log("Heap", "Foot %p = {Head:%p,Magic:%4C}", foot, foot->Head, &foot->Magic);
 769                 if(head->File) {
 770                         Log_Log("Heap", "%sowned by %s:%i",
 771                                 (head->Magic==MAGIC_FREE?"was ":""), head->File, head->Line);
 772                 }
 773                 Log_Log("Heap", "");
 774         }
 775
 776         badHead = head;
 777
 778         // Work backwards
 779         foot = Heap_PrevFoot(gHeapEnd);
 780         Log_Log("Heap", "==== Going Backwards ==== (from %p)", foot);
 781         head = foot->Head;
 782         while( (tVAddr)head >= (tVAddr)badHead )
 783         {
 784                 Log_Log("Heap", "%p (%P): 0x%08lx %i %4C",
 785                         head, MM_GetPhysAddr(head), head->Size, head->ValidSize, &head->Magic);
 786                 Log_Log("Heap", "%p %4C", foot->Head, &foot->Magic);
 787                 if(head->File)
 788                         Log_Log("Heap", "%sowned by %s:%i",
 789                                 (head->Magic!=MAGIC_USED?"was ":""),
 790                                 head->File, head->Line);
 791                 Log_Log("Heap", "");
 792
 793                 // Sanity Check Header
 794                 if(head->Size == 0) {
 795                         Log_Warning("Heap", "HALTED - Size is zero");
 796                         break;
 797                 }
 798                 if(head->Size & (MIN_SIZE-1)) {
 799                         Log_Warning("Heap", " - Size is malaligned (&0x%x)", ~(MIN_SIZE-1));
 800                         break ;
 801                 }
 802                 if(head->Magic != MAGIC_FREE && head->Magic != MAGIC_USED) {
 803                         Log_Warning("Heap", "HALTED - Head Magic is Bad");
 804                         break;
 805                 }
 806
 807                 // Check footer
 808                 if(foot->Magic != MAGIC_FOOT) {
 809                         Log_Warning("Heap", "HALTED - Foot Magic is Bad");
 810                         break;
 811                 }
 812                 if(head != foot->Head) {
 813                         Log_Warning("Heap", "HALTED - Footer backlink is invalid");
 814                         break;
 815                 }
 816
 817                 if(head == badHead)     break;
 818
 819                 foot = Heap_PrevFoot(head);
 820                 head = foot->Head;
 821                 Log_Debug("Heap", "head=%p", head);
 822         }
 823
 824         Panic("Heap_Dump - Heap is corrupted, kernel panic! (%p)", badHead);
 825 }
 826
 827 void Heap_Stats(void)
 828 {
 829          int    nBlocks = 0;
 830          int    nFree = 0;
 831          int    totalBytes = 0;
 832          int    freeBytes = 0;
 833          int    maxAlloc=0, minAlloc=-1;
 834          int    avgAlloc, frag, overhead;
 835
 836         for( tHeapHead *head = gHeapStart; head < gHeapEnd; head = Heap_NextHead(head) )
 837         {
 838                 nBlocks ++;
 839                 totalBytes += head->Size;
 840                 if( head->Magic == MAGIC_FREE )
 841                 {
 842                         nFree ++;
 843                         freeBytes += head->Size;
 844                 }
 845                 else if( head->Magic == MAGIC_USED) {
 846                         if(maxAlloc < head->Size)       maxAlloc = head->Size;
 847                         if(minAlloc == -1 || minAlloc > head->Size)
 848                                 minAlloc = head->Size;
 849                 }
 850                 else {
 851                         Log_Warning("Heap", "Magic on %p invalid, skipping remainder of heap", head);
 852                         break;
 853                 }
 854
 855                 // Print the block info?
 856                 #if 1
 857                 if( head->Magic == MAGIC_FREE )
 858                         Log_Debug("Heap", "%p (%P) - 0x%x free",
 859                                 head->Data, MM_GetPhysAddr(&head->Data), head->Size);
 860                 else
 861                         Log_Debug("Heap", "%p (%P) - 0x%x (%i) Owned by %s:%i (%lli ms old)",
 862                                 head->Data, MM_GetPhysAddr(&head->Data), head->Size,
 863                                 head->ValidSize, head->File, head->Line,
 864                                 now() - head->AllocateTime
 865                                 );
 866                 #endif
 867         }
 868
 869         Log_Log("Heap", "%i blocks (0x%x bytes)", nBlocks, totalBytes);
 870         Log_Log("Heap", "%i free blocks (0x%x bytes)", nFree, freeBytes);
 871         if(nBlocks != 0)
 872                 frag = (nFree-1)*10000/nBlocks;
 873         else
 874                 frag = 0;
 875         Log_Log("Heap", "%i.%02i%% Heap Fragmentation", frag/100, frag%100);
 876         if(nBlocks <= nFree)
 877                 avgAlloc = 0;
 878         else
 879                 avgAlloc = (totalBytes-freeBytes)/(nBlocks-nFree);
 880         if(avgAlloc != 0)
 881                 overhead = (sizeof(tHeapFoot)+sizeof(tHeapHead))*10000/avgAlloc;
 882         else
 883                 overhead = 0;
 884         Log_Log("Heap", "Average allocation: %i bytes, Average Overhead: %i.%02i%%",
 885                 avgAlloc, overhead/100, overhead%100
 886                 );
 887         Log_Log("Heap", "Smallest Block: %i bytes, Largest: %i bytes",
 888                 minAlloc, maxAlloc);
 889
 890         // Scan and get distribution
 891         #if 1
 892         if(nBlocks > 0)
 893         {
 894                 struct {
 895                         Uint    Size;
 896                         Uint    Count;
 897                 }       sizeCounts[nBlocks];
 898                  int    i;
 899
 900                 memset(sizeCounts, 0, nBlocks*sizeof(sizeCounts[0]));
 901
 902                 for(tHeapHead *head = gHeapStart; head < gHeapEnd; head = (void*)( (tVAddr)head + head->Size ) )
 903                 {
 904                         for( i = 0; i < nBlocks; i ++ ) {
 905                                 if( sizeCounts[i].Size == 0 )
 906                                         break;
 907                                 if( sizeCounts[i].Size == head->Size )
 908                                         break;
 909                         }
 910                         // Should never reach this part (in a non-concurrent case)
 911                         if( i == nBlocks )      continue;
 912                         sizeCounts[i].Size = head->Size;
 913                         sizeCounts[i].Count ++;
 914                         #if 0
 915                         Log("Heap_Stats: %i %p - 0x%x bytes (%s) (%i)", nBlocks, head,
 916                                 head->Size, (head->Magic==MAGIC_FREE?"FREE":"used"), i
 917                                 );
 918                         Log("Heap_Stats: sizeCounts[%i] = {Size:0x%x, Count: %i}", i,
 919                                 sizeCounts[i].Size, sizeCounts[i].Count);
 920                         #endif
 921                 }
 922
 923                 for( i = 0; i < nBlocks && sizeCounts[i].Count; i ++ )
 924                 {
 925                         Log("Heap_Stats: 0x%x - %i blocks",
 926                                 sizeCounts[i].Size, sizeCounts[i].Count
 927                                 );
 928                 }
 929         }
 930         #endif
 931 }
 932
 933 // === EXPORTS ===
 934 EXPORT(Heap_Allocate);
 935 EXPORT(Heap_AllocateZero);
 936 EXPORT(Heap_Reallocate);
 937 EXPORT(Heap_Deallocate);
 938 EXPORT(Heap_IsHeapAddr);