Kernel/arch/x86/mm_virt.c

   1 /*
   2  * AcessOS Microkernel Version
   3  * mm_virt.c
   4  *
   5  * Memory Map
   6  * 0xE0 - Kernel Base
   7  * 0xF0 - Kernel Stacks
   8  * 0xFD - Fractals
   9  * 0xFE - Unused
  10  * 0xFF - System Calls / Kernel's User Code
  11  */
  12 #define DEBUG   1
  13 #include <acess.h>
  14 #include <mm_phys.h>
  15 #include <proc.h>
  16
  17 #if USE_PAE
  18 # define TAB    21
  19 # define DIR    30
  20 #else
  21 # define TAB    22
  22 #endif
  23
  24 #define KERNEL_STACKS           0xF0000000
  25 #define KERNEL_STACK_SIZE       0x00008000
  26 #define KERNEL_STACKS_END       0xFC000000
  27 #define WORKER_STACKS           0x00100000      // Thread0 Only!
  28 #define WORKER_STACK_SIZE       KERNEL_STACK_SIZE
  29 #define WORKER_STACKS_END       0xB0000000
  30 #define NUM_WORKER_STACKS       ((WORKER_STACKS_END-WORKER_STACKS)/WORKER_STACK_SIZE)
  31
  32 #define PAE_PAGE_TABLE_ADDR     0xFC000000      // 16 MiB
  33 #define PAE_PAGE_DIR_ADDR       0xFCFC0000      // 16 KiB
  34 #define PAE_PAGE_PDPT_ADDR      0xFCFC3F00      // 32 bytes
  35 #define PAE_TMP_PDPT_ADDR       0xFCFC3F20      // 32 bytes
  36 #define PAE_TMP_DIR_ADDR        0xFCFE0000      // 16 KiB
  37 #define PAE_TMP_TABLE_ADDR      0xFD000000      // 16 MiB
  38
  39 #define PAGE_TABLE_ADDR 0xFC000000
  40 #define PAGE_DIR_ADDR   0xFC3F0000
  41 #define PAGE_CR3_ADDR   0xFC3F0FC0
  42 #define TMP_CR3_ADDR    0xFC3F0FC4      // Part of core instead of temp
  43 #define TMP_DIR_ADDR    0xFC3F1000      // Same
  44 #define TMP_TABLE_ADDR  0xFC400000
  45
  46 #define HW_MAP_ADDR             0xFE000000
  47 #define HW_MAP_MAX              0xFFEF0000
  48 #define NUM_HW_PAGES    ((HW_MAP_MAX-HW_MAP_ADDR)/0x1000)
  49 #define TEMP_MAP_ADDR   0xFFEF0000      // Allows 16 "temp" pages
  50 #define NUM_TEMP_PAGES  16
  51 #define LAST_BLOCK_ADDR 0xFFFF0000      // Free space for kernel provided user code/ *(-1) protection
  52
  53 #define PF_PRESENT      0x1
  54 #define PF_WRITE        0x2
  55 #define PF_USER         0x4
  56 #define PF_COW          0x200
  57 #define PF_PAGED        0x400
  58
  59 #define INVLPG(addr)    __asm__ __volatile__ ("invlpg (%0)"::"r"(addr))
  60
  61 #if USE_PAE
  62 typedef Uint64  tTabEnt;
  63 #else
  64 typedef Uint32  tTabEnt;
  65 #endif
  66
  67 // === IMPORTS ===
  68 extern Uint32   gaInitPageDir[1024];
  69 extern Uint32   gaInitPageTable[1024];
  70 extern void     Threads_SegFault(tVAddr Addr);
  71 extern void     Error_Backtrace(Uint eip, Uint ebp);
  72
  73 // === PROTOTYPES ===
  74 void    MM_PreinitVirtual();
  75 void    MM_InstallVirtual();
  76 void    MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs);
  77 void    MM_DumpTables(tVAddr Start, tVAddr End);
  78 tPAddr  MM_DuplicatePage(tVAddr VAddr);
  79
  80 // === GLOBALS ===
  81 #define gaPageTable     ((tTabEnt*)PAGE_TABLE_ADDR)
  82 #define gaPageDir       ((tTabEnt*)PAGE_DIR_ADDR)
  83 #define gaTmpTable      ((tTabEnt*)TMP_TABLE_ADDR)
  84 #define gaTmpDir        ((tTabEnt*)TMP_DIR_ADDR)
  85 #define gpPageCR3       ((tTabEnt*)PAGE_CR3_ADDR)
  86 #define gpTmpCR3        ((tTabEnt*)TMP_CR3_ADDR)
  87
  88 #define gaPAE_PageTable ((tTabEnt*)PAE_PAGE_TABLE_ADDR)
  89 #define gaPAE_PageDir   ((tTabEnt*)PAE_PAGE_DIR_ADDR)
  90 #define gaPAE_MainPDPT  ((tTabEnt*)PAE_PAGE_PDPT_ADDR)
  91 #define gaPAE_TmpTable  ((tTabEnt*)PAE_TMP_DIR_ADDR)
  92 #define gaPAE_TmpDir    ((tTabEnt*)PAE_TMP_DIR_ADDR)
  93 #define gaPAE_TmpPDPT   ((tTabEnt*)PAE_TMP_PDPT_ADDR)
  94  int    gbUsePAE = 0;
  95  int    gilTempMappings = 0;
  96  int    gilTempFractal = 0;
  97 Uint32  gWorkerStacks[(NUM_WORKER_STACKS+31)/32];
  98  int    giLastUsedWorker = 0;
  99
 100 // === CODE ===
 101 /**
 102  * \fn void MM_PreinitVirtual()
 103  * \brief Maps the fractal mappings
 104  */
 105 void MM_PreinitVirtual()
 106 {
 107         #if USE_PAE
 108         gaInitPDPT[ 0 ] = 0;
 109         gaInitPageDir[ ((PAGE_TABLE_ADDR >> TAB)-3*512+3)*2 ] = ((tTabEnt)&gaInitPageDir - KERNEL_BASE) | 3;
 110         #else
 111         gaInitPageDir[ 0 ] = 0;
 112         gaInitPageDir[ PAGE_TABLE_ADDR >> 22 ] = ((tTabEnt)&gaInitPageDir - KERNEL_BASE) | 3;
 113         #endif
 114         INVLPG( PAGE_TABLE_ADDR );
 115 }
 116
 117 /**
 118  * \fn void MM_InstallVirtual()
 119  * \brief Sets up the constant page mappings
 120  */
 121 void MM_InstallVirtual()
 122 {
 123          int    i;
 124
 125         #if USE_PAE
 126         // --- Pre-Allocate kernel tables
 127         for( i = KERNEL_BASE >> TAB; i < 1024*4; i ++ )
 128         {
 129                 if( gaPAE_PageDir[ i ] )        continue;
 130
 131                 // Skip stack tables, they are process unique
 132                 if( i > KERNEL_STACKS >> TAB && i < KERNEL_STACKS_END >> TAB) {
 133                         gaPAE_PageDir[ i ] = 0;
 134                         continue;
 135                 }
 136                 // Preallocate table
 137                 gaPAE_PageDir[ i ] = MM_AllocPhys() | 3;
 138                 INVLPG( &gaPAE_PageTable[i*512] );
 139                 memset( &gaPAE_PageTable[i*512], 0, 0x1000 );
 140         }
 141         #else
 142         // --- Pre-Allocate kernel tables
 143         for( i = KERNEL_BASE>>22; i < 1024; i ++ )
 144         {
 145                 if( gaPageDir[ i ] )    continue;
 146                 // Skip stack tables, they are process unique
 147                 if( i > KERNEL_STACKS >> 22 && i < KERNEL_STACKS_END >> 22) {
 148                         gaPageDir[ i ] = 0;
 149                         continue;
 150                 }
 151                 // Preallocate table
 152                 gaPageDir[ i ] = MM_AllocPhys() | 3;
 153                 INVLPG( &gaPageTable[i*1024] );
 154                 memset( &gaPageTable[i*1024], 0, 0x1000 );
 155         }
 156         #endif
 157 }
 158
 159 /**
 160  * \fn void MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs)
 161  * \brief Called on a page fault
 162  */
 163 void MM_PageFault(tVAddr Addr, Uint ErrorCode, tRegs *Regs)
 164 {
 165         //ENTER("xAddr bErrorCode", Addr, ErrorCode);
 166
 167         // -- Check for COW --
 168         if( gaPageDir  [Addr>>22] & PF_PRESENT
 169          && gaPageTable[Addr>>12] & PF_PRESENT
 170          && gaPageTable[Addr>>12] & PF_COW )
 171         {
 172                 tPAddr  paddr;
 173                 if(MM_GetRefCount( gaPageTable[Addr>>12] & ~0xFFF ) == 1)
 174                 {
 175                         gaPageTable[Addr>>12] &= ~PF_COW;
 176                         gaPageTable[Addr>>12] |= PF_PRESENT|PF_WRITE;
 177                 }
 178                 else
 179                 {
 180                         paddr = MM_DuplicatePage( Addr );
 181                         MM_DerefPhys( gaPageTable[Addr>>12] & ~0xFFF );
 182                         gaPageTable[Addr>>12] &= PF_USER;
 183                         gaPageTable[Addr>>12] |= paddr|PF_PRESENT|PF_WRITE;
 184                 }
 185
 186                 INVLPG( Addr & ~0xFFF );
 187                 //LEAVE('-')
 188                 return;
 189         }
 190
 191         // If it was a user, tell the thread handler
 192         if(ErrorCode & 4) {
 193                 Warning("%s %s %s memory%s",
 194                         (ErrorCode&4?"User":"Kernel"),
 195                         (ErrorCode&2?"write to":"read from"),
 196                         (ErrorCode&1?"bad/locked":"non-present"),
 197                         (ErrorCode&16?" (Instruction Fetch)":"")
 198                         );
 199                 Warning("User Pagefault: Instruction at %p accessed %p", Regs->eip, Addr);
 200                 __asm__ __volatile__ ("sti");   // Restart IRQs
 201                 Threads_SegFault(Addr);
 202                 return ;
 203         }
 204
 205         // -- Check Error Code --
 206         if(ErrorCode & 8)
 207                 Warning("Reserved Bits Trashed!");
 208         else
 209         {
 210                 Warning("%s %s %s memory%s",
 211                         (ErrorCode&4?"User":"Kernel"),
 212                         (ErrorCode&2?"write to":"read from"),
 213                         (ErrorCode&1?"bad/locked":"non-present"),
 214                         (ErrorCode&16?" (Instruction Fetch)":"")
 215                         );
 216         }
 217
 218         Log("Code at %p accessed %p", Regs->eip, Addr);
 219         // Print Stack Backtrace
 220         Error_Backtrace(Regs->eip, Regs->ebp);
 221
 222         Log("gaPageDir[0x%x] = 0x%x", Addr>>22, gaPageDir[Addr>>22]);
 223         if( gaPageDir[Addr>>22] & PF_PRESENT )
 224                 Log("gaPageTable[0x%x] = 0x%x", Addr>>12, gaPageTable[Addr>>12]);
 225
 226         //MM_DumpTables(0, -1);
 227
 228         Panic("Page Fault at 0x%x (Accessed 0x%x)", Regs->eip, Addr);
 229 }
 230
 231 /**
 232  * \fn void MM_DumpTables(tVAddr Start, tVAddr End)
 233  * \brief Dumps the layout of the page tables
 234  */
 235 void MM_DumpTables(tVAddr Start, tVAddr End)
 236 {
 237         tVAddr  rangeStart = 0;
 238         tPAddr  expected = 0;
 239         tVAddr  curPos;
 240         Uint    page;
 241         const tPAddr    MASK = ~0xF98;
 242
 243         Start >>= 12;   End >>= 12;
 244
 245         #if 0
 246         Log("Directory Entries:");
 247         for(page = Start >> 10;
 248                 page < (End >> 10)+1;
 249                 page ++)
 250         {
 251                 if(gaPageDir[page])
 252                 {
 253                         Log(" 0x%08x-0x%08x :: 0x%08x",
 254                                 page<<22, ((page+1)<<22)-1,
 255                                 gaPageDir[page]&~0xFFF
 256                                 );
 257                 }
 258         }
 259         #endif
 260
 261         Log("Table Entries:");
 262         for(page = Start, curPos = Start<<12;
 263                 page < End;
 264                 curPos += 0x1000, page++)
 265         {
 266                 if( !(gaPageDir[curPos>>22] & PF_PRESENT)
 267                 ||  !(gaPageTable[page] & PF_PRESENT)
 268                 ||  (gaPageTable[page] & MASK) != expected)
 269                 {
 270                         if(expected) {
 271                                 Log(" 0x%08x-0x%08x => 0x%08x-0x%08x (%s%s%s%s)",
 272                                         rangeStart, curPos - 1,
 273                                         gaPageTable[rangeStart>>12] & ~0xFFF,
 274                                         (expected & ~0xFFF) - 1,
 275                                         (expected & PF_PAGED ? "p" : "-"),
 276                                         (expected & PF_COW ? "C" : "-"),
 277                                         (expected & PF_USER ? "U" : "-"),
 278                                         (expected & PF_WRITE ? "W" : "-")
 279                                         );
 280                                 expected = 0;
 281                         }
 282                         if( !(gaPageDir[curPos>>22] & PF_PRESENT) )     continue;
 283                         if( !(gaPageTable[curPos>>12] & PF_PRESENT) )   continue;
 284
 285                         expected = (gaPageTable[page] & MASK);
 286                         rangeStart = curPos;
 287                 }
 288                 if(expected)    expected += 0x1000;
 289         }
 290
 291         if(expected) {
 292                 Log("0x%08x-0x%08x => 0x%08x-0x%08x (%s%s%s%s)",
 293                         rangeStart, curPos - 1,
 294                         gaPageTable[rangeStart>>12] & ~0xFFF,
 295                         (expected & ~0xFFF) - 1,
 296                         (expected & PF_PAGED ? "p" : "-"),
 297                         (expected & PF_COW ? "C" : "-"),
 298                         (expected & PF_USER ? "U" : "-"),
 299                         (expected & PF_WRITE ? "W" : "-")
 300                         );
 301                 expected = 0;
 302         }
 303 }
 304
 305 /**
 306  * \fn tPAddr MM_Allocate(tVAddr VAddr)
 307  */
 308 tPAddr MM_Allocate(tVAddr VAddr)
 309 {
 310         tPAddr  paddr;
 311         //ENTER("xVAddr", VAddr);
 312         //__asm__ __volatile__ ("xchg %bx,%bx");
 313         // Check if the directory is mapped
 314         if( gaPageDir[ VAddr >> 22 ] == 0 )
 315         {
 316                 // Allocate directory
 317                 paddr = MM_AllocPhys();
 318                 //LOG("paddr = 0x%llx (new table)", paddr);
 319                 if( paddr == 0 ) {
 320                         Warning("MM_Allocate - Out of Memory (Called by %p)", __builtin_return_address(0));
 321                         LEAVE('i',0);
 322                         return 0;
 323                 }
 324                 // Map
 325                 gaPageDir[ VAddr >> 22 ] = paddr | 3;
 326                 // Mark as user
 327                 if(VAddr < MM_USER_MAX) gaPageDir[ VAddr >> 22 ] |= PF_USER;
 328
 329                 INVLPG( &gaPageDir[ VAddr >> 22 ] );
 330                 //LOG("Clearing new table");
 331                 memsetd( &gaPageTable[ (VAddr >> 12) & ~0x3FF ], 0, 1024 );
 332         }
 333         // Check if the page is already allocated
 334         else if( gaPageTable[ VAddr >> 12 ] != 0 ) {
 335                 Warning("MM_Allocate - Allocating to used address (%p)", VAddr);
 336                 //LEAVE('X', gaPageTable[ VAddr >> 12 ] & ~0xFFF);
 337                 return gaPageTable[ VAddr >> 12 ] & ~0xFFF;
 338         }
 339
 340         // Allocate
 341         paddr = MM_AllocPhys();
 342         //LOG("paddr = 0x%llx", paddr);
 343         if( paddr == 0 ) {
 344                 Warning("MM_Allocate - Out of Memory when allocating at %p (Called by %p)",
 345                         VAddr, __builtin_return_address(0));
 346                 //LEAVE('i',0);
 347                 return 0;
 348         }
 349         // Map
 350         gaPageTable[ VAddr >> 12 ] = paddr | 3;
 351         // Mark as user
 352         if(VAddr < MM_USER_MAX) gaPageTable[ VAddr >> 12 ] |= PF_USER;
 353         // Invalidate Cache for address
 354         INVLPG( VAddr & ~0xFFF );
 355
 356         //LEAVE('X', paddr);
 357         return paddr;
 358 }
 359
 360 /**
 361  * \fn void MM_Deallocate(tVAddr VAddr)
 362  */
 363 void MM_Deallocate(tVAddr VAddr)
 364 {
 365         if( gaPageDir[ VAddr >> 22 ] == 0 ) {
 366                 Warning("MM_Deallocate - Directory not mapped");
 367                 return;
 368         }
 369
 370         if(gaPageTable[ VAddr >> 12 ] == 0) {
 371                 Warning("MM_Deallocate - Page is not allocated");
 372                 return;
 373         }
 374
 375         // Dereference page
 376         MM_DerefPhys( gaPageTable[ VAddr >> 12 ] & ~0xFFF );
 377         // Clear page
 378         gaPageTable[ VAddr >> 12 ] = 0;
 379 }
 380
 381 /**
 382  * \fn tPAddr MM_GetPhysAddr(tVAddr Addr)
 383  * \brief Checks if the passed address is accesable
 384  */
 385 tPAddr MM_GetPhysAddr(tVAddr Addr)
 386 {
 387         if( !(gaPageDir[Addr >> 22] & 1) )
 388                 return 0;
 389         if( !(gaPageTable[Addr >> 12] & 1) )
 390                 return 0;
 391         return (gaPageTable[Addr >> 12] & ~0xFFF) | (Addr & 0xFFF);
 392 }
 393
 394
 395 /**
 396  * \fn int MM_IsUser(tVAddr VAddr)
 397  * \brief Checks if a page is user accessable
 398  */
 399 int MM_IsUser(tVAddr VAddr)
 400 {
 401         if( !(gaPageDir[VAddr >> 22] & 1) )
 402                 return 0;
 403         if( !(gaPageTable[VAddr >> 12] & 1) )
 404                 return 0;
 405         if( !(gaPageTable[VAddr >> 12] & PF_USER) )
 406                 return 0;
 407         return 1;
 408 }
 409
 410 /**
 411  * \fn void MM_SetCR3(tPAddr CR3)
 412  * \brief Sets the current process space
 413  */
 414 void MM_SetCR3(tPAddr CR3)
 415 {
 416         __asm__ __volatile__ ("mov %0, %%cr3"::"r"(CR3));
 417 }
 418
 419 /**
 420  * \fn int MM_Map(tVAddr VAddr, tPAddr PAddr)
 421  * \brief Map a physical page to a virtual one
 422  */
 423 int MM_Map(tVAddr VAddr, tPAddr PAddr)
 424 {
 425         //ENTER("xVAddr xPAddr", VAddr, PAddr);
 426         // Sanity check
 427         if( PAddr & 0xFFF || VAddr & 0xFFF ) {
 428                 Warning("MM_Map - Physical or Virtual Addresses are not aligned");
 429                 //LEAVE('i', 0);
 430                 return 0;
 431         }
 432
 433         // Align addresses
 434         PAddr &= ~0xFFF;        VAddr &= ~0xFFF;
 435
 436         // Check if the directory is mapped
 437         if( gaPageDir[ VAddr >> 22 ] == 0 )
 438         {
 439                 gaPageDir[ VAddr >> 22 ] = MM_AllocPhys() | 3;
 440
 441                 // Mark as user
 442                 if(VAddr < MM_USER_MAX) gaPageDir[ VAddr >> 22 ] |= PF_USER;
 443
 444                 INVLPG( &gaPageTable[ (VAddr >> 12) & ~0x3FF ] );
 445                 memsetd( &gaPageTable[ (VAddr >> 12) & ~0x3FF ], 0, 1024 );
 446         }
 447         // Check if the page is already allocated
 448         else if( gaPageTable[ VAddr >> 12 ] != 0 ) {
 449                 Warning("MM_Map - Allocating to used address");
 450                 //LEAVE('i', 0);
 451                 return 0;
 452         }
 453
 454         // Map
 455         gaPageTable[ VAddr >> 12 ] = PAddr | 3;
 456         // Mark as user
 457         if(VAddr < MM_USER_MAX) gaPageTable[ VAddr >> 12 ] |= PF_USER;
 458
 459         //LOG("gaPageTable[ 0x%x ] = (Uint)%p = 0x%x",
 460         //      VAddr >> 12, &gaPageTable[ VAddr >> 12 ], gaPageTable[ VAddr >> 12 ]);
 461
 462         // Reference
 463         MM_RefPhys( PAddr );
 464
 465         //LOG("INVLPG( 0x%x )", VAddr);
 466         INVLPG( VAddr );
 467
 468         //LEAVE('i', 1);
 469         return 1;
 470 }
 471
 472 /**
 473  * \fn tVAddr MM_ClearUser()
 474  * \brief Clear user's address space
 475  */
 476 tVAddr MM_ClearUser()
 477 {
 478         Uint    i, j;
 479
 480         // Copy Directories
 481         for( i = 0; i < (MM_USER_MAX>>22); i ++ )
 482         {
 483                 // Check if directory is not allocated
 484                 if( !(gaPageDir[i] & PF_PRESENT) ) {
 485                         gaPageDir[i] = 0;
 486                         continue;
 487                 }
 488
 489
 490                 for( j = 0; j < 1024; j ++ )
 491                 {
 492                         if( gaPageTable[i*1024+j] & 1 )
 493                                 MM_DerefPhys( gaPageTable[i*1024+j] & ~0xFFF );
 494                         gaPageTable[i*1024+j] = 0;
 495                 }
 496
 497                 MM_DerefPhys( gaPageDir[i] & ~0xFFF );
 498                 gaPageDir[i] = 0;
 499                 INVLPG( &gaPageTable[i*1024] );
 500         }
 501         INVLPG( gaPageDir );
 502
 503         return *gpPageCR3;
 504 }
 505
 506 /**
 507  * \fn tPAddr MM_Clone()
 508  * \brief Clone the current address space
 509  */
 510 tPAddr MM_Clone()
 511 {
 512         Uint    i, j;
 513         tVAddr  ret;
 514         Uint    page = 0;
 515         tVAddr  kStackBase = Proc_GetCurThread()->KernelStack - KERNEL_STACK_SIZE;
 516         void    *tmp;
 517
 518         LOCK( &gilTempFractal );
 519
 520         // Create Directory Table
 521         *gpTmpCR3 = MM_AllocPhys() | 3;
 522         INVLPG( gaTmpDir );
 523         //LOG("Allocated Directory (%x)", *gpTmpCR3);
 524         memsetd( gaTmpDir, 0, 1024 );
 525
 526         // Copy Tables
 527         for(i=0;i<768;i++)
 528         {
 529                 // Check if table is allocated
 530                 if( !(gaPageDir[i] & PF_PRESENT) ) {
 531                         gaTmpDir[i] = 0;
 532                         page += 1024;
 533                         continue;
 534                 }
 535
 536                 // Allocate new table
 537                 gaTmpDir[i] = MM_AllocPhys() | (gaPageDir[i] & 7);
 538                 INVLPG( &gaTmpTable[page] );
 539                 // Fill
 540                 for( j = 0; j < 1024; j ++, page++ )
 541                 {
 542                         if( !(gaPageTable[page] & PF_PRESENT) ) {
 543                                 gaTmpTable[page] = 0;
 544                                 continue;
 545                         }
 546
 547                         // Refrence old page
 548                         MM_RefPhys( gaPageTable[page] & ~0xFFF );
 549                         // Add to new table
 550                         if(gaPageTable[page] & PF_WRITE) {
 551                                 gaTmpTable[page] = (gaPageTable[page] & ~PF_WRITE) | PF_COW;
 552                                 gaPageTable[page] = (gaPageTable[page] & ~PF_WRITE) | PF_COW;
 553                                 INVLPG( page << 12 );
 554                         }
 555                         else
 556                                 gaTmpTable[page] = gaPageTable[page];
 557                 }
 558         }
 559
 560         // Map in kernel tables (and make fractal mapping)
 561         for( i = 768; i < 1024; i ++ )
 562         {
 563                 // Fractal
 564                 if( i == (PAGE_TABLE_ADDR >> 22) ) {
 565                         gaTmpDir[ PAGE_TABLE_ADDR >> 22 ] = *gpTmpCR3;
 566                         continue;
 567                 }
 568
 569                 if( gaPageDir[i] == 0 ) {
 570                         gaTmpDir[i] = 0;
 571                         continue;
 572                 }
 573
 574                 //LOG("gaPageDir[%x/4] = 0x%x", i*4, gaPageDir[i]);
 575                 MM_RefPhys( gaPageDir[i] & ~0xFFF );
 576                 gaTmpDir[i] = gaPageDir[i];
 577         }
 578
 579         // Allocate kernel stack
 580         for(i = KERNEL_STACKS >> 22;
 581                 i < KERNEL_STACKS_END >> 22;
 582                 i ++ )
 583         {
 584                 // Check if directory is allocated
 585                 if( (gaPageDir[i] & 1) == 0 ) {
 586                         gaTmpDir[i] = 0;
 587                         continue;
 588                 }
 589
 590                 // We don't care about other kernel stacks, just the current one
 591                 if( i != kStackBase >> 22 ) {
 592                         MM_DerefPhys( gaPageDir[i] & ~0xFFF );
 593                         gaTmpDir[i] = 0;
 594                         continue;
 595                 }
 596
 597                 // Create a copy
 598                 gaTmpDir[i] = MM_AllocPhys() | 3;
 599                 INVLPG( &gaTmpTable[i*1024] );
 600                 for( j = 0; j < 1024; j ++ )
 601                 {
 602                         // Is the page allocated? If not, skip
 603                         if( !(gaPageTable[i*1024+j] & 1) ) {
 604                                 gaTmpTable[i*1024+j] = 0;
 605                                 continue;
 606                         }
 607
 608                         // We don't care about other kernel stacks
 609                         if( ((i*1024+j)*4096 & ~(KERNEL_STACK_SIZE-1)) != kStackBase ) {
 610                                 gaTmpTable[i*1024+j] = 0;
 611                                 continue;
 612                         }
 613
 614                         // Allocate page
 615                         gaTmpTable[i*1024+j] = MM_AllocPhys() | 3;
 616
 617                         MM_RefPhys( gaTmpTable[i*1024+j] & ~0xFFF );
 618
 619                         tmp = (void *) MM_MapTemp( gaTmpTable[i*1024+j] & ~0xFFF );
 620                         memcpy( tmp, (void *)( (i*1024+j)*0x1000 ), 0x1000 );
 621                         MM_FreeTemp( (Uint)tmp );
 622                 }
 623         }
 624
 625         ret = *gpTmpCR3 & ~0xFFF;
 626         RELEASE( &gilTempFractal );
 627
 628         //LEAVE('x', ret);
 629         return ret;
 630 }
 631
 632 /**
 633  * \fn tVAddr MM_NewKStack()
 634  * \brief Create a new kernel stack
 635  */
 636 tVAddr MM_NewKStack()
 637 {
 638         tVAddr  base = KERNEL_STACKS;
 639         Uint    i;
 640         for(;base<KERNEL_STACKS_END;base+=KERNEL_STACK_SIZE)
 641         {
 642                 if(MM_GetPhysAddr(base) != 0)   continue;
 643                 for(i=0;i<KERNEL_STACK_SIZE;i+=0x1000) {
 644                         MM_Allocate(base+i);
 645                 }
 646                 return base+KERNEL_STACK_SIZE;
 647         }
 648         Warning("MM_NewKStack - No address space left\n");
 649         return 0;
 650 }
 651
 652 /**
 653  * \fn tVAddr MM_NewWorkerStack()
 654  * \brief Creates a new worker stack
 655  */
 656 tVAddr MM_NewWorkerStack()
 657 {
 658         Uint    esp, ebp;
 659         Uint    oldstack;
 660         Uint    base, addr;
 661          int    i, j;
 662         Uint    *tmpPage;
 663         tPAddr  pages[WORKER_STACK_SIZE>>12];
 664
 665         // Get the old ESP and EBP
 666         __asm__ __volatile__ ("mov %%esp, %0": "=r"(esp));
 667         __asm__ __volatile__ ("mov %%ebp, %0": "=r"(ebp));
 668
 669         // Find a free worker stack address
 670         for(base = giLastUsedWorker; base < NUM_WORKER_STACKS; base++)
 671         {
 672                 // Used block
 673                 if( gWorkerStacks[base/32] == -1 ) {
 674                         base += 31;     base &= ~31;
 675                         base --;        // Counteracted by the base++
 676                         continue;
 677                 }
 678                 // Used stack
 679                 if( gWorkerStacks[base/32] & (1 << base) ) {
 680                         continue;
 681                 }
 682                 break;
 683         }
 684         if(base >= NUM_WORKER_STACKS) {
 685                 Warning("Uh-oh! Out of worker stacks");
 686                 return 0;
 687         }
 688
 689         // It's ours now!
 690         gWorkerStacks[base/32] |= (1 << base);
 691         // Make life easier for later calls
 692         giLastUsedWorker = base;
 693         // We have one
 694         base = WORKER_STACKS + base * WORKER_STACK_SIZE;
 695         //Log(" MM_NewWorkerStack: base = 0x%x", base);
 696
 697         // Acquire the lock for the temp fractal mappings
 698         LOCK(&gilTempFractal);
 699
 700         // Set the temp fractals to TID0's address space
 701         *gpTmpCR3 = ((Uint)gaInitPageDir - KERNEL_BASE) | 3;
 702         //Log(" MM_NewWorkerStack: *gpTmpCR3 = 0x%x", *gpTmpCR3);
 703         INVLPG( gaTmpDir );
 704
 705
 706         // Check if the directory is mapped (we are assuming that the stacks
 707         // will fit neatly in a directory)
 708         //Log(" MM_NewWorkerStack: gaTmpDir[ 0x%x ] = 0x%x", base>>22, gaTmpDir[ base >> 22 ]);
 709         if(gaTmpDir[ base >> 22 ] == 0) {
 710                 gaTmpDir[ base >> 22 ] = MM_AllocPhys() | 3;
 711                 INVLPG( &gaTmpTable[ (base>>12) & ~0x3FF ] );
 712         }
 713
 714         // Mapping Time!
 715         for( addr = 0; addr < WORKER_STACK_SIZE; addr += 0x1000 )
 716         {
 717                 pages[ addr >> 12 ] = MM_AllocPhys();
 718                 gaTmpTable[ (base + addr) >> 12 ] = pages[addr>>12] | 3;
 719         }
 720         *gpTmpCR3 = 0;
 721         // Release the temp mapping lock
 722         RELEASE(&gilTempFractal);
 723
 724         // Copy the old stack
 725         oldstack = (esp + KERNEL_STACK_SIZE-1) & ~(KERNEL_STACK_SIZE-1);
 726         esp = oldstack - esp;   // ESP as an offset in the stack
 727
 728         // Make `base` be the top of the stack
 729         base += WORKER_STACK_SIZE;
 730
 731         i = (WORKER_STACK_SIZE>>12) - 1;
 732         // Copy the contents of the old stack to the new one, altering the addresses
 733         // `addr` is refering to bytes from the stack base (mem downwards)
 734         for(addr = 0; addr < esp; addr += 0x1000)
 735         {
 736                 Uint    *stack = (Uint*)( oldstack-(addr+0x1000) );
 737                 tmpPage = (void*)MM_MapTemp( pages[i] );
 738                 // Copy old stack
 739                 for(j = 0; j < 1024; j++)
 740                 {
 741                         // Possible Stack address?
 742                         if(oldstack-esp < stack[j] && stack[j] < oldstack)
 743                                 tmpPage[j] = base - (oldstack - stack[j]);
 744                         else    // Seems not, best leave it alone
 745                                 tmpPage[j] = stack[j];
 746                 }
 747                 MM_FreeTemp((tVAddr)tmpPage);
 748                 i --;
 749         }
 750
 751         //Log("MM_NewWorkerStack: RETURN 0x%x", base);
 752         return base;
 753 }
 754
 755 /**
 756  * \fn void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask)
 757  * \brief Sets the flags on a page
 758  */
 759 void MM_SetFlags(tVAddr VAddr, Uint Flags, Uint Mask)
 760 {
 761         tTabEnt *ent;
 762         if( !(gaPageDir[VAddr >> 22] & 1) )     return ;
 763         if( !(gaPageTable[VAddr >> 12] & 1) )   return ;
 764
 765         ent = &gaPageTable[VAddr >> 12];
 766
 767         // Read-Only
 768         if( Mask & MM_PFLAG_RO )
 769         {
 770                 if( Flags & MM_PFLAG_RO )       *ent &= ~PF_WRITE;
 771                 else    *ent |= PF_WRITE;
 772         }
 773
 774         // Kernel
 775         if( Mask & MM_PFLAG_KERNEL )
 776         {
 777                 if( Flags & MM_PFLAG_KERNEL )   *ent &= ~PF_USER;
 778                 else    *ent |= PF_USER;
 779         }
 780
 781         // Copy-On-Write
 782         if( Mask & MM_PFLAG_COW )
 783         {
 784                 if( Flags & MM_PFLAG_COW ) {
 785                         *ent &= ~PF_WRITE;
 786                         *ent |= PF_COW;
 787                 }
 788                 else {
 789                         *ent &= ~PF_COW;
 790                         *ent |= PF_WRITE;
 791                 }
 792         }
 793 }
 794
 795 /**
 796  * \fn tPAddr MM_DuplicatePage(tVAddr VAddr)
 797  * \brief Duplicates a virtual page to a physical one
 798  */
 799 tPAddr MM_DuplicatePage(tVAddr VAddr)
 800 {
 801         tPAddr  ret;
 802         Uint    temp;
 803          int    wasRO = 0;
 804
 805         // Check if mapped
 806         if( !(gaPageDir  [VAddr >> 22] & PF_PRESENT) )  return 0;
 807         if( !(gaPageTable[VAddr >> 12] & PF_PRESENT) )  return 0;
 808
 809         // Page Align
 810         VAddr &= ~0xFFF;
 811
 812         // Allocate new page
 813         ret = MM_AllocPhys();
 814
 815         // Write-lock the page (to keep data constistent), saving its R/W state
 816         wasRO = (gaPageTable[VAddr >> 12] & PF_WRITE ? 0 : 1);
 817         gaPageTable[VAddr >> 12] &= ~PF_WRITE;
 818         INVLPG( VAddr );
 819
 820         // Copy Data
 821         temp = MM_MapTemp(ret);
 822         memcpy( (void*)temp, (void*)VAddr, 0x1000 );
 823         MM_FreeTemp(temp);
 824
 825         // Restore Writeable status
 826         if(!wasRO)      gaPageTable[VAddr >> 12] |= PF_WRITE;
 827         INVLPG(VAddr);
 828
 829         return ret;
 830 }
 831
 832 /**
 833  * \fn Uint MM_MapTemp(tPAddr PAddr)
 834  * \brief Create a temporary memory mapping
 835  * \todo Show Luigi Barone (C Lecturer) and see what he thinks
 836  */
 837 tVAddr MM_MapTemp(tPAddr PAddr)
 838 {
 839          int    i;
 840
 841         //ENTER("XPAddr", PAddr);
 842
 843         PAddr &= ~0xFFF;
 844
 845         //LOG("gilTempMappings = %i", gilTempMappings);
 846
 847         for(;;)
 848         {
 849                 LOCK( &gilTempMappings );
 850
 851                 for( i = 0; i < NUM_TEMP_PAGES; i ++ )
 852                 {
 853                         // Check if page used
 854                         if(gaPageTable[ (TEMP_MAP_ADDR >> 12) + i ] & 1)        continue;
 855                         // Mark as used
 856                         gaPageTable[ (TEMP_MAP_ADDR >> 12) + i ] = PAddr | 3;
 857                         INVLPG( TEMP_MAP_ADDR + (i << 12) );
 858                         //LEAVE('p', TEMP_MAP_ADDR + (i << 12));
 859                         RELEASE( &gilTempMappings );
 860                         return TEMP_MAP_ADDR + (i << 12);
 861                 }
 862                 RELEASE( &gilTempMappings );
 863                 Threads_Yield();
 864         }
 865 }
 866
 867 /**
 868  * \fn void MM_FreeTemp(tVAddr PAddr)
 869  * \brief Free's a temp mapping
 870  */
 871 void MM_FreeTemp(tVAddr VAddr)
 872 {
 873          int    i = VAddr >> 12;
 874         //ENTER("xVAddr", VAddr);
 875
 876         if(i >= (TEMP_MAP_ADDR >> 12))
 877                 gaPageTable[ i ] = 0;
 878
 879         //LEAVE('-');
 880 }
 881
 882 /**
 883  * \fn tVAddr MM_MapHWPage(tPAddr PAddr, Uint Number)
 884  * \brief Allocates a contigous number of pages
 885  */
 886 tVAddr MM_MapHWPage(tPAddr PAddr, Uint Number)
 887 {
 888          int    i, j;
 889
 890         PAddr &= ~0xFFF;
 891
 892         // Scan List
 893         for( i = 0; i < NUM_HW_PAGES; i ++ )
 894         {
 895                 // Check if addr used
 896                 if( gaPageTable[ (HW_MAP_ADDR >> 12) + i ] & 1 )
 897                         continue;
 898
 899                 // Check possible region
 900                 for( j = 0; j < Number && i + j < NUM_HW_PAGES; j ++ )
 901                 {
 902                         // If there is an allocated page in the region we are testing, break
 903                         if( gaPageTable[ (HW_MAP_ADDR >> 12) + i + j ] & 1 )    break;
 904                 }
 905                 // Is it all free?
 906                 if( j == Number )
 907                 {
 908                         // Allocate
 909                         for( j = 0; j < Number; j++ ) {
 910                                 MM_RefPhys( PAddr + (j<<12) );
 911                                 gaPageTable[ (HW_MAP_ADDR >> 12) + i + j ] = (PAddr + (j<<12)) | 3;
 912                         }
 913                         return HW_MAP_ADDR + (i<<12);
 914                 }
 915         }
 916         // If we don't find any, return NULL
 917         return 0;
 918 }
 919
 920 /**
 921  * \fn tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr)
 922  * \brief Allocates DMA physical memory
 923  * \param Pages Number of pages required
 924  * \param MaxBits       Maximum number of bits the physical address can have
 925  * \param PhysAddr      Pointer to the location to place the physical address allocated
 926  * \return Virtual address allocate
 927  */
 928 tVAddr MM_AllocDMA(int Pages, int MaxBits, tPAddr *PhysAddr)
 929 {
 930         tPAddr  maxCheck = (1 << MaxBits);
 931         tPAddr  phys;
 932         tVAddr  ret;
 933
 934         ENTER("iPages iMaxBits pPhysAddr", Pages, MaxBits, PhysAddr);
 935
 936         // Sanity Check
 937         if(MaxBits < 12 || !PhysAddr) {
 938                 LEAVE('i', 0);
 939                 return 0;
 940         }
 941
 942         // Bound
 943         if(MaxBits >= PHYS_BITS)        maxCheck = -1;
 944
 945         // Fast Allocate
 946         if(Pages == 1 && MaxBits >= PHYS_BITS)
 947         {
 948                 phys = MM_AllocPhys();
 949                 *PhysAddr = phys;
 950                 ret = MM_MapHWPage(phys, 1);
 951                 if(ret == 0) {
 952                         MM_DerefPhys(phys);
 953                         LEAVE('i', 0);
 954                         return 0;
 955                 }
 956                 LEAVE('x', ret);
 957                 return ret;
 958         }
 959
 960         // Slow Allocate
 961         phys = MM_AllocPhysRange(Pages, MaxBits);
 962         // - Was it allocated?
 963         if(phys == 0) {
 964                 LEAVE('i', 0);
 965                 return 0;
 966         }
 967
 968         // Allocated successfully, now map
 969         ret = MM_MapHWPage(phys, Pages);
 970         if( ret == 0 ) {
 971                 // If it didn't map, free then return 0
 972                 for(;Pages--;phys+=0x1000)
 973                         MM_DerefPhys(phys);
 974                 LEAVE('i', 0);
 975                 return 0;
 976         }
 977
 978         *PhysAddr = phys;
 979         LEAVE('x', ret);
 980         return ret;
 981 }
 982
 983 /**
 984  * \fn void MM_UnmapHWPage(tVAddr VAddr, Uint Number)
 985  * \brief Unmap a hardware page
 986  */
 987 void MM_UnmapHWPage(tVAddr VAddr, Uint Number)
 988 {
 989          int    i, j;
 990         // Sanity Check
 991         if(VAddr < HW_MAP_ADDR || VAddr-Number*0x1000 > HW_MAP_MAX)     return;
 992
 993         i = VAddr >> 12;
 994
 995         LOCK( &gilTempMappings );       // Temp and HW share a directory, so they share a lock
 996
 997         for( j = 0; j < Number; j++ )
 998         {
 999                 MM_DerefPhys( gaPageTable[ (HW_MAP_ADDR >> 12) + i + j ] );
1000                 gaPageTable[ (HW_MAP_ADDR >> 12) + i + j ] = 0;
1001         }
1002
1003         RELEASE( &gilTempMappings );
1004 }
1005
1006 // --- EXPORTS ---
1007 EXPORT(MM_GetPhysAddr);
1008 EXPORT(MM_Map);
1009 //EXPORT(MM_Unmap);
1010 EXPORT(MM_MapHWPage);
1011 EXPORT(MM_AllocDMA);
1012 EXPORT(MM_UnmapHWPage);