ntoskrnl/mm/freelist.c

   1 /*
   2  * COPYRIGHT:    See COPYING in the top level directory
   3  * PROJECT:      ReactOS kernel
   4  * FILE:         ntoskrnl/mm/freelist.c
   5  * PURPOSE:      Handle the list of free physical pages
   6  * PROGRAMMER:   David Welch (welch@cwcom.net)
   7  * UPDATE HISTORY:
   8  *               27/05/98: Created
   9  *               18/08/98: Added a fix from Robert Bergkvist
  10  */
  11
  12 /* INCLUDES ****************************************************************/
  13
  14 #include <ddk/ntddk.h>
  15 #include <internal/mm.h>
  16 #include <internal/ntoskrnl.h>
  17
  18 #define NDEBUG
  19 #include <internal/debug.h>
  20
  21 /* TYPES *******************************************************************/
  22
  23 #define MM_PHYSICAL_PAGE_FREE    (0x1)
  24 #define MM_PHYSICAL_PAGE_USED    (0x2)
  25 #define MM_PHYSICAL_PAGE_BIOS    (0x3)
  26
  27 typedef struct _PHYSICAL_PAGE
  28 {
  29   union
  30   {
  31     struct
  32     {
  33       ULONG Type:2;
  34       ULONG Consumer:3;
  35     }Flags;
  36     ULONG AllFlags;
  37   };
  38
  39   LIST_ENTRY ListEntry;
  40   ULONG ReferenceCount;
  41   SWAPENTRY SavedSwapEntry;
  42   ULONG LockCount;
  43   ULONG MapCount;
  44   struct _MM_RMAP_ENTRY* RmapListHead;
  45 } PHYSICAL_PAGE, *PPHYSICAL_PAGE;
  46
  47 /* GLOBALS ****************************************************************/
  48
  49 static PPHYSICAL_PAGE MmPageArray;
  50 static ULONG MmPageArraySize;
  51
  52 static KSPIN_LOCK PageListLock;
  53 static LIST_ENTRY UsedPageListHeads[MC_MAXIMUM];
  54 static LIST_ENTRY FreeZeroedPageListHead;
  55 static LIST_ENTRY FreeUnzeroedPageListHead;
  56 static LIST_ENTRY BiosPageListHead;
  57
  58 static HANDLE ZeroPageThreadHandle;
  59 static CLIENT_ID ZeroPageThreadId;
  60 static KEVENT ZeroPageThreadEvent;
  61
  62 static ULONG UnzeroedPageCount = 0;
  63
  64 /* FUNCTIONS *************************************************************/
  65
  66 VOID
  67 MmTransferOwnershipPage(PHYSICAL_ADDRESS PhysicalAddress, ULONG NewConsumer)
  68 {
  69   ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
  70   KIRQL oldIrql;
  71
  72   KeAcquireSpinLock(&PageListLock, &oldIrql);
  73   if (MmPageArray[Start].MapCount != 0)
  74     {
  75       DbgPrint("Transfering mapped page.\n");
  76       KEBUGCHECK(0);
  77     }
  78   RemoveEntryList(&MmPageArray[Start].ListEntry);
  79   InsertTailList(&UsedPageListHeads[NewConsumer],
  80     &MmPageArray[Start].ListEntry);
  81   MmPageArray[Start].Flags.Consumer = NewConsumer;
  82   KeReleaseSpinLock(&PageListLock, oldIrql);
  83   MiZeroPage(PhysicalAddress);
  84 }
  85
  86 PHYSICAL_ADDRESS
  87 MmGetLRUFirstUserPage(VOID)
  88 {
  89   PLIST_ENTRY NextListEntry;
  90   PHYSICAL_ADDRESS Next;
  91   PHYSICAL_PAGE* PageDescriptor;
  92   KIRQL oldIrql;
  93
  94   KeAcquireSpinLock(&PageListLock, &oldIrql);
  95   NextListEntry = UsedPageListHeads[MC_USER].Flink;
  96   if (NextListEntry == &UsedPageListHeads[MC_USER])
  97     {
  98       KeReleaseSpinLock(&PageListLock, oldIrql);
  99       return((LARGE_INTEGER)0LL);
 100     }
 101   PageDescriptor = CONTAINING_RECORD(NextListEntry, PHYSICAL_PAGE, ListEntry);
 102   Next.QuadPart = (ULONG)((ULONG)PageDescriptor - (ULONG)MmPageArray);
 103   Next.QuadPart = (Next.QuadPart / sizeof(PHYSICAL_PAGE)) * PAGE_SIZE;
 104   KeReleaseSpinLock(&PageListLock, oldIrql);
 105   return(Next);
 106 }
 107
 108 VOID
 109 MmSetLRULastPage(PHYSICAL_ADDRESS PhysicalAddress)
 110 {
 111   ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 112   KIRQL oldIrql;
 113
 114   KeAcquireSpinLock(&PageListLock, &oldIrql);
 115   if (MmPageArray[Start].Flags.Type == MM_PHYSICAL_PAGE_USED &&
 116       MmPageArray[Start].Flags.Consumer == MC_USER)
 117     {
 118       RemoveEntryList(&MmPageArray[Start].ListEntry);
 119       InsertTailList(&UsedPageListHeads[MC_USER],
 120                      &MmPageArray[Start].ListEntry);
 121     }
 122   KeReleaseSpinLock(&PageListLock, oldIrql);
 123 }
 124
 125 PHYSICAL_ADDRESS
 126 MmGetLRUNextUserPage(PHYSICAL_ADDRESS PreviousPhysicalAddress)
 127 {
 128   ULONG Start = PreviousPhysicalAddress.u.LowPart / PAGE_SIZE;
 129   PLIST_ENTRY NextListEntry;
 130   PHYSICAL_ADDRESS Next;
 131   PHYSICAL_PAGE* PageDescriptor;
 132   KIRQL oldIrql;
 133
 134   KeAcquireSpinLock(&PageListLock, &oldIrql);
 135   if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED ||
 136       MmPageArray[Start].Flags.Consumer != MC_USER)
 137     {
 138       NextListEntry = UsedPageListHeads[MC_USER].Flink;
 139     }
 140   else
 141     {
 142       NextListEntry = MmPageArray[Start].ListEntry.Flink;
 143     }
 144   if (NextListEntry == &UsedPageListHeads[MC_USER])
 145     {
 146       KeReleaseSpinLock(&PageListLock, oldIrql);
 147       return((LARGE_INTEGER)0LL);
 148     }
 149   PageDescriptor = CONTAINING_RECORD(NextListEntry, PHYSICAL_PAGE, ListEntry);
 150   Next.QuadPart = (ULONG)((ULONG)PageDescriptor - (ULONG)MmPageArray);
 151   Next.QuadPart = (Next.QuadPart / sizeof(PHYSICAL_PAGE)) * PAGE_SIZE;
 152   KeReleaseSpinLock(&PageListLock, oldIrql);
 153   return(Next);
 154 }
 155
 156 PHYSICAL_ADDRESS
 157 MmGetContinuousPages(ULONG NumberOfBytes,
 158                      PHYSICAL_ADDRESS HighestAcceptableAddress,
 159                      ULONG Alignment)
 160 {
 161    ULONG NrPages;
 162    ULONG i;
 163    LONG start;
 164    ULONG length;
 165    KIRQL oldIrql;
 166
 167    NrPages = PAGE_ROUND_UP(NumberOfBytes) / PAGE_SIZE;
 168
 169    KeAcquireSpinLock(&PageListLock, &oldIrql);
 170
 171    start = -1;
 172    length = 0;
 173    for (i = 0; i < (HighestAcceptableAddress.QuadPart / PAGE_SIZE); )
 174      {
 175         if (MmPageArray[i].Flags.Type ==  MM_PHYSICAL_PAGE_FREE)
 176           {
 177              if (start == -1)
 178                {
 179                   start = i;
 180                   length = 1;
 181                }
 182              else
 183                {
 184                   length++;
 185                }
 186              i++;
 187              if (length == NrPages)
 188                {
 189                   break;
 190                }
 191           }
 192         else
 193           {
 194              start = -1;
 195              /*
 196               * Fast forward to the base of the next aligned region
 197               */
 198              i = ROUND_UP((i + 1), (Alignment / PAGE_SIZE));
 199           }
 200      }
 201    if (start == -1 || length != NrPages)
 202      {
 203         KeReleaseSpinLock(&PageListLock, oldIrql);
 204         return((LARGE_INTEGER)(LONGLONG)0);
 205      }
 206    for (i = start; i < (start + length); i++)
 207      {
 208         RemoveEntryList(&MmPageArray[i].ListEntry);
 209         MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
 210         MmPageArray[i].Flags.Consumer = MC_NPPOOL;
 211         MmPageArray[i].ReferenceCount = 1;
 212         MmPageArray[i].LockCount = 0;
 213         MmPageArray[i].MapCount = 0;
 214         MmPageArray[i].SavedSwapEntry = 0;
 215         InsertTailList(&UsedPageListHeads[MC_NPPOOL],
 216                        &MmPageArray[i].ListEntry);
 217      }
 218    KeReleaseSpinLock(&PageListLock, oldIrql);
 219    return((LARGE_INTEGER)((LONGLONG)start * PAGE_SIZE));
 220 }
 221
 222 VOID
 223 MiParseRangeToFreeList(PADDRESS_RANGE Range)
 224 {
 225   ULONG i, first, last;
 226
 227   /* FIXME: Not 64-bit ready */
 228
 229   DPRINT("Range going to free list (Base 0x%X, Length 0x%X, Type 0x%X)\n",
 230          Range->BaseAddrLow,
 231          Range->LengthLow,
 232          Range->Type);
 233
 234   first = (Range->BaseAddrLow + PAGE_SIZE - 1) / PAGE_SIZE;
 235   last = first + ((Range->LengthLow + PAGE_SIZE - 1) / PAGE_SIZE);
 236   for (i = first; i < last; i++)
 237     {
 238       if (MmPageArray[i].Flags.Type == 0)
 239         {
 240           MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
 241           MmPageArray[i].ReferenceCount = 0;
 242           InsertTailList(&FreeUnzeroedPageListHead,
 243                          &MmPageArray[i].ListEntry);
 244           UnzeroedPageCount++;
 245         }
 246     }
 247 }
 248
 249 VOID
 250 MiParseRangeToBiosList(PADDRESS_RANGE Range)
 251 {
 252   ULONG i, first, last;
 253
 254   /* FIXME: Not 64-bit ready */
 255
 256   DPRINT("Range going to bios list (Base 0x%X, Length 0x%X, Type 0x%X)\n",
 257          Range->BaseAddrLow,
 258          Range->LengthLow,
 259          Range->Type);
 260
 261   first = (Range->BaseAddrLow + PAGE_SIZE - 1) / PAGE_SIZE;
 262   last = first + ((Range->LengthLow + PAGE_SIZE - 1) / PAGE_SIZE);
 263   for (i = first; i < last; i++)
 264     {
 265       /* Remove the page from the free list if it is there */
 266       if (MmPageArray[i].Flags.Type == MM_PHYSICAL_PAGE_FREE)
 267         {
 268           RemoveEntryList(&MmPageArray[i].ListEntry);
 269         }
 270
 271       if (MmPageArray[i].Flags.Type != MM_PHYSICAL_PAGE_BIOS)
 272         {
 273           MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
 274           MmPageArray[i].Flags.Consumer = MC_NPPOOL;
 275           MmPageArray[i].ReferenceCount = 1;
 276           InsertTailList(&BiosPageListHead,
 277                          &MmPageArray[i].ListEntry);
 278         }
 279     }
 280 }
 281
 282 VOID
 283 MiParseBIOSMemoryMap(ULONG MemorySizeInPages,
 284                      PADDRESS_RANGE BIOSMemoryMap,
 285                      ULONG AddressRangeCount)
 286 {
 287   PADDRESS_RANGE p;
 288   ULONG i;
 289
 290   p = BIOSMemoryMap;
 291   for (i = 0; i < AddressRangeCount; i++)
 292     {
 293       if (((p->BaseAddrLow + PAGE_SIZE - 1) / PAGE_SIZE) < MemorySizeInPages)
 294         {
 295           if (p->Type == 1)
 296             {
 297               MiParseRangeToFreeList(p);
 298             }
 299           else
 300             {
 301               MiParseRangeToBiosList(p);
 302             }
 303         }
 304       p += 1;
 305     }
 306 }
 307
 308 PVOID
 309 MmInitializePageList(PVOID FirstPhysKernelAddress,
 310                      PVOID LastPhysKernelAddress,
 311                      ULONG MemorySizeInPages,
 312                      ULONG LastKernelAddress,
 313                      PADDRESS_RANGE BIOSMemoryMap,
 314                      ULONG AddressRangeCount)
 315 /*
 316  * FUNCTION: Initializes the page list with all pages free
 317  * except those known to be reserved and those used by the kernel
 318  * ARGUMENTS:
 319  *         PageBuffer = Page sized buffer
 320  *         FirstKernelAddress = First physical address used by the kernel
 321  *         LastKernelAddress = Last physical address used by the kernel
 322  */
 323 {
 324    ULONG i;
 325    ULONG Reserved;
 326    NTSTATUS Status;
 327
 328    DPRINT("MmInitializePageList(FirstPhysKernelAddress %x, "
 329           "LastPhysKernelAddress %x, "
 330           "MemorySizeInPages %x, LastKernelAddress %x)\n",
 331           FirstPhysKernelAddress,
 332           LastPhysKernelAddress,
 333           MemorySizeInPages,
 334           LastKernelAddress);
 335
 336    for (i = 0; i < MC_MAXIMUM; i++)
 337      {
 338        InitializeListHead(&UsedPageListHeads[i]);
 339      }
 340    KeInitializeSpinLock(&PageListLock);
 341    InitializeListHead(&FreeUnzeroedPageListHead);
 342    InitializeListHead(&FreeZeroedPageListHead);
 343    InitializeListHead(&BiosPageListHead);
 344
 345    LastKernelAddress = PAGE_ROUND_UP(LastKernelAddress);
 346
 347    MmPageArraySize = MemorySizeInPages;
 348    Reserved =
 349      PAGE_ROUND_UP((MmPageArraySize * sizeof(PHYSICAL_PAGE))) / PAGE_SIZE;
 350    MmPageArray = (PHYSICAL_PAGE *)LastKernelAddress;
 351
 352    DPRINT("Reserved %d\n", Reserved);
 353
 354    LastKernelAddress = PAGE_ROUND_UP(LastKernelAddress);
 355    LastKernelAddress = ((ULONG)LastKernelAddress + (Reserved * PAGE_SIZE));
 356    LastPhysKernelAddress = (PVOID)PAGE_ROUND_UP(LastPhysKernelAddress);
 357    LastPhysKernelAddress = LastPhysKernelAddress + (Reserved * PAGE_SIZE);
 358
 359    MmStats.NrTotalPages = 0;
 360    MmStats.NrSystemPages = 0;
 361    MmStats.NrUserPages = 0;
 362    MmStats.NrReservedPages = 0;
 363    MmStats.NrFreePages = 0;
 364    MmStats.NrLockedPages = 0;
 365
 366    for (i = 0; i < Reserved; i++)
 367      {
 368        PVOID Address = (PVOID)(ULONG)MmPageArray + (i * PAGE_SIZE);
 369        if (!MmIsPagePresent(NULL, Address))
 370          {
 371            ULONG PhysicalAddress;
 372            PhysicalAddress = (ULONG)LastPhysKernelAddress -
 373              (Reserved * PAGE_SIZE) + (i * PAGE_SIZE);
 374            Status =
 375              MmCreateVirtualMappingUnsafe(NULL,
 376                                           Address,
 377                                           PAGE_READWRITE,
 378                                           (PHYSICAL_ADDRESS)(LONGLONG)PhysicalAddress,
 379                                           FALSE);
 380            if (!NT_SUCCESS(Status))
 381              {
 382                DbgPrint("Unable to create virtual mapping\n");
 383                KEBUGCHECK(0);
 384              }
 385          }
 386        memset((PVOID)MmPageArray + (i * PAGE_SIZE), 0, PAGE_SIZE);
 387      }
 388
 389
 390    /*
 391     * Page zero is reserved
 392     */
 393    MmPageArray[0].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
 394    MmPageArray[0].Flags.Consumer = MC_NPPOOL;
 395    MmPageArray[0].ReferenceCount = 0;
 396    InsertTailList(&BiosPageListHead,
 397                   &MmPageArray[0].ListEntry);
 398
 399    /*
 400     * Page one is reserved for the initial KPCR
 401     */
 402    MmPageArray[1].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
 403    MmPageArray[1].Flags.Consumer = MC_NPPOOL;
 404    MmPageArray[1].ReferenceCount = 0;
 405    InsertTailList(&BiosPageListHead,
 406       &MmPageArray[1].ListEntry);
 407
 408    i = 2;
 409    if ((ULONG)FirstPhysKernelAddress < 0xa0000)
 410      {
 411         MmStats.NrFreePages += (((ULONG)FirstPhysKernelAddress/PAGE_SIZE) - 2);
 412         for (; i<((ULONG)FirstPhysKernelAddress/PAGE_SIZE); i++)
 413           {
 414              MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
 415              MmPageArray[i].ReferenceCount = 0;
 416              InsertTailList(&FreeUnzeroedPageListHead,
 417                             &MmPageArray[i].ListEntry);
 418              UnzeroedPageCount++;
 419           }
 420         MmStats.NrSystemPages +=
 421           ((((ULONG)LastPhysKernelAddress) / PAGE_SIZE) - i);
 422         for (; i<((ULONG)LastPhysKernelAddress / PAGE_SIZE); i++)
 423           {
 424              MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
 425              MmPageArray[i].Flags.Consumer = MC_NPPOOL;
 426              MmPageArray[i].ReferenceCount = 1;
 427              MmPageArray[i].MapCount = 1;
 428              InsertTailList(&UsedPageListHeads[MC_NPPOOL],
 429                             &MmPageArray[i].ListEntry);
 430           }
 431         MmStats.NrFreePages += ((0xa0000/PAGE_SIZE) - i);
 432         for (; i<(0xa0000/PAGE_SIZE); i++)
 433           {
 434              MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
 435              MmPageArray[i].ReferenceCount = 0;
 436              InsertTailList(&FreeUnzeroedPageListHead,
 437                             &MmPageArray[i].ListEntry);
 438              UnzeroedPageCount++;
 439           }
 440         MmStats.NrReservedPages += ((0x100000/PAGE_SIZE) - i);
 441         for (; i<(0x100000 / PAGE_SIZE); i++)
 442           {
 443              MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
 444              MmPageArray[i].Flags.Consumer = MC_NPPOOL;
 445              MmPageArray[i].ReferenceCount = 1;
 446              InsertTailList(&BiosPageListHead,
 447                             &MmPageArray[i].ListEntry);
 448           }
 449      }
 450    else
 451      {
 452         MmStats.NrFreePages += ((0xa0000 / PAGE_SIZE) - 2);
 453         for (; i<(0xa0000 / PAGE_SIZE); i++)
 454           {
 455              MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
 456              MmPageArray[i].ReferenceCount = 0;
 457              InsertTailList(&FreeUnzeroedPageListHead,
 458                             &MmPageArray[i].ListEntry);
 459              UnzeroedPageCount++;
 460           }
 461         MmStats.NrReservedPages += (0x60000 / PAGE_SIZE);
 462         for (; i<(0x100000 / PAGE_SIZE); i++)
 463           {
 464              MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
 465              MmPageArray[i].Flags.Consumer = MC_NPPOOL;
 466              MmPageArray[i].ReferenceCount = 1;
 467              InsertTailList(&BiosPageListHead,
 468                             &MmPageArray[i].ListEntry);
 469           }
 470         MmStats.NrFreePages += (((ULONG)FirstPhysKernelAddress/PAGE_SIZE) - i);
 471         for (; i<((ULONG)FirstPhysKernelAddress/PAGE_SIZE); i++)
 472           {
 473              MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
 474              MmPageArray[i].ReferenceCount = 0;
 475              InsertTailList(&FreeUnzeroedPageListHead,
 476                             &MmPageArray[i].ListEntry);
 477              UnzeroedPageCount++;
 478           }
 479         MmStats.NrSystemPages +=
 480           (((ULONG)LastPhysKernelAddress/PAGE_SIZE) - i);
 481         for (; i<((ULONG)LastPhysKernelAddress/PAGE_SIZE); i++)
 482           {
 483              MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
 484              MmPageArray[i].Flags.Consumer = MC_NPPOOL;
 485              MmPageArray[i].ReferenceCount = 1;
 486              MmPageArray[i].MapCount = 1;
 487              InsertTailList(&UsedPageListHeads[MC_NPPOOL],
 488                             &MmPageArray[i].ListEntry);
 489           }
 490      }
 491
 492    MmStats.NrFreePages += (MemorySizeInPages - i);
 493    for (; i<MemorySizeInPages; i++)
 494      {
 495         MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
 496         MmPageArray[i].ReferenceCount = 0;
 497         InsertTailList(&FreeUnzeroedPageListHead,
 498                        &MmPageArray[i].ListEntry);
 499         UnzeroedPageCount++;
 500      }
 501
 502   if ((BIOSMemoryMap != NULL) && (AddressRangeCount > 0))
 503     {
 504       MiParseBIOSMemoryMap(
 505         MemorySizeInPages,
 506         BIOSMemoryMap,
 507         AddressRangeCount);
 508     }
 509
 510    KeInitializeEvent(&ZeroPageThreadEvent, NotificationEvent, TRUE);
 511
 512
 513    MmStats.NrTotalPages = MmStats.NrFreePages + MmStats.NrSystemPages +
 514      MmStats.NrReservedPages + MmStats.NrUserPages;
 515    MmInitializeBalancer(MmStats.NrFreePages);
 516    return((PVOID)LastKernelAddress);
 517 }
 518
 519 VOID
 520 MmSetFlagsPage(PHYSICAL_ADDRESS PhysicalAddress, ULONG Flags)
 521 {
 522    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 523    KIRQL oldIrql;
 524
 525    KeAcquireSpinLock(&PageListLock, &oldIrql);
 526    MmPageArray[Start].AllFlags = Flags;
 527    KeReleaseSpinLock(&PageListLock, oldIrql);
 528 }
 529
 530 VOID
 531 MmSetRmapListHeadPage(PHYSICAL_ADDRESS PhysicalAddress,
 532                       struct _MM_RMAP_ENTRY* ListHead)
 533 {
 534    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 535
 536    MmPageArray[Start].RmapListHead = ListHead;
 537 }
 538
 539 struct _MM_RMAP_ENTRY*
 540 MmGetRmapListHeadPage(PHYSICAL_ADDRESS PhysicalAddress)
 541 {
 542   ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 543
 544   return(MmPageArray[Start].RmapListHead);
 545 }
 546
 547 VOID
 548 MmMarkPageMapped(PHYSICAL_ADDRESS PhysicalAddress)
 549 {
 550   ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 551   KIRQL oldIrql;
 552
 553   if (Start < MmPageArraySize)
 554     {
 555       KeAcquireSpinLock(&PageListLock, &oldIrql);
 556       if (MmPageArray[Start].Flags.Type == MM_PHYSICAL_PAGE_FREE)
 557         {
 558           DbgPrint("Mapping non-used page\n");
 559           KEBUGCHECK(0);
 560         }
 561       MmPageArray[Start].MapCount++;
 562       KeReleaseSpinLock(&PageListLock, oldIrql);
 563     }
 564 }
 565
 566 VOID
 567 MmMarkPageUnmapped(PHYSICAL_ADDRESS PhysicalAddress)
 568 {
 569   ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 570   KIRQL oldIrql;
 571
 572   if (Start < MmPageArraySize)
 573     {
 574       KeAcquireSpinLock(&PageListLock, &oldIrql);
 575       if (MmPageArray[Start].Flags.Type == MM_PHYSICAL_PAGE_FREE)
 576         {
 577           DbgPrint("Unmapping non-used page\n");
 578           KEBUGCHECK(0);
 579         }
 580       if (MmPageArray[Start].MapCount == 0)
 581         {
 582           DbgPrint("Unmapping not mapped page\n");
 583           KEBUGCHECK(0);
 584         }
 585       MmPageArray[Start].MapCount--;
 586       KeReleaseSpinLock(&PageListLock, oldIrql);
 587     }
 588 }
 589
 590 ULONG
 591 MmGetFlagsPage(PHYSICAL_ADDRESS PhysicalAddress)
 592 {
 593    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 594    KIRQL oldIrql;
 595    ULONG Flags;
 596
 597    KeAcquireSpinLock(&PageListLock, &oldIrql);
 598    Flags = MmPageArray[Start].AllFlags;
 599    KeReleaseSpinLock(&PageListLock, oldIrql);
 600
 601    return(Flags);
 602 }
 603
 604
 605 VOID
 606 MmSetSavedSwapEntryPage(PHYSICAL_ADDRESS PhysicalAddress,
 607                         SWAPENTRY SavedSwapEntry)
 608 {
 609    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 610    KIRQL oldIrql;
 611
 612    KeAcquireSpinLock(&PageListLock, &oldIrql);
 613    MmPageArray[Start].SavedSwapEntry = SavedSwapEntry;
 614    KeReleaseSpinLock(&PageListLock, oldIrql);
 615 }
 616
 617 SWAPENTRY
 618 MmGetSavedSwapEntryPage(PHYSICAL_ADDRESS PhysicalAddress)
 619 {
 620    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 621    SWAPENTRY SavedSwapEntry;
 622    KIRQL oldIrql;
 623
 624    KeAcquireSpinLock(&PageListLock, &oldIrql);
 625    SavedSwapEntry = MmPageArray[Start].SavedSwapEntry;
 626    KeReleaseSpinLock(&PageListLock, oldIrql);
 627
 628    return(SavedSwapEntry);
 629 }
 630
 631 VOID
 632 MmReferencePage(PHYSICAL_ADDRESS PhysicalAddress)
 633 {
 634    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 635    KIRQL oldIrql;
 636
 637    DPRINT("MmReferencePage(PhysicalAddress %x)\n", PhysicalAddress);
 638
 639    if (PhysicalAddress.u.LowPart == 0)
 640      {
 641         KEBUGCHECK(0);
 642      }
 643
 644    KeAcquireSpinLock(&PageListLock, &oldIrql);
 645
 646    if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
 647      {
 648         DbgPrint("Referencing non-used page\n");
 649         KEBUGCHECK(0);
 650      }
 651
 652    MmPageArray[Start].ReferenceCount++;
 653    KeReleaseSpinLock(&PageListLock, oldIrql);
 654 }
 655
 656 ULONG
 657 MmGetReferenceCountPage(PHYSICAL_ADDRESS PhysicalAddress)
 658 {
 659    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 660    KIRQL oldIrql;
 661    ULONG RCount;
 662
 663    DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", PhysicalAddress);
 664
 665    if (PhysicalAddress.u.LowPart == 0)
 666      {
 667         KEBUGCHECK(0);
 668      }
 669
 670    KeAcquireSpinLock(&PageListLock, &oldIrql);
 671
 672    if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
 673      {
 674         DbgPrint("Getting reference count for free page\n");
 675         KEBUGCHECK(0);
 676      }
 677
 678    RCount = MmPageArray[Start].ReferenceCount;
 679
 680    KeReleaseSpinLock(&PageListLock, oldIrql);
 681    return(RCount);
 682 }
 683
 684 BOOLEAN
 685 MmIsUsablePage(PHYSICAL_ADDRESS PhysicalAddress)
 686 {
 687    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 688
 689    DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", PhysicalAddress);
 690
 691    if (PhysicalAddress.u.LowPart == 0)
 692      {
 693         KEBUGCHECK(0);
 694      }
 695
 696    if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED &&
 697        MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_BIOS)
 698      {
 699        return(FALSE);
 700      }
 701
 702    return(TRUE);
 703 }
 704
 705 VOID
 706 MmDereferencePage(PHYSICAL_ADDRESS PhysicalAddress)
 707 {
 708    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 709    KIRQL oldIrql;
 710
 711    DPRINT("MmDereferencePage(PhysicalAddress %I64x)\n", PhysicalAddress);
 712
 713    if (PhysicalAddress.u.LowPart == 0)
 714      {
 715         KEBUGCHECK(0);
 716      }
 717
 718    KeAcquireSpinLock(&PageListLock, &oldIrql);
 719
 720
 721    if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
 722      {
 723         DbgPrint("Dereferencing free page\n");
 724         KEBUGCHECK(0);
 725      }
 726
 727    MmPageArray[Start].ReferenceCount--;
 728    if (MmPageArray[Start].ReferenceCount == 0)
 729      {
 730        MmStats.NrFreePages++;
 731        MmStats.NrSystemPages--;
 732        RemoveEntryList(&MmPageArray[Start].ListEntry);
 733        if (MmPageArray[Start].RmapListHead != NULL)
 734          {
 735            DbgPrint("Freeing page with rmap entries.\n");
 736            KEBUGCHECK(0);
 737          }
 738        if (MmPageArray[Start].MapCount != 0)
 739          {
 740            DbgPrint("Freeing mapped page (0x%I64x count %d)\n",
 741                     PhysicalAddress, MmPageArray[Start].MapCount);
 742            KEBUGCHECK(0);
 743          }
 744        if (MmPageArray[Start].LockCount > 0)
 745          {
 746            DbgPrint("Freeing locked page\n");
 747            KEBUGCHECK(0);
 748          }
 749        if (MmPageArray[Start].SavedSwapEntry != 0)
 750          {
 751            DbgPrint("Freeing page with swap entry.\n");
 752            KEBUGCHECK(0);
 753          }
 754        if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
 755          {
 756            DbgPrint("Freeing page with flags %x\n",
 757                     MmPageArray[Start].Flags.Type);
 758            KEBUGCHECK(0);
 759          }
 760        MmPageArray[Start].Flags.Type = MM_PHYSICAL_PAGE_FREE;
 761        InsertTailList(&FreeUnzeroedPageListHead,
 762                       &MmPageArray[Start].ListEntry);
 763        UnzeroedPageCount++;
 764        if (UnzeroedPageCount > 8 && 0 == KeReadStateEvent(&ZeroPageThreadEvent))
 765          {
 766            KeSetEvent(&ZeroPageThreadEvent, IO_NO_INCREMENT, FALSE);
 767          }
 768      }
 769    KeReleaseSpinLock(&PageListLock, oldIrql);
 770 }
 771
 772 ULONG
 773 MmGetLockCountPage(PHYSICAL_ADDRESS PhysicalAddress)
 774 {
 775    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 776    KIRQL oldIrql;
 777    ULONG LockCount;
 778
 779    DPRINT("MmGetLockCountPage(PhysicalAddress %x)\n", PhysicalAddress);
 780
 781    if (PhysicalAddress.u.LowPart == 0)
 782      {
 783         KEBUGCHECK(0);
 784      }
 785
 786    KeAcquireSpinLock(&PageListLock, &oldIrql);
 787
 788    if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
 789      {
 790         DbgPrint("Getting lock count for free page\n");
 791         KEBUGCHECK(0);
 792      }
 793
 794    LockCount = MmPageArray[Start].LockCount;
 795    KeReleaseSpinLock(&PageListLock, oldIrql);
 796
 797    return(LockCount);
 798 }
 799
 800 VOID
 801 MmLockPage(PHYSICAL_ADDRESS PhysicalAddress)
 802 {
 803    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 804    KIRQL oldIrql;
 805
 806    DPRINT("MmLockPage(PhysicalAddress %x)\n", PhysicalAddress);
 807
 808    if (PhysicalAddress.u.LowPart == 0)
 809      {
 810         KEBUGCHECK(0);
 811      }
 812
 813    KeAcquireSpinLock(&PageListLock, &oldIrql);
 814
 815    if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
 816      {
 817         DbgPrint("Locking free page\n");
 818         KEBUGCHECK(0);
 819      }
 820
 821    MmPageArray[Start].LockCount++;
 822    KeReleaseSpinLock(&PageListLock, oldIrql);
 823 }
 824
 825 VOID
 826 MmUnlockPage(PHYSICAL_ADDRESS PhysicalAddress)
 827 {
 828    ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
 829    KIRQL oldIrql;
 830
 831    DPRINT("MmUnlockPage(PhysicalAddress %llx)\n", PhysicalAddress);
 832
 833    if (PhysicalAddress.u.LowPart == 0)
 834      {
 835         KEBUGCHECK(0);
 836      }
 837
 838    KeAcquireSpinLock(&PageListLock, &oldIrql);
 839
 840    if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
 841      {
 842         DbgPrint("Unlocking free page\n");
 843         KEBUGCHECK(0);
 844      }
 845
 846    MmPageArray[Start].LockCount--;
 847    KeReleaseSpinLock(&PageListLock, oldIrql);
 848 }
 849
 850 PHYSICAL_ADDRESS
 851 MmAllocPage(ULONG Consumer, SWAPENTRY SavedSwapEntry)
 852 {
 853   PHYSICAL_ADDRESS PageOffset;
 854   PLIST_ENTRY ListEntry;
 855   PPHYSICAL_PAGE PageDescriptor;
 856   KIRQL oldIrql;
 857   BOOLEAN NeedClear = FALSE;
 858
 859   DPRINT("MmAllocPage()\n");
 860
 861   KeAcquireSpinLock(&PageListLock, &oldIrql);
 862   if (IsListEmpty(&FreeZeroedPageListHead))
 863     {
 864       if (IsListEmpty(&FreeUnzeroedPageListHead))
 865         {
 866           DPRINT1("MmAllocPage(): Out of memory\n");
 867           KeReleaseSpinLock(&PageListLock, oldIrql);
 868           return((PHYSICAL_ADDRESS)0LL);
 869         }
 870       ListEntry = RemoveTailList(&FreeUnzeroedPageListHead);
 871       UnzeroedPageCount--;
 872
 873       PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
 874
 875       NeedClear = TRUE;
 876     }
 877   else
 878     {
 879       ListEntry = RemoveTailList(&FreeZeroedPageListHead);
 880
 881       PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
 882     }
 883
 884   if (PageDescriptor->Flags.Type != MM_PHYSICAL_PAGE_FREE)
 885     {
 886       DbgPrint("Got non-free page from freelist\n");
 887       KEBUGCHECK(0);
 888     }
 889   if (PageDescriptor->MapCount != 0)
 890     {
 891       DbgPrint("Got mapped page from freelist\n");
 892       KEBUGCHECK(0);
 893     }
 894   PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_USED;
 895   PageDescriptor->Flags.Consumer = Consumer;
 896   PageDescriptor->ReferenceCount = 1;
 897   PageDescriptor->LockCount = 0;
 898   PageDescriptor->MapCount = 0;
 899   PageDescriptor->SavedSwapEntry = SavedSwapEntry;
 900   InsertTailList(&UsedPageListHeads[Consumer], ListEntry);
 901
 902   MmStats.NrSystemPages++;
 903   MmStats.NrFreePages--;
 904
 905   KeReleaseSpinLock(&PageListLock, oldIrql);
 906
 907   PageOffset.QuadPart = (ULONG)((ULONG)PageDescriptor - (ULONG)MmPageArray);
 908   PageOffset.QuadPart =
 909     (PageOffset.QuadPart / sizeof(PHYSICAL_PAGE)) * PAGE_SIZE;
 910   if (NeedClear)
 911     {
 912       MiZeroPage(PageOffset);
 913     }
 914   if (PageDescriptor->MapCount != 0)
 915     {
 916       DbgPrint("Returning mapped page.\n");
 917       KEBUGCHECK(0);
 918     }
 919   return(PageOffset);
 920 }
 921
 922
 923 NTSTATUS STDCALL
 924 MmZeroPageThreadMain(PVOID Ignored)
 925 {
 926   NTSTATUS Status;
 927   KIRQL oldIrql;
 928   PLIST_ENTRY ListEntry;
 929   PPHYSICAL_PAGE PageDescriptor;
 930   PHYSICAL_ADDRESS PhysPage;
 931   static PVOID Address = NULL;
 932   ULONG Count;
 933
 934   while(1)
 935     {
 936       Status = KeWaitForSingleObject(&ZeroPageThreadEvent,
 937                                      0,
 938                                      KernelMode,
 939                                      FALSE,
 940                                      NULL);
 941       if (!NT_SUCCESS(Status))
 942         {
 943           DbgPrint("ZeroPageThread: Wait failed\n");
 944           KEBUGCHECK(0);
 945           return(STATUS_UNSUCCESSFUL);
 946         }
 947
 948       Count = 0;
 949       KeAcquireSpinLock(&PageListLock, &oldIrql);
 950       while (!IsListEmpty(&FreeUnzeroedPageListHead))
 951       {
 952          ListEntry = RemoveTailList(&FreeUnzeroedPageListHead);
 953          UnzeroedPageCount--;
 954          PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
 955          /* We set the page to used, because MmCreateVirtualMapping failed with unused pages */
 956          PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_USED;
 957          KeReleaseSpinLock(&PageListLock, oldIrql);
 958          Count++;
 959          PhysPage.QuadPart = (ULONG)((ULONG)PageDescriptor - (ULONG)MmPageArray);
 960          PhysPage.QuadPart = (PhysPage.QuadPart / sizeof(PHYSICAL_PAGE)) * PAGE_SIZE;
 961          if (Address == NULL)
 962          {
 963             Address = ExAllocatePageWithPhysPage(PhysPage);
 964          }
 965          else
 966          {
 967             Status = MmCreateVirtualMapping(NULL,
 968                                             Address,
 969                                             PAGE_READWRITE | PAGE_SYSTEM,
 970                                             PhysPage,
 971                                             FALSE);
 972             if (!NT_SUCCESS(Status))
 973             {
 974                DbgPrint("Unable to create virtual mapping\n");
 975                KEBUGCHECK(0);
 976             }
 977          }
 978          memset(Address, 0, PAGE_SIZE);
 979          MmDeleteVirtualMapping(NULL, (PVOID)Address, FALSE, NULL, NULL);
 980          KeAcquireSpinLock(&PageListLock, &oldIrql);
 981          if (PageDescriptor->MapCount != 0)
 982            {
 983              DbgPrint("Mapped page on freelist.\n");
 984              KEBUGCHECK(0);
 985            }
 986          PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_FREE;
 987          InsertHeadList(&FreeZeroedPageListHead, ListEntry);
 988       }
 989       DPRINT("Zeroed %d pages.\n", Count);
 990       KeResetEvent(&ZeroPageThreadEvent);
 991       KeReleaseSpinLock(&PageListLock, oldIrql);
 992     }
 993 }
 994
 995 NTSTATUS MmInitZeroPageThread(VOID)
 996 {
 997   KPRIORITY Priority;
 998   NTSTATUS Status;
 999
1000   Status = PsCreateSystemThread(&ZeroPageThreadHandle,
1001                                 THREAD_ALL_ACCESS,
1002                                 NULL,
1003                                 NULL,
1004                                 &ZeroPageThreadId,
1005                                 (PKSTART_ROUTINE) MmZeroPageThreadMain,
1006                                 NULL);
1007   if (!NT_SUCCESS(Status))
1008     {
1009       return(Status);
1010      }
1011
1012   Priority = 1;
1013   NtSetInformationThread(ZeroPageThreadHandle,
1014                          ThreadPriority,
1015                          &Priority,
1016                          sizeof(Priority));
1017
1018   return(STATUS_SUCCESS);
1019 }
1020
1021 /* EOF */