update for HEAD-2003091401

[reactos.git] / ntoskrnl / mm / freelist.c

diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c
index 5a037f4..861d577 100644 (file)
--- a/ntoskrnl/mm/freelist.c
+++ b/ntoskrnl/mm/freelist.c
@@ -24,11 +24,18 @@
 #define MM_PHYSICAL_PAGE_USED    (0x2)
 #define MM_PHYSICAL_PAGE_BIOS    (0x3)
 
-#define MM_PTYPE(x)              ((x) & 0x3)
-
 typedef struct _PHYSICAL_PAGE
 {
-  ULONG Flags;
+  union
+  {
+    struct
+    {
+      ULONG Type:2;
+      ULONG Consumer:3;
+    }Flags;
+    ULONG AllFlags;
+  };
+
   LIST_ENTRY ListEntry;
   ULONG ReferenceCount;
   SWAPENTRY SavedSwapEntry;
@@ -40,6 +47,7 @@ typedef struct _PHYSICAL_PAGE
 /* GLOBALS ****************************************************************/
 
 static PPHYSICAL_PAGE MmPageArray;
+static ULONG MmPageArraySize;
 
 static KSPIN_LOCK PageListLock;
 static LIST_ENTRY UsedPageListHeads[MC_MAXIMUM];
@@ -47,6 +55,12 @@ static LIST_ENTRY FreeZeroedPageListHead;
 static LIST_ENTRY FreeUnzeroedPageListHead;
 static LIST_ENTRY BiosPageListHead;
 
+static HANDLE ZeroPageThreadHandle;
+static CLIENT_ID ZeroPageThreadId;
+static KEVENT ZeroPageThreadEvent;
+
+static ULONG UnzeroedPageCount = 0;
+
 /* FUNCTIONS *************************************************************/
 
 VOID
@@ -56,9 +70,15 @@ MmTransferOwnershipPage(PHYSICAL_ADDRESS PhysicalAddress, ULONG NewConsumer)
   KIRQL oldIrql;
    
   KeAcquireSpinLock(&PageListLock, &oldIrql);
+  if (MmPageArray[Start].MapCount != 0)
+    {
+      DbgPrint("Transfering mapped page.\n");
+      KEBUGCHECK(0);
+    }
   RemoveEntryList(&MmPageArray[Start].ListEntry);
   InsertTailList(&UsedPageListHeads[NewConsumer], 
     &MmPageArray[Start].ListEntry);
+  MmPageArray[Start].Flags.Consumer = NewConsumer;
   KeReleaseSpinLock(&PageListLock, oldIrql);  
   MiZeroPage(PhysicalAddress);
 }
@@ -85,6 +105,23 @@ MmGetLRUFirstUserPage(VOID)
   return(Next);
 }
 
+VOID
+MmSetLRULastPage(PHYSICAL_ADDRESS PhysicalAddress)
+{
+  ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
+  KIRQL oldIrql;
+
+  KeAcquireSpinLock(&PageListLock, &oldIrql);
+  if (MmPageArray[Start].Flags.Type == MM_PHYSICAL_PAGE_USED &&
+      MmPageArray[Start].Flags.Consumer == MC_USER)
+    {
+      RemoveEntryList(&MmPageArray[Start].ListEntry);
+      InsertTailList(&UsedPageListHeads[MC_USER], 
+                    &MmPageArray[Start].ListEntry);
+    }
+  KeReleaseSpinLock(&PageListLock, oldIrql);
+}
+
 PHYSICAL_ADDRESS
 MmGetLRUNextUserPage(PHYSICAL_ADDRESS PreviousPhysicalAddress)
 {
@@ -95,7 +132,8 @@ MmGetLRUNextUserPage(PHYSICAL_ADDRESS PreviousPhysicalAddress)
   KIRQL oldIrql;
 
   KeAcquireSpinLock(&PageListLock, &oldIrql);
-  if (!(MmPageArray[Start].Flags & MM_PHYSICAL_PAGE_USED))
+  if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED || 
+      MmPageArray[Start].Flags.Consumer != MC_USER)
     {
       NextListEntry = UsedPageListHeads[MC_USER].Flink;
     }
@@ -134,7 +172,7 @@ MmGetContinuousPages(ULONG NumberOfBytes,
    length = 0;
    for (i = 0; i < (HighestAcceptableAddress.QuadPart / PAGE_SIZE); )
      {
-       if (MM_PTYPE(MmPageArray[i].Flags) ==  MM_PHYSICAL_PAGE_FREE)
+       if (MmPageArray[i].Flags.Type ==  MM_PHYSICAL_PAGE_FREE)
          {
             if (start == -1)
               {
@@ -168,7 +206,8 @@ MmGetContinuousPages(ULONG NumberOfBytes,
    for (i = start; i < (start + length); i++)
      {
        RemoveEntryList(&MmPageArray[i].ListEntry);
-       MmPageArray[i].Flags = MM_PHYSICAL_PAGE_USED;
+       MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
+       MmPageArray[i].Flags.Consumer = MC_NPPOOL;
        MmPageArray[i].ReferenceCount = 1;
        MmPageArray[i].LockCount = 0;
        MmPageArray[i].MapCount = 0;
@@ -177,7 +216,7 @@ MmGetContinuousPages(ULONG NumberOfBytes,
                       &MmPageArray[i].ListEntry);
      }
    KeReleaseSpinLock(&PageListLock, oldIrql);
-   return((LARGE_INTEGER)((LONGLONG)start * 4096));
+   return((LARGE_INTEGER)((LONGLONG)start * PAGE_SIZE));
 }
 
 VOID 
@@ -196,12 +235,13 @@ MiParseRangeToFreeList(PADDRESS_RANGE Range)
   last = first + ((Range->LengthLow + PAGE_SIZE - 1) / PAGE_SIZE);
   for (i = first; i < last; i++)
     {
-      if (MmPageArray[i].Flags == 0)
+      if (MmPageArray[i].Flags.Type == 0)
         {
-          MmPageArray[i].Flags = MM_PHYSICAL_PAGE_FREE;
+          MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
          MmPageArray[i].ReferenceCount = 0;
          InsertTailList(&FreeUnzeroedPageListHead,
                         &MmPageArray[i].ListEntry);
+         UnzeroedPageCount++;
         }
     }
 }
@@ -223,14 +263,15 @@ MiParseRangeToBiosList(PADDRESS_RANGE Range)
   for (i = first; i < last; i++)
     {
       /* Remove the page from the free list if it is there */
-      if (MmPageArray[i].Flags == MM_PHYSICAL_PAGE_FREE)
+      if (MmPageArray[i].Flags.Type == MM_PHYSICAL_PAGE_FREE)
         {
           RemoveEntryList(&MmPageArray[i].ListEntry);
         }
       
-      if (MmPageArray[i].Flags != MM_PHYSICAL_PAGE_BIOS)
+      if (MmPageArray[i].Flags.Type != MM_PHYSICAL_PAGE_BIOS)
         {
-          MmPageArray[i].Flags = MM_PHYSICAL_PAGE_BIOS;
+          MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
+         MmPageArray[i].Flags.Consumer = MC_NPPOOL;
          MmPageArray[i].ReferenceCount = 1;
          InsertTailList(&BiosPageListHead,
                         &MmPageArray[i].ListEntry);
@@ -303,8 +344,9 @@ MmInitializePageList(PVOID FirstPhysKernelAddress,
 
    LastKernelAddress = PAGE_ROUND_UP(LastKernelAddress);
    
+   MmPageArraySize = MemorySizeInPages;
    Reserved = 
-     PAGE_ROUND_UP((MemorySizeInPages * sizeof(PHYSICAL_PAGE))) / PAGE_SIZE;
+     PAGE_ROUND_UP((MmPageArraySize * sizeof(PHYSICAL_PAGE))) / PAGE_SIZE;
    MmPageArray = (PHYSICAL_PAGE *)LastKernelAddress;
    
    DPRINT("Reserved %d\n", Reserved);
@@ -338,16 +380,18 @@ MmInitializePageList(PVOID FirstPhysKernelAddress,
           if (!NT_SUCCESS(Status))
             {
               DbgPrint("Unable to create virtual mapping\n");
-              KeBugCheck(0);
+              KEBUGCHECK(0);
             }
         }
        memset((PVOID)MmPageArray + (i * PAGE_SIZE), 0, PAGE_SIZE);
      }
    
+
    /*
     * Page zero is reserved
     */
-   MmPageArray[0].Flags = MM_PHYSICAL_PAGE_BIOS;
+   MmPageArray[0].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
+   MmPageArray[0].Flags.Consumer = MC_NPPOOL;
    MmPageArray[0].ReferenceCount = 0;
    InsertTailList(&BiosPageListHead,
                  &MmPageArray[0].ListEntry); 
@@ -355,7 +399,8 @@ MmInitializePageList(PVOID FirstPhysKernelAddress,
    /*
     * Page one is reserved for the initial KPCR
     */
-   MmPageArray[1].Flags = MM_PHYSICAL_PAGE_BIOS;
+   MmPageArray[1].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
+   MmPageArray[1].Flags.Consumer = MC_NPPOOL;
    MmPageArray[1].ReferenceCount = 0;
    InsertTailList(&BiosPageListHead,
       &MmPageArray[1].ListEntry); 
@@ -363,35 +408,40 @@ MmInitializePageList(PVOID FirstPhysKernelAddress,
    i = 2;
    if ((ULONG)FirstPhysKernelAddress < 0xa0000)
      {
-       MmStats.NrFreePages += (((ULONG)FirstPhysKernelAddress/PAGE_SIZE) - 1);
+       MmStats.NrFreePages += (((ULONG)FirstPhysKernelAddress/PAGE_SIZE) - 2);
        for (; i<((ULONG)FirstPhysKernelAddress/PAGE_SIZE); i++)
          {
-            MmPageArray[i].Flags = MM_PHYSICAL_PAGE_FREE;
+            MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
             MmPageArray[i].ReferenceCount = 0;
             InsertTailList(&FreeUnzeroedPageListHead,
                            &MmPageArray[i].ListEntry);
+            UnzeroedPageCount++;
          }
        MmStats.NrSystemPages += 
          ((((ULONG)LastPhysKernelAddress) / PAGE_SIZE) - i);
        for (; i<((ULONG)LastPhysKernelAddress / PAGE_SIZE); i++)
          {
-            MmPageArray[i].Flags = MM_PHYSICAL_PAGE_USED;
+            MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
+            MmPageArray[i].Flags.Consumer = MC_NPPOOL;
             MmPageArray[i].ReferenceCount = 1;
+            MmPageArray[i].MapCount = 1;
             InsertTailList(&UsedPageListHeads[MC_NPPOOL],
                            &MmPageArray[i].ListEntry);
          }
        MmStats.NrFreePages += ((0xa0000/PAGE_SIZE) - i);
        for (; i<(0xa0000/PAGE_SIZE); i++)
          {
-            MmPageArray[i].Flags = MM_PHYSICAL_PAGE_FREE;
+            MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
             MmPageArray[i].ReferenceCount = 0;
             InsertTailList(&FreeUnzeroedPageListHead,
                            &MmPageArray[i].ListEntry);
+            UnzeroedPageCount++;
          }
        MmStats.NrReservedPages += ((0x100000/PAGE_SIZE) - i);
        for (; i<(0x100000 / PAGE_SIZE); i++)
          {
-            MmPageArray[i].Flags = MM_PHYSICAL_PAGE_BIOS;
+            MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
+            MmPageArray[i].Flags.Consumer = MC_NPPOOL;
             MmPageArray[i].ReferenceCount = 1;
             InsertTailList(&BiosPageListHead,
                            &MmPageArray[i].ListEntry);
@@ -399,18 +449,20 @@ MmInitializePageList(PVOID FirstPhysKernelAddress,
      }
    else
      {
-       MmStats.NrFreePages += ((0xa0000 / PAGE_SIZE) - 1);       
+       MmStats.NrFreePages += ((0xa0000 / PAGE_SIZE) - 2);       
        for (; i<(0xa0000 / PAGE_SIZE); i++)
          {
-            MmPageArray[i].Flags = MM_PHYSICAL_PAGE_FREE;
+            MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
             MmPageArray[i].ReferenceCount = 0;
             InsertTailList(&FreeUnzeroedPageListHead,
                            &MmPageArray[i].ListEntry);
+            UnzeroedPageCount++;
          }
        MmStats.NrReservedPages += (0x60000 / PAGE_SIZE);
        for (; i<(0x100000 / PAGE_SIZE); i++)
          {
-            MmPageArray[i].Flags = MM_PHYSICAL_PAGE_BIOS;
+            MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
+            MmPageArray[i].Flags.Consumer = MC_NPPOOL;
             MmPageArray[i].ReferenceCount = 1;
             InsertTailList(&BiosPageListHead,
                            &MmPageArray[i].ListEntry);
@@ -418,17 +470,20 @@ MmInitializePageList(PVOID FirstPhysKernelAddress,
        MmStats.NrFreePages += (((ULONG)FirstPhysKernelAddress/PAGE_SIZE) - i);
        for (; i<((ULONG)FirstPhysKernelAddress/PAGE_SIZE); i++)
          {
-            MmPageArray[i].Flags = MM_PHYSICAL_PAGE_FREE;
+            MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
             MmPageArray[i].ReferenceCount = 0;
             InsertTailList(&FreeUnzeroedPageListHead,
                            &MmPageArray[i].ListEntry);
+            UnzeroedPageCount++;
          }
        MmStats.NrSystemPages += 
          (((ULONG)LastPhysKernelAddress/PAGE_SIZE) - i);
        for (; i<((ULONG)LastPhysKernelAddress/PAGE_SIZE); i++)
          {
-            MmPageArray[i].Flags = MM_PHYSICAL_PAGE_USED;
+            MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
+            MmPageArray[i].Flags.Consumer = MC_NPPOOL;
             MmPageArray[i].ReferenceCount = 1;
+            MmPageArray[i].MapCount = 1;
             InsertTailList(&UsedPageListHeads[MC_NPPOOL],
                            &MmPageArray[i].ListEntry);
          }
@@ -437,10 +492,11 @@ MmInitializePageList(PVOID FirstPhysKernelAddress,
    MmStats.NrFreePages += (MemorySizeInPages - i);
    for (; i<MemorySizeInPages; i++)
      {
-       MmPageArray[i].Flags = MM_PHYSICAL_PAGE_FREE;
+       MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
        MmPageArray[i].ReferenceCount = 0;
        InsertTailList(&FreeUnzeroedPageListHead,
                       &MmPageArray[i].ListEntry);
+       UnzeroedPageCount++;
      }
 
   if ((BIOSMemoryMap != NULL) && (AddressRangeCount > 0))
@@ -450,6 +506,9 @@ MmInitializePageList(PVOID FirstPhysKernelAddress,
         BIOSMemoryMap,
         AddressRangeCount);
     }
+
+   KeInitializeEvent(&ZeroPageThreadEvent, NotificationEvent, TRUE);
+
   
    MmStats.NrTotalPages = MmStats.NrFreePages + MmStats.NrSystemPages +
      MmStats.NrReservedPages + MmStats.NrUserPages;
@@ -464,7 +523,7 @@ MmSetFlagsPage(PHYSICAL_ADDRESS PhysicalAddress, ULONG Flags)
    KIRQL oldIrql;
    
    KeAcquireSpinLock(&PageListLock, &oldIrql);
-   MmPageArray[Start].Flags = Flags;
+   MmPageArray[Start].AllFlags = Flags;
    KeReleaseSpinLock(&PageListLock, oldIrql);
 }
 
@@ -488,23 +547,44 @@ MmGetRmapListHeadPage(PHYSICAL_ADDRESS PhysicalAddress)
 VOID 
 MmMarkPageMapped(PHYSICAL_ADDRESS PhysicalAddress)
 {
-   ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
-   KIRQL oldIrql;
-   
-   KeAcquireSpinLock(&PageListLock, &oldIrql);
-   MmPageArray[Start].MapCount++;
-   KeReleaseSpinLock(&PageListLock, oldIrql);
+  ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
+  KIRQL oldIrql;
+
+  if (Start < MmPageArraySize)
+    {   
+      KeAcquireSpinLock(&PageListLock, &oldIrql);
+      if (MmPageArray[Start].Flags.Type == MM_PHYSICAL_PAGE_FREE)
+       {
+         DbgPrint("Mapping non-used page\n");
+         KEBUGCHECK(0);
+       }
+      MmPageArray[Start].MapCount++;
+      KeReleaseSpinLock(&PageListLock, oldIrql);
+    }
 }
 
 VOID 
 MmMarkPageUnmapped(PHYSICAL_ADDRESS PhysicalAddress)
 {
-   ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
-   KIRQL oldIrql;
-   
-   KeAcquireSpinLock(&PageListLock, &oldIrql);
-   MmPageArray[Start].MapCount--;
-   KeReleaseSpinLock(&PageListLock, oldIrql);
+  ULONG Start = PhysicalAddress.u.LowPart / PAGE_SIZE;
+  KIRQL oldIrql;
+
+  if (Start < MmPageArraySize)
+    {   
+      KeAcquireSpinLock(&PageListLock, &oldIrql);
+      if (MmPageArray[Start].Flags.Type == MM_PHYSICAL_PAGE_FREE)
+       {
+         DbgPrint("Unmapping non-used page\n");
+         KEBUGCHECK(0);
+       }
+      if (MmPageArray[Start].MapCount == 0)
+       {
+         DbgPrint("Unmapping not mapped page\n");
+         KEBUGCHECK(0);
+       }
+      MmPageArray[Start].MapCount--;
+      KeReleaseSpinLock(&PageListLock, oldIrql);
+    }
 }
 
 ULONG 
@@ -515,7 +595,7 @@ MmGetFlagsPage(PHYSICAL_ADDRESS PhysicalAddress)
    ULONG Flags;
    
    KeAcquireSpinLock(&PageListLock, &oldIrql);
-   Flags = MmPageArray[Start].Flags;
+   Flags = MmPageArray[Start].AllFlags;
    KeReleaseSpinLock(&PageListLock, oldIrql);
    
    return(Flags);
@@ -555,18 +635,18 @@ MmReferencePage(PHYSICAL_ADDRESS PhysicalAddress)
    KIRQL oldIrql;
    
    DPRINT("MmReferencePage(PhysicalAddress %x)\n", PhysicalAddress);
-   
+
    if (PhysicalAddress.u.LowPart == 0)
      {
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    KeAcquireSpinLock(&PageListLock, &oldIrql);
    
-   if (MM_PTYPE(MmPageArray[Start].Flags) != MM_PHYSICAL_PAGE_USED)
+   if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
      {
        DbgPrint("Referencing non-used page\n");
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    MmPageArray[Start].ReferenceCount++;
@@ -584,15 +664,15 @@ MmGetReferenceCountPage(PHYSICAL_ADDRESS PhysicalAddress)
 
    if (PhysicalAddress.u.LowPart == 0)
      {
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    KeAcquireSpinLock(&PageListLock, &oldIrql);
    
-   if (MM_PTYPE(MmPageArray[Start].Flags) != MM_PHYSICAL_PAGE_USED)
+   if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
      {
        DbgPrint("Getting reference count for free page\n");
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    RCount = MmPageArray[Start].ReferenceCount;
@@ -610,11 +690,11 @@ MmIsUsablePage(PHYSICAL_ADDRESS PhysicalAddress)
 
    if (PhysicalAddress.u.LowPart == 0)
      {
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
 
-   if (MM_PTYPE(MmPageArray[Start].Flags) != MM_PHYSICAL_PAGE_USED &&
-       MM_PTYPE(MmPageArray[Start].Flags) != MM_PHYSICAL_PAGE_BIOS)
+   if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED &&
+       MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_BIOS)
      {
        return(FALSE);
      }
@@ -632,16 +712,16 @@ MmDereferencePage(PHYSICAL_ADDRESS PhysicalAddress)
 
    if (PhysicalAddress.u.LowPart == 0)
      {
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    KeAcquireSpinLock(&PageListLock, &oldIrql);
   
 
-   if (MM_PTYPE(MmPageArray[Start].Flags) != MM_PHYSICAL_PAGE_USED)
+   if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
      {
        DbgPrint("Dereferencing free page\n");
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
   
    MmPageArray[Start].ReferenceCount--;
@@ -653,33 +733,38 @@ MmDereferencePage(PHYSICAL_ADDRESS PhysicalAddress)
        if (MmPageArray[Start].RmapListHead != NULL)
         {
           DbgPrint("Freeing page with rmap entries.\n");
-          KeBugCheck(0);
+          KEBUGCHECK(0);
         }
        if (MmPageArray[Start].MapCount != 0)
         {
           DbgPrint("Freeing mapped page (0x%I64x count %d)\n",
                    PhysicalAddress, MmPageArray[Start].MapCount);
-          KeBugCheck(0);
+          KEBUGCHECK(0);
         }
        if (MmPageArray[Start].LockCount > 0)
         {
           DbgPrint("Freeing locked page\n");
-          KeBugCheck(0);
+          KEBUGCHECK(0);
         }
        if (MmPageArray[Start].SavedSwapEntry != 0)
         {
           DbgPrint("Freeing page with swap entry.\n");
-          KeBugCheck(0);
+          KEBUGCHECK(0);
         }
-       if (MmPageArray[Start].Flags != MM_PHYSICAL_PAGE_USED)
+       if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
         {
           DbgPrint("Freeing page with flags %x\n",
-                   MmPageArray[Start].Flags);
-          KeBugCheck(0);
+                   MmPageArray[Start].Flags.Type);
+          KEBUGCHECK(0);
         }
-       MmPageArray[Start].Flags = MM_PHYSICAL_PAGE_FREE;
+       MmPageArray[Start].Flags.Type = MM_PHYSICAL_PAGE_FREE;
        InsertTailList(&FreeUnzeroedPageListHead, 
                      &MmPageArray[Start].ListEntry);
+       UnzeroedPageCount++;
+       if (UnzeroedPageCount > 8 && 0 == KeReadStateEvent(&ZeroPageThreadEvent))
+         {
+           KeSetEvent(&ZeroPageThreadEvent, IO_NO_INCREMENT, FALSE);
+         }
      }
    KeReleaseSpinLock(&PageListLock, oldIrql);
 }
@@ -695,15 +780,15 @@ MmGetLockCountPage(PHYSICAL_ADDRESS PhysicalAddress)
    
    if (PhysicalAddress.u.LowPart == 0)
      {
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    KeAcquireSpinLock(&PageListLock, &oldIrql);
    
-   if (MM_PTYPE(MmPageArray[Start].Flags) != MM_PHYSICAL_PAGE_USED)
+   if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
      {
        DbgPrint("Getting lock count for free page\n");
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    LockCount = MmPageArray[Start].LockCount;
@@ -722,15 +807,15 @@ MmLockPage(PHYSICAL_ADDRESS PhysicalAddress)
    
    if (PhysicalAddress.u.LowPart == 0)
      {
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    KeAcquireSpinLock(&PageListLock, &oldIrql);
    
-   if (MM_PTYPE(MmPageArray[Start].Flags) != MM_PHYSICAL_PAGE_USED)
+   if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
      {
        DbgPrint("Locking free page\n");
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    MmPageArray[Start].LockCount++;
@@ -747,15 +832,15 @@ MmUnlockPage(PHYSICAL_ADDRESS PhysicalAddress)
    
    if (PhysicalAddress.u.LowPart == 0)
      {
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    KeAcquireSpinLock(&PageListLock, &oldIrql);
    
-   if (MM_PTYPE(MmPageArray[Start].Flags) != MM_PHYSICAL_PAGE_USED)
+   if (MmPageArray[Start].Flags.Type != MM_PHYSICAL_PAGE_USED)
      {
        DbgPrint("Unlocking free page\n");
-       KeBugCheck(0);
+       KEBUGCHECK(0);
      }
    
    MmPageArray[Start].LockCount--;
@@ -783,36 +868,42 @@ MmAllocPage(ULONG Consumer, SWAPENTRY SavedSwapEntry)
          return((PHYSICAL_ADDRESS)0LL);
        }
       ListEntry = RemoveTailList(&FreeUnzeroedPageListHead);
+      UnzeroedPageCount--;
       
       PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
-      KeReleaseSpinLock(&PageListLock, oldIrql);
       
       NeedClear = TRUE;
     }
   else
     {
       ListEntry = RemoveTailList(&FreeZeroedPageListHead);
-      KeReleaseSpinLock(&PageListLock, oldIrql);
       
       PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
     }
   
-  if (PageDescriptor->Flags != MM_PHYSICAL_PAGE_FREE)
+  if (PageDescriptor->Flags.Type != MM_PHYSICAL_PAGE_FREE)
     {
       DbgPrint("Got non-free page from freelist\n");
-      KeBugCheck(0);
+      KEBUGCHECK(0);
+    }
+  if (PageDescriptor->MapCount != 0)
+    {
+      DbgPrint("Got mapped page from freelist\n");
+      KEBUGCHECK(0);
     }
-  PageDescriptor->Flags = MM_PHYSICAL_PAGE_USED;
+  PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_USED;
+  PageDescriptor->Flags.Consumer = Consumer;
   PageDescriptor->ReferenceCount = 1;
   PageDescriptor->LockCount = 0;
   PageDescriptor->MapCount = 0;
   PageDescriptor->SavedSwapEntry = SavedSwapEntry;
-  ExInterlockedInsertTailList(&UsedPageListHeads[Consumer], ListEntry, 
-                             &PageListLock);
+  InsertTailList(&UsedPageListHeads[Consumer], ListEntry);
   
   MmStats.NrSystemPages++;
   MmStats.NrFreePages--;
 
+  KeReleaseSpinLock(&PageListLock, oldIrql);
+
   PageOffset.QuadPart = (ULONG)((ULONG)PageDescriptor - (ULONG)MmPageArray);
   PageOffset.QuadPart = 
     (PageOffset.QuadPart / sizeof(PHYSICAL_PAGE)) * PAGE_SIZE;
@@ -820,5 +911,111 @@ MmAllocPage(ULONG Consumer, SWAPENTRY SavedSwapEntry)
     {
       MiZeroPage(PageOffset);
     }
+  if (PageDescriptor->MapCount != 0)
+    {
+      DbgPrint("Returning mapped page.\n");
+      KEBUGCHECK(0);
+    }
   return(PageOffset);
 }
+
+
+NTSTATUS STDCALL
+MmZeroPageThreadMain(PVOID Ignored)
+{
+  NTSTATUS Status;
+  KIRQL oldIrql;
+  PLIST_ENTRY ListEntry;
+  PPHYSICAL_PAGE PageDescriptor;
+  PHYSICAL_ADDRESS PhysPage;
+  static PVOID Address = NULL;
+  ULONG Count;
+    
+  while(1)
+    {
+      Status = KeWaitForSingleObject(&ZeroPageThreadEvent,
+                                    0,
+                                    KernelMode,
+                                    FALSE,
+                                    NULL);
+      if (!NT_SUCCESS(Status))
+       {
+         DbgPrint("ZeroPageThread: Wait failed\n");
+         KEBUGCHECK(0);
+         return(STATUS_UNSUCCESSFUL);
+       }
+
+      Count = 0;
+      KeAcquireSpinLock(&PageListLock, &oldIrql);
+      while (!IsListEmpty(&FreeUnzeroedPageListHead))
+      {
+         ListEntry = RemoveTailList(&FreeUnzeroedPageListHead);
+        UnzeroedPageCount--;
+         PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
+        /* We set the page to used, because MmCreateVirtualMapping failed with unused pages */
+        PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_USED;
+         KeReleaseSpinLock(&PageListLock, oldIrql);
+        Count++;
+         PhysPage.QuadPart = (ULONG)((ULONG)PageDescriptor - (ULONG)MmPageArray);
+         PhysPage.QuadPart = (PhysPage.QuadPart / sizeof(PHYSICAL_PAGE)) * PAGE_SIZE;
+        if (Address == NULL)
+        {
+           Address = ExAllocatePageWithPhysPage(PhysPage);
+        }
+        else
+        {
+            Status = MmCreateVirtualMapping(NULL, 
+                                           Address, 
+                                           PAGE_READWRITE | PAGE_SYSTEM, 
+                                           PhysPage,
+                                           FALSE);
+            if (!NT_SUCCESS(Status))
+            {
+               DbgPrint("Unable to create virtual mapping\n");
+              KEBUGCHECK(0);
+            }
+        }
+         memset(Address, 0, PAGE_SIZE);
+         MmDeleteVirtualMapping(NULL, (PVOID)Address, FALSE, NULL, NULL);
+         KeAcquireSpinLock(&PageListLock, &oldIrql);
+        if (PageDescriptor->MapCount != 0)
+          {
+            DbgPrint("Mapped page on freelist.\n");
+            KEBUGCHECK(0);
+          }
+        PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_FREE;
+        InsertHeadList(&FreeZeroedPageListHead, ListEntry);
+      }
+      DPRINT("Zeroed %d pages.\n", Count);
+      KeResetEvent(&ZeroPageThreadEvent);
+      KeReleaseSpinLock(&PageListLock, oldIrql);
+    }
+}
+ 
+NTSTATUS MmInitZeroPageThread(VOID)
+{
+  KPRIORITY Priority;
+  NTSTATUS Status;
+  
+  Status = PsCreateSystemThread(&ZeroPageThreadHandle,
+                               THREAD_ALL_ACCESS,
+                               NULL,
+                               NULL,
+                               &ZeroPageThreadId,
+                               (PKSTART_ROUTINE) MmZeroPageThreadMain,
+                               NULL);
+  if (!NT_SUCCESS(Status))
+    {
+      return(Status);
+     }
+  
+  Priority = 1;
+  NtSetInformationThread(ZeroPageThreadHandle,
+                        ThreadPriority,
+                        &Priority,
+                        sizeof(Priority));
+  
+  return(STATUS_SUCCESS);
+}
+ 
+/* EOF */