[MERGE #1635 @leirocks] fix xProc number allocator

Merge pull request #1635 from leirocks:xprocnumber

- some fix needed because of the IDL adjustment(numberSegment changing to pointer and Arena allocated)
- delay allocate the segment only when necessary(when there's number need to be allocated in JIT time)
- added test switch to make the integration and segment full code path easier to test in oop JIT (currently 0x20 pages hold 0x2000 numbers which is extremely hard to fullfil the segment)
- make recycler memory verify work for cross process number allocator

Author: leirocks

lib/Backend/CodeGenNumberAllocator.cpp

@@ -4,7 +4,7 @@
 //-------------------------------------------------------------------------------------------------------
 #include "Backend.h"
 
-
+#if !FLOATVAR
 CodeGenNumberThreadAllocator::CodeGenNumberThreadAllocator(Recycler * recycler)
     : recycler(recycler), currentNumberSegment(nullptr), currentChunkSegment(nullptr),
     numberSegmentEnd(nullptr), currentNumberBlockEnd(nullptr), nextNumber(nullptr), chunkSegmentEnd(nullptr),
@@ -245,7 +245,6 @@ CodeGenNumberAllocator::CodeGenNumberAllocator(CodeGenNumberThreadAllocator * th
 }
 
 // We should never call this function if we are using tagged float
-#if !FLOATVAR
 Js::JavascriptNumber *
 CodeGenNumberAllocator::Alloc()
 {
@@ -274,7 +273,7 @@ CodeGenNumberAllocator::Alloc()
     this->chunkTail->numbers[this->currentChunkNumberCount++] = newNumber;
     return newNumber;
 }
-#endif
+
 
 CodeGenNumberChunk *
 CodeGenNumberAllocator::Finalize()
@@ -290,16 +289,12 @@ CodeGenNumberAllocator::Finalize()
     return finalizedChunk;
 }
 
-/* static */
-uint
-XProcNumberPageSegmentImpl::GetSizeCat()
-{
-    return (uint)HeapInfo::GetAlignedSizeNoCheck(sizeof(Js::JavascriptNumber));
-}
-
 
-Js::JavascriptNumber* XProcNumberPageSegmentImpl::AllocateNumber(HANDLE hProcess, double value, Js::StaticType* numberTypeStatic, void* javascriptNumberVtbl)
+uint XProcNumberPageSegmentImpl::sizeCat = sizeof(Js::JavascriptNumber);
+Js::JavascriptNumber* XProcNumberPageSegmentImpl::AllocateNumber(Func* func, double value)
 {
+    HANDLE hProcess = func->GetThreadContextInfo()->GetProcessHandle();
+
     XProcNumberPageSegmentImpl* tail = this;
 
     if (this->pageAddress != 0)
@@ -309,39 +304,54 @@ Js::JavascriptNumber* XProcNumberPageSegmentImpl::AllocateNumber(HANDLE hProcess
             tail = (XProcNumberPageSegmentImpl*)tail->nextSegment;
         }
 
-        if (tail->pageAddress + tail->committedEnd - tail->allocEndAddress >= GetSizeCat())
+        if (tail->pageAddress + tail->committedEnd - tail->allocEndAddress >= sizeCat)
         {
             auto number = tail->allocEndAddress;
-            tail->allocEndAddress += GetSizeCat();
+            tail->allocEndAddress += sizeCat;
 
-            Js::JavascriptNumber localNumber(value, numberTypeStatic
 #if DBG
-                , true
+            Js::JavascriptNumber localNumber(value, (Js::StaticType*)func->GetScriptContextInfo()->GetNumberTypeStaticAddr(), true);
+#else
+            Js::JavascriptNumber localNumber(value, (Js::StaticType*)func->GetScriptContextInfo()->GetNumberTypeStaticAddr());
 #endif
-            );
+            Js::JavascriptNumber* pLocalNumber = &localNumber;
 
+#ifdef RECYCLER_MEMORY_VERIFY
+            if (func->GetScriptContextInfo()->IsRecyclerVerifyEnabled())
+            {
+                pLocalNumber = (Js::JavascriptNumber*)alloca(sizeCat);                
+                memset(pLocalNumber, Recycler::VerifyMemFill, sizeCat);
+                Recycler::FillPadNoCheck(pLocalNumber, sizeof(Js::JavascriptNumber), sizeCat, false);
+                pLocalNumber = new (pLocalNumber) Js::JavascriptNumber(localNumber);
+            }
+#endif
             // change vtable to the remote one
-            *(void**)&localNumber = javascriptNumberVtbl;
+            *(void**)pLocalNumber = (void*)func->GetScriptContextInfo()->GetVTableAddress(VTableValue::VtableJavascriptNumber);
 
             // initialize number by WriteProcessMemory
             SIZE_T bytesWritten;
-            WriteProcessMemory(hProcess, (void*)number, &localNumber, sizeof(localNumber), &bytesWritten);
+            if (!WriteProcessMemory(hProcess, (void*)number, pLocalNumber, sizeCat, &bytesWritten)
+                || bytesWritten != sizeCat)
+            {
+                Output::Print(_u("FATAL ERROR: WriteProcessMemory failed, GLE: %d\n"), GetLastError());
+                Js::Throw::FatalInternalError(); // TODO: don't bring down whole server process, but pass the last error to main process
+            }
 
             return (Js::JavascriptNumber*) number;
         }
 
         // alloc blocks
-        if ((void*)tail->committedEnd < tail->GetEndAddress())
+        if (tail->GetCommitEndAddress() < tail->GetEndAddress())
         {
-            Assert((unsigned int)((char*)tail->GetEndAddress() - (char*)tail->committedEnd) >= BlockSize);
+            Assert((unsigned int)((char*)tail->GetEndAddress() - (char*)tail->GetCommitEndAddress()) >= BlockSize);
             // TODO: implement guard pages (still necessary for OOP JIT?)
             auto ret = ::VirtualAllocEx(hProcess, tail->GetCommitEndAddress(), BlockSize, MEM_COMMIT, PAGE_READWRITE);
             if (!ret)
             {
                 Js::Throw::OutOfMemory();
             }
             tail->committedEnd += BlockSize;
-            return AllocateNumber(hProcess, value, numberTypeStatic, javascriptNumberVtbl);
+            return AllocateNumber(func, value);
         }
     }
 
@@ -354,12 +364,11 @@ Js::JavascriptNumber* XProcNumberPageSegmentImpl::AllocateNumber(HANDLE hProcess
 
     if (tail->pageAddress == 0)
     {
-        tail = new (tail) XProcNumberPageSegmentImpl();
         tail->pageAddress = (intptr_t)pages;
         tail->allocStartAddress = this->pageAddress;
         tail->allocEndAddress = this->pageAddress;
         tail->nextSegment = nullptr;
-        return AllocateNumber(hProcess, value, numberTypeStatic, javascriptNumberVtbl);
+        return AllocateNumber(func, value);
     }
     else
     {
@@ -370,15 +379,40 @@ Js::JavascriptNumber* XProcNumberPageSegmentImpl::AllocateNumber(HANDLE hProcess
         }
         seg = new (seg) XProcNumberPageSegmentImpl();
         tail->nextSegment = seg;
-        return seg->AllocateNumber(hProcess, value, numberTypeStatic, javascriptNumberVtbl);
+        return seg->AllocateNumber(func, value);
     }
 }
 
 
 XProcNumberPageSegmentImpl::XProcNumberPageSegmentImpl()
 {
-    this->blockIntegratedSize = 0;
-    this->pageSegment = 0;
+    memset(this, 0, sizeof(XProcNumberPageSegment));
+}
+
+void XProcNumberPageSegmentImpl::Initialize(bool recyclerVerifyEnabled, uint recyclerVerifyPad)
+{
+    size_t allocSize = sizeof(Js::JavascriptNumber);
+#ifdef ENABLE_DEBUG_CONFIG_OPTIONS
+    allocSize += Js::Configuration::Global.flags.NumberAllocPlusSize;
+#endif
+#ifdef RECYCLER_MEMORY_VERIFY
+    // TODO: share same pad size with main process
+    if (recyclerVerifyEnabled)
+    {
+        size_t padAllocSize = AllocSizeMath::Add(sizeof(Js::JavascriptNumber) + sizeof(size_t), recyclerVerifyPad);
+        allocSize = padAllocSize < allocSize ? allocSize : padAllocSize;
+    }
+#endif    
+
+    allocSize = (uint)HeapInfo::GetAlignedSizeNoCheck(allocSize);
+
+    if (BlockSize%allocSize != 0)
+    {
+        // align allocation sizeCat to be 2^n to make integration easier
+        allocSize = BlockSize / (1 << (Math::Log2(BlockSize / allocSize)));
+    }
+
+    sizeCat = allocSize;
 }
 
 Js::JavascriptNumber** ::XProcNumberPageSegmentManager::RegisterSegments(XProcNumberPageSegment* segments)
@@ -390,7 +424,7 @@ Js::JavascriptNumber** ::XProcNumberPageSegmentManager::RegisterSegments(XProcNu
     size_t totalCount = 0;
     while (temp)
     {
-        totalCount += (temp->allocEndAddress - temp->allocStartAddress) / XProcNumberPageSegmentImpl::GetSizeCat();
+        totalCount += (temp->allocEndAddress - temp->allocStartAddress) / XProcNumberPageSegmentImpl::sizeCat;
         temp = (XProcNumberPageSegmentImpl*)temp->nextSegment;
     }
 
@@ -400,18 +434,15 @@ Js::JavascriptNumber** ::XProcNumberPageSegmentManager::RegisterSegments(XProcNu
     int count = 0;
     while (temp)
     {
-        auto start = temp->allocStartAddress;
-        while (start < temp->allocEndAddress)
+        while (temp->allocStartAddress < temp->allocEndAddress)
         {
-            numbers[count] = (Js::JavascriptNumber*)start;
+            numbers[count] = (Js::JavascriptNumber*)temp->allocStartAddress;
             count++;
-            start += XProcNumberPageSegmentImpl::GetSizeCat();
+            temp->allocStartAddress += XProcNumberPageSegmentImpl::sizeCat;
         }
         temp = (XProcNumberPageSegmentImpl*)temp->nextSegment;
     }
 
-
-
     AutoCriticalSection autoCS(&cs);
     if (this->segmentsList == nullptr)
     {
@@ -430,16 +461,10 @@ Js::JavascriptNumber** ::XProcNumberPageSegmentManager::RegisterSegments(XProcNu
     return numbers;
 }
 
-void XProcNumberPageSegmentManager::GetFreeSegment(XProcNumberPageSegment * seg)
+XProcNumberPageSegment * XProcNumberPageSegmentManager::GetFreeSegment(Memory::ArenaAllocator* alloc)
 {
     AutoCriticalSection autoCS(&cs);
 
-    if (segmentsList == nullptr)
-    {
-        new (seg) XProcNumberPageSegmentImpl();
-        return;
-    }
-
     auto temp = segmentsList;
     auto prev = &segmentsList;
     while (temp)
@@ -449,13 +474,17 @@ void XProcNumberPageSegmentManager::GetFreeSegment(XProcNumberPageSegment * seg)
             *prev = (XProcNumberPageSegmentImpl*)temp->nextSegment;
 
             // remove from the list
-            memcpy(seg, temp, sizeof(XProcNumberPageSegment));
+            XProcNumberPageSegment * seg = (XProcNumberPageSegment *)AnewStructZ(alloc, XProcNumberPageSegmentImpl);
+            temp->nextSegment = 0;
+            memcpy(seg, temp, sizeof(XProcNumberPageSegment));            
             midl_user_free(temp);
-            return;
+            return seg;
         }
         prev = (XProcNumberPageSegmentImpl**)&temp->nextSegment;
         temp = (XProcNumberPageSegmentImpl*)temp->nextSegment;
     }
+
+    return nullptr;
 }
 
 void XProcNumberPageSegmentManager::Integrate()
@@ -466,36 +495,62 @@ void XProcNumberPageSegmentManager::Integrate()
     auto prev = &this->segmentsList;
     while (temp)
     {
-        if (temp->pageSegment == 0)
-        {
-            auto leafPageAllocator = recycler->GetRecyclerLeafPageAllocator();
-            DListBase<PageSegment> segmentList;
-            temp->pageSegment = (intptr_t)leafPageAllocator->AllocPageSegment(segmentList, leafPageAllocator,
-                (void*)temp->pageAddress, XProcNumberPageSegmentImpl::PageCount, temp->committedEnd / AutoSystemInfo::PageSize);
-            leafPageAllocator->IntegrateSegments(segmentList, 1, XProcNumberPageSegmentImpl::PageCount);
-
-            this->integratedSegmentCount++;
-        }
-
-        unsigned int minIntegrateSize = XProcNumberPageSegmentImpl::BlockSize;
-        for (; temp->pageAddress + temp->blockIntegratedSize + minIntegrateSize < (unsigned int)temp->allocEndAddress;
-            temp->blockIntegratedSize += minIntegrateSize)
+        if((uintptr_t)temp->allocEndAddress - (uintptr_t)temp->pageAddress > temp->blockIntegratedSize + XProcNumberPageSegmentImpl::BlockSize)
         {
-            TRACK_ALLOC_INFO(recycler, Js::JavascriptNumber, Recycler, 0, (size_t)-1);
+            if (temp->pageSegment == 0)
+            {
+                auto leafPageAllocator = recycler->GetRecyclerLeafPageAllocator();
+                DListBase<PageSegment> segmentList;
+                temp->pageSegment = (intptr_t)leafPageAllocator->AllocPageSegment(segmentList, leafPageAllocator,
+                    (void*)temp->pageAddress, XProcNumberPageSegmentImpl::PageCount, temp->committedEnd / AutoSystemInfo::PageSize);
+
+                if (temp->pageSegment)
+                {
+                    leafPageAllocator->IntegrateSegments(segmentList, 1, XProcNumberPageSegmentImpl::PageCount);
+                    this->integratedSegmentCount++;
+                }
+            }
 
-            if (!recycler->IntegrateBlock<LeafBit>((char*)temp->pageAddress + temp->blockIntegratedSize,
-                (PageSegment*)temp->pageSegment, XProcNumberPageSegmentImpl::GetSizeCat(), sizeof(Js::JavascriptNumber)))
+            if (temp->pageSegment)
             {
-                Js::Throw::OutOfMemory();
+                unsigned int minIntegrateSize = XProcNumberPageSegmentImpl::BlockSize;
+                for (; temp->pageAddress + temp->blockIntegratedSize + minIntegrateSize < (unsigned int)temp->allocEndAddress;
+                    temp->blockIntegratedSize += minIntegrateSize)
+                {
+                    TRACK_ALLOC_INFO(recycler, Js::JavascriptNumber, Recycler, 0, (size_t)-1);
+
+                    if (!recycler->IntegrateBlock<LeafBit>((char*)temp->pageAddress + temp->blockIntegratedSize,
+                        (PageSegment*)temp->pageSegment, XProcNumberPageSegmentImpl::sizeCat, sizeof(Js::JavascriptNumber)))
+                    {
+                        Js::Throw::OutOfMemory();
+                    }
+                }
+
+                if ((uintptr_t)temp->allocEndAddress + XProcNumberPageSegmentImpl::sizeCat 
+                    > (uintptr_t)temp->pageAddress + XProcNumberPageSegmentImpl::PageCount*AutoSystemInfo::PageSize)
+                {
+                    *prev = (XProcNumberPageSegmentImpl*)temp->nextSegment;
+                    midl_user_free(temp);
+                    temp = *prev;
+                    continue;
+                }
             }
         }
 
-        *prev = (XProcNumberPageSegmentImpl*)temp->nextSegment;
-        midl_user_free(temp);
-        temp = *prev;
+        temp = (XProcNumberPageSegmentImpl*)temp->nextSegment;
     }
 }
 
+XProcNumberPageSegmentManager::XProcNumberPageSegmentManager(Recycler* recycler)
+    :segmentsList(nullptr), recycler(recycler), integratedSegmentCount(0)
+{
+#ifdef RECYCLER_MEMORY_VERIFY
+    XProcNumberPageSegmentImpl::Initialize(recycler->VerifyEnabled() == TRUE, recycler->GetVerifyPad());
+#else
+    XProcNumberPageSegmentImpl::Initialize(false, 0);
+#endif
+}
+
 XProcNumberPageSegmentManager::~XProcNumberPageSegmentManager()
 {
     auto temp = segmentsList;
@@ -505,4 +560,5 @@ XProcNumberPageSegmentManager::~XProcNumberPageSegmentManager()
         midl_user_free(temp);
         temp = (XProcNumberPageSegmentImpl*)next;
     }
-}
+}
+#endif

lib/Backend/CodeGenNumberAllocator.h

@@ -4,6 +4,7 @@
 //-------------------------------------------------------------------------------------------------------
 #pragma once
 
+#if !FLOATVAR
 /****************************************************************************
  * CodeGenNumberThreadAllocator
  *
@@ -157,14 +158,15 @@ namespace Js
 struct XProcNumberPageSegmentImpl : public XProcNumberPageSegment
 {
     XProcNumberPageSegmentImpl();
-    Js::JavascriptNumber* AllocateNumber(HANDLE hProcess, double value, Js::StaticType* numberTypeStatic, void* javascriptNumberVtbl);
+    Js::JavascriptNumber* AllocateNumber(Func* func, double value);
     unsigned int GetTotalSize() { return PageCount * AutoSystemInfo::PageSize; }
     void* GetEndAddress() { return (void*)(this->pageAddress + PageCount * AutoSystemInfo::PageSize); }
     void* GetCommitEndAddress() { return (void*)(this->pageAddress + this->committedEnd); }
 
     static const uint BlockSize = SmallAllocationBlockAttributes::PageCount*AutoSystemInfo::PageSize;
     static const uint PageCount = Memory::IdleDecommitPageAllocator::DefaultMaxAllocPageCount;
-    static uint GetSizeCat();
+    static uint sizeCat;
+    static void Initialize(bool recyclerVerifyEnabled, uint recyclerVerifyPad);
 };
 
 static_assert(sizeof(XProcNumberPageSegmentImpl) == sizeof(XProcNumberPageSegment), "should not have data member in XProcNumberPageSegmentImpl");
@@ -175,16 +177,13 @@ struct XProcNumberPageSegmentManager
     XProcNumberPageSegmentImpl* segmentsList;
     Recycler* recycler;
     unsigned int integratedSegmentCount;
-    XProcNumberPageSegmentManager(Recycler* recycler)
-        :segmentsList(nullptr), recycler(recycler), integratedSegmentCount(0)
-    {
-    }
+    XProcNumberPageSegmentManager(Recycler* recycler);
 
     ~XProcNumberPageSegmentManager();
 
-    void GetFreeSegment(XProcNumberPageSegment * seg);
+    XProcNumberPageSegment * GetFreeSegment(Memory::ArenaAllocator* alloc);
     Js::JavascriptNumber** RegisterSegments(XProcNumberPageSegment* segments);
 
     void Integrate();
 };
-
+#endif