GPUDefragAllocator.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "CoreMinimal.h"
#include "HAL/ThreadSafeCounter.h"
#include "Stats/Stats.h"
#include "Misc/OutputDeviceRedirector.h"
#include "HAL/IConsoleManager.h"
#include "Containers/List.h"
#include "HAL/LowLevelMemTracker.h"
#include "ProfilingDebugging/MemoryTrace.h"
 
#define LOG_EVERY_ALLOCATION            0
#define DUMP_ALLOC_FREQUENCY            0 // 100
 
#define VALIDATE_SYNC_SIZE              0 //validates that the CPU is returning memory in an allocate that's part of an active GPU move.
#define VALIDATE_MOVES                  0 //validates that GPU moves in a given frame do not overlap destination areas.
#define TRACK_RELOCATIONS (VALIDATE_SYNC_SIZE || VALIDATE_MOVES)
#define VALIDATE_MEMORY_PROTECTION      0
 
#define USE_ALLOCATORFIXEDSIZEFREELIST  0
 
class FAsyncReallocationRequest;
class FScopedGPUDefragLock;
 
/*-----------------------------------------------------------------------------
Custom fixed size pool best fit texture memory allocator
-----------------------------------------------------------------------------*/
 
// Forward declaration
class FAsyncReallocationRequest;
 
class FScopedGPUDefragLock;
class FGPUDefragAllocator
{
public:
 
    typedef TDoubleLinkedList<FAsyncReallocationRequest*> FRequestList;
    typedef TDoubleLinkedList<FAsyncReallocationRequest*>::TDoubleLinkedListNode FRequestNode;
 
    struct FSettings
    {
        FSettings()
        : MaxDefragRelocations(128 * 1024)
        , MaxDefragDownShift(32 * 1024)
        , OverlappedBandwidthScale(1)
        {
        }
 
        int32       MaxDefragRelocations;
        int32       MaxDefragDownShift;
        int32       OverlappedBandwidthScale;
    };
 
    enum EMemoryElementType
    {
        MET_Allocated,
        MET_Free,
        MET_Locked,
        MET_Relocating,
        MET_Resizing,
        MET_Resized,
        MET_Max
    };
 
    struct FMemoryLayoutElement
    {
        FMemoryLayoutElement()
        : Size(0)
        , Type(MET_Allocated)
        {
        }
        FMemoryLayoutElement(int32 InSize, EMemoryElementType InType)
            : Size(InSize)
            , Type(InType)
        {
        }
        int32               Size;
        EMemoryElementType  Type;
 
        friend FArchive& operator<<(FArchive& Ar, FMemoryLayoutElement& Element)
        {
            Ar << Element.Size;
            uint32 ElementType = Element.Type;
            Ar << ElementType;
            Element.Type = EMemoryElementType(ElementType);
            return Ar;
        }
    };
 
    struct FRelocationStats
    {
        FRelocationStats()
        : NumBytesRelocated(0)
        , NumBytesDownShifted(0)
        , LargestHoleSize(0)
        , NumRelocations(0)     
        , NumHoles(0)
        , NumLockedChunks(0)
        {
        }
 
        int64 NumBytesRelocated;
 
        int64 NumBytesDownShifted;
 
        int64 LargestHoleSize;
 
        int32 NumRelocations;       
 
        int32 NumHoles;
 
        int32 NumLockedChunks;
    };
 
    class FMemoryChunk
    {
    public:
        FMemoryChunk(uint8* InBase, int64 InSize, FGPUDefragAllocator& InBestFitAllocator, FMemoryChunk*& ChunkToInsertAfter, TStatId InStat)
            : Base(InBase)
            , Size(InSize)
            , OrigSize(0)
            , bIsAvailable(false)
            , LockCount(0)
            , DefragCounter(0)
            , BestFitAllocator(InBestFitAllocator)
            , SyncIndex(0)
            , SyncSize(0)
            , UserPayload(0)            
            , Stat(InStat)
            , bTail(false)
        {
            Link(ChunkToInsertAfter);
            // This is going to change bIsAvailable.
            LinkFree(ChunkToInsertAfter);
        }
 
        ~FMemoryChunk()
        {
            // Remove from linked lists.
            Unlink();
            UnlinkFree();
            //todo: add check for numlocks == 0
        }
 
        bool IsLocked() const
        {           
            return (LockCount != 0);
        }
 
        void    Link(FMemoryChunk*& ChunkToInsertAfter)
        {
            if (ChunkToInsertAfter)
            {
                NextChunk = ChunkToInsertAfter->NextChunk;
                PreviousChunk = ChunkToInsertAfter;
                ChunkToInsertAfter->NextChunk = this;
                if (NextChunk)
                {
                    NextChunk->PreviousChunk = this;
                }
                else
                {
                    BestFitAllocator.LastChunk = this;
                }
            }
            else
            {
                PreviousChunk = nullptr;
                NextChunk = nullptr;
                ChunkToInsertAfter = this;
                BestFitAllocator.LastChunk = this;
            }
        }
 
        void    LinkFree(FMemoryChunk* FirstFreeChunkToSearch);
 
        void    Unlink()
        {
            if (PreviousChunk)
            {
                PreviousChunk->NextChunk = NextChunk;
            }
            else
            {
                BestFitAllocator.FirstChunk = NextChunk;
            }
 
            if (NextChunk)
            {
                NextChunk->PreviousChunk = PreviousChunk;
            }
            else
            {
                BestFitAllocator.LastChunk = PreviousChunk;
            }
 
            PreviousChunk = nullptr;
            NextChunk = nullptr;
        }
 
        void    UnlinkFree()
        {
            check(bIsAvailable);
            bIsAvailable = false;
 
            if (PreviousFreeChunk)
            {
                PreviousFreeChunk->NextFreeChunk = NextFreeChunk;
            }
            else
            {
                BestFitAllocator.FirstFreeChunk = NextFreeChunk;
            }
 
            if (NextFreeChunk)
            {
                NextFreeChunk->PreviousFreeChunk = PreviousFreeChunk;
            }
 
            PreviousFreeChunk = nullptr;
            NextFreeChunk = nullptr;
        }       
 
        bool    IsRelocating() const
        {
            return SyncIndex > BestFitAllocator.CompletedSyncIndex;
        }
 
        int64       GetAvailableSize() const
        {
            if (bIsAvailable)
            {
                return IsRelocating() ? (Size - SyncSize) : Size;
            }
            else
            {
                return 0;
            }
        }
 
        int64       GetFinalSize() const;
 
        void    SetSyncIndex(uint32 InSyncIndex, int64 InSyncSize)
        {           
            SyncIndex = InSyncIndex;
            SyncSize = InSyncSize;
        }
 
        uint32  GetSyncIndex() const
        {
            return SyncIndex;
        }
 
        static uint64 Compare(const FMemoryChunk* A, const FMemoryChunk* B)
        {
            return B->Base - A->Base;
        }
 
#if USE_ALLOCATORFIXEDSIZEFREELIST
        void* operator new(size_t Size);
        void operator delete(void *RawMemory);
#endif
 
        uint8*                  Base;
        int64                   Size;
        int64                   OrigSize;
        bool                    bIsAvailable;
        int32                   LockCount;
        uint16                  DefragCounter;
 
        FGPUDefragAllocator&        BestFitAllocator;
        FMemoryChunk*           PreviousChunk;
        FMemoryChunk*           NextChunk;
 
        FMemoryChunk*           PreviousFreeChunk;
        FMemoryChunk*           NextFreeChunk;
        uint32                  SyncIndex;
 
        int64                   SyncSize;
        void*                   UserPayload;        
 
        //stat associated with this allocation
        TStatId Stat;
        bool bTail;
    };
 
    FGPUDefragAllocator()
        : MemorySize(0)
        , MemoryBase(nullptr)
        , AllocationAlignment(0)
        , FirstChunk(nullptr)
        , LastChunk(nullptr)
        , FirstFreeChunk(nullptr)
        , TimeSpentInAllocator(0.0)
        , PaddingWasteSize(0)
        , AllocatedMemorySize(0)
        , AvailableMemorySize(0)
        , PendingMemoryAdjustment(0)
        , CurrentSyncIndex(1)
        , CompletedSyncIndex(0)
        , NumRelocationsInProgress(0)
        , PlatformSyncFence(0)
        , CurrentLargestHole(0)
        , CurrentNumHoles(0)
        , TotalNumRelocations(0)
        , TotalNumBytesRelocated(0)     
        , MinLargestHole(MAX_int64)
        , MaxNumHoles(0)
        , NumFinishedAsyncReallocations(0)
        , NumFinishedAsyncAllocations(0)
        , NumCanceledAsyncRequests(0)
        , BlockedCycles(0)
        , NumLockedChunks(0)
        , bBenchmarkMode(false)
    {}
 
    virtual ~FGPUDefragAllocator()
    {
    }
 
    virtual void    Initialize(uint8* InMemoryBase, int64 InMemorySize, int32 InAllocationAlignment)
    {
        // Update size, Pointer and alignment.
        MemoryBase = InMemoryBase;
        MemorySize = InMemorySize;
        AllocationAlignment = InAllocationAlignment;
        check(Align(MemoryBase, AllocationAlignment) == MemoryBase);
 
        // Update stats in a thread safe way.
        FPlatformAtomics::InterlockedExchange(&AvailableMemorySize, MemorySize);
        // Allocate initial chunk.
        FirstChunk = new FMemoryChunk(MemoryBase, MemorySize, *this, FirstChunk, TStatId());
        LastChunk = FirstChunk;
    }
 
    virtual void    Initialize(uint8* InMemoryBase, int64 InMemorySize)
    {
        Initialize(InMemoryBase, InMemorySize, 64);     
    }
 
    void    GetSettings(FSettings& OutSettings)
    {
        OutSettings = Settings;
    }
 
    void    SetSettings(const FSettings& InSettings)
    {
        Settings = InSettings;
    }
 
    bool    IsInitialized()
    {
        return MemoryBase != nullptr;
    }
 
    virtual void*   Allocate(int64 AllocationSize, int32 Alignment, TStatId InStat, bool bAllowFailure);    
 
    virtual void    Free(void* Pointer);
 
    virtual void    Lock(const void* Pointer);
 
    virtual void    Unlock(const void* Pointer);
 
    void    SetUserPayload(const void* Pointer, void* UserPayload);
 
    void*   GetUserPayload(const void* Pointer);
 
    int64       GetAllocatedSize(void* Pointer);
 
    void*   Reallocate(void* OldBaseAddress, int64 NewSize);
 
    bool IsValidPoolMemory(const void* Pointer) const
    {
        return (Pointer != nullptr) && (Pointer >= MemoryBase) && (Pointer < (MemoryBase + MemorySize));
    }
 
#if 0
    bool    AsyncReallocate(FAsyncReallocationRequest* ReallocationRequest, bool bForceRequest);
#endif
 
    void    DumpAllocs(FOutputDevice& Ar = *GLog);
 
    virtual void    GetMemoryStats(int64& OutAllocatedMemorySize, int64& OutAvailableMemorySize, int64& OutPendingMemoryAdjustment, int64& OutPaddingWasteSize)
    {
        OutAllocatedMemorySize = AllocatedMemorySize;
        OutAvailableMemorySize = AvailableMemorySize;
        OutPendingMemoryAdjustment = PendingMemoryAdjustment;
        OutPaddingWasteSize = PaddingWasteSize;
    }
 
    int64 GetTotalSize() const
    {
        return MemorySize;
    }
 
    int32       GetLargestAvailableAllocation(int32* OutNumFreeChunks = nullptr);
 
    uint32  GetBlockedCycles() const
    {
        return BlockedCycles;
    }
 
    bool    InBenchmarkMode() const
    {
        return bBenchmarkMode;
    }
 
    bool    GetTextureMemoryVisualizeData(FColor* TextureData, int32 SizeX, int32 SizeY, int32 Pitch, const int32 PixelSize);
 
    void    GetMemoryLayout(TArray<FMemoryLayoutElement>& MemoryLayout);
 
    EMemoryElementType  GetChunkType(FMemoryChunk* Chunk) const;
 
    void    DefragmentMemory(FRelocationStats& Stats);
 
    virtual int32       Tick(FRelocationStats& Stats, bool bPanicDefrag);
 
    bool    FinishAllRelocations();
 
    void    BlockOnAsyncReallocation(FAsyncReallocationRequest* Request);
 
    void    CancelAsyncReallocation(FAsyncReallocationRequest* Request, const void* CurrentBaseAddress);
 
    void    Benchmark(int32 MinChunkSize, int32 MaxChunkSize, float FreeRatio, float LockRatio, bool bFullDefrag, bool bSaveImages, const TCHAR* Filename);
    
    static inline bool IsAligned(const volatile void* Ptr, const uint32 Alignment)
    {
        return !(UPTRINT(Ptr) & (Alignment - 1));
    }
 
    int32 GetAllocationAlignment() const
    {
        return AllocationAlignment;
    }
 
protected:
 
#if TRACK_RELOCATIONS
    struct FRelocationEntry
    {
        FRelocationEntry(const uint8* InOldBase, const uint8* InNewBase, uint64 InSize, uint64 InSyncIndex);
 
        const uint8* OldBase;
        const uint8* NewBase;
        uint64 Size;
        uint64 SyncIndex;
    };
 
    void ValidateRelocations(uint8* UsedBaseAddress, uint64 Size);
    
    TArray<FRelocationEntry> Relocations;
#endif
 
#if VALIDATE_MEMORY_PROTECTION
    struct FMemProtectTracker
    {
        FMemProtectTracker(const void* InMemory, const void* InUserPayload, uint64 InBlockSize, uint32 InSyncIndex)
        : Memory(InMemory)
        , UserPayload(InUserPayload)
        , BlockSize(InBlockSize)
        , SyncIndex(InSyncIndex)
        {
 
        }
        const void* Memory;
        const void* UserPayload;
        uint64 BlockSize;
        uint32 SyncIndex;
    };
 
    TArray<FMemProtectTracker> BlocksToProtect;
    TArray<FMemProtectTracker> BlocksToUnProtect;
 
    static int32 GGPUDefragDumpRelocationsToTTY;
    static FAutoConsoleVariableRef CVarGPUDefragDumpRelocationsToTTY;
 
    void SetStaticMemoryPrivileges();
 
    virtual void    PlatformSetNoMemoryPrivileges(const FMemProtectTracker& Block) {};
 
    virtual void    PlatformSetStandardMemoryPrivileges(const FMemProtectTracker& Block) {};
    
    virtual void    PlatformSetStaticMemoryPrivileges(const FMemProtectTracker& BlocksToAllow) {};
 
    virtual void    PlatformSetRelocationMemoryPrivileges(const FMemProtectTracker& BlocksToAllow) {};
    virtual void    PlatformSetRelocationMemoryPrivileges(const TArray<FMemProtectTracker>& BlocksToRemove) {};
#endif
 
 
    virtual void    PlatformRelocate(void* Dest, const void* Source, int64 Size, void* UserPayload) = 0;
 
    
 
    virtual uint64  PlatformInsertFence() = 0;
 
    virtual void    PlatformBlockOnFence(uint64 Fence) = 0;
 
    virtual bool    PlatformCanRelocate(const void* Source, void* UserPayload) const = 0;
 
    virtual void    PlatformNotifyReallocationFinished(FAsyncReallocationRequest* FinishedRequest, void* UserPayload) = 0;
 
    void Relocate(FRelocationStats& Stats, FMemoryChunk* Dest, int64 DestOffset, const void* Source, int64 Size, void* UserPayload)
    {
        MemoryTrace_ReallocFree((uint64)Source);
        MemoryTrace_ReallocAlloc((uint64)Dest->Base, Size, 4);
        LLM_IF_ENABLED(FLowLevelMemTracker::Get().OnLowLevelAllocMoved(ELLMTracker::Default, Dest->Base, Source));
 
        uint8* DestAddr = Dest->Base + DestOffset;
        int64 MemDistance = (int64)(DestAddr) - (int64)(Source);
        int64 AbsDistance = FMath::Abs(MemDistance);
        bool bOverlappedMove = AbsDistance < Size;
 
        if (!bBenchmarkMode)
        {
#if VALIDATE_MEMORY_PROTECTION
            BlocksToProtect.Emplace(DestAddr, UserPayload, Size, CurrentSyncIndex);
            BlocksToProtect.Emplace(Source, UserPayload, Size, CurrentSyncIndex);
#endif
            PlatformRelocate(DestAddr, Source, Size, UserPayload);
        }
        int64 RelocateSize = bOverlappedMove ? (Size * Settings.MaxDefragRelocations) : Size;
        Dest->UserPayload = UserPayload;
        Stats.NumBytesRelocated += RelocateSize;
        Stats.NumRelocations++;     
    }
 
    uint32          GetCurrentSyncIndex() const
    {
        return CurrentSyncIndex;
    }
 
    void            PartialDefragmentationFast(FRelocationStats& Stats, double StartTime);
 
    void            PartialDefragmentationSlow(FRelocationStats& Stats, double StartTime);
 
    void            PartialDefragmentationDownshift(FRelocationStats& Stats, double StartTime);
 
    void            FullDefragmentation(FRelocationStats& Stats);
 
    FMemoryChunk*   Grow(FMemoryChunk* Chunk, int64 GrowAmount);
 
    FMemoryChunk*   Shrink(FMemoryChunk* Chunk, int64 ShrinkAmount);
 
    void            CheckForErrors(bool bCheckSortedFreeList);
 
    bool            CanRelocate(const FMemoryChunk* Chunk) const;
 
    void            InsertFence();
 
    void            BlockOnFence();
 
    void            BlockOnSyncIndex(uint32 SyncIndex);
 
    void            Split(FMemoryChunk* BaseChunk, int64 FirstSize)
    {
        check(BaseChunk);
        check(FirstSize < BaseChunk->Size);
        check(FirstSize > 0);
 
        // Don't make any assumptions on the following chunk. Because Reallocate() will make the 1st chunk free
        // and the 2nd chunk used, so it's ok to have the following chunk free. Note, this only happens when Reallocate()
        // is splitting the very first chunk in the pool.
        // Don't make any assumptions for the previous chunk either...
        //      check( !BaseChunk->NextChunk || !BaseChunk->NextChunk->bIsAvailable );
        //      check( !BaseChunk->PreviousChunk || !BaseChunk->PreviousChunk->bIsAvailable || !BaseChunk->bIsAvailable );
 
        // Calculate size of second chunk...
        int64 SecondSize = BaseChunk->Size - FirstSize;
        // ... and create it.
 
        //todo: fix stats
        //ensureMsgf(BaseChunk->Stat.IsNone(), TEXT("Free chunk has stat"));
        FMemoryChunk* NewFreeChunk = new FMemoryChunk(BaseChunk->Base + FirstSize, SecondSize, *this, BaseChunk, TStatId());
 
        // Keep the original sync index for the new chunk if necessary.
        if (BaseChunk->IsRelocating() && BaseChunk->SyncSize > FirstSize)
        {
            int64 SecondSyncSize = BaseChunk->SyncSize - FirstSize;
            NewFreeChunk->SetSyncIndex(BaseChunk->SyncIndex, SecondSyncSize);
        }
        BaseChunk->SetSyncIndex(BaseChunk->SyncIndex, FMath::Min((int64)FirstSize, BaseChunk->SyncSize));
 
        // Resize base chunk.
        BaseChunk->Size = FirstSize;
    } //-V773
 
    FMemoryChunk*   AllocateChunk(FMemoryChunk* FreeChunk, int64 AllocationSize, bool bAsync, bool bDoValidation = true);
 
    void            FreeChunk(FMemoryChunk* Chunk);
 
    void            LinkFreeChunk(FMemoryChunk* Chunk)
    {
        check(Chunk);
        // Mark chunk as available.
        Chunk->LinkFree(nullptr);
        // Kick of merge pass.
        Coalesce(Chunk);
    }
 
    void            Coalesce(FMemoryChunk* FreedChunk);
 
    void            SortFreeList(int32& NumFreeChunks, int64& LargestFreeChunk);
 
    FMemoryChunk*   FindAdjacent(FMemoryChunk* UsedChunk, bool bAnyChunkType);  
 
    FMemoryChunk*   FindAdjacentToHole(FMemoryChunk* FreeChunk);
 
    FMemoryChunk*   FindAny(FMemoryChunk* FreeChunk);
 
    FMemoryChunk*   RelocateIntoFreeChunk(FRelocationStats& Stats, FMemoryChunk* FreeChunk, FMemoryChunk* UsedChunk);
 
    FMemoryChunk* RelocateAllowed(FMemoryChunk* FreeChunk, FMemoryChunk* UsedChunk);    
 
    FCriticalSection    SynchronizationObject;
 
    uint64              MemorySize;
    uint8*          MemoryBase;
    int32               AllocationAlignment;
    FMemoryChunk*   FirstChunk;
    FMemoryChunk*   LastChunk;
    FMemoryChunk*   FirstFreeChunk;
    double          TimeSpentInAllocator;
    typedef int64 memsize_t;
 
    volatile memsize_t  PaddingWasteSize;
 
    volatile memsize_t  AllocatedMemorySize;
    volatile memsize_t  AvailableMemorySize;
    volatile memsize_t  PendingMemoryAdjustment;
 
    TMap<void*, FMemoryChunk*>  PointerToChunkMap;
 
    FSettings       Settings;
 
    uint64          CurrentSyncIndex;
    uint64          CompletedSyncIndex;
 
    int32               NumRelocationsInProgress;
    uint64          PlatformSyncFence;
 
    TDoubleLinkedList<FMemoryChunk*>    PendingFreeChunks;
 
    uint64 CurrentLargestHole;
    int32 CurrentNumHoles;
 
    // Stats
    uint64          TotalNumRelocations;
    uint64          TotalNumBytesRelocated;
    int64               MinLargestHole;
    int32               MaxNumHoles;    
    int32               NumFinishedAsyncReallocations;
    int32               NumFinishedAsyncAllocations;
    int32               NumCanceledAsyncRequests;
    uint32          BlockedCycles;
 
    int32 NumLockedChunks;
 
#if VALIDATE_MEMORY_PROTECTION
    double TimeInMemProtect;
#endif
 
    bool            bBenchmarkMode;
 
    friend FScopedGPUDefragLock;
};
 
//FScopedGPUDefragLock can't cover any scope that will add dcb commands or we might deadlock.
class FScopedGPUDefragLock
{
public:
    FScopedGPUDefragLock(FGPUDefragAllocator& InDefragAllocator)
        : DefragAllocator(InDefragAllocator)
    {
        DefragAllocator.SynchronizationObject.Lock();
    }
 
    ~FScopedGPUDefragLock()
    {
        DefragAllocator.SynchronizationObject.Unlock();
    }
private:
    FGPUDefragAllocator& DefragAllocator;
};
 
class FAsyncReallocationRequest
{
public:
    FAsyncReallocationRequest(void* InCurrentBaseAddress, int32 InNewSize, FThreadSafeCounter* InRequestStatus)
        : OldAddress(InCurrentBaseAddress)
        , NewAddress(nullptr)
        , OldSize(0)    // Set by AsyncReallocate()
        , NewSize(InNewSize)
        , int32ernalRequestStatus(1)
        , ExternalRequestStatus(InRequestStatus)
        , bIsCanceled(false)
        , MemoryChunk(nullptr)
    {
    }
 
    ~FAsyncReallocationRequest()
    {
        check(!HasStarted() || IsCanceled() || HasCompleted());
    }
 
    bool    IsAllocation() const
    {
        return OldAddress == nullptr && OldSize == 0;
    }
 
    bool    IsReallocation() const
    {
        return OldAddress != nullptr;
    }
 
    bool    IsCanceled() const
    {
        return bIsCanceled;
    }
 
    bool    HasCompleted() const
    {
        bool bHasCompleted = int32ernalRequestStatus.GetValue() == 0;
        check(!bHasCompleted || NewAddress || bIsCanceled);
        return bHasCompleted;
    }
 
    bool    HasStarted() const
    {
        return NewAddress ? true : false;
    }
 
    void*   GetOldBaseAddress() const
    {
        return OldAddress;
    }
 
    void*   GetNewBaseAddress() const
    {
        return NewAddress;
    }
 
    int32       GetNewSize() const
    {
        return NewSize;
    }
 
private:
    // Hidden on purpose since outside usage isn't necessarily thread-safe.
    FAsyncReallocationRequest(const FAsyncReallocationRequest& Other)   { FMemory::Memcpy(this, &Other, sizeof(FAsyncReallocationRequest)); }
    void operator=(const FAsyncReallocationRequest& Other) { FMemory::Memcpy(this, &Other, sizeof(FAsyncReallocationRequest)); }
 
    void    MarkCompleted()
    {
        check(int32ernalRequestStatus.GetValue() == 1);
        int32ernalRequestStatus.Decrement();
        if (ExternalRequestStatus)
        {
            ExternalRequestStatus->Decrement();
        }
    }
 
    void*               OldAddress;
    void*               NewAddress;
    int32                   OldSize;
    int32                   NewSize;
    FThreadSafeCounter  int32ernalRequestStatus;
    FThreadSafeCounter* ExternalRequestStatus;
    uint32              bIsCanceled : 1;
 
    class FGPUDefragAllocator::FMemoryChunk*        MemoryChunk;
 
    friend class FGPUDefragAllocator;
};
 
 
inline int64 FGPUDefragAllocator::FMemoryChunk::GetFinalSize() const
{
    return Size;
}
 
inline bool FGPUDefragAllocator::CanRelocate(const FMemoryChunk* Chunk) const
{
    if (Chunk->IsLocked())
    {
        return false;
    }
 
    if (!bBenchmarkMode)
    {
        return PlatformCanRelocate(Chunk->Base, Chunk->UserPayload);
    }
    else
    {
        return true;
    }
}
 
inline void FGPUDefragAllocator::BlockOnAsyncReallocation(FAsyncReallocationRequest* Request)
{
    check(Request->HasStarted());
    if (!Request->HasCompleted())
    {
        BlockOnSyncIndex(Request->MemoryChunk->SyncIndex);
    }
}