GenericPlatformMemory.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
 
/*=============================================================================================
    GenericPlatformMemory.h: Generic platform memory classes
==============================================================================================*/
 
#pragma once
 
#include "CoreFwd.h" // IWYU pragma: keep
#include "CoreTypes.h"
#include "HAL/PlatformString.h"
 
#include <string.h>
#include <wchar.h>
 
class FMalloc;
class FOutputDevice;
class FString;
struct FGenericMemoryStats;
struct FPlatformMemoryStats;
 
#ifndef UE_CHECK_LARGE_ALLOCATIONS
#define UE_CHECK_LARGE_ALLOCATIONS 0
#endif
 
#if UE_CHECK_LARGE_ALLOCATIONS
class FAutoConsoleVariableRef;
namespace UE::Memory::Private
{
    extern bool GEnableLargeAllocationChecks;
    extern int32 GLargeAllocationThreshold;
    extern FAutoConsoleVariableRef CVarEnableLargeAllocationChecks;
}
#endif
 
#define PLATFORM_MEMORY_SIZE_BUCKET_LIST(XBUCKET) \
    /* not used with texture LODs (you can't use bigger textures than what is cooked out, which is what Default should map to) */ \
    XBUCKET(Largest)    \
    XBUCKET(Larger)     \
    /* these are used by texture LODs */ \
    XBUCKET(Default)    \
    XBUCKET(Smaller)    \
    XBUCKET(Smallest)   \
    XBUCKET(Tiniest)    \
 
#define PLATFORM_MEMORY_SIZE_BUCKET_ENUM(Name) Name,
enum class EPlatformMemorySizeBucket
{
    PLATFORM_MEMORY_SIZE_BUCKET_LIST(PLATFORM_MEMORY_SIZE_BUCKET_ENUM)
};
#undef PLATFORM_MEMORY_SIZE_BUCKET_ENUM
 
inline const TCHAR* LexToString(EPlatformMemorySizeBucket Bucket)
{
#define PLATFORM_MEMORY_SIZE_BUCKET_LEXTOSTRING(Name) case EPlatformMemorySizeBucket::Name: return TEXT(#Name);
    switch (Bucket)
    {
        PLATFORM_MEMORY_SIZE_BUCKET_LIST(PLATFORM_MEMORY_SIZE_BUCKET_LEXTOSTRING)
    }
#undef PLATFORM_MEMORY_SIZE_BUCKET_LEXTOSTRING
 
    return TEXT("Unknown");
}
 
enum class EMemcpyCachePolicy : uint8
{
    // Writes to destination memory are cache-visible (default).
    // This should be used if copy results are immediately accessed by CPU.
    StoreCached,
 
    // Writes to destination memory bypass cache (avoiding pollution).
    // Optimizes for large copies that aren't read from soon after.
    StoreUncached,
};
 
struct FGenericPlatformMemoryConstants
{
    uint64 TotalPhysical = 0;
 
    uint64 TotalVirtual = 0;
 
    SIZE_T PageSize = 0;
 
    SIZE_T OsAllocationGranularity = 0;
 
    SIZE_T BinnedPageSize = 0;
 
    SIZE_T BinnedAllocationGranularity = 0;
 
    uint64 AddressStart = 0;
 
    uint64 AddressLimit = (uint64)0xffffffff + 1;
 
    uint32 TotalPhysicalGB = 1;
};
 
typedef FGenericPlatformMemoryConstants FPlatformMemoryConstants;
 
struct FGenericPlatformMemoryStats : public FPlatformMemoryConstants
{
    uint64 AvailablePhysical;
 
    uint64 AvailableVirtual;
 
    uint64 UsedPhysical;
 
    uint64 PeakUsedPhysical;
 
    uint64 UsedVirtual;
 
    uint64 PeakUsedVirtual;
    
    enum class EMemoryPressureStatus : uint8 
    { 
        Unknown,
        Nominal,
        Warning,
        Critical, // high risk of OOM conditions
    };
    CORE_API EMemoryPressureStatus GetMemoryPressureStatus() const;
 
    CORE_API FGenericPlatformMemoryStats();
 
    struct FPlatformSpecificStat
    {
        const TCHAR* Name;
        uint64 Value;
 
        FPlatformSpecificStat(const TCHAR* InName, uint64 InValue)
            : Name(InName)
            , Value(InValue)
        {}
    };
 
    CORE_API TArray<FPlatformSpecificStat> GetPlatformSpecificStats() const;
 
    CORE_API uint64 GetAvailablePhysical(bool bExcludeExtraDevMemory) const;
 
    void SetEndFrameCsvStats() const {}
};
 
// Contains shared/private information for a single page allocation from the kernel. A page
// allocation may contain many pages.
struct FForkedPageAllocation
{
    // Start/End virtual address for the allocation.
    uint64 PageStart;
    uint64 PageEnd;
 
    // The amount of memory in this allocation range that is shared across the forked
    // child processes.
    uint64 SharedCleanKiB;
    uint64 SharedDirtyKiB;
 
    // The amount of memory in this allocation range that has been written to by the child
    // process, and as a result has been made unique to the process.
    uint64 PrivateCleanKiB;
    uint64 PrivateDirtyKiB;
};
 
 
 
struct FPlatformMemoryStats;
 
#if PLATFORM_USES_MICROSOFT_LIBC_FUNCTIONS
#define __FMemory_Alloca_Func _alloca
#else
#define __FMemory_Alloca_Func alloca
#endif
 
#define FMemory_Alloca(Size) ((Size==0) ? 0 : (void*)(((PTRINT)__FMemory_Alloca_Func(Size + 15) + 15) & ~15))
 
 // Version that supports alignment requirement. However since the old alignment was always forced to be 16, this continues to enforce a min alignment of 16 but allows a larger value.
#define FMemory_Alloca_Aligned(Size, Alignment) ((Size==0) ? 0 : ((Alignment <= 16) ? FMemory_Alloca(Size) : (void*)(((PTRINT)__FMemory_Alloca_Func(Size + Alignment-1) + Alignment-1) & ~(Alignment-1))))
 
struct FGenericPlatformMemory
{
    static CORE_API bool bIsOOM;
 
    static CORE_API uint64 OOMAllocationSize;
 
    static CORE_API uint32 OOMAllocationAlignment;
 
    static CORE_API void* BackupOOMMemoryPool;
 
    static CORE_API uint32 BackupOOMMemoryPoolSize;
 
    enum EMemoryCounterRegion
    {
        MCR_Invalid, // not memory
        MCR_Physical, // main system memory
        MCR_GPU, // memory directly a GPU (graphics card, etc)
        MCR_GPUSystem, // system memory directly accessible by a GPU
        MCR_TexturePool, // presized texture pools
        MCR_StreamingPool, // amount of texture pool available for streaming.
        MCR_UsedStreamingPool, // amount of texture pool used for streaming.
        MCR_GPUDefragPool, // presized pool of memory that can be defragmented.
        MCR_PhysicalLLM, // total physical memory including CPU and GPU
        MCR_MAX
    };
 
    enum EMemoryAllocatorToUse
    {
        Ansi, // Default C allocator
        Stomp, // Allocator to check for memory stomping
        TBB, // Thread Building Blocks malloc
        Jemalloc, // Linux/FreeBSD malloc
        Binned, // Older binned malloc
        Binned2, // Newer binned malloc
        Binned3, // Newer VM-based binned malloc, 64 bit only
        Platform, // Custom platform specific allocator
        Mimalloc, // mimalloc
        Libpas, // libpas
    };
 
    static CORE_API EMemoryAllocatorToUse AllocatorToUse;
 
    enum ESharedMemoryAccess
    {
        Read    =       (1 << 1),
        Write   =       (1 << 2)
    };
 
    struct FSharedMemoryRegion
    {
        const TCHAR *   GetName() const         { return Name; }
 
        void *          GetAddress()            { return Address; }
 
        const void *    GetAddress() const      { return Address; }
 
        SIZE_T          GetSize() const         { return Size; }
    
        
        FSharedMemoryRegion(const FString& InName, uint32 InAccessMode, void* InAddress, SIZE_T InSize);
 
    protected:
 
        enum Limits
        {
            MaxSharedMemoryName     =   128
        };
 
        TCHAR           Name[MaxSharedMemoryName];
 
        uint32          AccessMode;
 
        void *          Address;
 
        SIZE_T          Size;
    };
 
 
 
    static CORE_API void Init();
    
    [[noreturn]] static CORE_API void OnOutOfMemory(uint64 Size, uint32 Alignment);
 
    static CORE_API void SetupMemoryPools();
 
    
    static uint32 GetBackMemoryPoolSize()
    {
        // by default, don't create a backup memory buffer
        return 0;
    }
 
    static CORE_API FMalloc* BaseAllocator();
 
    static CORE_API FPlatformMemoryStats GetStats();
 
    static CORE_API FPlatformMemoryStats GetStatsRaw();
 
    static CORE_API uint64 GetMemoryUsedFast();
 
    static CORE_API void GetStatsForMallocProfiler( FGenericMemoryStats& out_Stats );
 
    static CORE_API const FPlatformMemoryConstants& GetConstants();
    
    static CORE_API uint32 GetPhysicalGBRam();
 
    static CORE_API bool PageProtect(void* const Ptr, const SIZE_T Size, const bool bCanRead, const bool bCanWrite);
 
    static CORE_API void* BinnedAllocFromOS( SIZE_T Size );
    
    static CORE_API void BinnedFreeToOS( void* Ptr, SIZE_T Size );
    
    static void MiMallocInit()
    {
    }
 
    static void NanoMallocInit()
    {
        return;
    }
    
    static bool PtrIsOSMalloc( void* Ptr)
    {
        return false;
    }
    
    static bool IsNanoMallocAvailable()
    {
        return false;
    }
    
    static bool PtrIsFromNanoMalloc( void* Ptr)
    {
        return false;
    }
 
    class FBasicVirtualMemoryBlock
    {
    protected:
        void *Ptr;
        uint32 VMSizeDivVirtualSizeAlignment;
 
    public:
 
        FBasicVirtualMemoryBlock()
            : Ptr(nullptr)
            , VMSizeDivVirtualSizeAlignment(0)
        {
        }
 
        FBasicVirtualMemoryBlock(void *InPtr, uint32 InVMSizeDivVirtualSizeAlignment)
            : Ptr(InPtr)
            , VMSizeDivVirtualSizeAlignment(InVMSizeDivVirtualSizeAlignment)
        {
        }
 
        FBasicVirtualMemoryBlock(const FBasicVirtualMemoryBlock& Other) = default;
        FBasicVirtualMemoryBlock& operator=(const FBasicVirtualMemoryBlock& Other) = default;
 
        UE_FORCEINLINE_HINT uint32 GetActualSizeInPages() const
        {
            return VMSizeDivVirtualSizeAlignment;
        }
 
        UE_FORCEINLINE_HINT void* GetVirtualPointer() const
        {
            return Ptr;
        }
 
        
#if 0 // documentation, must be defined by the platform
        void Commit(size_t InOffset, size_t InSize);
        void Decommit(size_t InOffset, size_t InSize);
        void FreeVirtual();
 
        UE_FORCEINLINE_HINT void CommitByPtr(void *InPtr, size_t InSize)
        {
            Commit(size_t(((uint8*)InPtr) - ((uint8*)Ptr)), InSize);
        }
 
        UE_FORCEINLINE_HINT void DecommitByPtr(void *InPtr, size_t InSize)
        {
            Decommit(size_t(((uint8*)InPtr) - ((uint8*)Ptr)), InSize);
        }
 
        UE_FORCEINLINE_HINT void Commit()
        {
            Commit(0, GetActualSize());
        }
 
        UE_FORCEINLINE_HINT void Decommit()
        {
            Decommit(0, GetActualSize());
        }
 
        UE_FORCEINLINE_HINT size_t GetActualSize() const
        {
            return VMSizeDivVirtualSizeAlignment * GetVirtualSizeAlignment();
        }
 
        static FPlatformVirtualMemoryBlock AllocateVirtual(size_t Size, size_t InAlignment = FPlatformVirtualMemoryBlock::GetVirtualSizeAlignment());
        static size_t GetCommitAlignment();
        static size_t GetVirtualSizeAlignment();
#endif
    };
 
    static bool BinnedPlatformHasMemoryPoolForThisSize(SIZE_T Size)
    {
        return false;
    }
 
    static CORE_API void DumpStats( FOutputDevice& Ar );
 
    static CORE_API void DumpPlatformAndAllocatorStats( FOutputDevice& Ar );
 
    static CORE_API EPlatformMemorySizeBucket GetMemorySizeBucket();
 
    static UE_FORCEINLINE_HINT void* Memmove( void* Dest, const void* Src, SIZE_T Count )
    {
        return memmove( Dest, Src, Count );
    }
 
    static UE_FORCEINLINE_HINT int32 Memcmp( const void* Buf1, const void* Buf2, SIZE_T Count )
    {
        return memcmp( Buf1, Buf2, Count );
    }
 
    static UE_FORCEINLINE_HINT void* Memset(void* Dest, uint8 Char, SIZE_T Count)
    {
        return memset( Dest, Char, Count );
    }
 
    static UE_FORCEINLINE_HINT void* Memzero(void* Dest, SIZE_T Count)
    {
        return memset( Dest, 0, Count );
    }
 
    static UE_FORCEINLINE_HINT void* Memcpy(void* Dest, const void* Src, SIZE_T Count)
    {
        return memcpy( Dest, Src, Count );
    }
 
    static UE_FORCEINLINE_HINT void* BigBlockMemcpy(void* Dest, const void* Src, SIZE_T Count)
    {
        return memcpy(Dest, Src, Count);
    }
 
    static UE_FORCEINLINE_HINT void* StreamingMemcpy(void* Dest, const void* Src, SIZE_T Count)
    {
        return memcpy( Dest, Src, Count );
    }
 
    static inline void* ParallelMemcpy(void* Dest, const void* Src, SIZE_T Count, EMemcpyCachePolicy Policy = EMemcpyCachePolicy::StoreCached)
    {
        (void)Policy;
        return memcpy( Dest, Src, Count );
    }
 
private:
    template <typename T>
    static inline void Valswap(T& A, T& B)
    {
        // Usually such an implementation would use move semantics, but
        // we're only ever going to call it on fundamental types and MoveTemp
        // is not necessarily in scope here anyway, so we don't want to
        // #include it if we don't need to.
        T Tmp = A;
        A = B;
        B = Tmp;
    }
 
    static CORE_API void MemswapGreaterThan8( void* Ptr1, void* Ptr2, SIZE_T Size );
 
public:
    static inline void Memswap( void* Ptr1, void* Ptr2, SIZE_T Size )
    {
        switch (Size)
        {
            case 0:
                break;
 
            case 1:
                Valswap(*(uint8*)Ptr1, *(uint8*)Ptr2);
                break;
 
            case 2:
                Valswap(*(uint16*)Ptr1, *(uint16*)Ptr2);
                break;
 
            case 3:
                Valswap(*((uint16*&)Ptr1)++, *((uint16*&)Ptr2)++);
                Valswap(*(uint8*)Ptr1, *(uint8*)Ptr2);
                break;
 
            case 4:
                Valswap(*(uint32*)Ptr1, *(uint32*)Ptr2);
                break;
 
            case 5:
                Valswap(*((uint32*&)Ptr1)++, *((uint32*&)Ptr2)++);
                Valswap(*(uint8*)Ptr1, *(uint8*)Ptr2);
                break;
 
            case 6:
                Valswap(*((uint32*&)Ptr1)++, *((uint32*&)Ptr2)++);
                Valswap(*(uint16*)Ptr1, *(uint16*)Ptr2);
                break;
 
            case 7:
                Valswap(*((uint32*&)Ptr1)++, *((uint32*&)Ptr2)++);
                Valswap(*((uint16*&)Ptr1)++, *((uint16*&)Ptr2)++);
                Valswap(*(uint8*)Ptr1, *(uint8*)Ptr2);
                break;
 
            case 8:
                Valswap(*(uint64*)Ptr1, *(uint64*)Ptr2);
                break;
 
            case 16:
                Valswap(((uint64*)Ptr1)[0], ((uint64*)Ptr2)[0]);
                Valswap(((uint64*)Ptr1)[1], ((uint64*)Ptr2)[1]);
                break;
 
            default:
                MemswapGreaterThan8(Ptr1, Ptr2, Size);
                break;
        }
    }
 
    template <typename T>
    static inline T ReadUnaligned(const void* Ptr)
    {
        T AlignedT;
        memcpy((void*)&AlignedT, Ptr, sizeof(T));
        return AlignedT;
    }
 
    template <typename T>
    static UE_FORCEINLINE_HINT void WriteUnaligned(void* Ptr, const T& InValue)
    {
        memcpy(Ptr, &InValue, sizeof(T));
    }
 
    static CORE_API FSharedMemoryRegion* MapNamedSharedMemoryRegion(const FString& Name, bool bCreate, uint32 AccessMode, SIZE_T Size);
 
    static CORE_API bool UnmapNamedSharedMemoryRegion(FSharedMemoryRegion * MemoryRegion);
 
    static UE_FORCEINLINE_HINT bool SupportsFastVRAMMemory()
    {
        return false;
    }
 
    static CORE_API bool IsExtraDevelopmentMemoryAvailable();
 
    static CORE_API uint64 GetExtraDevelopmentMemorySize();
 
    static uint64 GetProgramSize()
    {
        return ProgramSize;
    }
 
    static void SetProgramSize(uint64 InProgramSize)
    {
        ProgramSize = InProgramSize;
    }
 
    static CORE_API bool GetLLMAllocFunctions(void*(*&OutAllocFunction)(size_t), void(*&OutFreeFunction)(void*, size_t), int32& OutAlignment);
 
    UE_FORCEINLINE_HINT static void OnLowLevelMemory_Alloc(void const* Pointer, uint64 Size, uint64 Tag) { }
    UE_FORCEINLINE_HINT static void OnLowLevelMemory_Free(void const* Pointer, uint64 Size, uint64 Tag) { }
 
    static void RegisterCustomLLMTags() { };
 
    static void UpdateCustomLLMTags() { };
 
    static bool TracksLLMAllocations() { return false; }
 
    static bool HasForkPageProtectorEnabled() { return false; }
 
    static bool GetForkedPageAllocationInfo(TArray<FForkedPageAllocation>& OutPageAllocationInfos)
    {
        return false; // Most platform do not implement this.
    }
 
    static CORE_API FString PrettyMemory( uint64 Memory );
 
    UE_FORCEINLINE_HINT static bool CanOverallocateVirtualMemory()
    {
        return true;
    }
 
    // This bit is always zero in user mode addresses and most likely won't be used by current or future
    // CPU features like ARM's PAC / Top-Byte Ignore or Intel's Linear Address Masking / 5-Level Paging
#if defined(__x86_64__) || defined(_M_X64)
    static constexpr uint32 KernelAddressBit = 63;
#elif defined(__aarch64__) || defined(_M_ARM64)
    static constexpr uint32 KernelAddressBit = 55;
#endif
 
protected:
    friend struct FGenericStatsUpdater;
 
    static CORE_API void InternalUpdateStats( const FPlatformMemoryStats& MemoryStats );
 
    static CORE_API uint64 ProgramSize;
};