PipelineFileCache.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "CoreMinimal.h"
#include "Containers/List.h"
#include "Containers/MpscQueue.h"
#include "Containers/StringView.h"
#include "RHI.h"
#include "RHIShaderBindingLayout.h"
 
DECLARE_STATS_GROUP(TEXT("ShaderPipelineCache"),STATGROUP_PipelineStateCache, STATCAT_Advanced);
 
DECLARE_DWORD_ACCUMULATOR_STAT_EXTERN(TEXT("Total Graphics Pipeline State Count"), STAT_TotalGraphicsPipelineStateCount, STATGROUP_PipelineStateCache, RHI_API);
DECLARE_DWORD_ACCUMULATOR_STAT_EXTERN(TEXT("Total Compute Pipeline State Count"), STAT_TotalComputePipelineStateCount, STATGROUP_PipelineStateCache, RHI_API);
DECLARE_DWORD_ACCUMULATOR_STAT_EXTERN(TEXT("Total RayTracing Pipeline State Count"), STAT_TotalRayTracingPipelineStateCount, STATGROUP_PipelineStateCache, RHI_API);
 
#define PIPELINE_CACHE_DEFAULT_ENABLED (!WITH_EDITOR)
 
#define PSO_COOKONLY_DATA (WITH_EDITOR || IS_PROGRAM)
 
struct FPipelineFileCacheRasterizerState
{
    FPipelineFileCacheRasterizerState() { FMemory::Memzero(*this); }
    FPipelineFileCacheRasterizerState(FRasterizerStateInitializerRHI const& Other) { operator=(Other); }
    
    float DepthBias;
    float SlopeScaleDepthBias;
    TEnumAsByte<ERasterizerFillMode> FillMode;
    TEnumAsByte<ERasterizerCullMode> CullMode;
    ERasterizerDepthClipMode DepthClipMode;
    bool bAllowMSAA;
    
    FPipelineFileCacheRasterizerState& operator=(FRasterizerStateInitializerRHI const& Other)
    {
        DepthBias = Other.DepthBias;
        SlopeScaleDepthBias = Other.SlopeScaleDepthBias;
        FillMode = Other.FillMode;
        CullMode = Other.CullMode;
        DepthClipMode = Other.DepthClipMode;
        bAllowMSAA = Other.bAllowMSAA;
        return *this;
    }
    
    operator FRasterizerStateInitializerRHI() const
    {
        FRasterizerStateInitializerRHI Initializer(FillMode, CullMode, DepthBias, SlopeScaleDepthBias, DepthClipMode, bAllowMSAA);
        return Initializer;
    }
    
    friend RHI_API FArchive& operator<<(FArchive& Ar, FPipelineFileCacheRasterizerState& RasterizerStateInitializer);
 
    friend uint32 GetTypeHash(const FPipelineFileCacheRasterizerState &Key)
    {
        uint32 KeyHash = (*((uint32*)&Key.DepthBias) ^ *((uint32*)&Key.SlopeScaleDepthBias));
        KeyHash ^= (Key.FillMode << 8);
        KeyHash ^= Key.CullMode;
        KeyHash ^= Key.DepthClipMode == ERasterizerDepthClipMode::DepthClamp ? 0x951f4c3b : 0; // crc32 "DepthClamp"
        KeyHash ^= Key.bAllowMSAA ? 0x694ea601 : 0; // crc32 "bAllowMSAA"
        return KeyHash;
    }
    RHI_API FString ToString() const;
    RHI_API void FromString(const FStringView& Src);
};
 
class FRayTracingPipelineStateInitializer;
class FRHIRayTracingShader;
enum class ERayTracingPipelineCacheFlags : uint8;
 
struct FPipelineStateStats
{
    FPipelineStateStats()
    : FirstFrameUsed(-1)
    , LastFrameUsed(-1)
    , CreateCount(0)
    , TotalBindCount(0)
    , PSOHash(0)
    {
    }
    
    ~FPipelineStateStats()
    {
    }
 
    RHI_API static void UpdateStats(FPipelineStateStats* Stats);
    
    friend FArchive& operator<<( FArchive& Ar, FPipelineStateStats& Info );
 
    int64 FirstFrameUsed;
    int64 LastFrameUsed;
    uint64 CreateCount;
    int64 TotalBindCount;
    uint32 PSOHash;
};
 
struct FPipelineCacheFileFormatPSO
{
    using TReadableStringBuilder = TStringBuilder<1024>;
 
    struct ComputeDescriptor
    {
        FSHAHash ComputeShader;
 
        RHI_API FString ToString() const;
        void AddToReadableString(TReadableStringBuilder& OutBuilder) const;
        static FString HeaderLine();
        RHI_API void FromString(const FStringView& Src);
    };
    struct GraphicsDescriptor
    {
        FSHAHash VertexShader;
        FSHAHash FragmentShader;
        FSHAHash GeometryShader;
        FSHAHash MeshShader;
        FSHAHash AmplificationShader;
 
        FVertexDeclarationElementList VertexDescriptor;
        FBlendStateInitializerRHI BlendState;
        FPipelineFileCacheRasterizerState RasterizerState;
        FDepthStencilStateInitializerRHI DepthStencilState;
        
        EPixelFormat RenderTargetFormats[MaxSimultaneousRenderTargets];
        ETextureCreateFlags RenderTargetFlags[MaxSimultaneousRenderTargets];
        uint32 RenderTargetsActive;
        uint32 MSAASamples;
        
        EPixelFormat DepthStencilFormat;
        ETextureCreateFlags DepthStencilFlags;
        ERenderTargetLoadAction DepthLoad;
        ERenderTargetLoadAction StencilLoad;
        ERenderTargetStoreAction DepthStore;
        ERenderTargetStoreAction StencilStore;
        
        EPrimitiveType PrimitiveType;
        
        uint8 SubpassHint;  
        uint8 SubpassIndex;
 
        uint8   MultiViewCount;
        bool    bHasFragmentDensityAttachment;
 
        bool    bAllowVariableRateShading;
        EVRSShadingRate ShadingRate;
 
        bool    bDepthBounds;
        
        RHI_API FString ToString() const;
        RHI_API void AddToReadableString(TReadableStringBuilder& OutBuilder) const;
        RHI_API static FString HeaderLine();
        RHI_API bool FromString(const FStringView& Src);
 
        RHI_API FString ShadersToString() const;
        RHI_API void AddShadersToReadableString(TReadableStringBuilder& OutBuilder) const;
 
        RHI_API static FString ShaderHeaderLine();
        RHI_API void ShadersFromString(const FStringView& Src);
 
        RHI_API FString StateToString() const;
        RHI_API void AddStateToReadableString(TReadableStringBuilder& OutBuilder) const;
        RHI_API static FString StateHeaderLine();
        RHI_API bool StateFromString(const FStringView& Src);
 
        static ETextureCreateFlags ReduceRTFlags(ETextureCreateFlags InFlags);
 
        static ETextureCreateFlags ReduceDSFlags(ETextureCreateFlags InFlags);
    };
    struct FPipelineFileCacheRayTracingDesc
    {
        FSHAHash ShaderHash;
        uint32 DeprecatedMaxPayloadSizeInBytes = 0;
        EShaderFrequency Frequency = SF_RayGen;
        FRHIShaderBindingLayout ShaderBindingLayout;
        
        FPipelineFileCacheRayTracingDesc() = default;
        FPipelineFileCacheRayTracingDesc(const FRayTracingPipelineStateInitializer& Initializer, const FRHIRayTracingShader* ShaderRHI);
 
        RHI_API FString ToString() const;
        void AddToReadableString(TReadableStringBuilder& OutBuilder) const;
        FString HeaderLine() const;
        RHI_API void FromString(const FString& Src);
 
        friend uint32 GetTypeHash(const FPipelineFileCacheRayTracingDesc& Desc)
        {
            return HashCombineFast(GetTypeHash(Desc.ShaderHash), HashCombineFast(GetTypeHash(Desc.Frequency), GetTypeHash(Desc.ShaderBindingLayout)));
        }
 
        bool operator == (const FPipelineFileCacheRayTracingDesc& Other) const
        {
            return ShaderHash == Other.ShaderHash &&
                Frequency == Other.Frequency &&
                ShaderBindingLayout == Other.ShaderBindingLayout;
        }
    };
    enum class DescriptorType : uint32
    {
        Compute = 0,
        Graphics = 1,
        RayTracing = 2,
    };
    
    DescriptorType Type;
    ComputeDescriptor ComputeDesc;
    GraphicsDescriptor GraphicsDesc;
    FPipelineFileCacheRayTracingDesc RayTracingDesc;
 
#if PSO_COOKONLY_DATA
    uint64 UsageMask;
    int64 BindCount;
#endif
    
    RHI_API FPipelineCacheFileFormatPSO();
    RHI_API ~FPipelineCacheFileFormatPSO();
    
    RHI_API FPipelineCacheFileFormatPSO& operator=(const FPipelineCacheFileFormatPSO& Other);
    RHI_API FPipelineCacheFileFormatPSO(const FPipelineCacheFileFormatPSO& Other);
    
    RHI_API bool operator==(const FPipelineCacheFileFormatPSO& Other) const;
 
    friend RHI_API uint32 GetTypeHash(const FPipelineCacheFileFormatPSO &Key);
    friend RHI_API FArchive& operator<<( FArchive& Ar, FPipelineCacheFileFormatPSO& Info );
    
    static bool Init(FPipelineCacheFileFormatPSO& PSO, FRHIComputeShader const* Init);
    static bool Init(FPipelineCacheFileFormatPSO& PSO, FGraphicsPipelineStateInitializer const& Init);
    static bool Init(FPipelineCacheFileFormatPSO & PSO, FPipelineFileCacheRayTracingDesc const& Desc);
 
    RHI_API static void SortVertexElements(FVertexDeclarationElementList& VertexElements);
 
    RHI_API FString CommonToString() const;
    RHI_API static FString CommonHeaderLine();
    RHI_API void CommonFromString(const FStringView& Src);
 
    RHI_API FString ToStringReadable() const;
    
    // Potential cases for seperating verify logic if requiired: RunTime-Logging, RunTime-UserCaching, RunTime-PreCompile, CommandLet-Cooking
    RHI_API bool Verify() const;
};
 
struct FPipelineCacheFileFormatPSORead
{   
    FPipelineCacheFileFormatPSORead()
    : Ar(nullptr)
    , Hash(0)
    , bReadCompleted(false)
    , bValid(false)
    {}
    
    ~FPipelineCacheFileFormatPSORead()
    {
        if(Ar != nullptr)
        {
            delete Ar;
            Ar = nullptr;
        }
    }
    
    TArray<uint8> Data;
    FArchive* Ar;
    
    uint32 Hash;
    bool bReadCompleted;
    bool bValid;
    
    // Note that the contract of IAsyncReadFileHandle and IAsyncReadRequest requires that we delete the ReadRequest before deleting its ParentFileHandle. 
    // We therefore require that ParentFileHandle is declared before ReadRequest, so that the class destructor tears down first ReadRequest then ParentFileHandle.
    TSharedPtr<class IAsyncReadFileHandle, ESPMode::ThreadSafe> ParentFileHandle;
    TSharedPtr<class IAsyncReadRequest, ESPMode::ThreadSafe> ReadRequest;
};
 
struct FPipelineCachePSOHeader
{
    TSet<FSHAHash> Shaders;
    uint32 Hash;
};
 
extern RHI_API const uint32 FPipelineCacheFileFormatCurrentVersion;
 
/*
 * User definable Mask Comparsion function:
 * @param ReferenceMask is the Current Bitmask set via SetGameUsageMask
 * @param PSOMask is the PSO UsageMask
 * @return Function should return true if this PSO is to be precompiled or false otherwise
 */
typedef bool(*FPSOMaskComparisonFn)(uint64 ReferenceMask, uint64 PSOMask);
 
struct FPSOUsageData
{
    FPSOUsageData(): UsageMask(0), PSOHash(0), EngineFlags(0) {}
    FPSOUsageData(uint32 InPSOHash, uint64 InUsageMask, uint16 InEngineFlags): UsageMask(InUsageMask), PSOHash(InPSOHash), EngineFlags(InEngineFlags) {}
    uint64 UsageMask;
    uint32 PSOHash;
    uint16 EngineFlags;
};
 
class FPipelineFileCacheManager
{
    friend class FPipelineCacheFile;
public:
    enum class SaveMode : uint32
    {
        Incremental = 0, // Fast(er) approach which saves new entries incrementally at the end of the file, replacing the table-of-contents, but leaves everything else alone.
        BoundPSOsOnly = 1, // Slower approach which consolidates and saves all PSOs used in this run of the program, removing any entry that wasn't seen, and sorted by the desired sort-mode.
    };
    
    enum class PSOOrder : uint32
    {
        Default = 0, // Whatever order they are already in.
        FirstToLatestUsed = 1, // Start with the PSOs with the lowest first-frame used and work toward those with the highest.
        MostToLeastUsed = 2 // Start with the most often used PSOs working toward the least.
    };
 
public:
    
    RHI_API static void Initialize(uint32 GameVersion);
    RHI_API static void Shutdown();
    
    RHI_API static bool LoadPipelineFileCacheInto(FString const& Path, TSet<FPipelineCacheFileFormatPSO>& PSOs);
    RHI_API static bool SavePipelineFileCacheFrom(uint32 GameVersion, EShaderPlatform Platform, FString const& Path, const TSet<FPipelineCacheFileFormatPSO>& PSOs);
    RHI_API static bool MergePipelineFileCaches(FString const& PathA, FString const& PathB, FPipelineFileCacheManager::PSOOrder Order, FString const& OutputPath);
                                                                                
    /* Open the pipeline file cache for the specfied name and platform. If successful, the GUID of the game file will be returned in OutGameFileGuid */
    RHI_API static bool OpenPipelineFileCache(const FString& Key, const FString& CacheName, EShaderPlatform Platform, FGuid& OutGameFileGuid);
 
    /* Open the user pipeline file cache for the specified name and platform. The user cache is always created even if the file was not present when opened.
    * Name is the name used when opening the file, the key value for the user cache is held within UserCacheName.
    * returns true if the file was opened.
    */
    RHI_API static bool OpenUserPipelineFileCache(const FString& Key, const FString& CacheName, EShaderPlatform Platform, FGuid& OutGameFileGuid);
 
    RHI_API static bool SavePipelineFileCache(SaveMode Mode);
 
    RHI_API static void CloseUserPipelineFileCache();
 
    RHI_API static void CacheGraphicsPSO(uint32 RunTimeHash, FGraphicsPipelineStateInitializer const& Initializer, bool bWasPSOPrecached, FPipelineStateStats** OutStats = nullptr);
    RHI_API static void CacheComputePSO(uint32 RunTimeHash, FRHIComputeShader const* Initializer, bool bWasPSOPrecached, FPipelineStateStats** OutStats = nullptr);
    RHI_API static void CacheRayTracingPSO(const FRayTracingPipelineStateInitializer& Initializer, ERayTracingPipelineCacheFlags Flags);
 
    // true if the named PSOFC is currently open.
    RHI_API static bool HasPipelineFileCache(const FString& PSOCacheKey);
 
    UE_DEPRECATED(5.6, "Do not call this function, please instead use the OutStats parameter in CacheGraphicsPSO/CacheComputePSO")
    RHI_API static FPipelineStateStats* RegisterPSOStats(uint32 RunTimeHash);
    
    /*
     * This PSO has failed compile and is invalid - this cache should not return this invalid PSO from subsequent calls for PreCompile requests.
     * Note: Not implementated for Compute that has no flag to say it came from this cache - don't want to consume failures that didn't propagate from this cache.
     */
    RHI_API static void RegisterPSOCompileFailure(uint32 RunTimeHash, FGraphicsPipelineStateInitializer const& Initializer);
    
    DECLARE_MULTICAST_DELEGATE_OneParam(FPipelineStateLoggedEvent, const FPipelineCacheFileFormatPSO&);
 
    RHI_API static FPipelineStateLoggedEvent& OnPipelineStateLogged();
 
    /*
     * If the delegate is set, broadcasts any new PSOs that were encountered since the last time the delegate was broadcast.
     * The broadcast is scheduled to be run on the game thread.
     */
    RHI_API static void BroadcastNewPSOsDelegate();
    
    RHI_API static void GetOrderedPSOHashes(const FString& PSOCacheKey, TArray<FPipelineCachePSOHeader>& PSOHashes, PSOOrder Order, int64 MinBindCount, TSet<uint32> const& AlreadyCompiledHashes);
    RHI_API static void FetchPSODescriptors(const FString& PSOCacheKey, TDoubleLinkedList<FPipelineCacheFileFormatPSORead*>& LoadedBatch);
 
    RHI_API static int32 GetTotalPSOCount(const FString& PSOCacheKey);
 
    RHI_API static uint32 NumPSOsLogged();
    
    RHI_API static bool IsPipelineFileCacheEnabled();
    RHI_API static bool LogPSOtoFileCache();
    RHI_API static bool ReportNewPSOs();
    /* Report additional data about new PSOs to the log. */
    RHI_API static bool LogPSODetails();
    
    RHI_API static uint64 SetGameUsageMaskWithComparison(uint64 GameUsageMask, FPSOMaskComparisonFn InComparisonFnPtr);
    static uint64 GetGameUsageMask() { return GameUsageMask; }
    static bool IsGameUsageMaskSet() { return GameUsageMaskSet; }
    
    RHI_API static void PreCompileComplete();
 
    /*
     * Enable or disable the logging of new PSOs (PSOs that were needed but not found in the file cache) to console and CSV.
     */
    static void SetNewPSOConsoleAndCSVLogging(bool bEnabled) { LogNewPSOsToConsoleAndCSV = bEnabled; }
private:
    
    static void RegisterPSOUsageDataUpdateForNextSave(FPSOUsageData& UsageData);
    static void ClearOSPipelineCache();
    static bool ShouldEnableFileCache();
    
    static bool IsBSSEquivalentPSOEntryCached(FPipelineCacheFileFormatPSO const& NewEntry);
    static bool IsPSOEntryCached(FPipelineCacheFileFormatPSO const& NewEntry, FPSOUsageData* EntryData = nullptr);
 
    static void LogNewGraphicsPSOToConsoleAndCSV(FPipelineCacheFileFormatPSO& PSO, uint32 PSOHash, bool bWasPSOPrecached);
    static void LogNewComputePSOToConsoleAndCSV(FPipelineCacheFileFormatPSO& PSO, uint32 PSOHash, bool bWasPSOPrecached);
    static void LogNewRaytracingPSOToConsole(FPipelineCacheFileFormatPSO& PSO, uint32 PSOHash, bool bIsNonBlockingPSO);
 
    static FPipelineStateStats* RegisterPSOStatsInternal(class FRWScopeLock& Lock, uint32 RunTimeHash);
    
private:
    static FRWLock FileCacheLock;
 
    // Containers for the multiple bundled PSOFCs
    // Name to PipelineCacheFile
    static TMap<FString, TUniquePtr<class FPipelineCacheFile>> FileCacheMap;
    // PipelineCacheFile GUID to Name
    static TMap<FGuid, FString> GameGuidToCacheKey;
    // User cache's key within FileCacheMap
    static FString UserCacheKey;
    // Helper for retrieving a file cache from the name.
    static class FPipelineCacheFile* GetPipelineCacheFileFromKey(const FString& PSOCacheKey)
    {
        TUniquePtr<FPipelineCacheFile>* FileCacheFound = FileCacheMap.Find(PSOCacheKey);
        return FileCacheFound ? FileCacheFound->Get() : nullptr;
    }
 
    // PSO recording
    static TMap<uint32, FPSOUsageData> RunTimeToPSOUsage;       // Fast check structure - Not saved (External state cache runtime hash to seen usage data)
    static TMap<uint32, FPSOUsageData> NewPSOUsage;             // For mask or engine updates - Merged + Saved (Our internal PSO hash to latest usage data) - temp working scratch, only holds updates since last "save" so is not the authority on state
    static TMap<uint32, FPipelineStateStats*> Stats;
    static TSet<FPipelineCacheFileFormatPSO> NewPSOs;
    static TMpscQueue<FPipelineCacheFileFormatPSO> NewPSOsToReport; // New PSOs that will be broadcast via a delegate (if bound). Cleared when the delegate is broadcast.
    static TSet<uint32> NewPSOHashes;
    static uint32 NumNewPSOs;
    static PSOOrder RequestedOrder;
    static bool FileCacheEnabled;
    static FPipelineStateLoggedEvent PSOLoggedEvent;
    RHI_API static uint64 GameUsageMask;
    RHI_API static bool GameUsageMaskSet;
    static FPSOMaskComparisonFn MaskComparisonFn;   
 
    RHI_API static bool LogNewPSOsToConsoleAndCSV; // Whether to log new PSOs to the log file and CSV.
};