ShaderCodeLibrary.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
/*=============================================================================
    ShaderCodeLibrary.h: 
=============================================================================*/
 
#pragma once
 
#include "Containers/Array.h"
#include "Containers/ArrayView.h"
#include "Containers/ContainerAllocationPolicies.h"
#include "Containers/Map.h"
#include "Containers/Set.h"
#include "Containers/SparseArray.h"
#include "Containers/StringFwd.h"
#include "Containers/UnrealString.h"
#include "Delegates/Delegate.h"
#include "HAL/Platform.h"
#include "HAL/PlatformCrt.h"
#include "Logging/LogMacros.h"
#include "Misc/Crc.h"
#include "Misc/SecureHash.h"
#include "RHI.h"
#include "RHIDefinitions.h"
#include "Serialization/Archive.h"
#include "Templates/RefCounting.h"
#include "UObject/NameTypes.h"
 
DECLARE_LOG_CATEGORY_EXTERN(LogShaderLibrary, Log, All);
 
class FShaderMapResource;
class FShaderMapResourceCode;
class FShaderPipeline;
 
using FShaderMapAssetPaths = TSet<FName>;
class FIoBuffer;
class FIoChunkId;
class IPlugin;
class UObject;
 
#if WITH_EDITOR
class FCbFieldView;
class FCbWriter;
class ICookArtifactReader;
#endif
 
struct FShaderCodeLibraryPipeline
{
    FSHAHash Shaders[SF_NumGraphicsFrequencies];
    mutable uint32 Hash;
    
    RENDERCORE_API void Initialize(const FShaderPipeline* Pipeline);
 
    FShaderCodeLibraryPipeline() : Hash(0) {}
    
    friend bool operator ==(const FShaderCodeLibraryPipeline& A,const FShaderCodeLibraryPipeline& B)
    {
        for (uint32 Frequency = 0u; Frequency < SF_NumGraphicsFrequencies; ++Frequency)
        {
            if (A.Shaders[Frequency] != B.Shaders[Frequency])
            {
                return false;
            }
        }
        return true;
    }
    
    friend uint32 GetTypeHash(const FShaderCodeLibraryPipeline &Key)
    {
        if(!Key.Hash)
        {
            for (uint32 Frequency = 0u; Frequency < SF_NumGraphicsFrequencies; ++Frequency)
            {
                Key.Hash = FCrc::MemCrc32(Key.Shaders[Frequency].Hash, sizeof(FSHAHash), Key.Hash);
            }
        }
        return Key.Hash;
    }
 
    RENDERCORE_API void GetPipelineHash(FSHAHash& Output);
    
    friend FArchive& operator<<( FArchive& Ar, FShaderCodeLibraryPipeline& Info )
    {
        for (uint32 Frequency = 0u; Frequency < SF_NumGraphicsFrequencies; ++Frequency)
        {
            Ar << Info.Shaders[Frequency];
        }
        return Ar << Info.Hash;
    }
};
 
struct FCompactFullName
{
    TArray<FName, TInlineAllocator<16>> ObjectClassAndPath;
 
    bool operator==(const FCompactFullName& Other) const
    {
        return ObjectClassAndPath == Other.ObjectClassAndPath;
    }
 
    RENDERCORE_API FString ToString() const;
    RENDERCORE_API FString ToStringPathOnly() const;
    RENDERCORE_API void AppendString(FStringBuilderBase& Out) const;
    RENDERCORE_API void AppendString(FAnsiStringBuilderBase& Out) const;
    RENDERCORE_API void ParseFromString(const FStringView& Src);
    friend RENDERCORE_API uint32 GetTypeHash(const FCompactFullName& A);
 
#if WITH_EDITOR
    RENDERCORE_API void SetCompactFullNameFromObject(UObject* InDepObject);
#endif
};
 
 
struct FStableShaderKeyAndValue
{
    FCompactFullName ClassNameAndObjectPath;
    FName ShaderType;
    FName ShaderClass;
    FName MaterialDomain;
    FName FeatureLevel;
    FName QualityLevel;
    FName TargetFrequency;
    FName TargetPlatform;
    FName VFType;
    FName PermutationId;
    FSHAHash PipelineHash;
 
    uint32 KeyHash;
 
    FSHAHash OutputHash;
 
    FStableShaderKeyAndValue()
        : KeyHash(0)
    {
    }
 
    RENDERCORE_API void ComputeKeyHash();
    RENDERCORE_API void ParseFromString(const FStringView& Src);
    RENDERCORE_API void ParseFromStringCached(const FStringView& Src, class TMap<uint32, FName>& NameCache);
    RENDERCORE_API FString ToString() const;
    RENDERCORE_API void ToString(FString& OutResult) const;
    RENDERCORE_API void AppendString(FAnsiStringBuilderBase& Out) const;
    static RENDERCORE_API FString HeaderLine();
 
    RENDERCORE_API void SetPipelineHash(const FShaderPipeline* Pipeline);
 
    friend bool operator ==(const FStableShaderKeyAndValue& A, const FStableShaderKeyAndValue& B)
    {
        return
            A.ClassNameAndObjectPath == B.ClassNameAndObjectPath &&
            A.ShaderType == B.ShaderType &&
            A.ShaderClass == B.ShaderClass &&
            A.MaterialDomain == B.MaterialDomain &&
            A.FeatureLevel == B.FeatureLevel &&
            A.QualityLevel == B.QualityLevel &&
            A.TargetFrequency == B.TargetFrequency &&
            A.TargetPlatform == B.TargetPlatform &&
            A.VFType == B.VFType &&
            A.PermutationId == B.PermutationId &&
            A.PipelineHash == B.PipelineHash;
    }
 
    friend uint32 GetTypeHash(const FStableShaderKeyAndValue &Key)
    {
        return Key.KeyHash;
    }
};
 
struct FStableShaderKeyFuncs : BaseKeyFuncs<TUniquePtr<FStableShaderKeyAndValue>, FStableShaderKeyAndValue>
{
    [[nodiscard]] static FORCEINLINE const FStableShaderKeyAndValue& GetSetKey(const TUniquePtr<FStableShaderKeyAndValue>& Element)
    {
        return *Element;
    }
 
    [[nodiscard]] static FORCEINLINE bool Matches(const FStableShaderKeyAndValue& A, const FStableShaderKeyAndValue& B)
    {
        return A == B;
    }
 
    [[nodiscard]] static FORCEINLINE uint32 GetKeyHash(const FStableShaderKeyAndValue& Key)
    {
        return GetTypeHash(Key);
    }
};
 
using FStableShaderSet = TSet<TUniquePtr<FStableShaderKeyAndValue>, FStableShaderKeyFuncs>;
 
#if WITH_EDITOR
RENDERCORE_API void WriteToCompactBinary(FCbWriter& Writer, const FStableShaderKeyAndValue& Key, 
    const TMap<FSHAHash, int32>& HashToIndex);
RENDERCORE_API bool LoadFromCompactBinary(FCbFieldView Field, FStableShaderKeyAndValue& Key,
    const TArray<FSHAHash>& IndexToHash);
#endif
 
 
DECLARE_DELEGATE_OneParam(FSharedShaderMapResourceExplicitRelease, const FShaderMapResource*);
RENDERCORE_API extern FSharedShaderMapResourceExplicitRelease OnSharedShaderMapResourceExplicitRelease;
 
using FSharedShaderCodeRequest = TTSMulticastDelegate<void(const FSHAHash&, FArchive*)>; // thread-safe because it's "broadcasted" concurrently
using FSharedShaderCodeRelease = TMulticastDelegate<void(const FSHAHash&)>;
 
// Collection of unique shader code
// Populated at cook time
struct FShaderCodeLibrary
{
    static RENDERCORE_API void PreInit();
 
    static RENDERCORE_API void InitForRuntime(EShaderPlatform ShaderPlatform);
    static RENDERCORE_API void Shutdown();
 
    static RENDERCORE_API bool IsEnabled();
 
    static RENDERCORE_API bool AreShaderMapsPreloadedAtLoadTime();
 
    static RENDERCORE_API void AddKnownChunkIDs(const int32* IDs, const int32 NumChunkIDs);
 
    static RENDERCORE_API bool OpenLibrary(FString const& Name, FString const& Directory, bool bMonolithicOnly = false);
 
    static RENDERCORE_API void CloseLibrary(FString const& Name);
 
    static RENDERCORE_API bool ContainsShaderCode(const FSHAHash& Hash);
    static RENDERCORE_API bool ContainsShaderCode(const FSHAHash& Hash, const FString& LogicalLibraryName);
 
    static RENDERCORE_API TRefCountPtr<FShaderMapResource> LoadResource(const FSHAHash& Hash, FArchive* Ar);
 
    static RENDERCORE_API bool PreloadShader(const FSHAHash& Hash, FArchive* Ar);
    static RENDERCORE_API bool ReleasePreloadedShader(const FSHAHash& Hash);
 
    static RENDERCORE_API FVertexShaderRHIRef CreateVertexShader(EShaderPlatform Platform, const FSHAHash& Hash);
    static RENDERCORE_API FPixelShaderRHIRef CreatePixelShader(EShaderPlatform Platform, const FSHAHash& Hash);
    static RENDERCORE_API FGeometryShaderRHIRef CreateGeometryShader(EShaderPlatform Platform, const FSHAHash& Hash);
    static RENDERCORE_API FComputeShaderRHIRef CreateComputeShader(EShaderPlatform Platform, const FSHAHash& Hash);
    static RENDERCORE_API FMeshShaderRHIRef CreateMeshShader(EShaderPlatform Platform, const FSHAHash& Hash);
    static RENDERCORE_API FAmplificationShaderRHIRef CreateAmplificationShader(EShaderPlatform Platform, const FSHAHash& Hash);
    static RENDERCORE_API FRayTracingShaderRHIRef CreateRayTracingShader(EShaderPlatform Platform, const FSHAHash& Hash, EShaderFrequency Frequency);
 
    // Total number of shader entries in the library
    static RENDERCORE_API uint32 GetShaderCount(void);
    
    // The shader platform that the library manages - at runtime this will only be one
    static RENDERCORE_API EShaderPlatform GetRuntimeShaderPlatform(void);
 
    // Safely assign the hash to a shader object
    static RENDERCORE_API void SafeAssignHash(FRHIShader* InShader, const FSHAHash& Hash);
 
    // Delegate called whenever shader code is requested.
    static RENDERCORE_API FDelegateHandle RegisterSharedShaderCodeRequestDelegate_Handle(const FSharedShaderCodeRequest::FDelegate& Delegate);
    static RENDERCORE_API void UnregisterSharedShaderCodeRequestDelegate_Handle(FDelegateHandle Handle);
 
    // Disables opening the specified plugin's shader library on mount
    static RENDERCORE_API void DontOpenPluginShaderLibraryOnMount(const FString& PluginName);
    
    // Open the plugin's shader library
    // @param bMonolithicOnly If true, only attempt to open a monolithic library (no chunks) - which is a default behavior for DLC plugins, see FShaderLibraryChunkDataGenerator.
    static RENDERCORE_API void OpenPluginShaderLibrary(IPlugin& Plugin, bool bMonolithicOnly = true);
};
 
#if WITH_EDITOR
class ITargetPlatform;
 
struct FShaderLibraryCooker
{
    // Initialize the library cooker
    static RENDERCORE_API void InitForCooking(bool bNativeFormat, ICookArtifactReader* CookArtifactReader);
    // Shutdown the library cooker
    static RENDERCORE_API void Shutdown();
 
    // Clean the cook directories
    static RENDERCORE_API void CleanDirectories(TArray<FName> const& ShaderFormats);
 
    struct FShaderFormatDescriptor
    {
        FName ShaderFormat;
        bool bNeedsStableKeys;
        bool bNeedsDeterministicOrder;
    };
 
    static RENDERCORE_API bool BeginCookingLibrary(FString const& Name);
 
    static RENDERCORE_API void EndCookingLibrary(FString const& Name);
 
    static RENDERCORE_API bool IsShaderLibraryEnabled();
 
    // Specify the shader formats to cook and which ones needs stable keys. Provide an array of FShaderFormatDescriptors
    static RENDERCORE_API void CookShaderFormats(TArray<FShaderFormatDescriptor> const& ShaderFormats);
 
    // At cook time, add shader code to collection
    static RENDERCORE_API bool AddShaderCode(EShaderPlatform ShaderPlatform, const FShaderMapResourceCode* Code, const FShaderMapAssetPaths& AssociatedAssets);
 
#if WITH_EDITOR
    static RENDERCORE_API void CopyToCompactBinaryAndClear(FCbWriter& Writer, bool& bOutHasData, bool& bOutRanOutOfRoom, int64 MaxShaderSize);
 
    static RENDERCORE_API bool AppendFromCompactBinary(FCbFieldView Field);
#endif
 
    // We check this early in the callstack to avoid creating a bunch of FName and keys and things we will never save anyway. 
    // Pass the shader platform to check or EShaderPlatform::SP_NumPlatforms to check if any of the registered types require
    // stable keys.
    static RENDERCORE_API bool NeedsShaderStableKeys(EShaderPlatform ShaderPlatform);
 
    // At cook time, add the human readable key value information
    static RENDERCORE_API void AddShaderStableKeyValue(EShaderPlatform ShaderPlatform, FStableShaderKeyAndValue& StableKeyValue);
 
    static RENDERCORE_API void FinishPopulateShaderLibrary(const ITargetPlatform* TargetPlatform, FString const& Name, FString const& SandboxDestinationPath,
        FString const& SandboxMetadataPath);
 
    static RENDERCORE_API bool MergeShaderCodeArchive(const TArray<FString>& CookedMetadataDirs, const FString& OutputDir, TArray<FString>& OutWrittenFiles);
 
    static RENDERCORE_API bool SaveShaderLibraryWithoutChunking(const ITargetPlatform* TargetPlatform, FString const& Name, FString const& SandboxDestinationPath,
        FString const& SandboxMetadataPath, TArray<FString>& PlatformSCLCSVPaths, FString& OutErrorMessage, bool& bOutHasData);
 
    static RENDERCORE_API bool SaveShaderLibraryChunk(int32 ChunkId, const TSet<FName>& InPackagesInChunk, const ITargetPlatform* TargetPlatform,
        const FString& SandboxDestinationPath, const FString& SandboxMetadataPath, TArray<FString>& OutChunkFilenames, bool& bOutHasData);
 
    // Dump collected stats for each shader platform
    UE_DEPRECATED(5.6, "DumpShaderCodeStats is deprecated due to being incorrect w.r.t shader duplication due to chunking.")
    static RENDERCORE_API void DumpShaderCodeStats();
 
    static RENDERCORE_API void DumpShaderTypeStats(const FString& DebugInfoPath, const FString& MetaDataPath);
 
    // Create a smaller 'patch' library that only contains data from 'NewMetaDataDir' not contained in any of 'OldMetaDataDirs'
    static RENDERCORE_API bool CreatePatchLibrary(TArray<FString> const& OldMetaDataDirs, FString const& NewMetaDataDir, FString const& OutDir, bool bNativeFormat, bool bNeedsDeterministicOrder);
};
#endif