ShaderCompilerCore.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
/*=============================================================================
    ShaderCompilerCore.h: Shader Compiler core module definitions.
=============================================================================*/
 
#pragma once
 
#include "CoreMinimal.h"
#include "HAL/FileManager.h"
#include "Hash/Blake3.h"
#include "Hash/xxhash.h"
#include "Stats/Stats.h"
#include "Templates/RefCounting.h"
#include "Misc/EnumClassFlags.h"
#include "Misc/SecureHash.h"
#include "Misc/Paths.h"
#include "Misc/PathViews.h"
#include "Misc/CoreStats.h"
#include "ShaderCore.h"
#include "ShaderParameterMetadata.h"
 
class Error;
class IShaderFormat;
class FShaderCommonCompileJob;
class FShaderCompileJob;
struct FShaderCompilerInput;
class FShaderPipelineCompileJob;
 
using FShaderSharedStringPtr = TSharedPtr<FString, ESPMode::ThreadSafe>;
typedef TSharedPtr<TArray<ANSICHAR>, ESPMode::ThreadSafe> FShaderSharedAnsiStringPtr;
 
// this is for the protocol, not the data, bump if FShaderCompilerInput/FShaderPreprocessOutput serialization, SerializeWorkerInput or ProcessInputFromArchive changes.
inline const int32 ShaderCompileWorkerInputVersion = 32;
// this is for the protocol, not the data, bump if FShaderCompilerOutput or WriteToOutputArchive changes.
inline const int32 ShaderCompileWorkerOutputVersion = 28;
// this is for the protocol, not the data.
inline const int32 ShaderCompileWorkerSingleJobHeader = 'S';
// this is for the protocol, not the data.
inline const int32 ShaderCompileWorkerPipelineJobHeader = 'P';
 
// Modify this in ShaderCompilerCore.cpp to invalidate _just_ the cache/DDC entries for individual shaders (will not cause
// shadermaps to rebuild if they are not otherwise out-of-date).
// This should be bumped for changes to the FShaderCompilerOutput data structure (in addition to ShaderCompileWorkerOutputVersion)
extern RENDERCORE_API const FGuid UE_SHADER_CACHE_VERSION;
 
namespace UE::ShaderCompiler
{
    RENDERCORE_API ERHIBindlessConfiguration GetBindlessConfiguration(EShaderPlatform ShaderPlatform);
    RENDERCORE_API bool ShouldCompileWithBindlessEnabled(EShaderPlatform ShaderPlatform, const FShaderCompilerInput& Input);
 
    UE_DEPRECATED(5.7, "GetBindlessResourcesConfiguration is now GetBindlessConfiguration")
    RENDERCORE_API ERHIBindlessConfiguration GetBindlessResourcesConfiguration(FName ShaderFormat);
 
    UE_DEPRECATED(5.7, "GetBindlessSamplersConfiguration is now GetBindlessConfiguration")
    RENDERCORE_API ERHIBindlessConfiguration GetBindlessSamplersConfiguration(FName ShaderFormat);
}
 
extern RENDERCORE_API const FString& GetBuildMachineArtifactBasePath();
 
extern RENDERCORE_API const FString& GetShaderDebugInfoPath();
 
extern RENDERCORE_API bool ShouldGenerateShaderSymbols(FName ShaderFormat);
 
extern RENDERCORE_API bool ShouldGenerateShaderSymbolsInfo(FName ShaderFormat);
 
extern RENDERCORE_API bool ShouldWriteShaderSymbols(FName ShaderFormat);
 
extern RENDERCORE_API bool GetShaderSymbolPathOverride(FString& OutPathOverride, FName ShaderFormat);
 
extern RENDERCORE_API bool GetShaderFileNameOverride(FString& OutFileNameOverride, const TCHAR* Cvar, FName ShaderFormat, FName PlatformName);
 
extern RENDERCORE_API bool ShouldAllowUniqueShaderSymbols(FName ShaderFormat);
 
enum class EWriteShaderSymbols : uint8
{
    Disable     = 0,
    UnCompress  = 1,
    Compress    = 2
};
 
extern RENDERCORE_API EWriteShaderSymbols GetWriteShaderSymbolsOptions(FName ShaderFormat);
 
extern RENDERCORE_API bool ShouldEnableExtraShaderData(FName ShaderFormat);
 
extern RENDERCORE_API bool ShouldOptimizeShaders(FName ShaderFormat);
 
extern RENDERCORE_API bool AllowShaderCompiling();
 
extern RENDERCORE_API bool AllowGlobalShaderLoad();
 
 
enum ECompilerFlags
{
    #define SHADER_COMPILER_FLAGS_ENTRY(Name) CFLAG_##Name,
    #define SHADER_COMPILER_FLAGS_ENTRY_DEPRECATED(Name, Version, Message) CFLAG_##Name UE_DEPRECATED(Version, Message),
    #include "ShaderCompilerFlags.inl"
    CFLAG_Max
};
static_assert(CFLAG_PreferFlowControl == 0, "First entry in ECompilerFlags must be 'CFLAG_PreferFlowControl' and assigned to 0");
static_assert(CFLAG_Max < 64, "Out of bitfield space! Modify FShaderCompilerFlags");
 
extern RENDERCORE_API void LexFromString(ECompilerFlags& OutValue, const TCHAR* InString);
extern RENDERCORE_API const TCHAR* LexToString(ECompilerFlags InValue);
 
struct FShaderCompilerResourceTable
{
    uint32 ResourceTableBits;
 
    uint32 MaxBoundResourceTable;
 
    TArray<uint32> TextureMap;
 
    TArray<uint32> ShaderResourceViewMap;
 
    TArray<uint32> SamplerMap;
 
    TArray<uint32> UnorderedAccessViewMap;
 
    TArray<uint32> ResourceCollectionMap;
 
    TArray<uint32> ResourceTableLayoutHashes;
 
    FShaderCompilerResourceTable()
        : ResourceTableBits(0)
        , MaxBoundResourceTable(0)
    {
    }
};
 
enum class EOfflineShaderCompilerType : uint8
{
    Mali,
    Adreno,
 
    Num
};
 
struct FExtraShaderCompilerSettings
{
    bool bExtractShaderSource = false;
    FString OfflineCompilerPath;
    EOfflineShaderCompilerType OfflineCompiler = EOfflineShaderCompilerType::Mali;
    FString GPUTarget;
    bool bDumpAll = false;
    bool bSaveCompilerStatsFiles = false;
    bool bMobileMultiView = false;
 
    friend FArchive& operator<<(FArchive& Ar, FExtraShaderCompilerSettings& StatsSettings)
    {
        // Note: this serialize is used to pass between UE and the shader compile worker, recompile both when modifying
        return Ar << StatsSettings.bExtractShaderSource << StatsSettings.OfflineCompilerPath 
            << StatsSettings.OfflineCompiler << StatsSettings.GPUTarget << StatsSettings.bDumpAll << StatsSettings.bSaveCompilerStatsFiles << StatsSettings.bMobileMultiView;
    }
};
 
namespace FOodleDataCompression
{
    enum class ECompressor : uint8;
    enum class ECompressionLevel : int8;
}
 
enum class EShaderDebugInfoFlags : uint8
{
    Default = 0,
    DirectCompileCommandLine UE_DEPRECATED(5.7, "DirectCompile has been renamed DebugCompile and DebugCompileArgs.txt is dumped by default") = 1 << 0,
    InputHash = 1 << 1,
    Diagnostics = 1 << 2,
    ShaderCodeBinary = 1 << 3,
    DetailedSource = 1 << 4,
    CompileFromDebugUSF = 1 << 5,
    ShaderCodePlatformHashes = 1 << 6,
};
ENUM_CLASS_FLAGS(EShaderDebugInfoFlags)
 
using FShaderCompilerInputHash = FBlake3Hash;
 
struct FShaderDebugDataContext
{
    bool bIsPipeline = false;
    TArray<TPair<EShaderFrequency, FString>> DebugSourceFiles;
};
 
struct FShaderCompilerInput
{
    FShaderTarget Target{ SF_NumFrequencies, SP_NumPlatforms };
    
    FName ShaderFormat;
    FName CompressionFormat;
    FName ShaderPlatformName;
 
    // Preferred output format for FShaderCode, e.g. when DXIL is preferred over DXBC while both would be supported.
    FName PreferredShaderCodeFormat;
    
    FString VirtualSourceFilePath;
    FString EntryPointName;
    FString ShaderName;
 
    uint32 SupportedHardwareMask = 0;
 
    // Indicates which additional debug outputs should be written for this compile job.
    EShaderDebugInfoFlags DebugInfoFlags = EShaderDebugInfoFlags::Default;
 
    // Array of symbols that should be maintained when deadstripping. If this is empty, entry
    // point name alone will be used.
    TArray<FString> RequiredSymbols;
    
    // Shader pipeline information
    TArray<FString> UsedOutputs;
 
    // Dump debug path (up to platform) e.g. "D:/Project/Saved/ShaderDebugInfo/PCD3D_SM5"
    FString DumpDebugInfoRootPath;
    // only used if enabled by r.DumpShaderDebugInfo (platform/groupname) e.g. ""
    FString DumpDebugInfoPath;
    // materialname or "Global" "for debugging and better error messages
    FString DebugGroupName;
 
    FString DebugExtension;
 
    // Description of the configuration used when compiling. 
    FString DebugDescription;
 
    // Hash of this input (used as the key for the shader job cache)
    FShaderCompilerInputHash Hash;
 
    // Compilation Environment
    FShaderCompilerEnvironment Environment;
    TRefCountPtr<FSharedShaderCompilerEnvironment> SharedEnvironment;
 
    // The root of the shader parameter structures / uniform buffers bound to this shader to generate shader resource table from.
    // This only set if a shader class is defining the 
    const FShaderParametersMetadata* RootParametersStructure = nullptr;
 
 
    // Additional compilation settings that can be filled by FMaterial::SetupExtraCompilationSettings
    // FMaterial::SetupExtraCompilationSettings is usually called by each (*)MaterialShaderType::BeginCompileShader() function
    FExtraShaderCompilerSettings ExtraSettings;
 
    FOodleDataCompression::ECompressor OodleCompressor;
 
    FOodleDataCompression::ECompressionLevel OodleLevel;
 
    bool bCompilingForShaderPipeline = false;
    bool bIncludeUsedOutputs = false;
 
    // Internal only flag to signal that bindless is enabled
    bool bBindlessEnabled = false;
 
    bool DumpDebugInfoEnabled() const 
    {
        return DumpDebugInfoPath != TEXT("") && IFileManager::Get().DirectoryExists(*DumpDebugInfoPath);
    }
 
    bool NeedsOriginalShaderSource() const
    {
        return DumpDebugInfoEnabled() || ExtraSettings.bExtractShaderSource;
    }
 
    // generate human readable name for debugging
    FString GenerateShaderName() const
    {
        FString Name;
 
        if(DebugGroupName == TEXT("Global"))
        {
            Name = VirtualSourceFilePath + TEXT("|") + EntryPointName;
        }
        else
        {
            Name = DebugGroupName + TEXT(":") + VirtualSourceFilePath + TEXT("|") + EntryPointName;
        }
 
        return Name;
    }
 
    FStringView GetSourceFilenameView() const
    {
        return FPathViews::GetCleanFilename(VirtualSourceFilePath);
    }
 
    FString GetSourceFilename() const
    {
        return FPaths::GetCleanFilename(VirtualSourceFilePath);
    }
 
    // Common code to generate a debug string to associate with platform-specific shader symbol files and hashes
    // Currently uses DebugGroupName, but can be updated to contain other important information as needed
    FString GenerateDebugInfo() const
    {
        return Environment.CompilerFlags.Contains(CFLAG_GenerateSymbolsInfo) ? DebugGroupName : FString();
    }
 
    void GatherSharedInputs(
        TArray<TRefCountPtr<FSharedShaderCompilerEnvironment>>& SharedEnvironments,
        TArray<const FShaderParametersMetadata*>& ParametersStructures)
    {
        check(!SharedEnvironment || SharedEnvironment->IncludeVirtualPathToSharedContentsMap.Num() == 0);
 
        if (SharedEnvironment)
        {
            SharedEnvironments.AddUnique(SharedEnvironment);
        }
 
        if (RootParametersStructure)
        {
            ParametersStructures.AddUnique(RootParametersStructure);
        }
    }
 
    void SerializeSharedInputs(FArchive& Ar, const TArray<TRefCountPtr<FSharedShaderCompilerEnvironment>>& SharedEnvironments, const TArray<const FShaderParametersMetadata*>& ParametersStructures)
    {
        check(Ar.IsSaving());
 
        int32 SharedEnvironmentIndex = SharedEnvironments.Find(SharedEnvironment);
        Ar << SharedEnvironmentIndex;
 
        int32 ShaderParameterStructureIndex = INDEX_NONE;
        if (RootParametersStructure)
        {
            ShaderParameterStructureIndex = ParametersStructures.Find(RootParametersStructure);
            check(ShaderParameterStructureIndex != INDEX_NONE);
        }
        Ar << ShaderParameterStructureIndex;
    }
 
    void DeserializeSharedInputs(
        FArchive& Ar,
        const TArray<FShaderCompilerEnvironment>& SharedEnvironments,
        const TArray<TUniquePtr<FShaderParametersMetadata>>& ShaderParameterStructures)
    {
        check(Ar.IsLoading());
 
        int32 SharedEnvironmentIndex = 0;
        Ar << SharedEnvironmentIndex;
 
        if (SharedEnvironments.IsValidIndex(SharedEnvironmentIndex))
        {
            Environment.Merge(SharedEnvironments[SharedEnvironmentIndex]);
        }
 
        int32 ShaderParameterStructureIndex = INDEX_NONE;
        Ar << ShaderParameterStructureIndex;
        if (ShaderParameterStructureIndex != INDEX_NONE)
        {
            RootParametersStructure = ShaderParameterStructures[ShaderParameterStructureIndex].Get();
        }
    }
 
    friend RENDERCORE_API FArchive& operator<<(FArchive& Ar, FShaderCompilerInput& Input);
 
    bool IsRayTracingShader() const
    {
        return IsRayTracingShaderFrequency(Target.GetFrequency());
    }
 
    bool IsWorkGraphShader() const
    {
        return IsWorkGraphShaderFrequency(Target.GetFrequency());
    }
 
    bool ShouldUseStableConstantBuffer() const
    {
        // stable constant buffer is for the FShaderParameterBindings::BindForLegacyShaderParameters() code path.
        // Ray tracing shaders use FShaderParameterBindings::BindForRootShaderParameters instead.
        if (IsRayTracingShader())
        {
            return false;
        }
 
        return RootParametersStructure != nullptr;
    }
 
    bool IsBindlessEnabled() const
    {
        return bBindlessEnabled;
    }
 
    RENDERCORE_API FString GetOrCreateShaderDebugInfoPath() const;
};
 
struct FShaderCompilerError
{
    FShaderCompilerError(const TCHAR* InStrippedErrorMessage = TEXT(""))
        : ErrorVirtualFilePath(TEXT(""))
        , ErrorLineString(TEXT(""))
        , StrippedErrorMessage(InStrippedErrorMessage)
        , HighlightedLine(TEXT(""))
        , HighlightedLineMarker(TEXT(""))
    {}
 
    FShaderCompilerError(const TCHAR* InVirtualFilePath, const TCHAR* InLineString, const TCHAR* InStrippedErrorMessage)
        : ErrorVirtualFilePath(InVirtualFilePath)
        , ErrorLineString(InLineString)
        , StrippedErrorMessage(InStrippedErrorMessage)
        , HighlightedLine(TEXT(""))
        , HighlightedLineMarker(TEXT(""))
    {}
 
    FShaderCompilerError(FString&& InStrippedErrorMessage)
        : ErrorVirtualFilePath(TEXT(""))
        , ErrorLineString(TEXT(""))
        , StrippedErrorMessage(MoveTemp(InStrippedErrorMessage))
        , HighlightedLine(TEXT(""))
        , HighlightedLineMarker(TEXT(""))
    {}
 
    FShaderCompilerError(FString&& InStrippedErrorMessage, FString&& InHighlightedLine, FString&& InHighlightedLineMarker)
        : ErrorVirtualFilePath(TEXT(""))
        , ErrorLineString(TEXT(""))
        , StrippedErrorMessage(MoveTemp(InStrippedErrorMessage))
        , HighlightedLine(MoveTemp(InHighlightedLine))
        , HighlightedLineMarker(MoveTemp(InHighlightedLineMarker))
    {}
 
    FString ErrorVirtualFilePath;
    FString ErrorLineString;
    FString StrippedErrorMessage;
    FString HighlightedLine;
    FString HighlightedLineMarker;
 
    FString RENDERCORE_API GetErrorStringWithSourceLocation() const;
    
    FString RENDERCORE_API GetErrorStringWithLineMarker() const;
 
    FString RENDERCORE_API GetErrorString(bool bOmitLineMarker = false) const;
 
    inline bool HasLineMarker() const
    {
        return !HighlightedLine.IsEmpty() && !HighlightedLineMarker.IsEmpty();
    }
 
    bool RENDERCORE_API ExtractSourceLocation();
 
    static void RENDERCORE_API ExtractSourceLocations(TArray<FShaderCompilerError>& InOutErrors);
 
    FString RENDERCORE_API GetShaderSourceFilePath(TArray<FShaderCompilerError>* InOutErrors = nullptr) const;
 
    friend FArchive& operator<<(FArchive& Ar,FShaderCompilerError& Error)
    {
        return Ar << Error.ErrorVirtualFilePath << Error.ErrorLineString << Error.StrippedErrorMessage << Error.HighlightedLine << Error.HighlightedLineMarker;
    }
};
 
struct FShaderCompilerOutput
{
    FShaderCompilerOutput()
    :   NumInstructions(0)
    ,   NumTextureSamplers(0)
    ,   CompileTime(0.0)
    ,   PreprocessTime(0.0)
    ,   bSucceeded(false)
    ,   bSupportsQueryingUsedAttributes(false)
    ,   bSerializingForCache(false)
    {
    }
 
    FShaderParameterMap ParameterMap;
    TArray<FShaderCompilerError> Errors;
    FShaderTarget Target;
    FShaderCode ShaderCode;
    FSHAHash OutputHash;
    FShaderCompilerInputHash ValidateInputHash;
    uint32 NumInstructions;
    uint32 NumTextureSamplers;
    double CompileTime;
    double PreprocessTime;
    bool bSucceeded;
    bool bSupportsQueryingUsedAttributes;
    UE_DEPRECATED(5.6, "bSerializeModifiedSource is no longer used")
    bool bSerializeModifiedSource;
    bool bSerializingForCache;
    TArray<FString> UsedAttributes;
 
    PRAGMA_DISABLE_DEPRECATION_WARNINGS
    // Explicitly-defaulted copy/move ctors & assignment operators are needed temporarily due to deprecation
    // of bSerializeModifiedSource field. These can be removed once the deprecation window for this field ends.
    FShaderCompilerOutput(FShaderCompilerOutput&&) = default;
    FShaderCompilerOutput(const FShaderCompilerOutput&) = default;
    FShaderCompilerOutput& operator=(FShaderCompilerOutput&&) = default;
    FShaderCompilerOutput& operator=(const FShaderCompilerOutput&) = default;
    PRAGMA_ENABLE_DEPRECATION_WARNINGS
 
    TArray<FShaderCodeValidationStride> ParametersStrideToValidate;
    TArray<FShaderCodeValidationType> ParametersSRVTypeToValidate;
    TArray<FShaderCodeValidationType> ParametersUAVTypeToValidate;
    TArray<FShaderCodeValidationUBSize> ParametersUBSizeToValidate;
 
    TArray<FShaderDiagnosticData> ShaderDiagnosticDatas;
 
    FString ModifiedShaderSource;
 
    FString ModifiedEntryPointName;
 
    UE_DEPRECATED(5.6, "Use symbols accessors (GetSymbolWriteAccess, GetSymbolReadView) on FShaderCode object instead (ShaderCode member)")
    TArray<uint8> PlatformDebugData;
 
    TArray<FGenericShaderStat> ShaderStatistics;
 
    RENDERCORE_API void GenerateOutputHash();
 
    RENDERCORE_API void CompressOutput(FName ShaderCompressionFormat, FOodleDataCompression::ECompressor OodleCompressor, FOodleDataCompression::ECompressionLevel OodleLevel);
 
    RENDERCORE_API void SerializeShaderCodeValidation();
 
    RENDERCORE_API void SerializeShaderDiagnosticData();
 
    template<typename TValue>
    void AddStatistic(const TCHAR* Name, TValue Value, FGenericShaderStat::EFlags Flags = FGenericShaderStat::EFlags::None, FName TagName = NAME_None)
    {
        // handle the case where a stat with the given name already exists; some backends compile iteratively and this is a simpler way to handle that
        // than requiring each backend to manually avoid setting until the "final" compilation. linear search is in practice fine here; there are low 
        // single digit numbers of stats for all existing shader formats at the time of writing.
        FName StatName(Name);
        FGenericShaderStat* Stat = nullptr;
        for (FGenericShaderStat& ExistingStat : ShaderStatistics)
        {
            if (ExistingStat.StatName == StatName)
            {
                Stat = &ExistingStat;
                break;
            }
        }
 
        if (!Stat)
        {
            Stat = &ShaderStatistics.AddZeroed_GetRef();
            Stat->StatName = StatName;
        }
 
        Stat->Value = FShaderStatVariant(TInPlaceType<TValue>(), Value);
        Stat->Flags = Flags;
        Stat->TagName = TagName;
    }
 
    const FShaderCodeResource& GetFinalizedCodeResource() const
    {
        return ShaderCode.GetFinalizedResource(Target.GetFrequency(), OutputHash);
    }
 
    void SetCodeFromResource(FShaderCodeResource&& Resource)
    {
        return ShaderCode.SetFromResource(MoveTemp(Resource));
    }
 
    // Bump ShaderCompileWorkerOutputVersion if FShaderCompilerOutput changes
    friend FArchive& operator<<(FArchive& Ar, FShaderCompilerOutput& Output)
    {
        // Note: this serialize is used to pass between UE and the shader compile worker, recompile both when modifying
        Ar << Output.ParameterMap;
        Ar << Output.Errors;
        Ar << Output.Target;
        Ar << Output.bSerializingForCache;
        if (!Output.bSerializingForCache)
        {
            // skip serializing these fields when saving to cache/DDC; only needed when reading back results from workers
            Ar << Output.ShaderCode;
            Ar << Output.ValidateInputHash;
            Ar << Output.CompileTime;
        }
        Ar << Output.OutputHash;
        Ar << Output.NumInstructions;
        Ar << Output.NumTextureSamplers;
        Ar << Output.bSucceeded;
        Ar << Output.ModifiedShaderSource;
        Ar << Output.ModifiedEntryPointName;
        Ar << Output.ShaderStatistics;
 
        // note: intentionally never serializing the following fields:
        // - PreprocessTime - it is always set in the cooker since we no longer run preprocessing in SCW
        // - bSupportsQueryingUsedAttributes - only used when compiling pipelines by subsequent stage compile steps, these are always executed in order in a single SCW job invocation
        // - UsedAttributes - as above
 
        return Ar;
    }
};
 
struct FShaderCompileWorkerDiagnostics
{
    int32 ErrorCode = 0;
 
    double EntryPointTimestamp = 0.0;
 
    double BatchPreparationTime = 0.0;
 
    double BatchProcessTime = 0.0;
 
    int32 BatchIndex = 0;
 
    friend FArchive& operator<<(FArchive& Ar, FShaderCompileWorkerDiagnostics& Output)
    {
        Ar << Output.ErrorCode << Output.EntryPointTimestamp << Output.BatchPreparationTime << Output.BatchProcessTime << Output.BatchIndex;
        return Ar;
    }
};
 
#if PLATFORM_WINDOWS
extern RENDERCORE_API int HandleShaderCompileException(Windows::LPEXCEPTION_POINTERS Info, FString& OutExMsg, FString& OutCallStack);
#endif
extern RENDERCORE_API const IShaderFormat* FindShaderFormat(FName Format, const TArray<const IShaderFormat*>& ShaderFormats);
 
// Executes preprocessing for the given job, if the job is marked to be preprocessed independently prior to compilation.
extern RENDERCORE_API bool PreprocessShader(FShaderCommonCompileJob* Job);
extern RENDERCORE_API void CompileShader(const TArray<const IShaderFormat*>& ShaderFormats, FShaderCompileJob& Job, const FString& WorkingDirectory, int32* CompileCount = nullptr);
extern RENDERCORE_API void CompileShaderPipeline(const TArray<const IShaderFormat*>& ShaderFormats, FShaderPipelineCompileJob* PipelineJob, const FString& WorkingDirectory, int32* CompileCount = nullptr);
 
extern RENDERCORE_API bool CheckVirtualShaderFilePath(FStringView VirtualPath, TArray<FShaderCompilerError>* CompileErrors = nullptr);
 
extern RENDERCORE_API void FixupShaderFilePath(FString& VirtualFilePath, EShaderPlatform ShaderPlatform, const FName* ShaderPlatformName);
 
enum class EConvertAndStripFlags : uint8
{
    None = 0,
    NoSimdPadding = 1 << 0,
};
ENUM_CLASS_FLAGS(EConvertAndStripFlags);
extern RENDERCORE_API void ShaderConvertAndStripComments(const FString& ShaderSource, TArray<ANSICHAR>& OutStripped, EConvertAndStripFlags Flags = EConvertAndStripFlags::None);
 
extern RENDERCORE_API bool LoadShaderSourceFile(const TCHAR* VirtualFilePath, EShaderPlatform ShaderPlatform, FString* OutFileContents, TArray<FShaderCompilerError>* OutCompileErrors, const FName* ShaderPlatformName = nullptr, FShaderSharedAnsiStringPtr* OutStrippedContents = nullptr);
 
extern RENDERCORE_API bool LoadCachedShaderSourceFile(const TCHAR* VirtualFilePath, EShaderPlatform ShaderPlatform, FShaderSharedStringPtr* OutFileContents, TArray<FShaderCompilerError>* OutCompileErrors, const FName* ShaderPlatformName = nullptr, FShaderSharedAnsiStringPtr* OutStrippedContents = nullptr);
 
 
struct FShaderPreprocessDependency
{
    FXxHash64                   PathInSourceHash;       // PathInSourceHash doesn't include PathInSource's null terminator, so hash computation can use a string view
    TArray<ANSICHAR>            PathInSource;           // Path as it appears in include directive in original shader source, allowing faster case sensitive hash
    TArray<ANSICHAR>            ParentPath;             // For relative paths, ResultPath is dependent on the parent file the include directive is found in
    TArray<ANSICHAR>            ResultPath;
    uint32                      ResultPathHash;         // Case insensitive hash of ResultPath (compatible with hash of corresponding FString)
    uint32                      ResultPathUniqueIndex;  // Index of first instance of a given result path in Dependencies array
    FShaderSharedAnsiStringPtr  StrippedSource;         // Source with comments stripped out, and converted to ANSICHAR (output of ShaderConvertAndStripComments)
 
    inline bool EqualsPathInSource(const ANSICHAR* InPathInSource, int32 InPathInSourceLen, FXxHash64 InPathInSourceHash, const ANSICHAR* InParentPath) const
    {
        // PathInSource is case sensitive, ParentPath is case insensitive.
        // If the path is absolute (starts with '/'), then the parent path isn't relevant, and shouldn't be checked.
        return
            PathInSourceHash == InPathInSourceHash &&
            (PathInSource[0] == '/' || !FCStringAnsi::Stricmp(ParentPath.GetData(), InParentPath)) &&
            !FCStringAnsi::Strncmp(PathInSource.GetData(), InPathInSource, InPathInSourceLen);
    }
 
    inline bool EqualsResultPath(const FString& InResultPath, uint32 InResultPathHash) const
    {
        return (ResultPathHash == InResultPathHash) && InResultPath.Equals(ResultPath.GetData(), ESearchCase::IgnoreCase);
    }
 
    inline bool EqualsResultPath(const ANSICHAR* InResultPath, uint32 InResultPathHash) const
    {
        return (ResultPathHash == InResultPathHash) && !FCStringAnsi::Stricmp(ResultPath.GetData(), InResultPath);
    }
};
 
 
// Structure that provides an array of #include dependencies for a given root shader file, including not just immediate
// dependencies, but recursive dependencies from children as well.  Not exhaustive, as it does not include platform
// specific or generated files, although it does include children of "/Engine/Generated/Material.ush", as derived from
// "/Engine/Private/MaterialTemplate.ush".  Take the example of ClearUAV.usf:
//
// /Engine/Private/Tools/ClearUAV.usf    #include "../Common.ush"
// /Engine/Private/Common.ush            #include "/Engine/Public/Platform.ush"
//                                       #include "PackUnpack.ush"
// /Engine/Public/Platform.ush           #include "FP16Math.ush"
//
// The above is a small subset, but the above (and many more) would all show up as elements in Dependencies:
//
//      PathInSource                   ParentPath                                ResultPath
//      --------------------------------------------------------------------------------------------------------
//      ../Common.ush                  /Engine/Private/Tools/ClearUAV.usf        /Engine/Private/Common.ush
//      /Engine/Public/Platform.ush    /Engine/Private/Common.ush                /Engine/Public/Platform.ush
//      PackUnpack.ush                 /Engine/Private/Common.ush                /Engine/Private/PackUnpack.ush
//      FP16Math.ush                   /Engine/Public/Platform.ush               /Engine/Public/FP16Math.ush
//
// The goal of this structure is to allow a shader preprocessor implementation to fetch most of the source dependencies in a
// single query of the loaded shader cache, and then efficiently search for dependencies encountered in the shader source
// code, without needing to do string operations to resolve paths (such as converting relative paths like "../Common.ush" to
// "/Engine/Private/Common.ush").  Besides that, the array organization can be used to manage encountered source files
// by index, rather than needing a map, and the "ResultPath" strings from this structure can be referenced by pointer,
// rather than needing to dynamically allocate a copy of the resolved path.  Lookups by PathInSource can use a much faster
// case sensitive hash, because PathInSource has verbatim capitalization from the source code files.  Altogether, this
// utility structure saves a bunch of shader cache query, hash, map, string, and memory allocation overhead.
//
struct FShaderPreprocessDependencies
{
    // First item in array contains stripped source for root file, and is not in the hash tables
    TArray<FShaderPreprocessDependency> Dependencies;
    FHashTable BySource;                                // Hash table by PathInSource
    FHashTable ByResult;                                // Hash table by ResultPath
};
 
typedef TSharedPtr<FShaderPreprocessDependencies, ESPMode::ThreadSafe> FShaderPreprocessDependenciesShared;
 
extern RENDERCORE_API bool GetShaderPreprocessDependencies(const TCHAR* VirtualFilePath, EShaderPlatform ShaderPlatform, FShaderPreprocessDependenciesShared& OutDependencies);
 
enum class EShaderCompilerWorkerType : uint8
{
    None,
    LocalThread,
    Distributed,
};
 
enum class EShaderCompileJobType : uint8
{
    Single,
    Pipeline,
    Num,
};
inline constexpr int32 NumShaderCompileJobTypes = (int32)EShaderCompileJobType::Num;