ShaderCompiler.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
/*=============================================================================
    ShaderCompiler.h: Platform independent shader compilation definitions.
=============================================================================*/
 
#pragma once
 
#include "Templates/RefCounting.h"
#include "HAL/PlatformProcess.h"
#include "ShaderCore.h"
#include "ShaderCompilerCore.h"
#include "ShaderCompilerJobTypes.h"
#include "Shader.h"
#include "HAL/RunnableThread.h"
#include "HAL/Runnable.h"
#include "Templates/Atomic.h"
#include "Templates/UniquePtr.h"
#include "HAL/ThreadSafeCounter.h"
#include "RHIDefinitions.h"
#include "GBufferInfo.h"
#include "ShaderMaterial.h"
#include "Misc/ScopeRWLock.h"
#include "IAssetCompilingManager.h"
#include "Containers/HashTable.h"
#include "Containers/List.h"
#include "Containers/Deque.h"
#include "Hash/Blake3.h"
#include "Hash/CityHash.h"
#include "SceneTypes.h"
#include "UObject/StrongObjectPtr.h"
 
#include "ShaderCompiler.generated.h"
 
class FAsyncCompilationNotification;
class FCbObjectView;
class FCbWriter;
class FVertexFactoryType;
class IDistributedBuildController;
class FMaterialShaderMap;
class FShaderCompileJob;
struct FShaderCompilerStats;
class FShaderKeyGenerator;
class FShaderPipelineCompileJob;
class UMaterialInterface;
class FJsonObject;
struct FAnalyticsEventAttribute;
struct FShaderCompilerMemoryLimits;
 
DECLARE_LOG_CATEGORY_EXTERN(LogShaderCompilers, Log, All);
 
bool AreShaderErrorsFatal();
bool CreateShadersOnLoad();
 
extern ENGINE_API bool IsShaderJobCacheDDCEnabled();
extern ENGINE_API bool IsMaterialMapDDCEnabled();
 
extern ENGINE_API void GetShaderCompilerCurrentMemoryLimits(FShaderCompilerMemoryLimits& limits);
extern ENGINE_API void SetShaderCompilerTargetMemoryLimitInMiB(int32 NewLimit);
 
extern ENGINE_API bool ShouldCompileODSCOnlyShaders();
 
struct FShaderCompilerJobMemoryLimitInfo;
 
struct FShaderJobCacheStoredOutput;
class FShaderJobCache;
 
class FShaderCompileJobCollection
{
public:
    FShaderCompileJobCollection(FCriticalSection& InCompileQueueSection);
 
    FShaderCompileJob* PrepareJob(uint32 InId, const FShaderCompileJobKey& InKey, EShaderCompileJobPriority InPriority);
    FShaderPipelineCompileJob* PrepareJob(uint32 InId, const FShaderPipelineCompileJobKey& InKey, EShaderCompileJobPriority InPriority);
    void RemoveJob(FShaderCommonCompileJob* InJob);
 
    int32 RemoveAllPendingJobsWithId(uint32 InId);
 
    void SubmitJobs(const TArray<FShaderCommonCompileJobPtr>& InJobs);
    
    void ProcessFinishedJob(FShaderCommonCompileJob* FinishedJob, EShaderCompileJobStatus Status);
 
    void AddToCacheAndProcessPending(FShaderCommonCompileJob* FinishedJob);
 
    void GetCachingStats(FShaderCompilerStats& OutStats) const;
 
    int32 GetNumPendingJobs(EShaderCompileJobPriority InPriority) const;
 
    int32 GetNumOutstandingJobs() const;
 
    int32 GetNumPendingJobs() const;
 
    int32 GetPendingJobs(EShaderCompilerWorkerType InWorkerType, EShaderCompileJobPriority InPriority, int32 MinNumJobs, int32 MaxNumJobs, TArray<FShaderCommonCompileJobPtr>& OutJobs);
 
private:
    void HandlePrintStats();
 
    TPimplPtr<FShaderJobCache> JobsCache;
 
    class IConsoleObject* PrintStatsCmd;
};
 
#if WITH_EDITOR
class FGlobalShaderTypeCompiler
{
public:
    ENGINE_API static void BeginCompileShader(const FGlobalShaderType* ShaderType, int32 PermutationId, EShaderPlatform Platform, EShaderPermutationFlags PermutationFlags, TArray<FShaderCommonCompileJobPtr>& NewJobs);
 
    ENGINE_API static void BeginCompileShaderPipeline(EShaderPlatform Platform, EShaderPermutationFlags PermutationFlags, const FShaderPipelineType* ShaderPipeline, TArray<FShaderCommonCompileJobPtr>& NewJobs);
 
    static FShader* FinishCompileShader(const FGlobalShaderType* ShaderType, const FShaderCompileJob& CompileJob, const FShaderPipelineType* ShaderPipelineType);
};
#endif // WITH_EDITOR
 
struct FShaderCompileMemoryUsage
{
    uint64 VirtualMemory;
    uint64 PhysicalMemory;
};
 
using FShaderCompileJobCallback = TFunction<bool(FShaderCommonCompileJob*)>;
 
class FShaderCompileThreadRunnableBase : public FRunnable
{
    friend class FShaderCompilingManager;
 
    uint64 WorkerStateHash;
 
    double WorkerStateChangeTimestamp;
 
protected:
    class FShaderCompilingManager* Manager;
    FRunnableThread* Thread;
 
    int32 MinPriorityIndex;
    int32 MaxPriorityIndex;
    
    TAtomic<bool> bForceFinish;
 
    virtual bool PrintWorkerMemoryUsage(bool bAllowToWaitForLock=true) { return false; }
    virtual FShaderCompileMemoryUsage GetExternalWorkerMemoryUsage() { return FShaderCompileMemoryUsage{}; }
 
    // Returns a name for this thread instance. Defaults to "ShaderCompilingThread".
    virtual const TCHAR* GetThreadName() const
    {
        return TEXT("ShaderCompilingThread");
    }
 
    bool WorkerStateHeartbeat(uint64 InWorkerStateHash);
 
public:
    FShaderCompileThreadRunnableBase(class FShaderCompilingManager* InManager);
    virtual ~FShaderCompileThreadRunnableBase()
    {}
 
    void SetPriorityRange(EShaderCompileJobPriority MinPriority, EShaderCompileJobPriority MaxPriority);
 
    void StartThread();
 
    // FRunnable interface.
    virtual void Stop() { bForceFinish = true; }
    virtual uint32 Run();
    inline void WaitForCompletion() const
    {
        if( Thread )
        {
            Thread->WaitForCompletion();
        }
    }
 
    virtual int32 CompilingLoop() = 0;
 
    virtual EShaderCompilerWorkerType GetWorkerType() const = 0;
 
    virtual void OnMachineResourcesChanged() {}
 
    virtual void ForEachPendingJob(const FShaderCompileJobCallback& PendingJobCallback) const {}
};
 
class FShaderCompileThreadRunnable : public FShaderCompileThreadRunnableBase
{
    friend class FShaderCompilingManager;
private:
 
    TArray<TUniquePtr<struct FShaderCompileWorkerInfo>> WorkerInfos;
    mutable FCriticalSection WorkerInfosLock;
 
    double LastCheckForWorkersTime = 0.0;
 
    bool bParallelizeIO = false;
 
    TArray<FShaderCommonCompileJobPtr> BackloggedJobs;
 
    struct FMemoryMonitoringState
    {
        double LastTimeOfMemoryLimitPoll = 0.0;
        double LastTimeOfSuspeningOrResumingWorkers = 0.0;
        bool bHasFailedToSuspendWorkers = false;
        bool bHasSuspendedWorkers = false;
    }
    MemoryMonitoringState;
 
public:
    FShaderCompileThreadRunnable(class FShaderCompilingManager* InManager);
    virtual ~FShaderCompileThreadRunnable();
 
    virtual EShaderCompilerWorkerType GetWorkerType() const override final
    {
        return EShaderCompilerWorkerType::LocalThread;
    }
 
protected:
    virtual bool PrintWorkerMemoryUsage(bool bAllowToWaitForLock = true) override;
    virtual FShaderCompileMemoryUsage GetExternalWorkerMemoryUsage() override;
 
private:
 
    int32 PullTasksFromQueue();
 
    void PushCompletedJobsToManager();
 
    void WriteNewTasks();
 
    bool LaunchWorkersIfNeeded();
 
    int32 ReadAvailableResults();
 
    void CompileDirectlyThroughDll();
 
    virtual int32 CompilingLoop() override;
 
    virtual void OnMachineResourcesChanged() override;
 
    virtual void ForEachPendingJob(const FShaderCompileJobCallback& PendingJobCallback) const override;
 
    void PrintWorkerMemoryUsageWithLockTaken();
 
    int32 GetNumberOfWorkers() const;
 
    int32 GetNumberOfAvailableWorkersUnsafe() const;
    int32 GetNumberOfAvailableWorkers() const;
 
    int32 GetNumberOfSuspendedWorkersUnsafe() const;
 
    int32 SuspendWorkersAndBacklogJobs(int32 NumWorkers, int32* OutNumBackloggedJobs = nullptr);
 
    int32 ResumeSuspendedWorkers(int32 NumWorkers);
 
    void DiscardWorkerOutputFile(int32 WorkerIndex);
 
    FString GetWorkingDirectoryForWorker(int32 WorkerIndex, bool bRelativePath = false) const;
 
    void CheckMemoryLimitViolation();
 
    bool QueryMemoryStatus(FShaderCompilerJobMemoryLimitInfo& OutInfo);
 
    bool QueryMemoryLimitViolationStatus(FShaderCompilerJobMemoryLimitInfo& OutInfo);
};
 
class FShaderCompileUtilities
{
public:
    static void ExecuteShaderCompileJob(FShaderCommonCompileJob& Job);
 
    static class FArchive* CreateFileHelper(const FString& Filename);
    static void MoveFileHelper(const FString& To, const FString& From);
    static void DeleteFileHelper(const FString& Filename);
 
    static ENGINE_API void GenerateBrdfHeaders(const EShaderPlatform Platform);
    static ENGINE_API void GenerateBrdfHeaders(const FName& ShaderFormat);
    static void ApplyDerivedDefines(FShaderCompilerEnvironment& OutEnvironment, FShaderCompilerEnvironment* SharedEnvironment, const EShaderPlatform Platform);
    static void AppendGBufferDDCKeyString(const EShaderPlatform Platform, FString& KeyString);
    static void AppendGBufferDDCKey(const EShaderPlatform Platform, FShaderKeyGenerator& KeyGen);
    static ENGINE_API void WriteGBufferInfoAutogen(EShaderPlatform TargetPlatform, ERHIFeatureLevel::Type FeatureLevel);
 
    static void ApplyFetchEnvironment(FShaderMaterialPropertyDefines& DefineData, const FShaderCompilerEnvironment& Environment);
    static void ApplyFetchEnvironment(FShaderGlobalDefines& DefineData, const FShaderCompilerEnvironment& Environment, const EShaderPlatform Platform);
    static void ApplyFetchEnvironment(FShaderLightmapPropertyDefines& DefineData, const FShaderCompilerEnvironment& Environment);
    static void ApplyFetchEnvironment(FShaderCompilerDefines& DefineData, const FShaderCompilerEnvironment& Environment);
 
    static ENGINE_API EGBufferLayout FetchGBufferLayout(const FShaderCompilerEnvironment& Environment);
 
    static ENGINE_API FGBufferParams FetchGBufferParamsRuntime(EShaderPlatform Platform, EGBufferLayout Layout); // this function is called from renderer
    static FGBufferParams FetchGBufferParamsPipeline(EShaderPlatform Platform, EGBufferLayout Layout);
};
 
class FShaderCompileDistributedThreadRunnable_Interface : public FShaderCompileThreadRunnableBase
{
    uint32 NumDispatchedJobs;
 
    TSparseArray<class FDistributedShaderCompilerTask*> DispatchedTasks;
 
    bool bIsHung;
 
public:
    FShaderCompileDistributedThreadRunnable_Interface(class FShaderCompilingManager* InManager, class IDistributedBuildController& InController);
    virtual ~FShaderCompileDistributedThreadRunnable_Interface();
 
    virtual EShaderCompilerWorkerType GetWorkerType() const override final
    {
        return EShaderCompilerWorkerType::Distributed;
    }
 
    virtual int32 CompilingLoop() override;
 
    static bool IsSupported();
 
    virtual void ForEachPendingJob(const FShaderCompileJobCallback& PendingJobCallback) const override;
 
protected:
    
    IDistributedBuildController& CachedController;
    TMap<EShaderPlatform, TArray<FString> > PlatformShaderInputFilesCache;
 
private:
 
    TArray<FString> GetDependencyFilesForJobs(TArray<FShaderCommonCompileJobPtr>& Jobs);
    void DispatchShaderCompileJobsBatch(TArray<FShaderCommonCompileJobPtr>& JobsToSerialize);
 
    virtual const TCHAR* GetThreadName() const override;
};
 
using FShaderMapFinalizeResults = FShaderMapCompileResults;
 
struct FDistributedBuildStats;
 
USTRUCT()
struct FShaderCompilerCounters
{
    GENERATED_BODY()
    UPROPERTY()
    double AccumulatedLocalWorkerIdleTime = 0.0;
 
    UPROPERTY()
    double TimesLocalWorkersWereIdle = 0;
 
    UPROPERTY()
    int64 JobsAssigned = 0;
 
    UPROPERTY()
    int64 JobsCompleted = 0;
 
    UPROPERTY()
    double AccumulatedPendingTime = 0;
 
    UPROPERTY()
    double MaxPendingTime = 0;
 
    UPROPERTY()
    double AccumulatedJobExecutionTime = 0;
 
    UPROPERTY()
    double MaxJobExecutionTime = 0;
 
    UPROPERTY()
    double AccumulatedJobLifeTime = 0;
 
    UPROPERTY()
    double MaxJobLifeTime = 0;
 
    UPROPERTY()
    double AccumulatedTaskSubmitJobs = 0.0;
    UPROPERTY()
    double AccumulatedTaskSubmitJobsStall = 0.0;
 
    UPROPERTY()
    int64 LocalJobBatchesSeen = 0;
 
    UPROPERTY()
    int64 TotalJobsReportedInLocalJobBatches = 0;
 
    UPROPERTY()
    int64 DistributedJobBatchesSeen = 0;
 
    UPROPERTY()
    int64 TotalJobsReportedInDistributedJobBatches = 0;
 
    UPROPERTY()
    int32 MinShaderCodeSize = 0;
 
    UPROPERTY()
    int32 MaxShaderCodeSize = 0;
 
    UPROPERTY()
    uint64 AccumulatedShaderCodeSize = 0;
 
    UPROPERTY()
    uint64 NumAccumulatedShaderCodes = 0;
 
    UPROPERTY()
    uint32 ShaderMapDDCMisses = 0;
 
    UPROPERTY()
    uint32 ShaderMapDDCHits = 0;
 
    UPROPERTY()
    uint64 TotalCacheSearchAttempts = 0;
 
    UPROPERTY()
    uint64 TotalCacheHits = 0;
 
    UPROPERTY()
    uint32 TotalCacheDuplicates = 0;
 
    UPROPERTY()
    uint32 TotalCacheDDCQueries = 0;
 
    UPROPERTY()
    uint32 TotalCacheDDCHits = 0;
 
    UPROPERTY()
    uint64 UniqueCacheInputHashes = 0;
 
    UPROPERTY()
    uint64 UniqueCacheOutputs = 0;
 
    UPROPERTY()
    uint64 CacheMemUsed = 0;
 
    UPROPERTY()
    uint64 CacheMemBudget = 0;
 
    UPROPERTY()
    uint32 MaxRemoteAgents = 0;
 
    UPROPERTY()
    uint32 MaxActiveAgentCores = 0;
 
    FShaderCompilerCounters& operator+=(const FShaderCompilerCounters& Other)
    {
        AccumulatedLocalWorkerIdleTime += Other.AccumulatedLocalWorkerIdleTime;
        TimesLocalWorkersWereIdle += Other.TimesLocalWorkersWereIdle;
        JobsAssigned += Other.JobsAssigned;
        JobsCompleted += Other.JobsCompleted;
        AccumulatedPendingTime += Other.AccumulatedPendingTime;
        MaxPendingTime = FMath::Max(Other.MaxPendingTime, MaxPendingTime);
        AccumulatedJobExecutionTime += Other.AccumulatedJobExecutionTime;
        MaxJobExecutionTime = FMath::Max(Other.MaxJobExecutionTime, MaxJobExecutionTime);
        AccumulatedJobLifeTime += Other.AccumulatedJobLifeTime;
        MaxJobLifeTime = FMath::Max(Other.MaxJobLifeTime, MaxJobLifeTime);
        AccumulatedTaskSubmitJobs += Other.AccumulatedTaskSubmitJobs;
        AccumulatedTaskSubmitJobsStall += Other.AccumulatedTaskSubmitJobsStall;
        LocalJobBatchesSeen += Other.LocalJobBatchesSeen;
        TotalJobsReportedInLocalJobBatches += Other.TotalJobsReportedInLocalJobBatches;
        DistributedJobBatchesSeen += Other.DistributedJobBatchesSeen;
        TotalJobsReportedInDistributedJobBatches += Other.TotalJobsReportedInDistributedJobBatches;
        if (Other.MinShaderCodeSize > 0)
        {
            MinShaderCodeSize = (MinShaderCodeSize > 0 ? FMath::Min(MinShaderCodeSize, Other.MinShaderCodeSize) : Other.MinShaderCodeSize);
        }
        MaxShaderCodeSize = FMath::Max(Other.MaxShaderCodeSize, MaxShaderCodeSize);
        AccumulatedShaderCodeSize += Other.AccumulatedShaderCodeSize;
        NumAccumulatedShaderCodes += Other.NumAccumulatedShaderCodes;
        ShaderMapDDCMisses += Other.ShaderMapDDCMisses;
        ShaderMapDDCHits += Other.ShaderMapDDCHits;
        TotalCacheSearchAttempts += Other.TotalCacheSearchAttempts;
        TotalCacheHits += Other.TotalCacheHits;
        TotalCacheDuplicates += Other.TotalCacheDuplicates;
        TotalCacheDDCQueries += Other.TotalCacheDDCQueries;
        TotalCacheDDCHits += Other.TotalCacheDDCHits;
        UniqueCacheInputHashes += Other.UniqueCacheInputHashes;
        UniqueCacheOutputs += Other.UniqueCacheOutputs;
        CacheMemUsed += Other.CacheMemUsed;
        CacheMemBudget += Other.CacheMemBudget;
        MaxRemoteAgents += Other.MaxRemoteAgents;
        MaxActiveAgentCores += Other.MaxActiveAgentCores;
 
        return *this;
    }
};
 
USTRUCT()
struct FShaderCompilerMaterialCounters
{
    GENERATED_BODY()
    int32 NumMaterialsCooked = 0;
 
    UPROPERTY()
    int32 MaterialTranslateCalls = 0;
 
    UPROPERTY()
    double MaterialTranslateTotalTimeSec = 0.0;
 
    UPROPERTY()
    double MaterialTranslateTranslationOnlyTimeSec = 0.0;
 
    UPROPERTY()
    double MaterialTranslateSerializationOnlyTimeSec = 0.0;
 
    UPROPERTY()
    int32 MaterialCacheHits = 0;
 
    FShaderCompilerMaterialCounters& operator+=(const FShaderCompilerMaterialCounters& Other)
    {
        NumMaterialsCooked += Other.NumMaterialsCooked;
        MaterialTranslateCalls += Other.MaterialTranslateCalls;
        MaterialTranslateTotalTimeSec += Other.MaterialTranslateTotalTimeSec;
        MaterialTranslateTranslationOnlyTimeSec += Other.MaterialTranslateTranslationOnlyTimeSec;
        MaterialTranslateSerializationOnlyTimeSec += Other.MaterialTranslateSerializationOnlyTimeSec;
        MaterialCacheHits += Other.MaterialCacheHits;
 
        return *this;
    }
 
    void WriteStatSummary(const TCHAR* AggregatedSuffix);
    void GatherAnalytics(TArray<FAnalyticsEventAttribute>& Attributes);
};
 
USTRUCT()
struct FShaderTimings
{
    GENERATED_BODY()
 
    UPROPERTY()
    float MinCompileTime = 0.0f;
    UPROPERTY()
    float MaxCompileTime = 0.0f;
    UPROPERTY()
    float TotalCompileTime = 0.0f;
    UPROPERTY()
    float TotalPreprocessTime = 0.0f;
    UPROPERTY()
    int32 NumCompiled = 0;
    UPROPERTY()
    float AverageCompileTime = 0.0f;    // stored explicitly as an optimization
 
    FShaderTimings& operator+=(const FShaderTimings& Other)
    {
        MinCompileTime = FMath::Min(MinCompileTime, Other.MinCompileTime);
        MaxCompileTime = FMath::Max(MaxCompileTime, Other.MaxCompileTime);
        TotalCompileTime += Other.TotalCompileTime;
        TotalPreprocessTime += Other.TotalPreprocessTime;
        NumCompiled += Other.NumCompiled;
        if (NumCompiled)
        {
            AverageCompileTime = TotalCompileTime / static_cast<float>(NumCompiled);
        }
        return *this;
    }
};
 
USTRUCT()
struct FShaderCompilerSinglePermutationStat
{
    GENERATED_BODY()
 
    FShaderCompilerSinglePermutationStat()
        : PermutationHash(0)
        , Compiled(0)
        , Cooked(0)
        , CompiledDouble(0)
        , CookedDouble(0)
    {}
 
    UE_DEPRECATED(5.6, "Use constructor accepting a u64 permutationstring hash instead of a string")
    FShaderCompilerSinglePermutationStat(FString PermutationString, uint32 Compiled, uint32 Cooked)
        : Compiled(Compiled)
        , Cooked(Cooked)
        , CompiledDouble(0)
        , CookedDouble(0)
 
    {
        PermutationHash = GetPermutationHash(PermutationString);
    }
 
    FShaderCompilerSinglePermutationStat(uint64 PermutationHash, uint32 Compiled, uint32 Cooked)
        : PermutationHash(PermutationHash)
        , Compiled(Compiled)
        , Cooked(Cooked)
        , CompiledDouble(0)
        , CookedDouble(0)
 
    {}
 
    PRAGMA_DISABLE_DEPRECATION_WARNINGS
    // Explicitly-defaulted ctors & assignment operators are needed temporarily due to deprecation
    // of PermutationString field. Can be removed once the deprecation window for said field ends.
    FShaderCompilerSinglePermutationStat(FShaderCompilerSinglePermutationStat&&) = default;
    FShaderCompilerSinglePermutationStat(const FShaderCompilerSinglePermutationStat&) = default;
    FShaderCompilerSinglePermutationStat& operator=(FShaderCompilerSinglePermutationStat&&) = default;
    FShaderCompilerSinglePermutationStat& operator=(const FShaderCompilerSinglePermutationStat&) = default;
    PRAGMA_ENABLE_DEPRECATION_WARNINGS
 
    static uint64 GetPermutationHash(const FString& PermutationString)
    {
        return CityHash64(reinterpret_cast<const char*>(*PermutationString), PermutationString.Len() * sizeof(TCHAR));
    }
 
    UPROPERTY()
    uint64 PermutationHash;
 
    UE_DEPRECATED(5.6, "PermutationString is no longer stored due to memory overhead; use PermutationHash to uniquely identify permutation stats")
    UPROPERTY()
    uint32 PermutationString = 0u;
 
    UPROPERTY()
    uint32 Compiled = 0u;
 
    UPROPERTY()
    uint32 Cooked = 0u;
 
    UPROPERTY()
    uint32 CompiledDouble = 0u;
 
    UPROPERTY()
    uint32 CookedDouble = 0u;
};
 
USTRUCT()
struct FShaderStats
{
    GENERATED_BODY()
 
    UPROPERTY()
    TArray<FShaderCompilerSinglePermutationStat> PermutationCompilations;
 
    UPROPERTY()
    uint32 Compiled = 0;
 
    UPROPERTY()
    uint32 Cooked = 0;
 
    UPROPERTY()
    uint32 CompiledDouble = 0;
 
    UPROPERTY()
    uint32 CookedDouble = 0;
 
    UPROPERTY()
    float CompileTime = 0.f;
 
    FShaderStats& operator+=(const FShaderStats& Other)
    {
        if (Compiled)
        {
            CompiledDouble += Other.Compiled;
        }
        else
        {
            Compiled += Other.Compiled;
        }
 
        if (Cooked)
        {
            CookedDouble += Other.Cooked;
        }
        else
        {
            Cooked += Other.Cooked;
        }
 
        CompiledDouble += Other.CompiledDouble;
        CookedDouble += Other.CookedDouble;
        CompileTime += Other.CompileTime;
 
        PermutationCompilations.Append(Other.PermutationCompilations);
 
        return *this;
    }
};
 
struct FShaderCompilerStats
{
public:
    using ShaderCompilerStats = TMap<FString, FShaderStats>;
 
    void IncrementMaterialCook();
    void IncrementMaterialTranslated(double InTotalTime, double InTranslationOnlyTime, double InSerializeTime);
    void IncrementMaterialCacheHit();
 
    ENGINE_API void RegisterCookedShaders(uint32 NumCooked, float CompileTime, EShaderPlatform Platform, const FString MaterialPath, FString PermutationString = FString(""));
    ENGINE_API void RegisterCompiledShaders(uint32 NumPermutations, EShaderPlatform Platform, const FString MaterialPath, FString PermutationString = FString(""));
    const TSparseArray<ShaderCompilerStats>& GetShaderCompilerStats() { return CompileStats; }
    ENGINE_API void WriteStats(class FOutputDevice* Ar = nullptr);
    ENGINE_API void WriteStatSummary();
    ENGINE_API uint32 GetTotalShadersCompiled();
 
    ENGINE_API void Aggregate(FShaderCompilerStats& Other);
    ENGINE_API void WriteToCompactBinary(FCbWriter& Writer);
    ENGINE_API void ReadFromCompactBinary(FCbObjectView& Reader);
    ENGINE_API TSharedPtr<FJsonObject> ToJson();
    inline void SetMultiProcessAggregated() { bMultiProcessAggregated = true; }
 
    void AddDDCMiss(uint32 NumMisses);
    uint32 GetDDCMisses() const;
    void AddDDCHit(uint32 NumHits);
    uint32 GetDDCHits() const;
    double GetTimeShaderCompilationWasActive();
 
    enum class EExecutionType
    {
        Local,
        Distributed
    };
 
    struct FWorkerDiagnosticsInfo
    {
        FShaderCompileWorkerDiagnostics WorkerDiagnosticsOutput;
        FString BatchLabel;
        int32 BatchSize = 0;
        uint32 WorkerId = 0; // Worker process ID or 0 if it came from a distributed job
    };
 
    void RegisterLocalWorkerIdleTime(double IdleTime);
 
    void RegisterNewPendingJob(FShaderCommonCompileJob& InOutJob);
 
    void RegisterAssignedJob(FShaderCommonCompileJob& InOutJob);
 
    void RegisterFinishedJob(FShaderCommonCompileJob& InOutJob);
 
    void RegisterJobBatch(int32 NumJobs, EExecutionType ExecType);
 
    void RegisterDistributedBuildStats(const FDistributedBuildStats& InStats);
 
    void RegisterWorkerDiagnostics(const FShaderCompileWorkerDiagnostics& InDiagnostics, FString&& InBatchLabel, int32 InBatchSize, uint32 InWorkerId);
 
    ENGINE_API void GatherAnalytics(const FString& BaseName, TArray<FAnalyticsEventAttribute>& Attributes);
 
private:
    friend class FShaderJobCache;
    FCriticalSection CompileStatsLock;
    TSparseArray<ShaderCompilerStats> CompileStats;
 
    FShaderCompilerCounters Counters;
    FShaderCompilerMaterialCounters MaterialCounters;
 
    TArray<TInterval<double>> JobLifeTimeIntervals;
 
    TMap<FString, FShaderTimings> ShaderTimings;
 
    TArray<FWorkerDiagnosticsInfo> WorkerDiagnostics;
 
    bool bMultiProcessAggregated = false;
};
 
struct FShaderCompilerMemoryLimits
{
    uint64 CurrentLimit;
    uint64 BaseLimit;
};
 
class FShaderCompilingManager : IAssetCompilingManager
{
    friend class FShaderCompileThreadRunnableBase;
    friend class FShaderCompileThreadRunnable;
 
#if PLATFORM_WINDOWS
    friend class FShaderCompileXGEThreadRunnable_XmlInterface;
#endif // PLATFORM_WINDOWS
    friend class FShaderCompileDistributedThreadRunnable_Interface;
    friend class FShaderCompileFASTBuildThreadRunnable;
 
public:
    ENGINE_API static FName GetStaticAssetTypeName();
 
private:
    FName GetAssetTypeName() const override;
    FTextFormat GetAssetNameFormat() const override;
    TArrayView<FName> GetDependentTypeNames() const override;
    int32 GetNumRemainingAssets() const override;
    void ProcessAsyncTasks(bool bLimitExecutionTime = false) override;
    void ProcessAsyncTasks(const AssetCompilation::FProcessAsyncTaskParams& Params) override;
 
    // Thread shared properties: These variables can only be read from or written to when a lock on CompileQueueSection is obtained, since they are used by both threads.
 
    bool bCompilingDuringGame;
 
    TMap<int32, FPendingShaderMapCompileResultsPtr> ShaderMapJobs;
 
    int32 NumExternalJobs;
 
    void ReleaseJob(FShaderCommonCompileJobPtr& Job);
    void ReleaseJob(FShaderCommonCompileJob* Job);
 
    FCriticalSection CompileQueueSection;
 
    FShaderCompileJobCollection AllJobs;
 
    // Main thread state - These are only accessed on the main thread and used to track progress
 
    TMap<int32, FShaderMapFinalizeResults> PendingFinalizeShaderMaps;
 
    TArray<TUniquePtr<FShaderCompileThreadRunnableBase>> Threads;
 
    // Configuration properties - these are set only on initialization and can be read from either thread
 
    uint32 NumShaderCompilingThreads;
    uint32 NumShaderCompilingThreadsDuringGame;
    int32 MaxShaderJobBatchSize;
    int32 NumSingleThreadedRunsBeforeRetry;
    std::atomic<int32> NumDumpedShaderSources = 0;
    uint32 ProcessId;
    bool bAllowCompilingThroughWorkers;
    bool bAllowAsynchronousShaderCompiling;
    bool bUseOnlyDistributedCompilationThread = false;
    bool bPromptToRetryFailedShaderCompiles;
    bool bDebugBreakOnPromptToRetryShaderCompile = false;
    bool bLogJobCompletionTimes;
    float ProcessGameThreadTargetTime;
    FString ShaderBaseWorkingDirectory;
    FString AbsoluteShaderBaseWorkingDirectory;
    FString AbsoluteShaderDebugInfoDirectory;
    FString ShaderCompileWorkerName;
    int32 LastNumRemainingAssets = 0;
    FString WorkerCrashLogBaseDirectory;
 
    double WorkersBusyTime;
 
    uint64 SuppressedShaderPlatforms;
 
    bool bIsEngineLoopInitialized;
 
    IDistributedBuildController* BuildDistributionController = nullptr;
 
    bool bNoShaderCompilation;
 
    bool bAllowForIncompleteShaderMaps;
 
    TUniquePtr<FAsyncCompilationNotification> Notification;
 
    FDelegateHandle OutOfMemoryDelegateHandle;
 
#if WITH_EDITOR
    TMap<FString, FDelegateHandle> DirectoryWatcherHandles;
#endif // WITH_EDITOR
 
    void CalculateNumberOfCompilingThreads(int32 NumberOfCores, int32 NumberOfCoresIncludingHyperthreads);
 
    FProcHandle LaunchWorker(const FString& WorkingDirectory, uint32 InParentProcessId, uint32 ThreadId, const FString& WorkerInputFile, const FString& WorkerOutputFile, uint32* OutWorkerProcessId = nullptr);
 
    void BlockOnShaderMapCompletion(const TArray<int32>& ShaderMapIdsToFinishCompiling, TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps);
 
    void BlockOnAllShaderMapCompletion(TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps);
 
    void AddCompiledResults(TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps, int32 ShaderMapIdx, const FShaderMapFinalizeResults& Results);
 
    void ProcessCompiledShaderMaps(TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps, float TimeBudget);
 
    void ProcessCompiledNiagaraShaderMaps(TMap<int32, FShaderMapFinalizeResults>& CompiledShaderMaps, float TimeBudget);
 
    void PropagateMaterialChangesToPrimitives(TMap<TRefCountPtr<FMaterial>, TRefCountPtr<FMaterialShaderMap>>& MaterialsToUpdate);
 
    bool HandlePotentialRetry(TMap<int32, FShaderMapFinalizeResults>& CompletedShaderMaps);
    
    bool AllTargetPlatformSupportsRemoteShaderCompiling();
 
    void UpdateNumRemainingAssets();
 
    IDistributedBuildController* FindRemoteCompilerController() const;
 
    void ReportMemoryUsage();
 
    FShaderCompileThreadRunnableBase* LaunchShaderCompilingThread(TUniquePtr<FShaderCompileThreadRunnableBase>&& NewShaderCompilingThread, bool bDelayThreadExecution = false);
 
    FShaderCompileThreadRunnableBase* LaunchRemoteShaderCompilingThread(bool bDelayThreadExecution = false);
 
    FShaderCompileThreadRunnableBase* LaunchLocalShaderCompilingThread(bool bDelayThreadExecution = false);
 
    FShaderCompileThreadRunnableBase* FindShaderCompilingThread(EShaderCompilerWorkerType InWorkerType);
 
public:
    
    ENGINE_API FShaderCompilingManager();
    ENGINE_API ~FShaderCompilingManager();
 
    ENGINE_API void OnMachineResourcesChanged(int32 NumberOfCores, int32 NumberOfCoresIncludingHyperthreads);
 
    ENGINE_API void OnDistributedShaderCompilingChanged();
 
    ENGINE_API int32 GetNumPendingJobs() const;
    ENGINE_API int32 GetNumOutstandingJobs() const;
    
    ENGINE_API FShaderCompilerMemoryLimits GetCurrentMemoryLimits() const;
    ENGINE_API void SetTargetMemoryLimitInMiB(int32 NewLimit);
 
    bool ShouldDisplayCompilingNotification() const 
    { 
        // Heuristic based on the number of jobs outstanding
        return GetNumOutstandingJobs() > 80 || GetNumPendingJobs() > 80 || NumExternalJobs > 10;
    }
 
    bool AllowAsynchronousShaderCompiling() const 
    {
        return bAllowAsynchronousShaderCompiling;
    }
 
    bool IsCompiling() const
    {
        return GetNumOutstandingJobs() > 0 || HasShaderJobs() || GetNumPendingJobs() > 0 || NumExternalJobs > 0;
    }
 
    bool IsRemoteCompilingEnabled() const
    {
        return BuildDistributionController != nullptr;
    }
 
    bool IsExclusiveDistributedCompilingEnabled() const
    {
        return bUseOnlyDistributedCompilationThread;
    }
 
    bool IgnoreAllThrottling() const
    {
        return !bIsEngineLoopInitialized;
    }
 
    bool HasShaderJobs() const
    {
        return ShaderMapJobs.Num() > 0 || PendingFinalizeShaderMaps.Num() > 0;
    }
 
    int32 GetNumRemainingJobs() const
    {
        return GetNumOutstandingJobs() + NumExternalJobs;
    }
 
    int32 GetNumLocalWorkers() const
    {
        return bCompilingDuringGame ? NumShaderCompilingThreadsDuringGame : NumShaderCompilingThreads;
    }
 
    void SetExternalJobs(int32 NumJobs)
    {
        NumExternalJobs = NumJobs;
    }
 
    enum class EDumpShaderDebugInfo : int32
    {
        Never               = 0,
        Always              = 1,
        OnError             = 2,
        OnErrorOrWarning    = 3
    };
 
    ENGINE_API EDumpShaderDebugInfo GetDumpShaderDebugInfo() const;
    ENGINE_API EShaderDebugInfoFlags GetDumpShaderDebugInfoFlags() const;
    ENGINE_API FString CreateShaderDebugInfoPath(const FShaderCompilerInput& ShaderCompilerInput) const;
    ENGINE_API bool ShouldRecompileToDumpShaderDebugInfo(const FShaderCompileJob& Job) const;
    ENGINE_API bool ShouldRecompileToDumpShaderDebugInfo(const FShaderCompilerInput& Input, const FShaderCompilerOutput& Output, bool bSucceeded) const;
 
 
    void IncrementNumDumpedShaderSources()
    {
        NumDumpedShaderSources++;
    }
 
    const FString& GetAbsoluteShaderDebugInfoDirectory() const
    {
        return GetShaderDebugInfoPath();
    }
 
    bool AreWarningsSuppressed(const EShaderPlatform Platform) const
    {
        return (SuppressedShaderPlatforms & (static_cast<uint64>(1) << Platform)) != 0;
    }
 
    void SuppressWarnings(const EShaderPlatform Platform)
    {
        SuppressedShaderPlatforms |= static_cast<uint64>(1) << Platform;
    }
 
    bool IsShaderCompilationSkipped() const
    {
        return bNoShaderCompilation;
    }
 
    void SkipShaderCompilation(bool toggle)
    {
        if (AllowShaderCompiling())
        {
            bNoShaderCompilation = toggle;
        }
    }
 
    void SetAllowForIncompleteShaderMaps(bool toggle)
    {
        bAllowForIncompleteShaderMaps = toggle;
    }
 
    ENGINE_API bool IsCompilingShaderMap(uint32 Id);
 
    ENGINE_API FShaderCompileJob* PrepareShaderCompileJob(uint32 Id, const FShaderCompileJobKey& Key, EShaderCompileJobPriority Priority);
    ENGINE_API FShaderPipelineCompileJob* PreparePipelineCompileJob(uint32 Id, const FShaderPipelineCompileJobKey& Key, EShaderCompileJobPriority Priority);
 
 
    UE_DEPRECATED(5.6, "ProcessFinishedJob must now be passed an EShaderCompileJobStatus")
    void ProcessFinishedJob(FShaderCommonCompileJob* FinishedJob) {}
 
    ENGINE_API void ProcessFinishedJob(FShaderCommonCompileJob* FinishedJob, EShaderCompileJobStatus Status);
 
    ENGINE_API void SubmitJobs(TArray<FShaderCommonCompileJobPtr>& NewJobs, const FString MaterialBasePath, FString PermutationString = FString(""));
 
    ENGINE_API void CancelCompilation(const TCHAR* MaterialName, const TArray<int32>& ShaderMapIdsToCancel);
 
    ENGINE_API void FinishCompilation(const TCHAR* MaterialName, const TArray<int32>& ShaderMapIdsToFinishCompiling);
 
    ENGINE_API void FinishAllCompilation() override;
 
    ENGINE_API void Shutdown() override;
 
    ENGINE_API void PrintStats();
 
    ENGINE_API void GetLocalStats(FShaderCompilerStats & OutStats) const;
 
    ENGINE_API FShaderCompileMemoryUsage GetExternalMemoryUsage();
 
    ENGINE_API void ProcessAsyncResults(float TimeSlice, bool bBlockOnGlobalShaderCompletion);
 
    ENGINE_API void ProcessAsyncResults(bool bLimitExecutionTime, bool bBlockOnGlobalShaderCompletion);
 
    ENGINE_API void DumpDebugInfoForEachPendingJob(const FShaderCompileJobCallback& PendingJobCallback) const;
 
    static bool IsShaderCompilerWorkerRunning(FProcHandle & WorkerHandle);
};
 
extern ENGINE_API FShaderCompilingManager* GShaderCompilingManager;
 
extern ENGINE_API FShaderCompilerStats* GShaderCompilerStats;
 
#if WITH_EDITOR
 
extern ENGINE_API void GlobalBeginCompileShader(
    const FString& DebugGroupName,
    const class FVertexFactoryType* VFType,
    const class FShaderType* ShaderType,
    const class FShaderPipelineType* ShaderPipelineType,
    int32 PermutationId,
    const TCHAR* SourceFilename,
    const TCHAR* FunctionName,
    FShaderTarget Target,
    FShaderCompilerInput& Input,
    bool bAllowDevelopmentShaderCompile,
    const FString& DebugDescription,
    const FString& DebugExtension
    );
 
extern ENGINE_API void GlobalBeginCompileShader(
    const FString& DebugGroupName,
    const class FVertexFactoryType* VFType,
    const class FShaderType* ShaderType,
    const class FShaderPipelineType* ShaderPipelineType,
    int32 PermutationId,
    const TCHAR* SourceFilename,
    const TCHAR* FunctionName,
    FShaderTarget Target,
    FShaderCompilerInput& Input,
    bool bAllowDevelopmentShaderCompile = true,
    const TCHAR* DebugDescription = nullptr,
    const TCHAR* DebugExtension = nullptr
);
 
#endif // WITH_EDITOR
 
extern void GetOutdatedShaderTypes(TArray<const FShaderType*>& OutdatedShaderTypes, TArray<const FShaderPipelineType*>& OutdatedShaderPipelineTypes, TArray<const FVertexFactoryType*>& OutdatedFactoryTypes);
 
extern bool RecompileShaders(const TCHAR* Cmd, FOutputDevice& Ar);
 
extern ENGINE_API bool IsGlobalShaderMapComplete(const TCHAR* TypeNameSubstring = nullptr);
 
#if WITH_EDITORONLY_DATA
DECLARE_MULTICAST_DELEGATE(FOnGlobalShadersCompilation);
extern ENGINE_API FOnGlobalShadersCompilation& GetOnGlobalShaderCompilation();
#endif // WITH_EDITORONLY_DATA
 
extern ENGINE_API void VerifyGlobalShaders(EShaderPlatform Platform, bool bLoadedFromCacheFile, const TArray<const FShaderType*>* OutdatedShaderTypes = nullptr, const TArray<const FShaderPipelineType*>* OutdatedShaderPipelineTypes = nullptr);
extern ENGINE_API void VerifyGlobalShaders(EShaderPlatform Platform, const ITargetPlatform* TargetPlatform, bool bLoadedFromCacheFile, const TArray<const FShaderType*>* OutdatedShaderTypes = nullptr, const TArray<const FShaderPipelineType*>* OutdatedShaderPipelineTypes = nullptr, const FShaderCompilerFlags& InExtraCompilerFlags = {});
 
extern ENGINE_API void PrecacheComputePipelineStatesForGlobalShaders(ERHIFeatureLevel::Type FeatureLevel, const ITargetPlatform* TargetPlatform);
 
extern ENGINE_API void RecompileGlobalShaders();
 
extern ENGINE_API bool RecompileChangedShadersForPlatform(const FString& PlatformName);
 
extern ENGINE_API void BeginRecompileGlobalShaders(const TArray<const FShaderType*>& OutdatedShaderTypes, const TArray<const FShaderPipelineType*>& OutdatedShaderPipelineTypes, EShaderPlatform ShaderPlatform, const ITargetPlatform* TargetPlatform = nullptr, const FShaderCompilerFlags& InExtraCompilerFlags = {});
 
extern ENGINE_API void FinishRecompileGlobalShaders();
 
#if WITH_EDITOR
extern ENGINE_API void ProcessCompiledGlobalShaders(const TArray<FShaderCommonCompileJobPtr>& CompilationResults);
 
extern ENGINE_API void SaveGlobalShadersForRemoteRecompile(FArchive& Ar, EShaderPlatform ShaderPlatform);
#endif // WITH_EDITOR
 
extern ENGINE_API void LoadGlobalShadersForRemoteRecompile(FArchive& Ar, EShaderPlatform ShaderPlatform);
 
extern ENGINE_API FString SaveGlobalShaderFile(EShaderPlatform Platform, FString SavePath, class ITargetPlatform* TargetPlatform = nullptr);
 
struct FODSCRequestPayload
{
    EShaderPlatform ShaderPlatform;
 
    ERHIFeatureLevel::Type FeatureLevel;
 
    EMaterialQualityLevel::Type QualityLevel;
 
    FString MaterialName;
 
    FString VertexFactoryName;
 
    FString PipelineName;
 
    TArray<FString> ShaderTypeNames;
 
    int32 PermutationId;
 
    FString RequestHash;
 
    FODSCRequestPayload() {};
    ENGINE_API FODSCRequestPayload(
        EShaderPlatform InShaderPlatform,
        ERHIFeatureLevel::Type InFeatureLevel,
        EMaterialQualityLevel::Type InQualityLevel,
        const FString& InMaterialName,
        const FString& InVertexFactoryName,
        const FString& InPipelineName,
        const TArray<FString>& InShaderTypeNames,
        int32 InPermutationId,
        const FString& InRequestHash
    );
 
    ENGINE_API friend FArchive& operator<<(FArchive& Ar, FODSCRequestPayload& Elem);
};
 
enum class ODSCRecompileCommand
{
    None,
    Changed,
    Global,
    Material,
    SingleShader,
    ResetMaterialCache
};
 
extern ENGINE_API const TCHAR* ODSCCmdEnumToString(ODSCRecompileCommand Cmd);
 
struct FShaderRecompileData
{
    FString PlatformName;
 
    EShaderPlatform ShaderPlatform = SP_NumPlatforms;
 
    ERHIFeatureLevel::Type FeatureLevel = ERHIFeatureLevel::SM5;
 
    EMaterialQualityLevel::Type QualityLevel = EMaterialQualityLevel::High;
 
    FShaderCompilerFlags ExtraCompilerFlags = FShaderCompilerFlags();
 
    TArray<FString>* ModifiedFiles = nullptr;
 
    TArray<uint8>* MeshMaterialMaps = nullptr;
 
    TArray<FString> MaterialsToLoad;
 
    FString ShaderTypesToLoad;
 
    ODSCRecompileCommand CommandType = ODSCRecompileCommand::Changed;
 
    TArray<uint8>* GlobalShaderMap = nullptr;
 
    TArray<FODSCRequestPayload> ShadersToRecompile;
 
    TArray<TStrongObjectPtr<UMaterialInterface>>* LoadedMaterialsToRecompile = nullptr;
 
#if WITH_EDITOR
    TFunction<UMaterialInterface*(const FString&)> ODSCCustomLoadMaterial;
#endif
 
    FShaderRecompileData() {};
 
    ENGINE_API FShaderRecompileData(const FString& InPlatformName, TArray<FString>* OutModifiedFiles, TArray<uint8>* OutMeshMaterialMaps, TArray<uint8>* OutGlobalShaderMap);
 
    ENGINE_API FShaderRecompileData(const FString& InPlatformName, EShaderPlatform InShaderPlatform, ODSCRecompileCommand InCommandType, TArray<FString>* OutModifiedFiles, TArray<uint8>* OutMeshMaterialMaps, TArray<uint8>* OutGlobalShaderMap);
 
    ENGINE_API friend FArchive& operator<<(FArchive& Ar, FShaderRecompileData& Elem);
};
 
#if WITH_EDITOR
 
extern ENGINE_API void RecompileShadersForRemote(FShaderRecompileData& Args, const FString& OutputDirectory);
 
extern ENGINE_API void ShutdownShaderCompilers(TConstArrayView<const ITargetPlatform*> TargetPlatforms);
 
#endif // WITH_EDITOR
 
extern ENGINE_API void CompileGlobalShaderMap(bool bRefreshShaderMap=false);
extern ENGINE_API void CompileGlobalShaderMap(ERHIFeatureLevel::Type InFeatureLevel, bool bRefreshShaderMap=false);
extern ENGINE_API void CompileGlobalShaderMap(EShaderPlatform Platform, bool bRefreshShaderMap = false);
extern ENGINE_API void CompileGlobalShaderMap(EShaderPlatform Platform, const ITargetPlatform* TargetPlatform, bool bRefreshShaderMap);
extern ENGINE_API void ShutdownGlobalShaderMap();
 
extern ENGINE_API const FGuid& GetGlobalShaderMapDDCGuid();
extern ENGINE_API const FGuid& GetMaterialShaderMapDDCGuid();
 
extern ENGINE_API bool ShouldDumpShaderDDCKeys();
extern ENGINE_API void DumpShaderDDCKeyToFile(const EShaderPlatform InPlatform, bool bWithEditor, const TCHAR* DebugGroupName, const FString& DDCKey);
 
extern ENGINE_API void ProcessCookOnTheFlyShaders(bool bReloadGlobalShaders, const TArray<uint8>& MeshMaterialMaps, const TArray<FString>& MaterialsToLoad, const TArray<uint8>& GlobalShaderMap);