GPUScene.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "RenderResource.h"
#include "RendererInterface.h"
#include "PrimitiveUniformShaderParameters.h"
#include "PrimitiveSceneInfo.h"
#include "SpanAllocator.h"
#include "GrowOnlySpanAllocator.h"
#include "InstanceCulling/InstanceCullingLoadBalancer.h"
#include "MeshBatch.h"
#include "LightSceneData.h"
#include "SceneUniformBuffer.h"
#include "UnifiedBuffer.h"
#include "ScenePrimitiveUpdates.h"
#include "GPUSceneWriter.h"
#include "RendererPrivateUtils.h"
 
class FRDGExternalAccessQueue;
class FRHICommandList;
class FScene;
class FViewInfo;
class FLightSceneInfoCompact;
class FGPUScene;
class FGPUSceneDynamicContext;
class FViewUniformShaderParameters;
class FInstanceCullingOcclusionQueryRenderer;
class FScenePreUpdateChangeSet;
class FScenePostUpdateChangeSet;
class FPrimitiveSceneProxy;
struct FLightSceneChangeSet;
class FGPUSceneWriterParameters;
 
namespace UE::Renderer::Private
{
    class IShadowInvalidatingInstances;
}
 
DECLARE_GPU_STAT_NAMED_EXTERN(GPUSceneUpdate, TEXT("GPUSceneUpdate"));
 
#if !ENABLE_SCENE_DATA_DX11_UB_ERROR_WORKAROUND
// Note: this is declared in GPUSceneWriter.h (a public header) until such a time as the UB path works.
// Parameter sub struct that is common to all parameter use-cases
BEGIN_SHADER_PARAMETER_STRUCT(FGPUSceneCommonParameters, RENDERER_API)
    SHADER_PARAMETER(uint32, GPUSceneInstanceDataSOAStride)
    SHADER_PARAMETER(uint32, GPUSceneFrameNumber)
    SHADER_PARAMETER(int32, GPUSceneMaxAllocatedInstanceId)
    SHADER_PARAMETER(int32, GPUSceneMaxPersistentPrimitiveIndex)
END_SHADER_PARAMETER_STRUCT()
 
#endif
 
BEGIN_SHADER_PARAMETER_STRUCT(FGPUSceneResourceParameters, RENDERER_API)
    SHADER_PARAMETER_STRUCT_INCLUDE(FGPUSceneCommonParameters, CommonParameters)
    SHADER_PARAMETER_RDG_BUFFER_SRV(StructuredBuffer<float4>, GPUSceneInstanceSceneData)
    SHADER_PARAMETER_RDG_BUFFER_SRV(StructuredBuffer<float4>, GPUSceneInstancePayloadData)
    SHADER_PARAMETER_RDG_BUFFER_SRV(StructuredBuffer<float4>, GPUScenePrimitiveSceneData)
    SHADER_PARAMETER_RDG_BUFFER_SRV(StructuredBuffer<float4>, GPUSceneLightmapData)
    SHADER_PARAMETER_RDG_BUFFER_SRV(ByteAddressBuffer, GPUSceneLightData)
END_SHADER_PARAMETER_STRUCT()
 
//Non-RDG parameters, untracked, use with care (or preferably, don't)
BEGIN_SHADER_PARAMETER_STRUCT(FGPUSceneResourceParametersRHI, RENDERER_API)
    SHADER_PARAMETER_STRUCT_INCLUDE(FGPUSceneCommonParameters, CommonParameters)
    SHADER_PARAMETER_SRV(StructuredBuffer<float4>, GPUSceneInstanceSceneData)
    SHADER_PARAMETER_SRV(StructuredBuffer<float4>, GPUSceneInstancePayloadData)
    SHADER_PARAMETER_SRV(StructuredBuffer<float4>, GPUScenePrimitiveSceneData)
    SHADER_PARAMETER_SRV(StructuredBuffer<float4>, GPUSceneLightmapData)
    SHADER_PARAMETER_SRV(ByteAddressBuffer, GPUSceneLightData)
END_SHADER_PARAMETER_STRUCT()
 
DECLARE_SCENE_UB_STRUCT(FGPUSceneResourceParameters, GPUScene, RENDERER_API)
 
class FGPUScenePrimitiveCollector
{
public:
    FGPUScenePrimitiveCollector(FGPUSceneDynamicContext* InGPUSceneDynamicContext = nullptr) :
        GPUSceneDynamicContext(InGPUSceneDynamicContext)
    {};
 
    virtual void Add(
        const FMeshBatchDynamicPrimitiveData* MeshBatchData,
        const FPrimitiveUniformShaderParameters& PrimitiveShaderParams,
        uint32 NumInstances,
        uint32& OutPrimitiveIndex,
        uint32& OutInstanceSceneDataOffset);
 
    RENDERER_API void Commit();
 
    FORCEINLINE const TRange<int32>& GetPrimitiveIdRange() const
    {
        check(bCommitted || UploadData == nullptr);
        return PrimitiveIdRange;
    }
 
    FORCEINLINE int32 GetInstanceSceneDataOffset() const
    {
        check(bCommitted || UploadData == nullptr);
 
        return UploadData != nullptr ? UploadData->InstanceSceneDataOffset : 0;
    }
 
    FORCEINLINE int32 GetInstancePayloadDataOffset() const
    {
        check(bCommitted || UploadData == nullptr);
 
        return UploadData != nullptr ? UploadData->InstancePayloadDataOffset : 0;
    }
 
    int32 Num() const { return UploadData != nullptr ? UploadData->PrimitiveData.Num() : 0; }
    int32 NumInstances() const { return UploadData != nullptr ? UploadData->TotalInstanceCount : 0; }
    int32 NumPayloadDataSlots() const { return UploadData != nullptr ? UploadData->InstancePayloadDataFloat4Count : 0; }
    const FPrimitiveUniformShaderParameters* GetPrimitiveShaderParameters(int32 PrimitiveId) const;
 
#if DO_CHECK
    void CheckPrimitiveProcessed(uint32 PrimitiveIndex, const FGPUScene& GPUScene) const;
#endif // DO_CHECK
 
private:
 
    friend class FGPUScene;
    friend class FGPUSceneDynamicContext;
    friend struct FUploadDataSourceAdapterDynamicPrimitives;
 
    struct FPrimitiveData
    {
        FMeshBatchDynamicPrimitiveData SourceData;
        const FPrimitiveUniformShaderParameters* ShaderParams = nullptr;
        uint32 NumInstances = 0;
        uint32 LocalInstanceSceneDataOffset = INDEX_NONE;
        uint32 LocalPayloadDataOffset = INDEX_NONE;
    };
 
    struct FUploadData
    {
        TArray<FPrimitiveData, TInlineAllocator<8>> PrimitiveData;
        TArray<uint32> GPUWritePrimitives;
 
        uint32 InstanceSceneDataOffset = INDEX_NONE;
        uint32 TotalInstanceCount = 0;
        uint32 InstancePayloadDataOffset = INDEX_NONE;
        uint32 InstancePayloadDataFloat4Count = 0;
        bool bIsUploaded = false;
    };
 
    FUploadData* AllocateUploadData();
 
    TRange<int32> PrimitiveIdRange = TRange<int32>::Empty();
    FUploadData* UploadData = nullptr; // Owned by FGPUSceneDynamicContext
    bool bCommitted = false;
    FGPUSceneDynamicContext* GPUSceneDynamicContext = nullptr;
};
 
// TODO: move to own header
class FInstanceProcessingGPULoadBalancer;
 
class FGPUSceneDynamicContext
{
public:
    FGPUSceneDynamicContext(FGPUScene& InGPUScene) : GPUScene(InGPUScene) {}
    ~FGPUSceneDynamicContext();
 
    void Release();
 
private:
    friend class FGPUScene;
    friend class FGPUScenePrimitiveCollector;
 
    FGPUScenePrimitiveCollector::FUploadData* AllocateDynamicPrimitiveData();
    UE::FMutex DymamicPrimitiveUploadDataMutex;
    TArray<FGPUScenePrimitiveCollector::FUploadData*, TInlineAllocator<128, SceneRenderingAllocator> > DymamicPrimitiveUploadData;
    FGPUScene& GPUScene;
};
 
struct FGPUSceneInstanceRange
{
    FGPUSceneInstanceRange(FPersistentPrimitiveIndex InPrimitive, uint32 InInstanceSceneDataOffset, uint32 InNumInstanceSceneDataEntries)
        : Primitive(InPrimitive)
        , InstanceSceneDataOffset(InInstanceSceneDataOffset)
        , NumInstanceSceneDataEntries(InNumInstanceSceneDataEntries)
    {}
 
    FPersistentPrimitiveIndex Primitive;
    uint32 InstanceSceneDataOffset;
    uint32 NumInstanceSceneDataEntries;
};
 
class FGPUScene
{
public:
    FGPUScene(FScene &InScene);
    ~FGPUScene();
 
    using FUpdateFromComputeCommands = FScenePrimitiveUpdates::TPayloadRangeView<FUpdateInstanceFromComputeCommand>;
 
    void SetEnabled(ERHIFeatureLevel::Type InFeatureLevel);
    bool IsEnabled() const { return bIsEnabled; }
    void BeginRender(FRDGBuilder& GraphBuilder, FGPUSceneDynamicContext &GPUSceneDynamicContext);
    inline bool IsRendering() const { return bInBeginEndBlock; }
    void EndRender();
 
    ERHIFeatureLevel::Type GetFeatureLevel() const { return FeatureLevel; }
    EShaderPlatform GetShaderPlatform() const { return GShaderPlatformForFeatureLevel[FeatureLevel]; }
 
    int32 AllocateInstanceSceneDataSlots(FPersistentPrimitiveIndex PersistentPrimitiveIndex, int32 NumInstanceSceneDataEntries);
    
    void FreeInstanceSceneDataSlots(int32 InstanceSceneDataOffset, int32 NumInstanceSceneDataEntries);
 
    int32 AllocateInstancePayloadDataSlots(int32 NumInstancePayloadFloat4Entries);
    void FreeInstancePayloadDataSlots(int32 InstancePayloadDataOffset, int32 NumInstancePayloadFloat4Entries);
 
    void UploadDynamicPrimitiveShaderDataForView(FRDGBuilder& GraphBuilder, FViewInfo& View, bool bRayTracing = false, UE::Renderer::Private::IShadowInvalidatingInstances *ShadowInvalidatingInstances = nullptr);
 
    bool FillSceneUniformBuffer(FRDGBuilder& GraphBuilder, FSceneUniformBuffer& SceneUB) const;
 
    void Update(FRDGBuilder& GraphBuilder, FSceneUniformBuffer& SceneUB, FRDGExternalAccessQueue& ExternalAccessQueue, const FUpdateFromComputeCommands& UpdatesFromCompute, const UE::Tasks::FTask& UpdateTaskPrerequisites = {});
 
    void RENDERER_API AddPrimitiveToUpdate(FPersistentPrimitiveIndex PersistentPrimitiveIndex, EPrimitiveDirtyState DirtyState = EPrimitiveDirtyState::ChangedAll);
 
    FORCEINLINE EPrimitiveDirtyState GetPrimitiveDirtyState(FPersistentPrimitiveIndex PersistentPrimitiveIndex) const 
    { 
        if (!PrimitiveDirtyState.IsValidIndex(PersistentPrimitiveIndex.Index))
        {
            return EPrimitiveDirtyState::None;
        }
        return PrimitiveDirtyState[PersistentPrimitiveIndex.Index]; 
    }
 
    uint32 GetSceneFrameNumber() const { return SceneFrameNumber; }
 
    int32 GetNumInstances() const { return InstanceSceneDataAllocator.GetMaxSize(); }
    int32 GetNumPrimitives() const { return DynamicPrimitivesOffset; }
    int32 GetMaxLightId() const { return NumGPULights; }
    int32 GetNumLightmapDataItems() const { return LightmapDataAllocator.GetMaxSize(); }
    FGPUSceneDynamicContext* GetCurrentDynamicContext() const { return CurrentDynamicContext; }
    uint32 GetInstanceIdUpperBoundGPU() const;
 
    const FSpanAllocator& GetInstanceSceneDataAllocator() const { return InstanceSceneDataAllocator; }
 
    FGPUSceneResourceParameters GetShaderParameters(FRDGBuilder& GraphBuilder) const;
 
    void DebugRender(FRDGBuilder& GraphBuilder, FSceneUniformBuffer& SceneUniformBuffer, FViewInfo& View);
 
    void ConsolidateInstanceDataAllocations();
 
    bool ExecuteDeferredGPUWritePass(FRDGBuilder& GraphBuilder, TArray<FViewInfo>& Views, EGPUSceneGPUWritePass Pass);
 
    bool HasPendingGPUWrite(uint32 PrimitiveId) const;
 
    void OnPreSceneUpdate(FRDGBuilder& GraphBuilder, const FScenePreUpdateChangeSet& ScenePreUpdateData);
 
    void OnPostSceneUpdate(FRDGBuilder& GraphBuilder, const FScenePostUpdateChangeSet& ScenePostUpdateData);
 
    void OnPostLightSceneInfoUpdate(FRDGBuilder& OnPostLightSceneInfoUpdate, const FLightSceneChangeSet& LightsPostUpdateData);
 
    bool bUpdateAllPrimitives;
 
    TRefCountPtr<FRDGPooledBuffer> PrimitiveBuffer;
    FRDGAsyncScatterUploadBuffer   PrimitiveUploadBuffer;
 
    TRefCountPtr<FRDGPooledBuffer> InstanceSceneDataBuffer;
    FRDGAsyncScatterUploadBuffer   InstanceSceneUploadBuffer;
 
    FSpanAllocator                 InstancePayloadDataAllocator;
    TRefCountPtr<FRDGPooledBuffer> InstancePayloadDataBuffer;
    FRDGAsyncScatterUploadBuffer   InstancePayloadUploadBuffer;
 
    FSpanAllocator                 LightmapDataAllocator;
    TRefCountPtr<FRDGPooledBuffer> LightmapDataBuffer;
    FRDGAsyncScatterUploadBuffer   LightmapUploadBuffer;
 
    using FInstanceRange = FGPUSceneInstanceRange;
    using FInstanceGPULoadBalancer = TInstanceCullingLoadBalancer<SceneRenderingAllocator>;
 
    inline const FScene &GetScene() const { return Scene; }
 
    SIZE_T GetAllocatedSize() const;
 
    FGPUSceneResourceParametersRHI GetShaderParametersRHI() const;
 
private:
    TRDGUniformBufferRef<FGPUSceneWriterUniformParameters> GetWriteParameters(FRDGBuilder& GraphBuilder, FGPUSceneWriterParameters* LegacyParameters = nullptr, bool bSkipBarriers = true);
 
    // set to true if the requested (cvar) tile size is >0, otherwise the tile size is re-derived from the instance count
    bool bUseTiledInstanceDataLayout = true;
 
    // log2(TileSize) in number of instances, thus the tile-size in memory is #float4s per instance * tilesize
    int32 InstanceDataTileSizeLog2 = -1;
 
    uint32 GetInstanceDataTileSizeLog2() const { return InstanceDataTileSizeLog2; }
    uint32 GetInstanceDataTileSize() const { return 1u << InstanceDataTileSizeLog2; }
    uint32 GetInstanceDataTileSizeMask() const;
    uint32 GetInstanceDataTileStride() const;
 
    static constexpr int32 InitialBufferSize = 256;
 
    TPersistentByteAddressBuffer<FLightSceneData> LightDataBuffer;
    int32 NumGPULights = 0;
    UE::Tasks::FTask LightDataSetupTask;
 
#if DO_CHECK
    TArray<FLightSceneData> LightDataArray;
#endif
 
    FGPUSceneCommonParameters CommonParameters;
 
    FGPUSceneCommonParameters GetCommonParameters() const;
 
 
    // Buffers used by GPU-Scene, since they can be resized during updates AND the render passes must retain the 
    // right copy (this is chiefly because the init of shadow views after pre-pass means we need to be able to set 
    // up GPU-Scene before pre-pass, but then may discover new primitives etc. As there is no way to know how many
    // dynamic primitives will turn up after Pre-pass, we can't guarantee a resize won't happen).
    struct FRegisteredBuffers
    {
        bool IsValid() const { return PrimitiveBuffer != nullptr; }
 
        FRDGBuffer* PrimitiveBuffer = nullptr;
        FRDGBuffer* InstanceSceneDataBuffer = nullptr;
        FRDGBuffer* InstancePayloadDataBuffer = nullptr;
        FRDGBuffer* LightmapDataBuffer = nullptr;
        FRDGBuffer* LightDataBuffer = nullptr;
    };
 
 
 
    FScene &Scene;
    FSpanAllocator                 InstanceSceneDataAllocator;
 
    FORCEINLINE void ResizeDirtyState(int32 NewSizeIn)
    {
        if (IsEnabled() && NewSizeIn > PrimitiveDirtyState.Num())
        {
            const int32 NewSize = Align(NewSizeIn, 64);
            static_assert(static_cast<uint32>(EPrimitiveDirtyState::None) == 0U, "Using AddZeroed to ensure efficent add, requires None == 0");
            PrimitiveDirtyState.AddZeroed(NewSize - PrimitiveDirtyState.Num());
        }
    }
 
    TArray<FPersistentPrimitiveIndex> PrimitivesToUpdate;
 
    FRegisteredBuffers CachedRegisteredBuffers;
 
    TArray<EPrimitiveDirtyState> PrimitiveDirtyState;
 
    TArray<FInstanceRange> InstanceRangesToClear;
 
    struct FDeferredGPUWrite
    {
        FGPUSceneWriteDelegateRef DataWriterGPU;
        int32 ViewId = INDEX_NONE;
        uint32 PrimitiveId = INDEX_NONE;
        uint32 InstanceSceneDataOffset = INDEX_NONE;
        uint32 NumCustomDataFloats = INDEX_NONE;
        uint32 PackedInstanceSceneDataFlags = 0u;
    };
 
    static constexpr uint32 NumDeferredGPUWritePasses = uint32(EGPUSceneGPUWritePass::Num);
    TArray<FDeferredGPUWrite> DeferredGPUWritePassDelegates[NumDeferredGPUWritePasses];
    EGPUSceneGPUWritePass LastDeferredGPUWritePass = EGPUSceneGPUWritePass::None;
 
    TRange<int32> CommitPrimitiveCollector(FGPUScenePrimitiveCollector& PrimitiveCollector);
 
 
    friend class FGPUScenePrimitiveCollector;
 
    uint32 SceneFrameNumber = 0xFFFFFFFF;
 
    int32 DynamicPrimitivesOffset = 0;
    
    int32 PrevDynamicPrimitivesMaxSize = 0;
    int32 PrevDynamicPrimitivePayloadMaxSize = 0;
    int32 PrevDynamicPrimitiveInstanceMaxSize = 0;
 
    bool bIsEnabled = false;
    bool bInBeginEndBlock = false;
    FGPUSceneDynamicContext* CurrentDynamicContext = nullptr;
 
    ERHIFeatureLevel::Type FeatureLevel;
 
    FRegisteredBuffers UpdateBufferAllocations(FRDGBuilder& GraphBuilder, FSceneUniformBuffer& SceneUB);
 
    FRegisteredBuffers RegisterBuffers(FRDGBuilder& GraphBuilder) const;
 
    template<typename FUploadDataSourceAdapter>
    void UploadGeneral(FRDGBuilder& GraphBuilder, const FRegisteredBuffers& BufferState, FRDGExternalAccessQueue* ExternalAccessQueue, const FUploadDataSourceAdapter& UploadDataSourceAdapter, const UE::Tasks::FTask& PrerequisiteTask);
 
    void UpdateGPULights(FRDGBuilder& GraphBuilder, const UE::Tasks::FTask& PrerequisiteTask, const FLightSceneChangeSet& LightsPostUpdateData);
 
    static void InitLightData(const FLightSceneInfoCompact& LightInfoCompact, bool bAllowStaticLighting, uint32 LightShaderParameterFlags, FLightSceneData& DataOut);
 
    void UploadDynamicPrimitiveShaderDataForViewInternal(FRDGBuilder& GraphBuilder, FViewInfo& View, bool bRayTracing, UE::Renderer::Private::IShadowInvalidatingInstances *ShadowInvalidatingInstances);
 
    void UpdateInternal(FRDGBuilder& GraphBuilder, FSceneUniformBuffer& SceneUB, FRDGExternalAccessQueue& ExternalAccessQueue, const UE::Tasks::FTask& UpdateTaskPrerequisites, const FUpdateFromComputeCommands& UpdatesFromCompute);
 
    void AddUpdatePrimitiveIdsPass(FRDGBuilder& GraphBuilder, FInstanceGPULoadBalancer& IdOnlyUpdateItems);
 
    void AddClearInstancesPass(FRDGBuilder& GraphBuilder, FInstanceCullingOcclusionQueryRenderer* OcclusionQueryRenderer = nullptr);
 
 
#if !UE_BUILD_SHIPPING
    FDelegateHandle ScreenMessageDelegate;
    bool bLoggedInstanceOverflow = false;
    uint32 MaxInstancesDuringPrevUpdate = 0;
    bool bLoggedLightOverflow = false;
    int32 MaxLightsDuringPrevUpdate = 0;
    bool bLoggedPayloadBufferOverflow = false;
#endif // UE_BUILD_SHIPPING
    uint64 InstancePayloadDataSizeReserveUnclamped = 0;
};
 
class FGPUSceneScopeBeginEndHelper
{
public:
    FGPUSceneScopeBeginEndHelper(FRDGBuilder& GraphBuilder, FGPUScene& InGPUScene, FGPUSceneDynamicContext &GPUSceneDynamicContext) :
        GPUScene(InGPUScene)
    {
        GPUScene.BeginRender(GraphBuilder, GPUSceneDynamicContext);
    }
 
    ~FGPUSceneScopeBeginEndHelper()
    {
        GPUScene.EndRender();
    }
 
private:
    FGPUSceneScopeBeginEndHelper() = delete;
    FGPUSceneScopeBeginEndHelper(const FGPUSceneScopeBeginEndHelper&) = delete;
    FGPUScene& GPUScene;
};
 
struct FBatchedPrimitiveShaderData
{
    static const uint32 DataStrideInFloat4s = BATCHED_PRIMITIVE_DATA_STRIDE_FLOAT4;
 
    TStaticArray<FVector4f, DataStrideInFloat4s> Data;
 
    FBatchedPrimitiveShaderData()
        : Data(InPlace, NoInit)
    {
        Setup(GetIdentityPrimitiveParameters());
    }
 
    explicit FBatchedPrimitiveShaderData(const FPrimitiveUniformShaderParameters& PrimitiveUniformShaderParameters)
        : Data(InPlace, NoInit)
    {
        Setup(PrimitiveUniformShaderParameters);
    }
 
    explicit FBatchedPrimitiveShaderData(const FPrimitiveSceneProxy* Proxy);
    
    static void Emplace(FBatchedPrimitiveShaderData* Dest, const FPrimitiveUniformShaderParameters& ShaderParams);
    static void Emplace(FBatchedPrimitiveShaderData* Dest, const FPrimitiveSceneProxy* Proxy);
 
private:
    void Setup(const FPrimitiveUniformShaderParameters& PrimitiveUniformShaderParameters);
};