ShaderCodeArchive.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "Async/TaskGraphInterfaces.h"
#include "Containers/Array.h"
#include "Containers/HashTable.h"
#include "Containers/Map.h"
#include "Containers/Set.h"
#include "Containers/UnrealString.h"
#include "CoreMinimal.h"
#include "FileCache/FileCache.h"
#include "HAL/CriticalSection.h"
#include "HAL/Platform.h"
#include "IO/IoDispatcher.h"
#include "Misc/AssertionMacros.h"
#include "Misc/CoreDelegates.h"
#include "Misc/MemoryReadStream.h"
#include "Misc/SecureHash.h"
#include "RHI.h"
#include "RHIDefinitions.h"
#include "Serialization/Archive.h"
#include "Shader.h"
#include "ShaderCodeLibrary.h"
#include "Templates/RefCounting.h"
#include "UObject/NameTypes.h"
 
#if WITH_EDITOR
class FCbFieldView;
class FCbWriter;
#endif
 
#ifndef USE_MMAPPED_SHADERARCHIVE
    // If enabled FSerializedShaderArchive will use a mmapping of the on-disk shader archive instead of normal serialization. The on-disk format must match the expected in memory layout.
   #define USE_MMAPPED_SHADERARCHIVE 0
#endif
 
// enable visualization in the desktop Development builds only as it has a memory hit and writes files
#define UE_SCA_VISUALIZE_SHADER_USAGE           (!WITH_EDITOR && UE_BUILD_DEVELOPMENT && PLATFORM_DESKTOP)
 
// enable deep, manual debugging of leaked preload groups. This level of information slows the engine down and is only needed when chasing tricky bugs
#define UE_SCA_DEBUG_PRELOADING                 (0)
 
struct FShaderMapEntry
{
    uint32 ShaderIndicesOffset = 0u;
    uint32 NumShaders = 0u;
    uint32 FirstPreloadIndex = 0u;
    uint32 NumPreloadEntries = 0u;
 
    friend FArchive& operator <<(FArchive& Ar, FShaderMapEntry& Ref)
    {
        return Ar << Ref.ShaderIndicesOffset << Ref.NumShaders << Ref.FirstPreloadIndex << Ref.NumPreloadEntries;
    }
};
 
static FArchive& operator <<(FArchive& Ar, FFileCachePreloadEntry& Ref)
{
    return Ar << Ref.Offset << Ref.Size;
}
 
 
#if USE_MMAPPED_SHADERARCHIVE
#pragma pack(push, 1)
#endif
struct FShaderCodeEntry
{
    uint64 Offset = 0;
    uint32 Size = 0;
    uint32 UncompressedSize = 0;
    uint8 Frequency;
 
    EShaderFrequency GetFrequency() const
    {
        return (EShaderFrequency)Frequency;
    }
 
    friend FArchive& operator <<(FArchive& Ar, FShaderCodeEntry& Ref)
    {
        return Ar << Ref.Offset << Ref.Size << Ref.UncompressedSize << Ref.Frequency;
    }
};
#if USE_MMAPPED_SHADERARCHIVE
#pragma pack(pop)
#endif
 
// Portion of shader code archive that's serialize to disk
class FSerializedShaderArchive
{
public:
#if USE_MMAPPED_SHADERARCHIVE
    template<typename T> using TArrayType = TArrayView<T>;
    // Ensure use of the const Get functions when mmapping is used.
    private:
#else
    template<typename T> using TArrayType = TArray<T>;
#endif
 
    TArrayType<FSHAHash> ShaderMapHashes;
 
    TArrayType<FSHAHash> ShaderHashes;
 
    TArrayType<FShaderMapEntry> ShaderMapEntries;
 
    TArrayType<FShaderCodeEntry> ShaderEntries;
    TArrayType<FFileCachePreloadEntry> PreloadEntries;
 
    TArrayType<uint32> ShaderIndices;
 
public:
    const TArrayView<const uint32> GetShaderIndices() const { return ShaderIndices; }
    const TArrayView<const FFileCachePreloadEntry> GetPreloadEntries() const { return PreloadEntries; }
    const TArrayView<const FShaderCodeEntry> GetShaderEntries() const { return ShaderEntries; }
    const TArrayView<const FShaderMapEntry> GetShaderMapEntries() const { return ShaderMapEntries; }
    const TArrayView<const FSHAHash> GetShaderHashes() const { return ShaderHashes; }
    const TArrayView<const FSHAHash> GetShaderMapHashes() const { return ShaderMapHashes; }
 
    FHashTable ShaderMapHashTable;
    FHashTable ShaderHashTable;
 
#if WITH_EDITOR
    TMap<FSHAHash, FShaderMapAssetPaths> ShaderCodeToAssets;
 
    struct FShaderTypeHashes
    {
        TArray<uint64> Data;
 
        friend FArchive& operator<<(FArchive& Ar, FShaderTypeHashes& Ref)
        {
            Ar << Ref.Data;
            return Ar;
        }
    };
    
    TArray<FShaderTypeHashes> ShaderTypes;
 
    enum class EAssetInfoVersion : uint8
    {
        CurrentVersion = 2
    };
 
    struct FDebugStats
    {
        int32 NumAssets;
        int64 ShadersSize;
        int64 ShadersUniqueSize;
        int32 NumShaders;
        int32 NumUniqueShaders;
        int32 NumShaderMaps;
    };
 
    struct FExtendedDebugStats
    {
        FString TextualRepresentation;
 
        uint32 MinNumberOfShadersPerSM;
 
        uint32 MedianNumberOfShadersPerSM;
 
        uint32 MaxNumberofShadersPerSM;
 
        TArray<int32> TopShaderUsages;
 
        int32 NumShadersPerFrequency[SF_NumFrequencies];
 
        uint64 UncompressedSizePerFrequency[SF_NumFrequencies];
 
        uint64 CompressedSizePerFrequency[SF_NumFrequencies];
    };
#endif
 
    FSerializedShaderArchive()
    {
    }
 
    int64 GetAllocatedSize() const
    {
#if USE_MMAPPED_SHADERARCHIVE
        return ShaderHashes.Num() * ShaderHashes.GetTypeSize() +
            ShaderEntries.Num() * ShaderEntries.GetTypeSize() +
            ShaderMapHashes.Num() * ShaderMapHashes.GetTypeSize() +
            ShaderMapEntries.Num() * ShaderMapEntries.GetTypeSize() +
            PreloadEntries.Num() * PreloadEntries.GetTypeSize() +
            ShaderIndices.Num() * ShaderIndices.GetTypeSize()
#else
        return ShaderHashes.GetAllocatedSize() +
            ShaderEntries.GetAllocatedSize() +
            ShaderMapHashes.GetAllocatedSize() +
            ShaderMapEntries.GetAllocatedSize() +
            PreloadEntries.GetAllocatedSize() +
            ShaderIndices.GetAllocatedSize()
#endif
#if WITH_EDITOR
            + ShaderCodeToAssets.GetAllocatedSize()
            + ShaderTypes.GetAllocatedSize()
#endif // WITH_EDITOR
            ;
    }
 
    void Empty()
    {
        EmptyShaderMaps();
#if !USE_MMAPPED_SHADERARCHIVE
        ShaderHashes.Empty();
        ShaderEntries.Empty();
#endif
        ShaderHashTable.Clear();
#if WITH_EDITOR
        ShaderTypes.Empty();
#endif
    }
 
    void EmptyShaderMaps()
    {
#if !USE_MMAPPED_SHADERARCHIVE
        ShaderMapHashes.Empty();
        ShaderMapEntries.Empty();
        PreloadEntries.Empty();
        ShaderIndices.Empty();
#endif
        ShaderMapHashTable.Clear();
#if WITH_EDITOR
        ShaderCodeToAssets.Empty();
#endif
    }
 
    int32 GetNumShaderMaps() const
    {
        return ShaderMapEntries.Num();
    }
 
    int32 GetNumShaders() const
    {
        return ShaderEntries.Num();
    }
 
    bool IsEmpty() const
    {
        return ShaderMapEntries.IsEmpty() && ShaderEntries.IsEmpty() && PreloadEntries.IsEmpty()
#if WITH_EDITOR
            && ShaderCodeToAssets.IsEmpty()
            && ShaderTypes.IsEmpty()
#endif
            ;
    }
 
    RENDERCORE_API int32 FindShaderMapWithKey(const FSHAHash& Hash, uint32 Key) const;
    RENDERCORE_API int32 FindShaderMap(const FSHAHash& Hash) const;
    RENDERCORE_API int32 FindShaderWithKey(const FSHAHash& Hash, uint32 Key) const;
    RENDERCORE_API int32 FindShader(const FSHAHash& Hash) const;
 
#if !USE_MMAPPED_SHADERARCHIVE
    RENDERCORE_API bool FindOrAddShader(const FSHAHash& Hash, int32& OutIndex);
    RENDERCORE_API bool FindOrAddShaderMap(const FSHAHash& Hash, int32& OutIndex, const FShaderMapAssetPaths* AssociatedAssets);
    RENDERCORE_API void RemoveLastAddedShader();
    RENDERCORE_API void Finalize();
#endif
 
    RENDERCORE_API void DecompressShader(int32 Index, const TArray<FSharedBuffer>& ShaderCode, TArray<uint8>& OutDecompressedShader) const;
 
    RENDERCORE_API void Serialize(FArchive& Ar);
#if WITH_EDITOR
    RENDERCORE_API void SaveAssetInfo(FArchive& Ar);
    RENDERCORE_API bool LoadAssetInfo(const FString& Filename);
    RENDERCORE_API bool LoadAssetInfo(FArchive* Ar);
    RENDERCORE_API void CreateAsChunkFrom(const FSerializedShaderArchive& Parent, const TSet<FName>& PackagesInChunk, TArray<int32>& OutShaderCodeEntriesNeeded);
    RENDERCORE_API void CollectStatsAndDebugInfo(FDebugStats& OutDebugStats, FExtendedDebugStats* OutExtendedDebugStats);
    RENDERCORE_API void DumpContentsInPlaintext(FString& OutText) const;
    RENDERCORE_API friend FCbWriter& operator<<(FCbWriter& Writer, const FSerializedShaderArchive& Archive);
    RENDERCORE_API friend bool LoadFromCompactBinary(FCbFieldView Field, FSerializedShaderArchive& OutArchive);
#endif
 
    friend FArchive& operator<<(FArchive& Ar, FSerializedShaderArchive& Ref)
    {
        Ref.Serialize(Ar);
        return Ar;
    }
};
 
// run-time only debugging facility
struct FShaderUsageVisualizer
{
#if UE_SCA_VISUALIZE_SHADER_USAGE
    FCriticalSection VisualizeLock;
 
    int32 NumShaders;
 
    TSet<int32> ExplicitlyPreloadedShaders;
 
    TSet<int32> PreloadedShaders;
 
    TSet<int32> DecompressedShaders;
 
    TSet<int32> CreatedShaders;
 
    void Initialize(const int32 InNumShaders);
 
    inline void MarkExplicitlyPreloadedForVisualization(int32 ShaderIndex)
    {
        extern int32 GShaderCodeLibraryVisualizeShaderUsage;
        if (LIKELY(GShaderCodeLibraryVisualizeShaderUsage))
        {
            FScopeLock Lock(&VisualizeLock);
            ExplicitlyPreloadedShaders.Add(ShaderIndex);
        }
    }
 
    inline void MarkPreloadedForVisualization(int32 ShaderIndex)
    {
        extern int32 GShaderCodeLibraryVisualizeShaderUsage;
        if (LIKELY(GShaderCodeLibraryVisualizeShaderUsage))
        {
            FScopeLock Lock(&VisualizeLock);
            PreloadedShaders.Add(ShaderIndex);
        }
    }
 
    inline void MarkDecompressedForVisualization(int32 ShaderIndex)
    {
        extern int32 GShaderCodeLibraryVisualizeShaderUsage;
        if (LIKELY(GShaderCodeLibraryVisualizeShaderUsage))
        {
            FScopeLock Lock(&VisualizeLock);
            DecompressedShaders.Add(ShaderIndex);
        }
    }
 
    inline void MarkCreatedForVisualization(int32 ShaderIndex)
    {
        extern int32 GShaderCodeLibraryVisualizeShaderUsage;
        if (LIKELY(GShaderCodeLibraryVisualizeShaderUsage))
        {
            FScopeLock Lock(&VisualizeLock);
            CreatedShaders.Add(ShaderIndex);
        }
    }
 
    void SaveShaderUsageBitmap(const FString& Name, EShaderPlatform ShaderPlatform);
#else
    inline void Initialize(const int32 InNumShaders) {}
    inline void MarkPreloadedForVisualization(int32 ShaderIndex) {}
    inline void MarkExplicitlyPreloadedForVisualization(int32 ShaderIndex) {}
    inline void MarkDecompressedForVisualization(int32 ShaderIndex) {}
    inline void MarkCreatedForVisualization(int32 ShaderIndex) {}
    inline void SaveShaderUsageBitmap(const FString& Name, EShaderPlatform ShaderPlatform) {}
#endif // UE_SCA_VISUALIZE_SHADER_USAGE
};
 
class FShaderCodeArchive : public FRHIShaderLibrary
{
public:
    static FShaderCodeArchive* Create(EShaderPlatform InPlatform, FArchive& Ar, const FString& InDestFilePath, const FString& InLibraryDir, const FString& InLibraryName);
 
    virtual ~FShaderCodeArchive();
 
    virtual bool IsNativeLibrary() const override { return false; }
 
    virtual uint32 GetSizeBytes() const override
    {
        return sizeof(*this) +
            SerializedShaders.GetAllocatedSize() +
            ShaderPreloads.GetAllocatedSize();
    }
 
    virtual int32 GetNumShaders() const override { return SerializedShaders.GetShaderEntries().Num(); }
    virtual int32 GetNumShaderMaps() const override { return SerializedShaders.GetShaderMapEntries().Num(); }
    virtual int32 GetNumShadersForShaderMap(int32 ShaderMapIndex) const override { return SerializedShaders.GetShaderMapEntries()[ShaderMapIndex].NumShaders; }
 
    virtual int32 GetShaderIndex(int32 ShaderMapIndex, int32 i) const override
    {
        const FShaderMapEntry& ShaderMapEntry = SerializedShaders.GetShaderMapEntries()[ShaderMapIndex];
        return SerializedShaders.GetShaderIndices()[ShaderMapEntry.ShaderIndicesOffset + i];
    }
 
    virtual void GetAllShaderIndices(int32 ShaderMapIndex, TArray<int32>& ShaderIndices)
    {
        const FShaderMapEntry& ShaderMapEntry = SerializedShaders.GetShaderMapEntries()[ShaderMapIndex];
        for (uint32 i = 0u; i < ShaderMapEntry.NumShaders; ++i)
        {
            ShaderIndices.AddUnique(ShaderIndices[ShaderMapEntry.ShaderIndicesOffset + i]);
        }
    }
 
    virtual int32 FindShaderMapIndex(const FSHAHash& Hash) override
    {
        return SerializedShaders.FindShaderMap(Hash);
    }
 
    virtual int32 FindShaderIndex(const FSHAHash& Hash) override
    {
        return SerializedShaders.FindShader(Hash);
    }
 
    virtual FSHAHash GetShaderHash(int32 ShaderMapIndex, int32 ShaderIndex) override
    {
        return SerializedShaders.GetShaderHashes()[GetShaderIndex(ShaderMapIndex, ShaderIndex)];
    };
 
    virtual FSHAHash GetShaderMapHash(int32 ShaderMapIndex) const override
    {
        return SerializedShaders.GetShaderMapHashes()[ShaderMapIndex];
    };
 
    virtual bool PreloadShader(int32 ShaderIndex, FGraphEventArray& OutCompletionEvents) override;
 
    virtual bool PreloadShaderMap(int32 ShaderMapIndex, FGraphEventArray& OutCompletionEvents) override;
 
    virtual void ReleasePreloadedShader(int32 ShaderIndex) override;
 
    virtual TRefCountPtr<FRHIShader> CreateShader(int32 Index, bool bRequired = true) override;
    virtual void Teardown() override;
 
    void OnShaderPreloadFinished(int32 ShaderIndex, const IMemoryReadStreamRef& PreloadData);
 
protected:
    FShaderCodeArchive(EShaderPlatform InPlatform, const FString& InLibraryDir, const FString& InLibraryName);
 
    FORCENOINLINE void CheckShaderCreation(void* ShaderPtr, int32 Index)
    {
    }
 
    struct FShaderPreloadEntry
    {
        FGraphEventRef PreloadEvent;
        void* Code = nullptr;
        uint32 FramePreloadStarted = ~0u;
        uint32 NumRefs : 31;
        uint32 bNeverToBePreloaded : 1;
 
        FShaderPreloadEntry()
            : NumRefs(0)
            , bNeverToBePreloaded(0)
        {
        }
    };
 
    bool WaitForPreload(FShaderPreloadEntry& ShaderPreloadEntry);
 
    // Library directory
    FString LibraryDir;
 
    // Offset at where shader code starts in a code library
    int64 LibraryCodeOffset;
 
    // Library file handle for async reads
    IFileCacheHandle* FileCacheHandle;
 
    // The shader code present in the library
    FSerializedShaderArchive SerializedShaders;
 
    TArray<FShaderPreloadEntry> ShaderPreloads;
    FRWLock ShaderPreloadLock;
 
    FShaderUsageVisualizer DebugVisualizer;
};
 
 
 
namespace ShaderCodeArchive
{
    RENDERCORE_API void DecompressShaderWithOodle(uint8* OutDecompressedShader, int64 UncompressedSize, const uint8* CompressedShaderCode, int64 CompressedSize);
    
    // We decompression, group, and recompress shaders when they are added to iostore containers in UnrealPak, where we don't have access to cvars - so there's no way 
    // to configure - so we force Oodle and allow specification of the parameters here.
    RENDERCORE_API bool CompressShaderWithOodle(uint8* OutCompressedShader, int64& OutCompressedSize, const uint8* InUncompressedShaderCode, int64 InUncompressedSize, FOodleDataCompression::ECompressor InOodleCompressor, FOodleDataCompression::ECompressionLevel InOodleLevel);
}
 
struct FIoStoreShaderMapEntry
{
    uint32 ShaderIndicesOffset = 0u;
    uint32 NumShaders = 0u;
 
    friend FArchive& operator <<(FArchive& Ar, FIoStoreShaderMapEntry& Ref)
    {
        return Ar << Ref.ShaderIndicesOffset << Ref.NumShaders;
    }
};
 
struct FIoStoreShaderCodeEntry
{
    union
    {
        uint64 Packed;
        struct
        {
            uint64 Frequency : SF_NumBits;  // 4 as of now
 
            uint64 ShaderGroupIndex : 30;
 
            uint64 UncompressedOffsetInGroup : 30;
        };
    };
 
    FIoStoreShaderCodeEntry()
        : Packed(0)
    {
    }
 
    EShaderFrequency GetFrequency() const
    {
        return (EShaderFrequency)Frequency;
    }
 
    friend FArchive& operator <<(FArchive& Ar, FIoStoreShaderCodeEntry& Ref)
    {
        return Ar << Ref.Packed;
    }
};
 
static_assert(sizeof(FIoStoreShaderCodeEntry) == sizeof(uint64), "To reduce memory footprint, shader code entries should be as small as possible");
 
struct FIoStoreShaderGroupEntry
{
    uint32 ShaderIndicesOffset = 0u;
    uint32 NumShaders = 0u;
 
    uint32 UncompressedSize = 0;
    uint32 CompressedSize = 0;
 
    friend FArchive& operator <<(FArchive& Ar, FIoStoreShaderGroupEntry& Ref)
    {
        return Ar << Ref.ShaderIndicesOffset << Ref.NumShaders << Ref.UncompressedSize << Ref.CompressedSize;
    }
 
    inline bool IsGroupCompressed() const
    {
        return CompressedSize != UncompressedSize;
    }
};
 
struct FIoStoreShaderCodeArchiveHeader
{
public:
 
    TArray<FSHAHash> ShaderMapHashes;
 
    TArray<FSHAHash> ShaderHashes;
 
    TArray<FIoChunkId> ShaderGroupIoHashes;
 
    TArray<FIoStoreShaderMapEntry> ShaderMapEntries;
 
    TArray<FIoStoreShaderCodeEntry> ShaderEntries;
 
    TArray<FIoStoreShaderGroupEntry> ShaderGroupEntries;
 
    TArray<uint32> ShaderIndices;
 
    friend RENDERCORE_API FArchive& operator <<(FArchive& Ar, FIoStoreShaderCodeArchiveHeader& Ref);
 
    inline uint64 GetShaderUncompressedSize(int ShaderIndex) const
    {
        const FIoStoreShaderCodeEntry& ThisShaderEntry = ShaderEntries[ShaderIndex];
        const FIoStoreShaderGroupEntry& GroupEntry = ShaderGroupEntries[ThisShaderEntry.ShaderGroupIndex];
 
        for (uint32 ShaderIdxIdx = GroupEntry.ShaderIndicesOffset, StopBeforeIdxIdx = GroupEntry.ShaderIndicesOffset + GroupEntry.NumShaders; ShaderIdxIdx < StopBeforeIdxIdx; ++ShaderIdxIdx)
        {
            int32 GroupMemberShaderIndex = ShaderIndices[ShaderIdxIdx];
            if (ShaderIndex == GroupMemberShaderIndex)
            {
                // found ourselves, now find our size by subtracting from the next neighbor or the group size
                if (LIKELY(ShaderIdxIdx < StopBeforeIdxIdx - 1))
                {
                    const FIoStoreShaderCodeEntry& NextShaderEntry = ShaderEntries[ShaderIndices[ShaderIdxIdx + 1]];
                    return NextShaderEntry.UncompressedOffsetInGroup - ThisShaderEntry.UncompressedOffsetInGroup;
                }
                else
                {
                    return GroupEntry.UncompressedSize - ThisShaderEntry.UncompressedOffsetInGroup;
                }
            }
        }
 
        checkf(false, TEXT("Could not find shader index %d in its own group %" UINT64_FMT " - library is corrupted."), ShaderIndex, ThisShaderEntry.ShaderGroupIndex);
        return 0;
    }
 
    uint64 GetAllocatedSize() const
    {
        return sizeof(*this) +
            ShaderMapHashes.GetAllocatedSize() +
            ShaderHashes.GetAllocatedSize() +
            ShaderGroupIoHashes.GetAllocatedSize() +
            ShaderMapEntries.GetAllocatedSize() +
            ShaderEntries.GetAllocatedSize() +
            ShaderGroupEntries.GetAllocatedSize() +
            ShaderIndices.GetAllocatedSize();
    }
};
 
class FIoStoreShaderCodeArchive : public FRHIShaderLibrary
{
public:
    RENDERCORE_API static FIoChunkId GetShaderCodeArchiveChunkId(const FString& LibraryName, FName FormatName);
    RENDERCORE_API static FIoChunkId GetShaderCodeChunkId(const FSHAHash& ShaderHash);
    RENDERCORE_API static void CreateIoStoreShaderCodeArchiveHeader(const FName& Format, const FSerializedShaderArchive& SerializedShaders, FIoStoreShaderCodeArchiveHeader& OutHeader);
    RENDERCORE_API static void SaveIoStoreShaderCodeArchive(const FIoStoreShaderCodeArchiveHeader& Header, FArchive& OutLibraryAr);
    static FIoStoreShaderCodeArchive* Create(EShaderPlatform InPlatform, const FString& InLibraryName, FIoDispatcher& InIoDispatcher);
 
    virtual ~FIoStoreShaderCodeArchive();
 
    virtual bool IsNativeLibrary() const override { return false; }
 
    virtual uint32 GetSizeBytes() const override
    {
        return sizeof(*this) +
            Header.GetAllocatedSize() +
            PreloadedShaderGroups.GetAllocatedSize();
    }
 
    virtual uint32 GetShaderSizeBytes(int32 ShaderIndex) const override;
 
    virtual int32 GetNumShaders() const override { return Header.ShaderEntries.Num(); }
    virtual int32 GetNumShaderMaps() const override { return Header.ShaderMapEntries.Num(); }
    virtual int32 GetNumShadersForShaderMap(int32 ShaderMapIndex) const override { return Header.ShaderMapEntries[ShaderMapIndex].NumShaders; }
 
    virtual int32 GetShaderIndex(int32 ShaderMapIndex, int32 i) const override
    {
        const FIoStoreShaderMapEntry& ShaderMapEntry = Header.ShaderMapEntries[ShaderMapIndex];
        return Header.ShaderIndices[ShaderMapEntry.ShaderIndicesOffset + i];
    }
 
    virtual void GetAllShaderIndices(int32 ShaderMapIndex, TArray<int32>& ShaderIndices)
    {
        const FIoStoreShaderMapEntry& ShaderMapEntry = Header.ShaderMapEntries[ShaderMapIndex];
        for (uint32 i = 0u; i < ShaderMapEntry.NumShaders; ++i)
        {
            ShaderIndices.AddUnique(Header.ShaderIndices[ShaderMapEntry.ShaderIndicesOffset + i]);
        }
    }
 
    virtual int32 FindShaderMapIndex(const FSHAHash& Hash) override;
    virtual int32 FindShaderIndex(const FSHAHash& Hash) override;
    virtual FSHAHash GetShaderHash(int32 ShaderMapIndex, int32 ShaderIndex) override
    {
        return Header.ShaderHashes[GetShaderIndex(ShaderMapIndex, ShaderIndex)];
    }
 
    virtual FSHAHash GetShaderMapHash(int32 ShaderMapIndex) const override
    {
        return Header.ShaderMapHashes[ShaderMapIndex];
    }
 
    virtual bool IsPreloading(int32 ShaderIndex, FGraphEventArray& OutCompletionEvents) override;
    virtual bool PreloadShader(int32 ShaderIndex, FGraphEventArray& OutCompletionEvents) override;
    virtual void AddRefPreloadedShaderGroup(int32 ShaderGroupIndex) override;
    virtual void ReleasePreloadedShaderGroup(int32 ShaderGroupIndex) override;
 
    virtual int32 GetGroupIndexForShader(int32 ShaderIndex) const override
    {
        return Header.ShaderEntries[ShaderIndex].ShaderGroupIndex;
    }
    
    virtual bool PreloadShaderMap(int32 ShaderMapIndex, FGraphEventArray& OutCompletionEvents) override;
    virtual bool PreloadShaderMap(int32 ShaderMapIndex, FCoreDelegates::FAttachShaderReadRequestFunc AttachShaderReadRequestFunc) override;
    virtual void ReleasePreloadedShader(int32 ShaderIndex) override;
    virtual TRefCountPtr<FRHIShader> CreateShader(int32 Index, bool bRequired = true) override;
    virtual void Teardown() override;
 
private:
    static constexpr uint32 CurrentVersion = 1;
 
    struct FShaderGroupPreloadEntry
    {
        FGraphEventRef PreloadEvent;
        FIoRequest IoRequest;
        uint32 FramePreloadStarted = ~0u;
        uint32 NumRefs : 31;
        uint32 bNeverToBePreloaded : 1;
 
#if UE_SCA_DEBUG_PRELOADING
        FString DebugInfo;
#endif
 
        FShaderGroupPreloadEntry()
            : NumRefs(0)
            , bNeverToBePreloaded(0)
        {
        }
 
        ~FShaderGroupPreloadEntry()
        {
        }
    };
 
    FIoStoreShaderCodeArchive(EShaderPlatform InPlatform, const FString& InLibraryName, FIoDispatcher& InIoDispatcher);
 
    FIoDispatcher& IoDispatcher;
 
    bool PreloadShaderGroup(int32 ShaderGroupIndex, FGraphEventArray& OutCompletionEvents, 
#if UE_SCA_DEBUG_PRELOADING
        const FString& CallsiteInfo,
#endif
        FCoreDelegates::FAttachShaderReadRequestFunc* AttachShaderReadRequestFuncPtr = nullptr);
 
    void SetupPreloadEntryForLoading(int32 ShaderGroupIndex, FShaderGroupPreloadEntry& PreloadEntry);
 
    void MarkPreloadEntrySkipped(int32 ShaderGroupIndex
#if UE_SCA_DEBUG_PRELOADING
        , const FString& CallsiteInfo
#endif
    );
 
    void ReleasePreloadEntry(int32 ShaderGroupIndex
#if UE_SCA_DEBUG_PRELOADING
        , const FString& CallsiteInfo
#endif
    );
 
    inline FShaderGroupPreloadEntry* FindOrAddPreloadEntry(int32 ShaderGroupIndex)
    {
        FShaderGroupPreloadEntry*& Ptr = PreloadedShaderGroups.FindOrAdd(ShaderGroupIndex);
        if (UNLIKELY(Ptr == nullptr))
        {
            Ptr = new FShaderGroupPreloadEntry;
        }
        return Ptr;
    }
 
    inline FShaderGroupPreloadEntry* FindExistingPreloadEntry(int32 ShaderGroupIndex)
    {
        FShaderGroupPreloadEntry** PtrPtr = PreloadedShaderGroups.Find(ShaderGroupIndex);
        checkf(PtrPtr != nullptr, TEXT("The preload entry for a shader group we assumed to exist does not exist!"));
        return *PtrPtr;
    }
 
    bool GroupOnlyContainsRaytracingShaders(int32 ShaderGroupIndex);
 
    FIoStoreShaderCodeArchiveHeader Header;
 
    FHashTable ShaderMapHashTable;
    FHashTable ShaderHashTable;
 
    TMap<int32, FShaderGroupPreloadEntry*> PreloadedShaderGroups;
    FRWLock PreloadedShaderGroupsLock;
 
    FShaderUsageVisualizer DebugVisualizer;
};