IPlatformFilePak.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "Containers/Array.h"
#include "Containers/ArrayView.h"
#include "Containers/DirectoryTree.h"
#include "Containers/Map.h"
#include "Containers/Set.h"
#include "Containers/SparseArray.h"
#include "Containers/StringConv.h"
#include "Containers/Ticker.h"
#include "Containers/Utf8String.h"
#include "Containers/UnrealString.h"
#include "CoreMinimal.h"
#include "Delegates/Delegate.h"
#include "GenericPlatform/GenericPlatformChunkInstall.h"
#include "GenericPlatform/GenericPlatformFile.h"
#include "HAL/CriticalSection.h"
#include "HAL/PlatformCrt.h"
#include "HAL/PlatformTime.h"
#include "HAL/UnrealMemory.h"
#include "Logging/LogCategory.h"
#include "Logging/LogMacros.h"
#include "Math/BigInt.h"
#include "Math/NumericLimits.h"
#include "Math/UnrealMathSSE.h"
#include "Misc/AES.h"
#include "Misc/AssertionMacros.h"
#include "Misc/CString.h"
#include "Misc/CompressionFlags.h"
#include "Misc/DateTime.h"
#include "Misc/Guid.h"
#include "Misc/IEngineCrypto.h"
#include "Misc/Optional.h"
#include "Misc/Paths.h"
#include "Misc/ScopeLock.h"
#include "Misc/ScopeRWLock.h"
#include "Misc/SecureHash.h"
#include "Misc/TransactionallySafeCriticalSection.h"
#include "RSA.h"
#include "Serialization/Archive.h"
#include "Serialization/MemoryImage.h"
#include "Stats/Stats.h"
#include "Templates/Function.h"
#include "Templates/RefCounting.h"
#include "Templates/SharedPointer.h"
#include "Templates/UniquePtr.h"
#include "Templates/UnrealTemplate.h"
#include "Trace/Detail/Channel.h"
#include "UObject/NameTypes.h"
#include "UObject/UnrealNames.h"
 
#ifndef ENABLE_PAKFILE_USE_DIRECTORY_TREE
#define ENABLE_PAKFILE_USE_DIRECTORY_TREE 1
#endif
 
class FChunkCacheWorker;
class FFileIoStore;
class FIoStatus;
class FFilePackageStoreBackend;
class FOutputDevice;
class IAsyncReadFileHandle;
class IMappedFileHandle;
namespace UE::PakFile::Private 
{ 
    class FPakFileDirectoryVisitorBase;
    struct FHandleMountPaksExDelegate;
 
    template<bool bInAllowDuplicateKeys = false>
    struct FPakEntryFilenameSetKeyFuncs;
}
namespace UE::IoStore { class IFileIoDispatcherBackend; }
struct FIoContainerHeader;
 
PAKFILE_API DECLARE_LOG_CATEGORY_EXTERN(LogPakFile, Log, All);
DECLARE_FLOAT_ACCUMULATOR_STAT_EXTERN(TEXT("Total pak file read time"), STAT_PakFile_Read, STATGROUP_PakFile, PAKFILE_API);
 
DECLARE_DWORD_ACCUMULATOR_STAT_EXTERN(TEXT("Num open pak file handles"), STAT_PakFile_NumOpenHandles, STATGROUP_PakFile, PAKFILE_API);
 
#define PAK_TRACKER 0
 
// ENABLE_PAKFILE_RUNTIME_PRUNING allows pruning the DirectoryIndex at runtime after all Paks have loaded rather than loading only the already-pruned DirectoryIndex
// This requires extra cputime to make reads of the DirectoryIndex ThreadSafe, and will be removed in a future version
#ifndef ENABLE_PAKFILE_RUNTIME_PRUNING
#define ENABLE_PAKFILE_RUNTIME_PRUNING 1
#endif
#define ENABLE_PAKFILE_RUNTIME_PRUNING_VALIDATE ENABLE_PAKFILE_RUNTIME_PRUNING && !UE_BUILD_SHIPPING
 
// Define the type of a chunk hash. Currently selectable between SHA1 and CRC32.
#define PAKHASH_USE_CRC 1
#if PAKHASH_USE_CRC
typedef uint32 TPakChunkHash;
#else
typedef FSHAHash TPakChunkHash;
#endif
 
PAKFILE_API TPakChunkHash ComputePakChunkHash(const void* InData, int64 InDataSizeInBytes);
inline FString ChunkHashToString(const TPakChunkHash& InHash)
{
#if PAKHASH_USE_CRC
    return FString::Printf(TEXT("%08X"), InHash);
#else
    return LexToString(InHash);
#endif
}
 
struct FPakChunkSignatureCheckFailedData
{
    FPakChunkSignatureCheckFailedData(const FString& InPakFilename, const TPakChunkHash& InExpectedHash, const TPakChunkHash& InReceivedHash, int32 InChunkIndex)
        : PakFilename(InPakFilename)
        , ChunkIndex(InChunkIndex)
        , ExpectedHash(InExpectedHash)
        , ReceivedHash(InReceivedHash)
    {
    }
    FString PakFilename;
    int32 ChunkIndex;
    TPakChunkHash ExpectedHash;
    TPakChunkHash ReceivedHash;
 
    FPakChunkSignatureCheckFailedData() : ChunkIndex(0) {}
};
DECLARE_DELEGATE_RetVal_OneParam(bool, FFilenameSecurityDelegate, const TCHAR* /*InFilename*/);
DECLARE_DELEGATE_ThreeParams(FPakCustomEncryptionDelegate, uint8* /*InData*/, uint32 /*InDataSize*/, FGuid /*InEncryptionKeyGuid*/);
DECLARE_MULTICAST_DELEGATE_OneParam(FPakChunkSignatureCheckFailedHandler, const FPakChunkSignatureCheckFailedData&);
DECLARE_MULTICAST_DELEGATE_OneParam(FPakPrincipalSignatureTableCheckFailureHandler, const FString&);
DECLARE_DELEGATE_ThreeParams(FPakSetIndexSettings, bool& /* bKeepFullDirectory*/, bool& /* bValidatePruning */, bool& /* bDelayPruning */);
 
UE_DEPRECATED("5.1", "Use FPakPrincipalSignatureTableCheckFailureHandler instead")
typedef FPakPrincipalSignatureTableCheckFailureHandler FPakMasterSignatureTableCheckFailureHandler;
struct FPakInfo
{
    enum 
    {
        PakFile_Magic = 0x5A6F12E1,
        MaxChunkDataSize = 64*1024,
        CompressionMethodNameLen = 32,
        MaxNumCompressionMethods=5, // when we remove patchcompatibilitymode421 we can reduce this to 4
    };
 
    enum
    {
        PakFile_Version_Initial = 1,
        PakFile_Version_NoTimestamps = 2,
        PakFile_Version_CompressionEncryption = 3,
        PakFile_Version_IndexEncryption = 4,
        PakFile_Version_RelativeChunkOffsets = 5,
        PakFile_Version_DeleteRecords = 6,
        PakFile_Version_EncryptionKeyGuid = 7,
        PakFile_Version_FNameBasedCompressionMethod = 8,
        PakFile_Version_FrozenIndex = 9,
        PakFile_Version_PathHashIndex = 10,
        PakFile_Version_Fnv64BugFix = 11,
        PakFile_Version_Utf8PakDirectory = 12,
 
 
        PakFile_Version_Last,
        PakFile_Version_Invalid,
        PakFile_Version_Latest = PakFile_Version_Last - 1
    };
 
    uint32 Magic;
    int32 Version;
    int64 IndexOffset;
    int64 IndexSize;
    FSHAHash IndexHash;
    uint8 bEncryptedIndex;
    FGuid EncryptionKeyGuid;
    TArray<FName> CompressionMethods;
 
    FPakInfo()
        : Magic(PakFile_Magic)
        , Version(PakFile_Version_Latest)
        , IndexOffset(-1)
        , IndexSize(0)
        , bEncryptedIndex(0)
    {
        // we always put in a NAME_None entry as index 0, so that an uncompressed PakEntry will have CompressionMethodIndex of 0 and can early out easily
        CompressionMethods.Add(NAME_None);
    }
 
    int64 GetSerializedSize(int32 InVersion = PakFile_Version_Latest) const
    {
        int64 Size = sizeof(Magic) + sizeof(Version) + sizeof(IndexOffset) + sizeof(IndexSize) + sizeof(IndexHash) + sizeof(bEncryptedIndex);
        if (InVersion >= PakFile_Version_EncryptionKeyGuid) Size += sizeof(EncryptionKeyGuid);
        if (InVersion >= PakFile_Version_FNameBasedCompressionMethod) Size += CompressionMethodNameLen * MaxNumCompressionMethods;
        if (InVersion >= PakFile_Version_FrozenIndex && InVersion < PakFile_Version_PathHashIndex) Size += sizeof(bool);
 
        return Size;
    }
 
    int64 HasRelativeCompressedChunkOffsets() const
    {
        return Version >= PakFile_Version_RelativeChunkOffsets;
    }
 
    void Serialize(FArchive& Ar, int32 InVersion)
    {
        if (Ar.IsLoading() && Ar.TotalSize() < (Ar.Tell() + GetSerializedSize(InVersion)))
        {
            Magic = 0;
            return;
        }
 
        if (Ar.IsSaving() || InVersion >= PakFile_Version_EncryptionKeyGuid)
        {
            Ar << EncryptionKeyGuid;
        }
        Ar << bEncryptedIndex;
        Ar << Magic;
        if (Magic != PakFile_Magic)
        {
            // handle old versions by failing out now (earlier versions will be attempted)
            Magic = 0;
            return;
        }
 
        Ar << Version;
        Ar << IndexOffset;
        Ar << IndexSize;
        Ar << IndexHash;
 
        if (Ar.IsLoading())
        {
            if (Version < PakFile_Version_IndexEncryption)
            {
                bEncryptedIndex = false;
            }
 
            if (Version < PakFile_Version_EncryptionKeyGuid)
            {
                EncryptionKeyGuid.Invalidate();
            }
        }
 
        if (Version >= PakFile_Version_FrozenIndex && Version < PakFile_Version_PathHashIndex)
        {
            bool bIndexIsFrozen = false;
            Ar << bIndexIsFrozen;
            if (bIndexIsFrozen)
            {
                UE_LOG(LogPakFile, Fatal, TEXT("PakFile was frozen with version FPakInfo::PakFile_Version_FrozenIndex, which is no longer supported. Regenerate Paks."));
            }
        }
 
        if (Version < PakFile_Version_FNameBasedCompressionMethod)
        {
            // for old versions, put in some known names that we may have used
            CompressionMethods.Add(NAME_Zlib);
            CompressionMethods.Add(NAME_Gzip);
            CompressionMethods.Add(TEXT("Oodle"));
        }
        else
        {
            // we need to serialize a known size, so make a buffer of "strings"
            const int32 BufferSize = CompressionMethodNameLen * MaxNumCompressionMethods;
            ANSICHAR Methods[BufferSize];
            if (Ar.IsLoading())
            {
                Ar.Serialize(Methods, BufferSize);
                for (int32 Index = 0; Index < MaxNumCompressionMethods; Index++)
                {
                    ANSICHAR* MethodString = &Methods[Index * CompressionMethodNameLen];
                    if (MethodString[0] != 0)
                    {
                        // Make sure if the file has garbage we don't read off in to lala land.
                        MethodString[CompressionMethodNameLen-1] = 0;
                        CompressionMethods.Add(FName(MethodString));
                    }
                }
            }
            else
            {
                // we always zero out fully what we write out so that reading in is simple
                FMemory::Memzero(Methods, BufferSize);
 
                for (int32 Index = 1; Index < CompressionMethods.Num(); Index++)
                {
                    ANSICHAR* MethodString = &Methods[(Index - 1) * CompressionMethodNameLen];
                    FCStringAnsi::Strncpy(MethodString, TCHAR_TO_ANSI(*CompressionMethods[Index].ToString()), CompressionMethodNameLen);
                }
                Ar.Serialize(Methods, BufferSize);
            }
        }
    }
 
    int32 GetCompressionMethodIndex(FName CompressionMethod)
    {
        // look for existing method
        for (uint8 Index = 0; Index < CompressionMethods.Num(); Index++)
        {
            if (CompressionMethods[Index] == CompressionMethod)
            {
                return Index;
            }
        }
 
        checkf(CompressionMethod.ToString().Len() < CompressionMethodNameLen, TEXT("Compression method name, %s, is too long for pak file serialization. You can increase CompressionMethodNameLen, but then will have to handle version management."), *CompressionMethod.ToString());
        // CompressionMethods always has None at Index 0, that we don't serialize, so we can allow for one more in the array
        checkf(CompressionMethods.Num() <= MaxNumCompressionMethods, TEXT("Too many unique compression methods in one pak file. You can increase MaxNumCompressionMethods, but then will have to handle version management."));
 
        // add it if it didn't exist
        return CompressionMethods.Add(CompressionMethod);
    }
 
    FName GetCompressionMethod(uint32 Index) const
    {
        return CompressionMethods[Index];
    }
    
    TOptional<FName> TryGetCompressionMethod(uint32 Index) const
    {
        if (IntFitsIn<int32>(Index) == false)
        {
            return TOptional<FName>();
        }
        return (int32)Index >= CompressionMethods.Num() ? TOptional<FName>() : TOptional<FName>(CompressionMethods[(int32)Index]);
    }
};
 
struct FPakCompressedBlock
{
    int64 CompressedStart;
    int64 CompressedEnd;
 
    bool operator == (const FPakCompressedBlock& B) const
    {
        return CompressedStart == B.CompressedStart && CompressedEnd == B.CompressedEnd;
    }
 
    bool operator != (const FPakCompressedBlock& B) const
    {
        return !(*this == B);
    }
};
 
inline FArchive& operator<<(FArchive& Ar, FPakCompressedBlock& Block)
{
    Ar << Block.CompressedStart;
    Ar << Block.CompressedEnd;
    return Ar;
}
 
struct FPakEntry
{
    static const uint8 Flag_None = 0x00;
    static const uint8 Flag_Encrypted = 0x01;
    static const uint8 Flag_Deleted = 0x02;
 
    int64 Offset;
    int64 Size;
    int64 UncompressedSize;
    uint8 Hash[20];
    TArray<FPakCompressedBlock> CompressionBlocks;
    uint32 CompressionBlockSize;
    uint32 CompressionMethodIndex;
    uint8 Flags;
    mutable bool  Verified;
 
    FPakEntry()
    {
        Reset();
    }
 
    int64 GetSerializedSize(int32 Version) const
    {
        int64 SerializedSize = sizeof(Offset) + sizeof(Size) + sizeof(UncompressedSize) + sizeof(Hash);
 
        if (Version >= FPakInfo::PakFile_Version_FNameBasedCompressionMethod)
        {
            SerializedSize += sizeof(CompressionMethodIndex);
        }
        else
        {
            SerializedSize += sizeof(int32); // Old CompressedMethod var from pre-fname based compression methods
        }
 
        if (Version >= FPakInfo::PakFile_Version_CompressionEncryption)
        {
            SerializedSize += sizeof(Flags) + sizeof(CompressionBlockSize);
            if(CompressionMethodIndex != 0)
            {
                SerializedSize += sizeof(FPakCompressedBlock) * CompressionBlocks.Num() + sizeof(int32);
            }
        }
        if (Version < FPakInfo::PakFile_Version_NoTimestamps)
        {
            // Timestamp
            SerializedSize += sizeof(int64);
        }
        return SerializedSize;
    }
 
    bool operator == (const FPakEntry& B) const
    {
        return IndexDataEquals(B) &&
            FMemory::Memcmp(Hash, B.Hash, sizeof(Hash)) == 0;
    }
 
    bool operator != (const FPakEntry& B) const
    {
        return !(*this == B);
    }
 
    bool IndexDataEquals(const FPakEntry& B) const
    {
        // Offset are only in the Index and so are not compared
        // Hash is only in the payload and so are not compared
        // Verified is only in the payload and is mutable and so is not compared
        return Size == B.Size &&
            UncompressedSize == B.UncompressedSize &&
            CompressionMethodIndex == B.CompressionMethodIndex &&
            Flags == B.Flags &&
            CompressionBlockSize == B.CompressionBlockSize &&
            CompressionBlocks == B.CompressionBlocks;
    }
 
    void Reset()
    {
        Offset = -1;
        Size = 0;
        UncompressedSize = 0;
        FMemory::Memset(Hash, 0, sizeof(Hash));
        CompressionBlocks.Reset();
        CompressionBlockSize = 0;
        CompressionMethodIndex = 0;
        Flags = Flag_None;
        Verified = false;
    }
 
    void Serialize(FArchive& Ar, int32 Version)
    {
        Ar << Offset;
        Ar << Size;
        Ar << UncompressedSize;
        if (Version < FPakInfo::PakFile_Version_FNameBasedCompressionMethod)
        {
            int32 LegacyCompressionMethod;
            Ar << LegacyCompressionMethod;
            if (LegacyCompressionMethod == COMPRESS_None)
            {
                CompressionMethodIndex = 0;
            }
            else if (LegacyCompressionMethod & COMPRESS_ZLIB_DEPRECATED)
            {
                CompressionMethodIndex = 1;
            }
            else if (LegacyCompressionMethod & COMPRESS_GZIP_DEPRECATED)
            {
                CompressionMethodIndex = 2;
            }
            else if (LegacyCompressionMethod & COMPRESS_Custom_DEPRECATED)
            {
                CompressionMethodIndex = 3;
            }
            else
            {
                UE_LOG(LogPakFile, Fatal, TEXT("Found an unknown compression type in pak file, will need to be supported for legacy files"));
            }
        }
        else
        {
            Ar << CompressionMethodIndex;
        }
        if (Version <= FPakInfo::PakFile_Version_Initial)
        {
            FDateTime Timestamp;
            Ar << Timestamp;
        }
        Ar.Serialize(Hash, sizeof(Hash));
        if (Version >= FPakInfo::PakFile_Version_CompressionEncryption)
        {
            if(CompressionMethodIndex != 0)
            {
                Ar << CompressionBlocks;
            }
            Ar << Flags;
            Ar << CompressionBlockSize;
        }
    }
 
    inline void SetFlag( uint8 InFlag, bool bValue )
    {
        if( bValue )
        {
            Flags |= InFlag;
        }
        else
        {
            Flags &= ~InFlag;
        }
    }
 
    inline bool GetFlag( uint8 InFlag ) const
    {
        return (Flags & InFlag) == InFlag;
    }
    
    inline bool IsEncrypted() const             { return GetFlag(Flag_Encrypted); }
    inline void SetEncrypted( bool bEncrypted ) { SetFlag( Flag_Encrypted, bEncrypted ); }
 
    inline bool IsDeleteRecord() const                { return GetFlag(Flag_Deleted); }
    inline void SetDeleteRecord( bool bDeleteRecord ) { SetFlag(Flag_Deleted, bDeleteRecord ); }
 
 
    static bool VerifyPakEntriesMatch(const FPakEntry& FileEntryA, const FPakEntry& FileEntryB);
};
 
struct FPakEntryLocation
{
public:
    /*
     * 0x00000000 - 0x7ffffffe: EncodedOffset from 0 to MaxIndex
     * 0x7fffffff: Unused, interpreted as Invalid
     * 0x80000000: Invalid
     * 0x80000001 - 0xffffffff: FileIndex from MaxIndex to 0
    */
    static const int32 Invalid = MIN_int32;
    static const int32 MaxIndex = MAX_int32 - 1;
 
    FPakEntryLocation() : Index(Invalid)
    {
    }
    FPakEntryLocation(const FPakEntryLocation& Other) = default;
    FPakEntryLocation& operator=(const FPakEntryLocation& other) = default;
 
    static FPakEntryLocation CreateInvalid()
    {
        return FPakEntryLocation();
    }
 
    static FPakEntryLocation CreateFromOffsetIntoEncoded(int32 Offset)
    {
        check(0 <= Offset && Offset <= MaxIndex);
        return FPakEntryLocation(Offset);
    }
 
    static FPakEntryLocation CreateFromListIndex(int32 ListIndex)
    {
        check(0 <= ListIndex && ListIndex <= MaxIndex);
        return FPakEntryLocation(-ListIndex - 1);
    }
 
    bool IsInvalid() const
    {
        return Index <= Invalid || MaxIndex < Index;
    }
 
    bool IsOffsetIntoEncoded() const
    {
        return 0 <= Index && Index <= MaxIndex;
    }
 
    bool IsListIndex() const
    {
        return (-MaxIndex - 1) <= Index && Index <= -1;
    }
 
    int32 GetAsOffsetIntoEncoded() const
    {
        if (IsOffsetIntoEncoded())
        {
            return Index;
        }
        else
        {
            return -1;
        }
    }
    int32 GetAsListIndex() const
    {
        if (IsListIndex())
        {
            return -(Index + 1);
        }
        else
        {
            return -1;
        }
    }
 
    void Serialize(FArchive& Ar)
    {
        Ar << Index;
    }
 
    bool operator==(const FPakEntryLocation& Other) const
    {
        return Index == Other.Index;
    }
private:
    explicit FPakEntryLocation(int32 InIndex) : Index(InIndex)
    {
    }
 
    int32 Index;
};
inline FArchive& operator<<(FArchive& Ar, FPakEntryLocation& PakEntryLocation)
{
    PakEntryLocation.Serialize(Ar);
    return Ar;
}
 
 
class FPakFile;
 
// Wrapper for a pointer to a shared pak reader archive that has been temporarily acquired. 
class FSharedPakReader final 
{
    friend class FPakFile;
 
    FArchive* Archive = nullptr;
    FPakFile* PakFile = nullptr; // Pak file to return ownership to on destruction
 
    PAKFILE_API FSharedPakReader(FArchive* InArchive, FPakFile* InPakFile);
 
public:
    PAKFILE_API ~FSharedPakReader();
 
    FSharedPakReader(const FSharedPakReader& Other) = delete;
    FSharedPakReader& operator=(const FSharedPakReader& Other) = delete;
    PAKFILE_API FSharedPakReader(FSharedPakReader&& Other);
    PAKFILE_API FSharedPakReader& operator=(FSharedPakReader&& Other);
 
    explicit operator bool() const { return Archive != nullptr; }
    bool operator==(TYPE_OF_NULLPTR) { return Archive == nullptr; }
    bool operator!=(TYPE_OF_NULLPTR) { return Archive != nullptr; }
    FArchive* operator->() { return Archive; }
 
    
    // USE WITH CARE, the FSharedPakReader must live longer than this reference to prevent the archive being used by another thread. Do not call on a temporary return value!
    FArchive& GetArchive() { return *Archive; } 
 
};
 
typedef TMap<FUtf8String, FPakEntryLocation> FPakDirectory;
 
namespace UE::PakFile::Private
{
#if ENABLE_PAKFILE_USE_DIRECTORY_TREE
    typedef TDirectoryTree<FPakDirectory> FDirectoryTreeIndex;
#else
    struct FNullDirectoryTreeIndex
    {
    };
    typedef FNullDirectoryTreeIndex FDirectoryTreeIndex;
#endif
}
 
/* Convenience struct for building FPakFile indexes from an enumeration of (Filename,FPakEntry) pairs */
struct FPakEntryPair
{
    FString Filename;
    FPakEntry Info;
};
 
class FPakFile : FNoncopyable, public FRefCountBase, public IPakFile
{
public:
    typedef TMap<uint64, FPakEntryLocation> FPathHashIndex;
    typedef TMap<FString, FPakDirectory> FDirectoryIndex;
 
    enum class ECacheType : uint8
    {
        Shared,
        Individual,
    };
 
    struct FScopedPakDirectoryIndexAccess
    {
        FScopedPakDirectoryIndexAccess(const FPakFile& InPakFile)
#if ENABLE_PAKFILE_RUNTIME_PRUNING
            : PakFile(InPakFile)
            , bRequiresDirectoryIndexLock(PakFile.RequiresDirectoryIndexLock())
#endif
        {
#if ENABLE_PAKFILE_RUNTIME_PRUNING
            if (bRequiresDirectoryIndexLock)
            {
                PakFile.DirectoryIndexLock.ReadLock();
            }
#endif
        }
#if ENABLE_PAKFILE_RUNTIME_PRUNING
        ~FScopedPakDirectoryIndexAccess()
        {
            if (bRequiresDirectoryIndexLock)
            {
                PakFile.DirectoryIndexLock.ReadUnlock();
            }
        }
        const FPakFile& PakFile;
        bool bRequiresDirectoryIndexLock;
#endif
    };
 
    PAKFILE_API bool RecreatePakReaders(IPlatformFile* LowerLevel);
 
    struct FArchiveAndLastAccessTime 
    {
        TUniquePtr<FArchive> Archive;
        double LastAccessTime;
    };
 
private:
    friend class FPakPlatformFile;
 
    FString PakFilename;
    FName PakFilenameName;
    TUniquePtr<class FChunkCacheWorker> Decryptor;
    TArray<FArchiveAndLastAccessTime> Readers;
    std::atomic<int32> CurrentlyUsedReaders = 0;
    FCriticalSection ReadersCriticalSection;
    FPakInfo Info;
    FString MountPoint;
    TArray<FPakEntry> Files;
    FDirectoryIndex DirectoryIndex;
 
    using FDirectoryTreeIndex = UE::PakFile::Private::FDirectoryTreeIndex;
 
    FDirectoryTreeIndex DirectoryTreeIndex;
 
#if ENABLE_PAKFILE_RUNTIME_PRUNING
    FDirectoryIndex PrunedDirectoryIndex;
    FDirectoryTreeIndex PrunedDirectoryTreeIndex;
    mutable FRWLock DirectoryIndexLock;
#endif
 
    FPathHashIndex PathHashIndex;
    /* FPakEntries that have been serialized into a compacted format in an array of bytes. */
    TArray<uint8> EncodedPakEntries;
    /* The seed passed to the hash function for hashing filenames in this pak.  Differs per pack so that the same filename in different paks has different hashes */
    uint64 PathHashSeed;
 
    int32 NumEntries;
    FDateTime Timestamp;    
    int64 CachedTotalSize;
    bool bSigned;
    bool bIsValid;
    /* True if the PathHashIndex has been populated for this PakFile */
    bool bHasPathHashIndex;
    /* True if the DirectoryIndex has not been pruned and still contains a Filename for every FPakEntry in this PakFile */
    bool bHasFullDirectoryIndex;
#if ENABLE_PAKFILE_RUNTIME_PRUNING
    /* True if we have a FullDirectoryIndex that we will modify in OptimizeMemoryUsageForMountedPaks and therefore need to guard access with DirectoryIndexLock */
    bool bWillPruneDirectoryIndex;
    /* True if the Index of this PakFile was a legacy index that did not have the precomputed Pruned DirectoryIndex and we need to compute it before swapping the Pruned DirectoryIndex*/
    bool bNeedsLegacyPruning;
#endif
#if ENABLE_PAKFILE_USE_DIRECTORY_TREE
    bool bUseDirectoryTree;
#endif
    int32 PakchunkIndex;
 
    TSharedPtr<IMappedFileHandle> MappedFileHandle;
    FCriticalSection MappedFileHandleCriticalSection;
 
    ECacheType  CacheType;
    int32       CacheIndex;
    bool UnderlyingCacheTrimDisabled;
    bool bIsMounted;
 
    TUniquePtr<FIoContainerHeader> IoContainerHeader;
#if WITH_EDITOR
    TUniquePtr<FIoContainerHeader> OptionalSegmentIoContainerHeader;
#endif
 
    static inline int32 CDECL CompareFilenameHashes(const void* Left, const void* Right)
    {
        const uint64* LeftHash = (const uint64*)Left;
        const uint64* RightHash = (const uint64*)Right;
        if (*LeftHash < *RightHash)
        {
            return -1;
        }
        if (*LeftHash > *RightHash)
        {
            return 1;
        }
        return 0;
    }
 
    PAKFILE_API FArchive* CreatePakReader(IPlatformFile* LowerLevel, const TCHAR* Filename);
    PAKFILE_API FArchive* SetupSignedPakReader(FArchive* Reader, const TCHAR* Filename);
 
 
public:
    // IPakFile interface, for users of PakFiles that cannot have a dependency on this header
    virtual const FString& PakGetPakFilename() const override
    {
        return PakFilename;
    }
 
    virtual bool PakContains(const FString& FullPath) const override
    {
        return Find(FullPath, nullptr) == EFindResult::Found;
    }
 
    virtual int32 PakGetPakchunkIndex() const override
    {
        return PakchunkIndex;
    }
 
    virtual void PakVisitPrunedFilenames(IPlatformFile::FDirectoryVisitor& Visitor) const override
    {
        for (FFilenameIterator It(*this); It; ++It)
        {
            Visitor.CallShouldVisitAndVisit(*It.Filename(), false);
        }
    }
 
    virtual const FString& PakGetMountPoint() const override
    {
        return MountPoint;
    }
 
 
    void SetUnderlyingCacheTrimDisabled(bool InUnderlyingCacheTrimDisabled) { UnderlyingCacheTrimDisabled = InUnderlyingCacheTrimDisabled; }
    bool GetUnderlyingCacheTrimDisabled(void) { return UnderlyingCacheTrimDisabled; }
 
    void SetCacheType(ECacheType InCacheType) { CacheType = InCacheType; }
    ECacheType GetCacheType(void) { return CacheType; }
    void SetCacheIndex(int32 InCacheIndex) { CacheIndex = InCacheIndex; }
    int32 GetCacheIndex(void) { return CacheIndex; }
    void SetIsMounted(bool bInIsMounted) { bIsMounted = bInIsMounted; }
    bool GetIsMounted() const { return bIsMounted; }
#if IS_PROGRAM
    PAKFILE_API FPakFile(const TCHAR* Filename, bool bIsSigned);
#endif
 
    PAKFILE_API FPakFile(IPlatformFile* LowerLevel, const TCHAR* Filename, bool bIsSigned, bool bLoadIndex = true);
 
#if WITH_EDITOR
    PAKFILE_API FPakFile(FArchive* Archive);
#endif
 
private:
    PAKFILE_API virtual ~FPakFile();
    friend class FRefCountBase;
 
public:
 
    bool IsValid() const
    {
        return bIsValid;
    }
 
    PAKFILE_API bool PassedSignatureChecks() const;
 
    const FString& GetFilename() const
    {
        return PakFilename;
    }
    FName GetFilenameName() const
    {
        return PakFilenameName;
    }
 
    int64 TotalSize() const
    {
        return CachedTotalSize;
    }
 
    virtual int32 GetNumFiles() const override
    {
        return NumEntries;
    }
 
    PAKFILE_API void GetPrunedFilenames(TArray<FString>& OutFileList) const;
 
    PAKFILE_API void GetPrunedFilenamesInChunk(const TArray<int32>& InChunkIDs, TArray<FString>& OutFileList) const;
 
    PAKFILE_API FSharedPakReader GetSharedReader(IPlatformFile* LowerLevel);
 
    // Return a shared pak reader. Should only be called from the FSharedPakReader's destructor.
    PAKFILE_API void ReturnSharedReader(FArchive* SharedReader);
 
    // Delete all readers that haven't been used in MaxAgeSeconds.
    PAKFILE_API void ReleaseOldReaders(double MaxAgeSeconds);
 
    enum class EFindResult : uint8
    {
        NotFound,
        Found,
        FoundDeleted,
    };
    PAKFILE_API EFindResult Find(const FString& FullPath, FPakEntry* OutEntry) const;
 
    void SetMountPoint(const TCHAR* Path)
    {
        MountPoint = Path;
        MakeDirectoryFromPath(MountPoint);
    }
 
    const FString& GetMountPoint() const
    {
        return MountPoint;
    }
 
    template <class ContainerType>
    UE_DEPRECATED(5.4, "Use version that takes a TArray<FString>& instead.")
    void FindPrunedFilesAtPath(ContainerType& OutFiles, const TCHAR* InPath,
        bool bIncludeFiles = true, bool bIncludeDirectories = false, bool bRecursive = false) const
    {
        TArray<FString> LocalFiles;
        FindPrunedFilesAtPath(InPath, LocalFiles, bIncludeFiles, bIncludeDirectories, bRecursive);
        for (FString& File : LocalFiles)
        {
            OutFiles.Add(File);
        }
    }
 
    PAKFILE_API void FindPrunedFilesAtPath(const TCHAR* InPath, TArray<FString>& OutFiles,
        bool bIncludeFiles = true, bool bIncludeDirectories = false, bool bRecursive = false) const;
 
    const FPakDirectory* FindPrunedDirectory(const TCHAR* InPath) const
    {
        // Caller holds FScopedPakDirectoryIndexAccess
        FString RelativePathFromMount;
        if (!NormalizeDirectoryQuery(InPath, RelativePathFromMount))
        {
            return nullptr;
        }
 
        return FindPrunedDirectoryInternal(RelativePathFromMount);
    }
 
    bool DirectoryExistsInPruned(const TCHAR* InPath) const
    {
        FString RelativePathFromMount;
        if (!NormalizeDirectoryQuery(InPath, RelativePathFromMount))
        {
            return false;
        }
 
        FScopedPakDirectoryIndexAccess ScopeAccess(*this);
        return FindPrunedDirectoryInternal(RelativePathFromMount) != nullptr;
    }
 
    PAKFILE_API bool Check();
 
    enum class EIteratorType : uint8
    {
        PathHash,
        DirectoryIndex,
        DirectoryTree,
    };
    class FBaseIterator
    {
    private:
        const FPakFile* PakFile;
        TOptional<FDirectoryIndex::TConstIterator> DirectoryIndexIt;
#if ENABLE_PAKFILE_USE_DIRECTORY_TREE
        TUniquePtr<FDirectoryTreeIndex::FConstIterator> TreeIt;
#endif
        TOptional<FPakDirectory::TConstIterator> FileIt;
        TOptional<FPathHashIndex::TConstIterator> PathHashIt;
        mutable FString CachedFilename;
        /* The PakEntry for return in Info */
        mutable FPakEntry PakEntry;
        /* Which type of internal iterator this iterator and its pakfile are using. */
        EIteratorType IteratorType;
        bool bIncludeDeleted;
#if ENABLE_PAKFILE_RUNTIME_PRUNING
        bool bRequiresDirectoryIndexLock;
#endif
 
    public:
        FBaseIterator(FBaseIterator&&) = default;
        FBaseIterator& operator=(FBaseIterator&&) = default;
        // PakFile iterators can be large, because they have to walk a tree, so do not copy them, move construct only.
        FBaseIterator(const FBaseIterator&) = delete;
        FBaseIterator& operator=(const FBaseIterator&) = delete;
 
        PAKFILE_API FBaseIterator& operator++();
 
        PAKFILE_API explicit operator bool() const;
 
        PAKFILE_API bool operator !() const;
 
        PAKFILE_API const FPakEntry& Info() const;
 
        PAKFILE_API bool HasFilename() const;
 
    protected:
        PAKFILE_API FBaseIterator(const FPakFile& InPakFile, bool bInIncludeDeleted, bool bUsePathHash);
 
#if ENABLE_PAKFILE_RUNTIME_PRUNING
        PAKFILE_API ~FBaseIterator();
#endif
 
        PAKFILE_API const FString& Filename() const;
 
        PAKFILE_API FPakEntryLocation GetPakEntryIndex() const;
 
    private:
 
        /* Skips over deleted records and moves to the next Directory in the DirectoryIndex when necessary. */
        void AdvanceToValid();
 
        FPathHashIndex::TConstIterator& GetPathHashIt();
        const FPathHashIndex::TConstIterator& GetPathHashIt() const;
 
        bool IsDirectoryItValid() const;
        void IncrementDirectoryIt();
        FStringView GetDirectoryItKey() const;
        const FPakDirectory& GetDirectoryItValue() const;
        FPakDirectory::TConstIterator& GetFileIt();
        const FPakDirectory::TConstIterator& GetFileIt() const;
 
        FDirectoryIndex::TConstIterator& GetDirectoryIndexIt();
        const FDirectoryIndex::TConstIterator& GetDirectoryIndexIt() const;
 
#if ENABLE_PAKFILE_USE_DIRECTORY_TREE
        FDirectoryTreeIndex::FConstIterator& GetTreeIt();
        const FDirectoryTreeIndex::FConstIterator& GetTreeIt() const;
#endif
    };
 
    class FPakEntryIterator : public FBaseIterator
    {
    public:
        FPakEntryIterator(const FPakFile& InPakFile, bool bInIncludeDeleted = false)
            : FBaseIterator(InPakFile, bInIncludeDeleted, !InPakFile.bHasFullDirectoryIndex /* bUsePathHash */)
        {
        }
 
        const FString* TryGetFilename() const
        {
            if (HasFilename())
            {
                return &Filename();
            }
            else
            {
                return nullptr;
            }
        }
    };
 
    class FFilenameIterator : public FBaseIterator
    {
    public:
        FFilenameIterator(const FPakFile& InPakFile, bool bInIncludeDeleted = false)
            : FBaseIterator(InPakFile, bInIncludeDeleted, false /* bUsePathHash */)
        {
        }
 
        using FBaseIterator::Filename;
    };
 
    const FPakInfo& GetInfo() const
    {
        return Info;
    }
 
    const FDateTime& GetTimestamp() const
    {
        return Timestamp;
    }
 
    bool HasFilenames() const
    {
        return bHasFullDirectoryIndex;
    }
 
    // FPakFile helper functions shared between the runtime and UnrealPak.exe
    /*
     * Given a FPakEntry from the index, seek to the payload and read the hash of the payload out of the payload entry
     * Warning: Slow function, do not use in performance critical operations
     *
     * @param PakEntry the FPakEntry from the index, which has the Offset to read to
     * @param OutBuffer a buffer at least sizeof(FPakEntry::Hash) in size, into which the hash will be copied
     */
    void ReadHashFromPayload(const FPakEntry& PakEntry, uint8* OutBuffer)
    {
        if (PakEntry.IsDeleteRecord())
        {
            FMemory::Memset(OutBuffer, 0, sizeof(FPakEntry::Hash));
        }
        else
        {
            TUniquePtr<FArchive> Reader {CreatePakReader(nullptr, *GetFilename())};
            Reader->Seek(PakEntry.Offset);
            FPakEntry SerializedEntry;
            SerializedEntry.Serialize(*Reader, GetInfo().Version);
            FMemory::Memcpy(OutBuffer, &SerializedEntry.Hash, sizeof(SerializedEntry.Hash));
        }
    }
 
    static PAKFILE_API uint64 HashPath(const TCHAR* RelativePathFromMount, uint64 Seed, int32 PakFileVersion);
 
    typedef TFunction<FPakEntryPair & ()> ReadNextEntryFunction;
    static PAKFILE_API void EncodePakEntriesIntoIndex(int32 InNumEntries, const ReadNextEntryFunction& InReadNextEntry, const TCHAR* InPakFilename, const FPakInfo& InPakInfo, const FString& MountPoint,
        int32& OutNumEncodedEntries, int32& OutNumDeletedEntries, uint64* OutPathHashSeed,
        FDirectoryIndex* OutDirectoryIndex, FPathHashIndex* OutPathHashIndex, TArray<uint8>& OutEncodedPakEntries, TArray<FPakEntry>& OutNonEncodableEntries, TMap<uint64, FString>* InOutCollisionDetection,
        int32 PakFileVersion);
 
    static PAKFILE_API const FPakEntryLocation* FindLocationFromIndex(const FString& FullPath, const FString& MountPoint, const FPathHashIndex& PathHashIndex, uint64 PathHashSeed, int32 PakFileVersion);
 
    static PAKFILE_API const FPakEntryLocation* FindLocationFromIndex(const FString& FullPath, const FString& MountPoint, const FDirectoryIndex& DirectoryIndex);
 
    static PAKFILE_API EFindResult GetPakEntry(const FPakEntryLocation& FPakEntryLocation, FPakEntry* OutEntry, const TArray<uint8>& EncodedPakEntries, const TArray<FPakEntry>& Files, const FPakInfo& Info);
 
    static PAKFILE_API void PruneDirectoryIndex(FDirectoryIndex& InOutDirectoryIndex, FDirectoryIndex* PrunedDirectoryIndex, const FString& MountPoint);
 
    /* Helper function to modify the given string to append '/' at the end of path to normalize directory names for hash and string compares */
    static void MakeDirectoryFromPath(FString& Path)
    {
        if (Path.Len() > 0 && Path[Path.Len() - 1] != '/')
        {
            Path += TEXT("/");
        }
    }
    /* Helper function to check that the given string is in our directory format (ends with '/') */
    static bool IsPathInDirectoryFormat(const FString& Path)
    {
        return Path.Len() > 0 && Path[Path.Len() - 1] == TEXT('/');
    }
 
    /* Helper function to join two path strings that are in the PakPath format */
    static FString PakPathCombine(FStringView Parent, FStringView Child);
    static FString PakPathCombine(FStringView Parent, FUtf8StringView Child);
 
    static bool SplitPathInline(FStringView& InOutPath, FStringView& OutFilename);
 
    static bool SplitPathInline(FString& InOutPath, FString& OutFilename)
    {
        FStringView Path(InOutPath), Filename;
        const bool bResult = SplitPathInline(Path, Filename);
        OutFilename = Filename;
        InOutPath = Path;
        return bResult;
    }
 
    static bool GetRelativePathFromMountInline(FString& Child, const FString& MountPoint)
    {
        if (!Child.StartsWith(MountPoint))
        {
            return false;
        }
        Child = Child.Mid(MountPoint.Len());
        if (Child.IsEmpty())
        {
            // Child equals the MountPoint
            Child = TEXT("/");
        }
        return true;
    }
 
    static const TCHAR* GetRelativeFilePathFromMountPointer(const FString& Child, const FString& MountPoint)
    {
        if (!Child.StartsWith(MountPoint))
        {
            return nullptr;
        }
        const TCHAR* RelativePathFromMount = (*Child) + MountPoint.Len();
        if (RelativePathFromMount[0] == TEXT('\0'))
        {
            // Child is equal to MountPoint, invalid
            return nullptr;
        }
        return RelativePathFromMount;
    }
 
    /* Returns the global,const flag for whether the current process is allowing PakFiles to keep their entire DirectoryIndex (if it exists in the PakFile on disk) rather than pruning it */
    static PAKFILE_API bool IsPakKeepFullDirectory();
 
    /* Returns the global,const flag for whether UnrealPak should write a copy of the full PathHashIndex and Pruned DirectoryIndex to the PakFile */
    static PAKFILE_API bool IsPakWritePathHashIndex();
 
    /* Returns the global,const flag for whether UnrealPak should write a copy of the full DirectoryIndex to the PakFile */
    static PAKFILE_API bool IsPakWriteFullDirectoryIndex();
 
private:
 
    void Initialize(FArchive& Reader, bool bLoadIndex = true);
 
    void LoadIndex(FArchive& Reader);
 
    PAKFILE_API EFindResult GetPakEntry(const FPakEntryLocation& FPakEntryLocation, FPakEntry* OutEntry) const;
 
    struct FIndexSettings;
 
    static FIndexSettings& GetIndexSettings();
 
    static bool IsPakValidatePruning();
    static bool IsPakDelayPruning();
 
#if ENABLE_PAKFILE_RUNTIME_PRUNING
    static bool bSomePakNeedsPruning;
#endif
 
    PAKFILE_API bool RequiresDirectoryIndexLock() const;
 
    PAKFILE_API bool ShouldValidatePrunedDirectory() const;
 
    PAKFILE_API bool ShouldUseDirectoryTree() const;
 
    static void AddEntryToIndex(const FString& Filename, const FPakEntryLocation& EntryLocation,
        const FString& MountPoint, uint64 PathHashSeed, FDirectoryIndex* DirectoryIndex,
        FDirectoryTreeIndex* DirectoryTreeIndex, FPathHashIndex* PathHashIndex,
        TMap<uint64, FString>* CollisionDetection, int32 PakFileVersion);
 
    /* Encodes a pak entry as an array of bytes into the given archive.  Returns true if encoding succeeded.  If encoding did not succeed, caller will need to store the InPakEntry in an unencoded list */
    static bool EncodePakEntry(FArchive& Ar, const FPakEntry& InPakEntry, const FPakInfo& InInfo);
 
    /* Decodes a bit-encoded pak entry from a pointer to the start of its encoded bytes into the given OutEntry */
    static void DecodePakEntry(const uint8* SourcePtr, FPakEntry& OutEntry, const FPakInfo& InInfo);
 
    /* Internal index loading function that returns false if index loading fails due to an intermittent IO error. Allows LoadIndex to retry or throw a fatal as required */
    bool LoadIndexInternal(FArchive& Reader, FDirectoryIndex& OutDirectoryTMap,
        FDirectoryTreeIndex& OutDirectoryTree, FDirectoryIndex& OutPrunedDirectoryTMap,
        FDirectoryTreeIndex& OutPrunedDirectoryTree, bool bStoreDirectoryTMap, bool bStoreDirectoryTree);
    void LoadIndexInternal_DirectoryIndex(FArchive& Ar, FDirectoryIndex& OutDirectoryTMap,
        FDirectoryTreeIndex& OutDirectoryTree, bool bLoadIntoDirectoryTMap, bool bLoadIntoDirectoryTree);
    // This is an internal function that should only be called by FPakFile or by PakFileUtilities.cpp; it can change without deprecation.
public:
    PAKFILE_API static void SaveIndexInternal_DirectoryIndex(FArchive& Ar, const FDirectoryIndex& DirectoryTMap);
private:
 
    /* Legacy index loading function for PakFiles saved before FPakInfo::PakFile_Version_PathHashIndex */
    bool LoadLegacyIndex(FArchive& Reader, FDirectoryIndex& OutDirectoryTMap);
 
    /* Helper function for LoadIndexInternal; each array of Index bytes read from the file needs to be independently decrypted and checked for corruption */
    bool DecryptAndValidateIndex(FArchive& Reader, TArray<uint8>& IndexData, FSHAHash& InExpectedHash, FSHAHash& OutActualHash);
 
    /* Manually add a file to a pak file */
    void AddSpecialFile(const FPakEntry& Entry, const FString& Filename);
 
    static void PruneDirectoryIndexInternal(FDirectoryIndex* InOutDirectoryIndex,
        FDirectoryTreeIndex* InOutDirectoryTreeIndex, FDirectoryIndex* PrunedDirectoryIndex,
        FDirectoryTreeIndex* PrunedDirectoryTreeIndex, const FString& MountPoint);
 
    template <typename ShouldVisitFunc>
    struct FVisitFilter
    {
        FVisitFilter(const ShouldVisitFunc& InShouldVisit, bool bInIncludeFiles, bool bInIncludeDirectories, bool bInRecursive);
        const ShouldVisitFunc& ShouldVisit;
        bool bIncludeFiles = true;
        bool bIncludeDirectories = false;
        bool bRecursive = false;
    };
 
    template <typename ShouldVisitFunc, class ContainerType>
    void FindPrunedFilesAtPathInternal(const TCHAR* InPath, ContainerType& OutFiles, const FVisitFilter<ShouldVisitFunc>& VisitFilter) const;
 
    PAKFILE_API const FPakEntryLocation* FindLocationFromIndex(const FString& FullPath,
        const FDirectoryIndex& InDirectoryIndex, const FDirectoryTreeIndex& InDirectoryTreeIndex) const;
 
    template <typename ShouldVisitFunc, class ContainerType>
    void FindFilesAtPathInIndex(const FDirectoryIndex& TargetIndex, const FDirectoryTreeIndex& TargetTreeIndex,
        ContainerType& OutFiles, const FString& FullSearchPath,
        const FVisitFilter<ShouldVisitFunc>& VisitFilter) const;
 
#if ENABLE_PAKFILE_USE_DIRECTORY_TREE
    template <typename ShouldVisitFunc, class ContainerType>
    void FindFilesAtPathInTreeIndexInternal(FStringView RelSearchPath, const FDirectoryTreeIndex& TargetTreeIndex,
        ContainerType& OutFiles, TArray<FString>& OutDirectories,
        const FString& FullSearchPath, const FVisitFilter<ShouldVisitFunc>& VisitFilter) const;
 
#if !UE_BUILD_SHIPPING
    template <class ContainerType>
    bool ValidateDirectoryTreeSearchConsistency(const ContainerType& FilesTree, const TArray<FString>& DirectoriesInPakTree, const ContainerType& FilesIndexed, const TArray<FString>& DirectoriesInPakIndexed) const;
#endif  // UE_BUILD_SHIPPING
#endif  // ENABLE_PAKFILE_USE_DIRECTORY_TREE
 
    template <typename ShouldVisitFunc, class ContainerType>
    void FindFilesAtPathInIndexInternal(const FStringView& RelSearchPath, const FDirectoryIndex& TargetIndex,
        ContainerType& OutFiles, TArray<FString>& OutDirectories,
        const FString& FullSearchPath, const FVisitFilter<ShouldVisitFunc>& VisitFilter) const;
 
    template <typename ShouldVisitFunc, class ContainerType>
    static void FindFilesAtPathInPakDirectoryInternal(const FString& MountPoint, FStringView RelPathInIndex,
        const FPakDirectory& PakDirectory, ContainerType& OutFiles, TArray<FString>& OutDirectoriesInPak,
        const FString& FullSearchPath, const FVisitFilter<ShouldVisitFunc>& VisitFilter);
 
    PAKFILE_API bool NormalizeDirectoryQuery(const TCHAR* InPath, FString& OutRelativePathFromMount) const;
 
    PAKFILE_API const FPakDirectory* FindPrunedDirectoryInternal(const FString& RelativePathFromMount) const;
    const FPakDirectory* FindPrunedDirectoryInIndexInternal(const FString& RelativePathFromMount,
        const FDirectoryIndex& InDirectoryIndex, const FDirectoryTreeIndex& InTreeIndex) const;
 
#if ENABLE_PAKFILE_RUNTIME_PRUNING_VALIDATE
    /* Logs an error if the two sets are not identical after removing all config-specified ignore paths */
    PAKFILE_API void ValidateDirectorySearch(const TSet<FString>& FoundFullFiles, const TSet<FString>& PrunedFoundFiles, const TCHAR* InPath) const;
#endif
};
 
class FPakNoEncryption
{
public:
    enum 
    {
        Alignment = 1,
    };
 
    static inline int64 AlignReadRequest(int64 Size) 
    {
        return Size;
    }
 
    static inline void DecryptBlock(void* Data, int64 Size, const FGuid& EncryptionKeyGuid)
    {
        // Nothing needs to be done here
    }
};
 
typedef TFunction<FSharedPakReader()> TAcquirePakReaderFunction;
 
template< typename EncryptionPolicy = FPakNoEncryption >
class FPakReaderPolicy
{
public:
    const FPakFile&     PakFile;
    FPakEntry           PakEntry;
    TAcquirePakReaderFunction AcquirePakReader;
    int64               OffsetToFile;
 
    FPakReaderPolicy(const FPakFile& InPakFile,const FPakEntry& InPakEntry, TAcquirePakReaderFunction& InAcquirePakReader)
        : PakFile(InPakFile)
        , PakEntry(InPakEntry)
        , AcquirePakReader(InAcquirePakReader)
    {
        OffsetToFile = PakEntry.Offset + PakEntry.GetSerializedSize(PakFile.GetInfo().Version);
    }
 
    inline int64 FileSize() const 
    {
        return PakEntry.Size;
    }
 
    void Serialize(int64 DesiredPosition, void* V, int64 Length) const
    {
        FGuid EncryptionKeyGuid = PakFile.GetInfo().EncryptionKeyGuid;
        const constexpr int64 Alignment = (int64)EncryptionPolicy::Alignment;
        const constexpr int64 AlignmentMask = ~(Alignment - 1);
        uint8 TempBuffer[Alignment];
        FSharedPakReader PakReader = AcquirePakReader();
        if (EncryptionPolicy::AlignReadRequest(DesiredPosition) != DesiredPosition)
        {
            int64 Start = DesiredPosition & AlignmentMask;
            int64 Offset = DesiredPosition - Start;
            int64 CopySize = FMath::Min(Alignment - Offset, Length);
            PakReader->Seek(OffsetToFile + Start);
            PakReader->Serialize(TempBuffer, Alignment);
            EncryptionPolicy::DecryptBlock(TempBuffer, Alignment, EncryptionKeyGuid);
            FMemory::Memcpy(V, TempBuffer + Offset, CopySize);
            V = (void*)((uint8*)V + CopySize);
            DesiredPosition += CopySize;
            Length -= CopySize;
            check(Length == 0 || DesiredPosition % Alignment == 0);
        }
        else
        {
            PakReader->Seek(OffsetToFile + DesiredPosition);
        }
        
        int64 CopySize = Length & AlignmentMask;
        PakReader->Serialize(V, CopySize);
        EncryptionPolicy::DecryptBlock(V, CopySize, EncryptionKeyGuid);
        Length -= CopySize;
        V = (void*)((uint8*)V + CopySize);
 
        if (Length > 0)
        {
            PakReader->Serialize(TempBuffer, Alignment);
            EncryptionPolicy::DecryptBlock(TempBuffer, Alignment, EncryptionKeyGuid);
            FMemory::Memcpy(V, TempBuffer, Length);
        }
    }
};
 
template< typename ReaderPolicy = FPakReaderPolicy<> >
class FPakFileHandle : public IFileHandle
{
    int64 ReadPos;
    ReaderPolicy Reader;
    TRefCountPtr<const FPakFile> PakFile;
 
public:
 
    FPakFileHandle(const TRefCountPtr<const FPakFile>& InPakFile, const FPakEntry& InPakEntry, TAcquirePakReaderFunction& InAcquirePakReaderFunction)
        : ReadPos(0)
        , Reader(*InPakFile, InPakEntry, InAcquirePakReaderFunction)
        , PakFile(InPakFile)
    {
        INC_DWORD_STAT(STAT_PakFile_NumOpenHandles);
    }
 
    FPakFileHandle(const TRefCountPtr<const FPakFile>& InPakFile, const FPakEntry& InPakEntry, FArchive* InPakReader)
        : ReadPos(0)
        , Reader(*InPakFile, InPakEntry, InPakReader)
        , PakFile(InPakFile)
    {
        INC_DWORD_STAT(STAT_PakFile_NumOpenHandles);
    }
 
    virtual ~FPakFileHandle()
    {
        DEC_DWORD_STAT(STAT_PakFile_NumOpenHandles);
    }
 
    //~ Begin IFileHandle Interface
    virtual int64 Tell() override
    {
        return ReadPos;
    }
    virtual bool Seek(int64 NewPosition) override
    {
        if (NewPosition > Reader.FileSize() || NewPosition < 0)
        {
            return false;
        }
        ReadPos = NewPosition;
        return true;
    }
    virtual bool SeekFromEnd(int64 NewPositionRelativeToEnd) override
    {
        return Seek(Reader.FileSize() - NewPositionRelativeToEnd);
    }
    virtual bool Read(uint8* Destination, int64 BytesToRead) override
    {
        if (ReadInternal(Destination, BytesToRead, ReadPos))
        {
            ReadPos += BytesToRead;
            return true;
        }
        else
        {
            return false;
        }
    }
    virtual bool ReadAt(uint8* Destination, int64 BytesToRead, int64 Offset) override
    {
        if (BytesToRead < 0 || Offset < 0 || (BytesToRead + Offset) > Reader.FileSize())
        {
            return false;
        }
 
        return ReadInternal(Destination, BytesToRead, Offset);
    }
    virtual bool Write(const uint8* Source, int64 BytesToWrite) override
    {
        // Writing in pak files is not allowed.
        return false;
    }
    virtual int64 Size() override
    {
        return Reader.FileSize();
    }
    virtual bool Flush(const bool bFullFlush = false) override
    {
        // pak files are read only, so don't need to support flushing
        return false;
    }
    virtual bool Truncate(int64 NewSize) override
    {
        // pak files are read only, so don't need to support truncation
        return false;
    }
private:
    bool ReadInternal(uint8* Destination, int64 BytesToRead, int64 Offset)
    {
        SCOPE_SECONDS_ACCUMULATOR(STAT_PakFile_Read);
 
        if (!Reader.PakEntry.Verified)
        {
            FPakEntry FileHeader;
            FSharedPakReader PakReader = Reader.AcquirePakReader();
            PakReader->Seek(Reader.PakEntry.Offset);
            FileHeader.Serialize(PakReader.GetArchive(), Reader.PakFile.GetInfo().Version);
            if (FPakEntry::VerifyPakEntriesMatch(Reader.PakEntry, FileHeader))
            {
                Reader.PakEntry.Verified = true;
            }
            else
            {
                return false;
            }
        }
 
        if (Reader.FileSize() >= (Offset + BytesToRead))
        {
            Reader.Serialize(Offset, Destination, BytesToRead);
            return true;
        }
        else
        {
            return false;
        }
    }
};
 
struct FPakMountArgs
{
    // Pak filename
    const TCHAR* PakFilename = nullptr;
    // Search order
    uint32 PakOrder = 0;
    // Path to mount the pak at
    const TCHAR* Path = nullptr;
    // Mount options
    FPakMountOptions MountOptions;
    // Flag to load Index
    bool bLoadIndex = true;
};
 
class FPakPlatformFile : public IPlatformFile
{
    struct FPakListEntry
    {
        FPakListEntry()
            : ReadOrder(0)
            , PakFile(nullptr)
        {}
 
        uint32                  ReadOrder;
        TRefCountPtr<FPakFile>  PakFile;
 
        inline bool operator < (const FPakListEntry& RHS) const
        {
            return ReadOrder > RHS.ReadOrder;
        }
    };
 
    template<bool bInAllowDuplicateKeys>
    friend struct UE::PakFile::Private::FPakEntryFilenameSetKeyFuncs;
 
    struct FPakListDeferredEntry
    {
        FString Filename;
        FString Path;
        uint32 ReadOrder;
        FGuid EncryptionKeyGuid;
        int32 PakchunkIndex;
    };
    
    IPlatformFile* LowerLevel;
    TArray<FPakListEntry> PakFiles;
    TArray<FPakListDeferredEntry> PendingEncryptedPakFiles;
    bool bSigned;
    mutable FTransactionallySafeCriticalSection PakListCritical;
    TSet<FName> ExcludedNonPakExtensions;
    FString IniFileExtension;
    FString GameUserSettingsIniFilename;
    TSharedPtr<UE::IoStore::IFileIoDispatcherBackend> IoDispatcherFileBackend;
    TSharedPtr<FFilePackageStoreBackend> PackageStoreBackend;
 
    FTSTicker::FDelegateHandle RetireReadersHandle;
 
#if !UE_BUILD_SHIPPING
    // if true (via -looklocalfirst) then loose/non-ufs files will be looked for before looking in the .pak file
    // this respects IsNonPakFilenameAllowed()
    bool bLookLooseFirst = false;
#endif
 
    inline void GetMountedPaks(TArray<FPakListEntry>& Paks)
    {
        UE::TScopeLock ScopedLock(PakListCritical);
        Paks.Append(PakFiles);
    }
 
    PAKFILE_API bool DirectoryExistsInPrunedPakFiles(const TCHAR* Directory);
 
    PAKFILE_API bool BufferedCopyFile(IFileHandle& Dest, IFileHandle& Source, const int64 FileSize, uint8* Buffer, const int64 BufferSize) const;
 
    PAKFILE_API IFileHandle* CreatePakFileHandle(const TCHAR* Filename, const TRefCountPtr<FPakFile>& PakFile, const FPakEntry* FileEntry);
 
    PAKFILE_API static int32 GetPakOrderFromPakFilePath(const FStringView PakFilePath);
 
    PAKFILE_API IPakFile* HandleMountPakDelegate(const FString& PakFilePath, int32 PakOrder);
 
    friend struct UE::PakFile::Private::FHandleMountPaksExDelegate;
 
    PAKFILE_API bool HandleUnmountPakDelegate(const FString& PakFilePath);
 
    PAKFILE_API static void FindPakFilesInDirectory(IPlatformFile* LowLevelFile, const TCHAR* Directory, const FString& WildCard, TArray<FString>& OutPakFiles);
 
    PAKFILE_API static void FindAllPakFiles(IPlatformFile* LowLevelFile, const TArray<FString>& PakFolders, const FString& WildCard, TArray<FString>& OutPakFiles);
 
    PAKFILE_API bool IsNonPakFilenameAllowed(const FString& InFilename);
 
    PAKFILE_API void RegisterEncryptionKey(const FGuid& InEncryptionKeyGuid, const FAES::FAESKey& InKey);
 
    PAKFILE_API static bool IsPakFileInstalled(const FString& InFilename);
 
public:
 
    static PAKFILE_API bool bMountFailOnMissingUtoc;
 
    //~ For visibility of overloads we don't override
    using IPlatformFile::IterateDirectory;
    using IPlatformFile::IterateDirectoryRecursively;
    using IPlatformFile::IterateDirectoryStat;
    using IPlatformFile::IterateDirectoryStatRecursively;
 
    static const TCHAR* GetTypeName()
    {
        return TEXT("PakFile");
    }
 
    PAKFILE_API static const TCHAR* GetMountStartupPaksWildCard();
 
    PAKFILE_API static void SetMountStartupPaksWildCard(const FString& WildCard);
 
    PAKFILE_API EChunkLocation::Type GetPakChunkLocation(int32 InPakchunkIndex) const;
 
    PAKFILE_API bool AnyChunksAvailable() const;
 
    inline void GetMountedPakFilenames(TArray<FString>& PakFilenames)
    {
        UE::TScopeLock ScopedLock(PakListCritical);
        PakFilenames.Empty(PakFiles.Num());
        for (FPakListEntry& Entry : PakFiles)
        {
            PakFilenames.Add(Entry.PakFile->GetFilename());
        }
    }
 
    inline void GetMountedPakFilenames(TSet<FString>& PakFilenames)
    {
        UE::TScopeLock ScopedLock(PakListCritical);
        PakFilenames.Empty(PakFiles.Num());
        for (FPakListEntry& Entry : PakFiles)
        {
            PakFilenames.Add(Entry.PakFile->GetFilename());
        }
    }
 
    inline void GetMountedChunkIds(TSet<int32>& OutChunkIds)
    {
        UE::TScopeLock ScopedLock(PakListCritical);
        OutChunkIds.Empty(PakFiles.Num());
        for (FPakListEntry& Entry : PakFiles)
        {
            OutChunkIds.Add(Entry.PakFile->PakGetPakchunkIndex());
        }
    }
 
    PAKFILE_API static bool CheckIfPakFilesExist(IPlatformFile* LowLevelFile, const TArray<FString>& PakFolders);
 
    PAKFILE_API static void GetPakFolders(const TCHAR* CmdLine, TArray<FString>& OutPakFolders);
 
    PAKFILE_API static void GetPakEncryptionKey(FAES::FAESKey& OutKey, const FGuid& InEncryptionKeyGuid);
 
    PAKFILE_API static TSharedPtr<const struct FPakSignatureFile, ESPMode::ThreadSafe> GetPakSignatureFile(const TCHAR* InFilename);
 
    PAKFILE_API static void RemoveCachedPakSignaturesFile(const TCHAR* InFilename);
 
    PAKFILE_API FPakPlatformFile();
 
    PAKFILE_API virtual ~FPakPlatformFile();
 
    PAKFILE_API virtual bool ShouldBeUsed(IPlatformFile* Inner, const TCHAR* CmdLine) const override;
    PAKFILE_API virtual bool Initialize(IPlatformFile* Inner, const TCHAR* CommandLineParam) override;
    PAKFILE_API virtual void InitializeNewAsyncIO() override;
 
    PAKFILE_API void OptimizeMemoryUsageForMountedPaks();
 
    virtual IPlatformFile* GetLowerLevel() override
    {
        return LowerLevel;
    }
    virtual void SetLowerLevel(IPlatformFile* NewLowerLevel) override
    {
        LowerLevel = NewLowerLevel;
    }
 
    virtual const TCHAR* GetName() const override
    {
        return FPakPlatformFile::GetTypeName();
    }
 
    PAKFILE_API void Tick() override;
 
    PAKFILE_API bool Mount(const TCHAR* InPakFilename, uint32 PakOrder, const TCHAR* InPath = nullptr, bool bLoadIndex = true, FPakListEntry* OutPakListEntry = nullptr);
 
    PAKFILE_API bool Mount(const FPakMountArgs& MountArgs, FIoStatus* OutIoMountStatus = nullptr, FPakListEntry* OutPakListEntry = nullptr);
 
    PAKFILE_API bool Unmount(const TCHAR* InPakFilename);
 
    PAKFILE_API int32 MountAllPakFiles(const TArray<FString>& PakFolders);
    PAKFILE_API int32 MountAllPakFiles(const TArray<FString>& PakFolders, const FString& WildCard);
 
    PAKFILE_API bool ReloadPakReaders();
 
    PAKFILE_API virtual void MakeUniquePakFilesForTheseFiles(const TArray<TArray<FString>>& InFiles);
 
    PAKFILE_API static bool FindFileInPakFiles(TArray<FPakListEntry>& Paks, const TCHAR* Filename,
        TRefCountPtr<FPakFile>* OutPakFile, FPakEntry* OutEntry = nullptr);
 
    PAKFILE_API bool FindFileInPakFiles(const TCHAR* Filename, TRefCountPtr<FPakFile>* OutPakFile = nullptr,
        FPakEntry* OutEntry = nullptr);
 
    //~ Begin IPlatformFile Interface
    virtual bool FileExists(const TCHAR* Filename) override
    {
        // Check pak files first.
        if (FindFileInPakFiles(Filename))
        {
            return true;
        }
        // File has not been found in any of the pak files, continue looking in inner platform file.
        bool Result = false;
        if (IsNonPakFilenameAllowed(Filename))
        {
            Result = LowerLevel->FileExists(Filename);
        }
        return Result;
    }
 
    virtual int64 FileSize(const TCHAR* Filename) override
    {
        // Check pak files first
        FPakEntry FileEntry;
        if (FindFileInPakFiles(Filename, nullptr, &FileEntry))
        {
            return FileEntry.CompressionMethodIndex != 0 ? FileEntry.UncompressedSize : FileEntry.Size;
        }
        // First look for the file in the user dir.
        int64 Result = INDEX_NONE;
        if (IsNonPakFilenameAllowed(Filename))
        {
            Result = LowerLevel->FileSize(Filename);
        }
        return Result;
    }
 
    virtual bool DeleteFile(const TCHAR* Filename) override
    {
        // If file exists in pak file it will never get deleted.
        if (FindFileInPakFiles(Filename))
        {
            return false;
        }
        // The file does not exist in pak files, try LowerLevel->
        bool Result = false;
        if (IsNonPakFilenameAllowed(Filename))
        {
            Result = LowerLevel->DeleteFile(Filename);
        }
        return Result;
    }
 
    virtual bool IsReadOnly(const TCHAR* Filename) override
    {
        // Files in pak file are always read-only.
        if (FindFileInPakFiles(Filename))
        {
            return true;
        }
        // The file does not exist in pak files, try LowerLevel->
        bool Result = false;
        if (IsNonPakFilenameAllowed(Filename))
        {
            Result = LowerLevel->IsReadOnly(Filename);
        }
        return Result;
    }
 
    virtual bool MoveFile(const TCHAR* To, const TCHAR* From) override
    {
        // Files which exist in pak files can't be moved
        if (FindFileInPakFiles(From))
        {
            return false;
        }
        // Files not in pak are allowed to be moved.
        bool Result = false;
        if (IsNonPakFilenameAllowed(From))
        {
            Result = LowerLevel->MoveFile(To, From);
        }
        return Result;
    }
 
    virtual bool SetReadOnly(const TCHAR* Filename, bool bNewReadOnlyValue) override
    {
        // Files in pak file will never change their read-only flag.
        if (FindFileInPakFiles(Filename))
        {
            // This fails if soemone wants to make files from pak writable.
            return bNewReadOnlyValue;
        }
        // Try lower level
        bool Result = bNewReadOnlyValue;
        if (IsNonPakFilenameAllowed(Filename))
        {
            Result = LowerLevel->SetReadOnly(Filename, bNewReadOnlyValue);
        }
        return Result;
    }
 
    virtual FDateTime GetTimeStamp(const TCHAR* Filename) override
    {
        // Check pak files first.
        TRefCountPtr<FPakFile> PakFile = NULL;
        if (FindFileInPakFiles(Filename, &PakFile))
        {
            return PakFile->GetTimestamp();
        }
        // Fall back to lower level.
        FDateTime Result = FDateTime::MinValue();
        if (IsNonPakFilenameAllowed(Filename))
        {
            double StartTime = (UE_LOG_ACTIVE(LogPakFile, Verbose)) ? FPlatformTime::Seconds() : 0.0;
            Result = LowerLevel->GetTimeStamp(Filename);
            UE_LOG(LogPakFile, Verbose, TEXT("GetTimeStamp on disk (!!) for %s took %6.2fms."), Filename, float(FPlatformTime::Seconds() - StartTime) * 1000.0f);
        }
        return Result;
    }
 
    virtual void GetTimeStampPair(const TCHAR* FilenameA, const TCHAR* FilenameB, FDateTime& OutTimeStampA, FDateTime& OutTimeStampB) override
    {
        TRefCountPtr<FPakFile> PakFileA;
        TRefCountPtr<FPakFile> PakFileB;
        FindFileInPakFiles(FilenameA, &PakFileA);
        FindFileInPakFiles(FilenameB, &PakFileB);
 
        // If either file exists, we'll assume both should exist here and therefore we can skip the
        // request to the lower level platform file.
        if (PakFileA != nullptr || PakFileB != nullptr)
        {
            OutTimeStampA = PakFileA != nullptr ? PakFileA->GetTimestamp() : FDateTime::MinValue();
            OutTimeStampB = PakFileB != nullptr ? PakFileB->GetTimestamp() : FDateTime::MinValue();
        }
        else
        {
            // Fall back to lower level.
            if (IsNonPakFilenameAllowed(FilenameA) && IsNonPakFilenameAllowed(FilenameB))
            {
                LowerLevel->GetTimeStampPair(FilenameA, FilenameB, OutTimeStampA, OutTimeStampB);
            }
            else
            {
                OutTimeStampA = FDateTime::MinValue();
                OutTimeStampB = FDateTime::MinValue();
            }
        }
    }
 
    virtual void SetTimeStamp(const TCHAR* Filename, FDateTime DateTime) override
    {
        // No modifications allowed on files from pak (although we could theoretically allow this one).
        if (!FindFileInPakFiles(Filename))
        {
            if (IsNonPakFilenameAllowed(Filename))
            {
                LowerLevel->SetTimeStamp(Filename, DateTime);
            }
        }
    }
 
    virtual FDateTime GetAccessTimeStamp(const TCHAR* Filename) override
    {
        // AccessTimestamp not yet supported in pak files (although it is possible).
        TRefCountPtr<FPakFile> PakFile;
        if (FindFileInPakFiles(Filename, &PakFile))
        {
            return PakFile->GetTimestamp();
        }
        // Fall back to lower level.
        FDateTime Result = false;
        if (IsNonPakFilenameAllowed(Filename))
        {
            Result = LowerLevel->GetAccessTimeStamp(Filename);
        }
        return Result;
    }
 
    virtual FString GetFilenameOnDisk(const TCHAR* Filename) override
    {
        FPakEntry FileEntry;
        TRefCountPtr<FPakFile> PakFile;
        if (FindFileInPakFiles(Filename, &PakFile, &FileEntry))
        {
            const FString Path(FPaths::GetPath(Filename));
            FPakFile::FScopedPakDirectoryIndexAccess ScopeAccess(*PakFile);
 
            const FPakDirectory* PakDirectory = PakFile->FindPrunedDirectory(*Path);
            if (PakDirectory != nullptr)
            {
                for (FPakDirectory::TConstIterator FileIt(*PakDirectory); FileIt; ++FileIt)
                {
                    FPakEntry PakEntry;
                    if (PakFile->GetPakEntry(FileIt.Value(), &PakEntry) != FPakFile::EFindResult::NotFound && PakEntry.Offset == FileEntry.Offset)
                    {
                        const FUtf8String& RealFilename = FileIt.Key();
                        return Path / FString(RealFilename);
                    }
                }
            }
 
#if ENABLE_PAKFILE_RUNTIME_PRUNING_VALIDATE
            // The File exists in the Pak but has been pruned from its DirectoryIndex; log an error if we are validating pruning and return the original Filename.
            if (PakFile->ShouldValidatePrunedDirectory())
            {
                TSet<FString> FullFoundFiles;
                TSet<FString> PrunedFoundFiles;
                FullFoundFiles.Add(Filename);
                PakFile->ValidateDirectorySearch(FullFoundFiles, PrunedFoundFiles, Filename);
            }
#endif
 
            return Filename;
        }
 
        // Fall back to lower level.
        if (IsNonPakFilenameAllowed(Filename))
        {
            return LowerLevel->GetFilenameOnDisk(Filename);
        }
        else
        {
            return Filename;
        }
    }
 
    virtual ESymlinkResult IsSymlink(const TCHAR* Filename) override
    {
        return LowerLevel->IsSymlink(Filename);
    }
 
    PAKFILE_API virtual IFileHandle* OpenRead(const TCHAR* Filename, bool bAllowWrite = false) override;
    PAKFILE_API virtual IFileHandle* OpenWrite(const TCHAR* Filename, bool bAppend = false,
        bool bAllowRead = false) override;
 
    PAKFILE_API virtual bool DirectoryExists(const TCHAR* Directory) override;
    PAKFILE_API virtual bool CreateDirectory(const TCHAR* Directory) override;
    PAKFILE_API virtual bool DeleteDirectory(const TCHAR* Directory) override;
    PAKFILE_API virtual FFileStatData GetStatData(const TCHAR* FilenameOrDirectory) override;
    PAKFILE_API virtual bool IterateDirectory(const TCHAR* Directory,
        IPlatformFile::FDirectoryVisitor& Visitor) override;
    PAKFILE_API virtual bool IterateDirectoryRecursively(const TCHAR* Directory,
        IPlatformFile::FDirectoryVisitor& Visitor) override;
    PAKFILE_API virtual bool IterateDirectoryStat(const TCHAR* Directory,
        IPlatformFile::FDirectoryStatVisitor& Visitor) override;
    PAKFILE_API virtual bool IterateDirectoryStatRecursively(const TCHAR* Directory,
        IPlatformFile::FDirectoryStatVisitor& Visitor) override;
    PAKFILE_API virtual void FindFiles(TArray<FString>& FoundFiles,
        const TCHAR* Directory, const TCHAR* FileExtension) override;
    PAKFILE_API virtual void FindFilesRecursively(TArray<FString>& FoundFiles,
        const TCHAR* Directory, const TCHAR* FileExtension) override;
    PAKFILE_API virtual bool DeleteDirectoryRecursively(const TCHAR* Directory) override;
    PAKFILE_API virtual bool CreateDirectoryTree(const TCHAR* Directory) override;
 
    PAKFILE_API virtual bool CopyFile(const TCHAR* To, const TCHAR* From, EPlatformFileRead ReadFlags = EPlatformFileRead::None, EPlatformFileWrite WriteFlags = EPlatformFileWrite::None) override;
 
    PAKFILE_API virtual IAsyncReadFileHandle* OpenAsyncRead(const TCHAR* Filename, bool bAllowWrite = false) override;
    PAKFILE_API virtual void SetAsyncMinimumPriority(EAsyncIOPriorityAndFlags Priority) override;
 
    virtual FOpenMappedResult OpenMappedEx(const TCHAR* Filename, EOpenReadFlags OpenOptions = EOpenReadFlags::None, int64 MaximumSize = 0) override;
    FString ConvertToPakRelativePath(const TCHAR* Filename, const FPakFile* Pak)
    {
        FString RelativeFilename(Filename);
        return RelativeFilename.Mid(Pak->GetMountPoint().Len());
    }
 
    FString ConvertToAbsolutePathForExternalAppForRead(const TCHAR* Filename) override
    {
        // Check in Pak file first
        TRefCountPtr<FPakFile> Pak;
        if (FindFileInPakFiles(Filename, &Pak))
        {
            return FString::Printf(TEXT("Pak: %s/%s"), *Pak->GetFilename(), *ConvertToPakRelativePath(Filename, Pak));
        }
        else
        {
            return LowerLevel->ConvertToAbsolutePathForExternalAppForRead(Filename);
        }
    }
 
    FString ConvertToAbsolutePathForExternalAppForWrite(const TCHAR* Filename) override
    {
        // Check in Pak file first
        TRefCountPtr<FPakFile> Pak;
        if (FindFileInPakFiles(Filename, &Pak))
        {
            return FString::Printf(TEXT("Pak: %s/%s"), *Pak->GetFilename(), *ConvertToPakRelativePath(Filename, Pak));
        }
        else
        {
            return LowerLevel->ConvertToAbsolutePathForExternalAppForWrite(Filename);
        }
    }
    //~ End IPlatformFile Interface
 
    PAKFILE_API void RefreshPakChunkIndicies();
 
    // Access static delegate for loose file security
    PAKFILE_API static FFilenameSecurityDelegate& GetFilenameSecurityDelegate();
 
    // Access static delegate for custom encryption
    PAKFILE_API static FPakCustomEncryptionDelegate& GetPakCustomEncryptionDelegate();
 
    struct FPakSigningFailureHandlerData
    {
        PRAGMA_DISABLE_DEPRECATION_WARNINGS
        FCriticalSection& GetLock() { return Lock; }
        FPakChunkSignatureCheckFailedHandler& GetPakChunkSignatureCheckFailedDelegate() { return ChunkSignatureCheckFailedDelegate; }
        FPakPrincipalSignatureTableCheckFailureHandler& GetPrincipalSignatureTableCheckFailedDelegate() { return MasterSignatureTableCheckFailedDelegate; }
        PRAGMA_ENABLE_DEPRECATION_WARNINGS
 
        UE_DEPRECATED("5.1", "Use GetLock instead")
        FCriticalSection Lock;
 
        UE_DEPRECATED("5.1", "Use GetPakChunkSignatureCheckFailedDelegate instead")
        FPakChunkSignatureCheckFailedHandler ChunkSignatureCheckFailedDelegate;
 
        UE_DEPRECATED("5.1", "Use GetPrincipalSignatureTableCheckFailureDelegate instead")
        FPakPrincipalSignatureTableCheckFailureHandler MasterSignatureTableCheckFailedDelegate;
    };
 
    // Access static delegate for handling a Pak signature check failure
    PAKFILE_API static FPakSigningFailureHandlerData& GetPakSigningFailureHandlerData();
    
    // Broadcast a signature check failure through any registered delegates in a thread safe way
    PAKFILE_API static void BroadcastPakChunkSignatureCheckFailure(const FPakChunkSignatureCheckFailedData& InData);
 
    // Broadcast a principal signature table failure through any registered delegates in a thread safe way
    PAKFILE_API static void BroadcastPakPrincipalSignatureTableCheckFailure(const FString& InFilename);
 
    UE_DEPRECATED("5.1", "Use BroadcastPakPrincipalSignatureTableCheckFailure instead")
    PAKFILE_API static void BroadcastPakMasterSignatureTableCheckFailure(const FString& InFilename);
 
    // Access static delegate for setting PakIndex settings.
    PAKFILE_API static FPakSetIndexSettings& GetPakSetIndexSettingsDelegate();
 
    /* Get a list of RelativePathFromMount for every file in the given Pak that lives in any of the given chunks.  Only searches the Pruned DirectoryIndex */
    PAKFILE_API void GetPrunedFilenamesInChunk(const FString& InPakFilename, const TArray<int32>& InChunkIDs, TArray<FString>& OutFileList);
 
    PAKFILE_API void GetPrunedFilenamesInPakFile(const FString& InPakFilename, TArray<FString>& OutFileList);
 
    PAKFILE_API static void GetFilenamesFromIostoreContainer(const FString& InContainerName, TArray<FString>& OutFileList);
 
    PAKFILE_API static void GetFilenamesFromIostoreByBlockIndex(const FString& InContainerName, const TArray<int32>& InBlockIndex, TArray<FString>& OutFileList);
 
    PAKFILE_API static void ForeachPackageInIostoreWhile(TFunctionRef<bool(FName)> Predicate);
 
    PAKFILE_API void ReleaseOldReaders();
 
    // BEGIN Console commands
#if !UE_BUILD_SHIPPING
    PAKFILE_API void HandlePakListCommand(const TCHAR* Cmd, FOutputDevice& Ar);
    PAKFILE_API void HandleMountCommand(const TCHAR* Cmd, FOutputDevice& Ar);
    PAKFILE_API void HandleUnmountCommand(const TCHAR* Cmd, FOutputDevice& Ar);
    PAKFILE_API void HandlePakCorruptCommand(const TCHAR* Cmd, FOutputDevice& Ar);
    PAKFILE_API void HandleReloadPakReadersCommand(const TCHAR* Cmd, FOutputDevice& Ar);
#endif
    // END Console commands
    
#if PAK_TRACKER
    PAKFILE_API static TMap<FString, int32> GPakSizeMap;
    PAKFILE_API static void TrackPak(const TCHAR* Filename, const FPakEntry* PakEntry);
    static TMap<FString, int32>& GetPakMap() { return GPakSizeMap; }
#endif
 
    // Internal cache of pak signature files
    PAKFILE_API static TMap<FName, TSharedPtr<const struct FPakSignatureFile, ESPMode::ThreadSafe>> PakSignatureFileCache;
    PAKFILE_API static FCriticalSection PakSignatureFileCacheLock;
 
private:
    bool IterateDirectoryInternal(const TCHAR* Directory, IPlatformFile::FDirectoryVisitor& Visitor, bool bRecursive);
    bool IterateDirectoryInPakFiles(const TCHAR* Directory,
        UE::PakFile::Private::FPakFileDirectoryVisitorBase& Visitor, bool bRecursive,
        TSet<FString>& FilesVisitedInPak);
    bool IterateDirectoryStatInternal(const TCHAR* Directory,
        IPlatformFile::FDirectoryStatVisitor& Visitor, bool bRecursive);
    void FindFilesInternal(TArray<FString>& FoundFiles,
        const TCHAR* Directory, const TCHAR* FileExtension, bool bRecursive);
};
 
struct FPakSignatureFile
{
    // Magic number that tells us we're dealing with the new format sig files
    static const uint32 Magic = 0x73832DAA;
 
    enum class EVersion
    {
        Invalid,
        First,
 
        Last,
        Latest = Last - 1
    };
 
    // Sig file version. Set to Legacy if the sig file is of an old version
    EVersion Version = EVersion::Latest;
 
    // RSA encrypted hash
    TArray<uint8> EncryptedHash;
 
    // SHA1 hash of the chunk CRC data. Only valid after calling DecryptSignatureAndValidate
    FSHAHash DecryptedHash;
 
    // The actual array of data that was encrypted in the RSA block. Contains the chunk table hash and also other custom data related to the pak file
    TArray<uint8> SignatureData;
 
    // CRCs of each contiguous 64kb block of the pak file
    TArray<TPakChunkHash> ChunkHashes;
    
    void SetChunkHashesAndSign(const TArray<TPakChunkHash>& InChunkHashes, const TArrayView<uint8>& InSignatureData, const FRSAKeyHandle InKey)
    {
        ChunkHashes = InChunkHashes;
        SignatureData = InSignatureData;
        DecryptedHash = ComputeCurrentPrincipalHash();
 
        TArray<uint8> NewSignatureData;
        NewSignatureData.Append(SignatureData);
        NewSignatureData.Append(DecryptedHash.Hash, UE_ARRAY_COUNT(FSHAHash::Hash));
        FRSA::EncryptPrivate(NewSignatureData, EncryptedHash, InKey);
    }
 
    void Serialize(FArchive& Ar)
    {
        uint32 FileMagic = Magic;
        Ar << FileMagic;
 
        if (Ar.IsLoading() && FileMagic != Magic)
        {
            Version = EVersion::Invalid;
            EncryptedHash.Empty();
            ChunkHashes.Empty();
            return;
        }
 
        Ar << Version;
        Ar << EncryptedHash;
        Ar << ChunkHashes;
    }
 
    bool DecryptSignatureAndValidate(const FRSAKeyHandle InKey, const FString& InFilename)
    {
        if (Version == EVersion::Invalid)
        {
            UE_LOG(LogPakFile, Warning, TEXT("Pak signature file for '%s' was invalid"), *InFilename);
        }
        else
        {
            int32 BytesDecrypted = FRSA::DecryptPublic(EncryptedHash, SignatureData, InKey);
            if (BytesDecrypted > (int32)UE_ARRAY_COUNT(FSHAHash::Hash))
            {
                FMemory::Memcpy(DecryptedHash.Hash, SignatureData.GetData() + SignatureData.Num() - UE_ARRAY_COUNT(FSHAHash::Hash), UE_ARRAY_COUNT(FSHAHash::Hash));
                SignatureData.SetNum(SignatureData.Num() - UE_ARRAY_COUNT(FSHAHash::Hash));
                FSHAHash CurrentHash = ComputeCurrentPrincipalHash();
                if (DecryptedHash == CurrentHash)
                {
                    return true;
                }
                else
                {
                    UE_LOG(LogPakFile, Warning, TEXT("Pak signature table validation failed for '%s'! Expected %s, Received %s"), *InFilename, *DecryptedHash.ToString(), *CurrentHash.ToString());
                }
            }
            else
            {
                UE_LOG(LogPakFile, Warning, TEXT("Pak signature table validation failed for '%s'! Failed to decrypt signature"), *InFilename);
            }
        }
 
        FPakPlatformFile::BroadcastPakPrincipalSignatureTableCheckFailure(InFilename);
        return false;
    }
 
    FSHAHash ComputeCurrentPrincipalHash() const
    {
        FSHAHash CurrentHash;
        FSHA1::HashBuffer(ChunkHashes.GetData(), ChunkHashes.Num() * sizeof(TPakChunkHash), CurrentHash.Hash);
        return CurrentHash;
    }
 
    UE_DEPRECATED("5.1", "Use ComputeCurrentPrincipalHash instead")
    FSHAHash ComputeCurrentMasterHash() const
    {
        return ComputeCurrentPrincipalHash();
    }
};