PackageTrailer.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "Compression/CompressedBuffer.h"
#include "Containers/Array.h"
#include "Containers/Map.h"
#include "Containers/UnrealString.h"
#include "CoreTypes.h"
#include "IO/IoHash.h"
#include "Misc/EnumClassFlags.h"
#include "Templates/Function.h"
#include "Templates/UniquePtr.h"
#include "UObject/NameTypes.h"
#include "Virtualization/VirtualizationTypes.h"
 
class FArchive;
class FLinkerSave;
class FPackagePath;
 
namespace UE
{
 
enum class EPayloadStorageType : uint8
{
    Any,
    Local,
    Referenced,
    Virtualized
};
 
enum class EPayloadFilter
{
    CanVirtualize
};
 
// TODO: Could consider merging EPayloadStorageType and EPayloadStatus, only difference is Any vs NotFound
 
enum class EPayloadStatus
{
    NotFound = 0,
    StoredLocally,
    StoredAsReference,
    StoredVirtualized,
};
 
enum class EPayloadAccessMode : uint8
{
    Local = 0,
    Referenced,
    Virtualized
};
 
enum class EPayloadFlags : uint16
{
    None = 0,
};
 
ENUM_CLASS_FLAGS(EPayloadFlags);
 
enum class EPackageTrailerVersion : uint32;
 
namespace Private
{
 
struct FLookupTableEntry
{
    static constexpr uint32 SizeOnDisk = 49;    // Identifier       | 20 bytes
                                                // OffsetInFile     | 8 bytes
                                                // CompressedSize   | 8 bytes
                                                // RawSize          | 8 bytes
                                                // Flags            | 4 bytes
                                                // AccessMode       | 1 byte
 
    FLookupTableEntry() = default;
    FLookupTableEntry(const FIoHash& InIdentifier, uint64 InRawSize);
 
    void Serialize(FArchive& Ar, EPackageTrailerVersion PackageTrailerVersion);
    
    [[nodiscard]] bool IsLocal() const
    {
        return AccessMode == EPayloadAccessMode::Local;
    }
 
    [[nodiscard]] bool IsReferenced() const
    {
        return AccessMode == EPayloadAccessMode::Referenced;
    }
 
    [[nodiscard]] bool IsVirtualized() const
    {
        return AccessMode == EPayloadAccessMode::Virtualized;
    }
 
    FIoHash Identifier;
    int64 OffsetInFile = INDEX_NONE;
    uint64 CompressedSize = INDEX_NONE;
    uint64 RawSize = INDEX_NONE;
    EPayloadFlags Flags = EPayloadFlags::None;
    Virtualization::EPayloadFilterReason FilterFlags = Virtualization::EPayloadFilterReason::None;
 
    EPayloadAccessMode AccessMode = EPayloadAccessMode::Local;
};
 
} // namespace Private
 
//** Info about a payload stored in the trailer
struct FPayloadInfo
{
    int64 OffsetInFile = INDEX_NONE;
 
    uint64 CompressedSize = INDEX_NONE;
    uint64 RawSize = INDEX_NONE;
 
    EPayloadAccessMode AccessMode = EPayloadAccessMode::Local;
    EPayloadFlags Flags = EPayloadFlags::None;
    Virtualization::EPayloadFilterReason FilterFlags = Virtualization::EPayloadFilterReason::None;
};
 
class FPackageTrailerBuilder
{
public:
    using AdditionalDataCallback = TFunction<void(FLinkerSave& LinkerSave, const class FPackageTrailer& Trailer)>;
 
    [[nodiscard]] static COREUOBJECT_API FPackageTrailerBuilder CreateFromTrailer(const class FPackageTrailer& Trailer, FArchive& Ar, FString DebugContext);
 
    UE_DEPRECATED(5.1, "Use the overload that takes a FString instead of an FName for the last parameter")
    static FPackageTrailerBuilder CreateFromTrailer(const class FPackageTrailer& Trailer, FArchive& Ar, const FName& PackageName)
    {
        return FPackageTrailerBuilder::CreateFromTrailer(Trailer, Ar, PackageName.ToString());
    }
 
    [[nodiscard]] static COREUOBJECT_API TUniquePtr<UE::FPackageTrailerBuilder> CreateReferenceToTrailer(const class FPackageTrailer& Trailer, FString DebugContext);
    
    UE_DEPRECATED(5.1, "Use the overload that takes a FString instead of an FName for the last parameter")
    static TUniquePtr<UE::FPackageTrailerBuilder> CreateReferenceToTrailer(const class FPackageTrailer& Trailer, const FName& PackageName)
    {
        return FPackageTrailerBuilder::CreateReferenceToTrailer(Trailer, PackageName.ToString());
    }
 
    FPackageTrailerBuilder() = default;
    UE_DEPRECATED(5.1, "Use the overload that takes a FString instead of an FName")
    COREUOBJECT_API FPackageTrailerBuilder(const FName& InPackageName);
    COREUOBJECT_API FPackageTrailerBuilder(FString&& DebugContext);
    ~FPackageTrailerBuilder() = default;
 
    // Methods that can be called while building the trailer
 
    COREUOBJECT_API void AddPayload(const FIoHash& Identifier, FCompressedBuffer Payload, UE::Virtualization::EPayloadFilterReason Filter, AdditionalDataCallback&& Callback);
    COREUOBJECT_API void AddPayload(const FIoHash& Identifier, FCompressedBuffer Payload, UE::Virtualization::EPayloadFilterReason Filter);
    COREUOBJECT_API void AddVirtualizedPayload(const FIoHash& Identifier, int64 RawSize);
 
    COREUOBJECT_API bool UpdatePayloadAsLocal(const FIoHash& Identifier, FCompressedBuffer Payload);
    
    [[nodiscard]] COREUOBJECT_API bool BuildAndAppendTrailer(FLinkerSave* Linker, FArchive& DataArchive);
 
    [[nodiscard]] COREUOBJECT_API bool BuildAndAppendTrailer(FLinkerSave* Linker, FArchive& DataArchive, int64& InOutPackageFileOffset);
 
    [[nodiscard]] COREUOBJECT_API bool IsEmpty() const;
 
    [[nodiscard]] COREUOBJECT_API bool IsLocalPayloadEntry(const FIoHash& Identifier) const;
    [[nodiscard]] COREUOBJECT_API bool IsReferencedPayloadEntry(const FIoHash& Identifier) const;
    [[nodiscard]] COREUOBJECT_API bool IsVirtualizedPayloadEntry(const FIoHash& Identifier) const;
 
    [[nodiscard]] COREUOBJECT_API uint64 CalculateTrailerLength();
 
    [[nodiscard]] COREUOBJECT_API int32 GetNumPayloads() const;
    
    [[nodiscard]] COREUOBJECT_API int32 GetNumLocalPayloads() const;
    [[nodiscard]] COREUOBJECT_API int32 GetNumReferencedPayloads() const;
    [[nodiscard]] COREUOBJECT_API int32 GetNumVirtualizedPayloads() const;
 
    [[nodiscard]] const FString& GetDebugContext() const
    {
        return DebugContext;
    }
 
private:
    
    struct LocalEntry
    {
        LocalEntry() = default;
        LocalEntry(FCompressedBuffer&& InPayload, Virtualization::EPayloadFilterReason InFilterFlags)
            : Payload(InPayload)
            , FilterFlags(InFilterFlags)
        {
 
        }
        ~LocalEntry() = default;
 
        FCompressedBuffer Payload;
        Virtualization::EPayloadFilterReason FilterFlags = Virtualization::EPayloadFilterReason::None;
    };
 
    struct ReferencedEntry
    {
        ReferencedEntry() = default;
        ReferencedEntry(int64 InOffset, int64 InCompressedSize, int64 InRawSize)
            : Offset(InOffset)
            , CompressedSize(InCompressedSize)
            , RawSize(InRawSize)
        {
 
        }
        ~ReferencedEntry() = default;
 
        int64 Offset = INDEX_NONE;
        int64 CompressedSize = INDEX_NONE;
        int64 RawSize = INDEX_NONE;
    };
 
 
    struct VirtualizedEntry
    {
        VirtualizedEntry() = default;
        VirtualizedEntry(int64 InRawSize)
            : RawSize(InRawSize)
        {
 
        }
        ~VirtualizedEntry() = default;
 
        int64 RawSize = INDEX_NONE;
    };
 
    COREUOBJECT_API uint32 CalculatePotentialHeaderSize() const;
    COREUOBJECT_API uint64 CalculatePotentialPayloadSize() const;
 
    COREUOBJECT_API void RemoveDuplicateEntries();
 
    // Members used when building the trailer
 
    FString DebugContext;
 
    TMap<FIoHash, LocalEntry> LocalEntries;
    TMap<FIoHash, ReferencedEntry> ReferencedEntries;
    TMap<FIoHash, VirtualizedEntry> VirtualizedEntries;
    TArray<AdditionalDataCallback> Callbacks;
};
 
class FPackageTrailer
{
public:
    UE_DEPRECATED(5.3, "FPackageTrailer::IsEnabled will always return true as the system is no longer optional")
    [[nodiscard]] static bool IsEnabled()
    {
        return true;
    }
 
    [[nodiscard]] static COREUOBJECT_API bool TryLoadFromPackage(const FPackagePath& PackagePath, FPackageTrailer& OutTrailer);
 
    [[nodiscard]] static COREUOBJECT_API bool TryLoadFromFile(const FString& Path, FPackageTrailer& OutTrailer);
 
    [[nodiscard]] static COREUOBJECT_API bool TryLoadFromArchive(FArchive& Ar, FPackageTrailer& OutTrailer);
 
    FPackageTrailer() = default;
    ~FPackageTrailer() = default;
 
    FPackageTrailer(const FPackageTrailer& Other) = default;
    FPackageTrailer& operator=(const FPackageTrailer & Other) = default;
 
    FPackageTrailer(FPackageTrailer&& Other) = default;
    FPackageTrailer& operator=(FPackageTrailer&& Other) = default;
 
    [[nodiscard]] bool IsValid() const
    {
        return Header.Tag == FHeader::HeaderTag;
    }
 
    [[nodiscard]] COREUOBJECT_API bool TryLoad(FArchive& Ar);
 
    [[nodiscard]] COREUOBJECT_API bool TryLoadBackwards(FArchive& Ar);
 
    [[nodiscard]] COREUOBJECT_API FCompressedBuffer LoadLocalPayload(const FIoHash& Id, FArchive& Ar) const;
 
    [[nodiscard]] COREUOBJECT_API bool UpdatePayloadAsVirtualized(const FIoHash& Identifier);
 
    COREUOBJECT_API void ForEachPayload(TFunctionRef<void(const FIoHash&, uint64, uint64, EPayloadAccessMode, UE::Virtualization::EPayloadFilterReason)> Callback) const;
 
    [[nodiscard]] COREUOBJECT_API EPayloadStatus FindPayloadStatus(const FIoHash& Id) const;
 
    [[nodiscard]] COREUOBJECT_API int64 FindPayloadOffsetInFile(const FIoHash& Id) const;
 
    [[nodiscard]] COREUOBJECT_API int64 FindPayloadSizeOnDisk(const FIoHash& Id) const;
 
    [[nodiscard]] COREUOBJECT_API int64 GetTrailerLength() const;
 
    [[nodiscard]] COREUOBJECT_API FPayloadInfo GetPayloadInfo(const FIoHash& Id) const;
 
    [[nodiscard]] COREUOBJECT_API TArray<FIoHash> GetPayloads(EPayloadStorageType StorageType) const;
 
    [[nodiscard]] COREUOBJECT_API int32 GetNumPayloads(EPayloadStorageType Type) const;
 
    [[nodiscard]] COREUOBJECT_API TArray<FIoHash> GetPayloads(EPayloadFilter Filter) const;
 
    [[nodiscard]] COREUOBJECT_API int32 GetNumPayloads(EPayloadFilter Filter) const;
 
    struct FHeader
    {
        static constexpr uint64 HeaderTag = 0xD1C43B2E80A5F697;
 
        static constexpr uint32 StaticHeaderSizeOnDisk = 28;    // HeaderTag            | 8 bytes
                                                                // Version              | 4 bytes
                                                                // HeaderLength         | 4 bytes
                                                                // PayloadsDataLength   | 8 bytes
                                                                // NumPayloads          | 4 bytes
 
        uint64 Tag = 0;
        int32 Version = INDEX_NONE;
        uint32 HeaderLength = 0;
        uint64 PayloadsDataLength = 0;
        TArray<Private::FLookupTableEntry> PayloadLookupTable;
 
        friend FArchive& operator<<(FArchive& Ar, FHeader& Header);
    };
 
    struct FFooter
    {
        static constexpr uint64 FooterTag = 0x29BFCA045138DE76;
 
        static constexpr uint32 SizeOnDisk = 20;    // Tag              | 8 bytes
                                                    // TrailerLength    | 8 bytes
                                                    // PackageTag       | 4 bytes
 
        uint64 Tag = 0;
        uint64 TrailerLength = 0;   
        uint32 PackageTag = 0;
 
        friend FArchive& operator<<(FArchive& Ar, FFooter& Footer);
    };
 
private:
    friend class FPackageTrailerBuilder;
 
    COREUOBJECT_API FFooter CreateFooter() const;
 
    int64 TrailerPositionInFile = INDEX_NONE;
 
    FHeader Header;
};
 
[[nodiscard]] COREUOBJECT_API bool FindPayloadsInPackageFile(const FPackagePath& PackagePath, EPayloadStorageType Filter, TArray<FIoHash>& OutPayloadIds);
 
template <typename CharType>
inline TStringBuilderBase<CharType>& operator<<(TStringBuilderBase<CharType>& Builder, EPayloadAccessMode Mode)
{
    switch (Mode)
    {
        case UE::EPayloadAccessMode::Local:
            Builder << TEXT("Local");
            break;
        case UE::EPayloadAccessMode::Referenced:
            Builder << TEXT("Referenced");
            break;
        case UE::EPayloadAccessMode::Virtualized:
            Builder << TEXT("Virtualized");
            break;
        default:
            Builder << TEXT("Invalid");
            break;
    }
 
    return Builder;
}
 
 
} //namespace UE
 
[[nodiscard]] COREUOBJECT_API FString LexToString(UE::Virtualization::EPayloadFilterReason FilterFlags);