EditorBulkData.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "Async/Mutex.h"
#include "Async/RecursiveMutex.h"
#include "Async/Future.h"
#include "Compression/CompressedBuffer.h"
#include "Containers/Array.h"
#include "CoreTypes.h"
#include "HAL/Platform.h"
#include "IO/IoHash.h"
#include "Internationalization/Text.h"
#include "Memory/SharedBuffer.h"
#include "Misc/EnumClassFlags.h"
#include "Misc/Guid.h"
#include "Misc/PackagePath.h"
#include "Misc/PackageSegment.h"
#include "Serialization/CustomVersion.h"
#include "Serialization/StructuredArchive.h"
 
class FArchive;
class FBulkData;
class FLinkerSave;
class UObject;
struct FPackageFileVersion;
 
namespace UE::BulkDataRegistry { enum class ERegisterResult : uint8; }
 
// When enabled it will be possible for specific editor bulkdata objects to opt out of being virtualized
// This is a development feature and not expected to be used!
#define UE_ENABLE_VIRTUALIZATION_TOGGLE 1
 
//TODO: At some point it might be a good idea to uncomment this to make sure that FEditorBulkData is
//      never used at runtime (requires a little too much reworking of some assets for now though)
//#if WITH_EDITORONLY_DATA
 
namespace UE::Serialization
{
 
namespace Private
{
struct FCompressionSettings
{
    COREUOBJECT_API FCompressionSettings();
    FCompressionSettings(const FCompressedBuffer& Buffer);
 
    [[nodiscard]] bool operator ==(const FCompressionSettings& Other) const;
    [[nodiscard]] bool operator != (const FCompressionSettings& Other) const;
 
    void Reset();
 
    void Set(ECompressedBufferCompressor InCompressor, ECompressedBufferCompressionLevel InCompressionLevel);
    void SetToDefault();
    void SetToDisabled();
 
    [[nodiscard]] bool IsSet() const;
    [[nodiscard]] bool IsCompressed() const;
 
    [[nodiscard]] ECompressedBufferCompressor GetCompressor() const;
    [[nodiscard]] ECompressedBufferCompressionLevel GetCompressionLevel();
 
private:
 
    ECompressedBufferCompressor Compressor;
    ECompressedBufferCompressionLevel CompressionLevel;
 
    bool bIsSet;
};
} // namespace Private
 
enum class ECompressionOptions : uint8
{
    Default,
    Disabled,
};
 
class FEditorBulkData final
{
public:
    FEditorBulkData() = default;
    COREUOBJECT_API ~FEditorBulkData();
 
    COREUOBJECT_API FEditorBulkData(FEditorBulkData&& Other);
    COREUOBJECT_API FEditorBulkData& operator=(FEditorBulkData&& Other);
 
    COREUOBJECT_API FEditorBulkData(const FEditorBulkData& Other);
    COREUOBJECT_API FEditorBulkData& operator=(const FEditorBulkData& Other);
 
    COREUOBJECT_API void CreateFromBulkData(FBulkData& BulkData, const FGuid& Guid, UObject* Owner);
    
    COREUOBJECT_API void CreateLegacyUniqueIdentifier(UObject* Owner);
 
    COREUOBJECT_API void Serialize(FArchive& Ar, UObject* Owner, bool bAllowRegister=true);
 
    COREUOBJECT_API void Reset();
 
    // TODO: Not sure if there is a scenario where we can reload the payload off disk but it is also
    // in memory.and so this might not really do anything anymore.
 
    COREUOBJECT_API void UnloadData();
 
    COREUOBJECT_API void DetachFromDisk(FArchive* Ar, bool bEnsurePayloadIsLoaded);
 
    COREUOBJECT_API FGuid GetIdentifier() const;
 
    COREUOBJECT_API const FIoHash& GetPayloadId() const;
 
    int64 GetPayloadSize() const
    { 
        return PayloadSize; 
    }
 
    bool HasPayloadData() const
    { 
        return PayloadSize > 0; 
    }
 
    COREUOBJECT_API bool DoesPayloadNeedLoading() const;
 
    COREUOBJECT_API TFuture<FSharedBuffer> GetPayload() const;
 
    COREUOBJECT_API TFuture<FCompressedBuffer> GetCompressedPayload() const;
 
    COREUOBJECT_API void UpdatePayload(FSharedBuffer InPayload, UObject* Owner = nullptr);
 
    COREUOBJECT_API void UpdatePayload(FCompressedBuffer InPayload, UObject* Owner = nullptr);
    
    struct FSharedBufferWithID
    {
        COREUOBJECT_API FSharedBufferWithID(FSharedBuffer InPayload);
 
        FSharedBufferWithID() = default;
        FSharedBufferWithID(FSharedBufferWithID&&) = default;
        FSharedBufferWithID& operator=(FSharedBufferWithID&&) = default;
 
        FSharedBufferWithID(const FSharedBufferWithID&) = delete;
        FSharedBufferWithID& operator=(const FSharedBufferWithID&) = delete;
 
        FSharedBuffer GetPayload() { return Payload; }
 
    private:
        friend FEditorBulkData;
 
        FSharedBuffer Payload;
        FIoHash PayloadId;
    };
 
    COREUOBJECT_API void UpdatePayload(FSharedBufferWithID InPayload, UObject* Owner = nullptr);
 
    COREUOBJECT_API void SetCompressionOptions(ECompressionOptions Option);
 
    COREUOBJECT_API void SetCompressionOptions(ECompressedBufferCompressor Compressor, ECompressedBufferCompressionLevel CompressionLevel);
 
    UE_DEPRECATED(5.1, "Call GetBulkDataVersions instead.")
    COREUOBJECT_API FCustomVersionContainer GetCustomVersions(FArchive& InlineArchive);
 
    COREUOBJECT_API void GetBulkDataVersions(FArchive& InlineArchive, FPackageFileVersion& OutUEVersion, int32& OutLicenseeUEVersion,
        FCustomVersionContainer& OutCustomVersions) const;
 
    COREUOBJECT_API void TearOff();
 
    FEditorBulkData CopyTornOff() const
    { 
        return FEditorBulkData(*this, ETornOff());
    }
 
    // Functions used by the BulkDataRegistry
 
    COREUOBJECT_API void SerializeForRegistry(FArchive& Ar);
    
    COREUOBJECT_API bool CanSaveForRegistry() const;
    
    bool HasPlaceholderPayloadId() const
    {
        return EnumHasAnyFlags(Flags, EFlags::LegacyKeyWasGuidDerived);
    }
    
    COREUOBJECT_API bool IsMemoryOnlyPayload() const;
    
    COREUOBJECT_API void UpdatePayloadId();
    
    COREUOBJECT_API bool LocationMatches(const FEditorBulkData& Other) const;
 
    COREUOBJECT_API void UpdateRegistrationOwner(UObject* Owner);
 
#if UE_ENABLE_VIRTUALIZATION_TOGGLE
    UE_DEPRECATED(5.0, "SetVirtualizationOptOut is an internal feature for development and will be removed without warning!")
    COREUOBJECT_API void SetVirtualizationOptOut(bool bOptOut);
#endif //UE_ENABLE_VIRTUALIZATION_TOGGLE
 
protected:
    enum class ETornOff {};
    COREUOBJECT_API FEditorBulkData(const FEditorBulkData& Other, ETornOff);
 
private:
    friend struct FTocEntry;
 
    enum class EFlags : uint32
    {
        None                        = 0,
        IsVirtualized               = 1 << 0,
        HasPayloadSidecarFile       = 1 << 1,
        ReferencesLegacyFile        = 1 << 2,
        LegacyFileIsCompressed      = 1 << 3,
        DisablePayloadCompression   = 1 << 4,
        LegacyKeyWasGuidDerived     = 1 << 5,
        HasRegistered               = 1 << 6,
        IsTornOff                   = 1 << 7,
        ReferencesWorkspaceDomain   = 1 << 8,
        StoredInPackageTrailer      = 1 << 9,
        IsCooked                    = 1 << 10,
        WasDetached                 = 1 << 11
    };
 
    FRIEND_ENUM_CLASS_FLAGS(EFlags);
 
    static constexpr EFlags TransientFlags = EFlags((uint32)EFlags::HasRegistered | (uint32)EFlags::IsTornOff | (uint32)EFlags::WasDetached);
 
    enum ErrorVerbosity
    {
        None = 0,
        All
    };
 
    COREUOBJECT_API void SerializeToLegacyPath(FLinkerSave& LinkerSave, FCompressedBuffer PayloadToSerialize, EFlags UpdatedFlags, UObject* Owner);
    COREUOBJECT_API void SerializeToPackageTrailer(FLinkerSave& LinkerSave, FCompressedBuffer PayloadToSerialize, EFlags UpdatedFlags, UObject* Owner);
 
    COREUOBJECT_API void UpdatePayloadImpl(FSharedBuffer&& InPayload, const FIoHash& InPayloadID, UObject* Owner);
 
    COREUOBJECT_API FCompressedBuffer GetDataInternal() const;
 
    COREUOBJECT_API FCompressedBuffer LoadFromDisk() const;
    COREUOBJECT_API FCompressedBuffer LoadFromPackageFile() const;
    COREUOBJECT_API FCompressedBuffer LoadFromPackageTrailer() const;
    COREUOBJECT_API FCompressedBuffer LoadFromSidecarFile() const;
    COREUOBJECT_API FCompressedBuffer LoadFromSidecarFileInternal(ErrorVerbosity Verbosity) const;
 
    COREUOBJECT_API bool SerializeData(FArchive& Ar, FCompressedBuffer& Payload, const EFlags PayloadFlags) const;
 
    COREUOBJECT_API void PushData(const FPackagePath& InPackagePath);
    COREUOBJECT_API FCompressedBuffer PullData() const;
 
    COREUOBJECT_API bool CanUnloadData() const;
    COREUOBJECT_API bool CanLoadDataFromDisk() const;
 
    COREUOBJECT_API void UpdateKeyIfNeeded();
    COREUOBJECT_API void UpdateKeyIfNeeded(FCompressedBuffer InPayload) const;
 
    COREUOBJECT_API void RecompressForSerialization(FCompressedBuffer& InOutPayload, EFlags PayloadFlags) const;
    COREUOBJECT_API EFlags BuildFlagsForSerialization(FArchive& Ar, bool bKeepFileDataByReference) const;
 
    bool IsDataVirtualized() const 
    { 
        return IsDataVirtualized(Flags);
    }
 
    static bool IsDataVirtualized(EFlags InFlags)
    {
        return EnumHasAnyFlags(InFlags, EFlags::IsVirtualized);
    }
 
    bool HasPayloadSidecarFile() const 
    { 
        return EnumHasAnyFlags(Flags, EFlags::HasPayloadSidecarFile); 
    }
 
    bool IsReferencingOldBulkData() const
    { 
        return IsReferencingOldBulkData(Flags);
    }
    static bool IsReferencingOldBulkData(EFlags InFlags)
    {
        return EnumHasAnyFlags(InFlags, EFlags::ReferencesLegacyFile);
    }
 
    bool IsReferencingWorkspaceDomain() const
    {
        return IsReferencingWorkspaceDomain(Flags);
    }
    static bool IsReferencingWorkspaceDomain(EFlags InFlags)
    {
        return EnumHasAnyFlags(InFlags, EFlags::ReferencesWorkspaceDomain);
    }
 
    bool IsReferencingByPackagePath() const
    {
        return IsReferencingByPackagePath(Flags);
    }
    static bool IsReferencingByPackagePath(EFlags InFlags)
    {
        return EnumHasAnyFlags(InFlags, EFlags::ReferencesLegacyFile | EFlags::ReferencesWorkspaceDomain);
    }
 
    bool IsStoredInPackageTrailer() const
    {
        return IsStoredInPackageTrailer(Flags);
    }
    static bool IsStoredInPackageTrailer(EFlags InFlags)
    {
        return EnumHasAnyFlags(InFlags, EFlags::StoredInPackageTrailer);
    }
 
    bool HasAttachedArchive() const
    {
        return AttachedAr != nullptr;
    }
 
    COREUOBJECT_API void Register(UObject* Owner, const TCHAR* LogCallerName, bool bAllowUpdateId);
    COREUOBJECT_API void Unregister();
    COREUOBJECT_API void OnExitMemory();
    COREUOBJECT_API void UpdateRegistrationData(UObject* Owner, const TCHAR* LogCallerName, bool bAllowUpdateId);
    COREUOBJECT_API void LogRegisterError(UE::BulkDataRegistry::ERegisterResult Value, UObject* Owner, const FGuid& FailedBulkDataId,
        const TCHAR* CallerName, bool bHandledbyCreateUniqueGuid) const;
 
    COREUOBJECT_API bool TryPayloadValidationForSaving(const FCompressedBuffer& PayloadForSaving, FLinkerSave* LinkerSave) const;
 
    COREUOBJECT_API FText GetCorruptedPayloadErrorMsgForSave(FLinkerSave* Linker) const;
 
    static COREUOBJECT_API void ValidatePackageTrailerBuilder(const FLinkerSave* LinkerSave, const FIoHash& Id, EFlags PayloadFlags);
 
    COREUOBJECT_API bool ShouldUseLegacySerialization(const FLinkerSave* LinkerSave) const;
 
    FGuid BulkDataId;
 
    FIoHash PayloadContentId;
 
    FSharedBuffer Payload;
 
    int64 PayloadSize = 0;
 
    //---- The remaining members are used when the payload is not virtualized.
    
    FArchive* AttachedAr = nullptr;
 
    int64 OffsetInFile = INDEX_NONE;
 
    FPackagePath PackagePath;
 
    EFlags Flags = EFlags::None;
 
    // Use of IBulkDataRegistry requires a recursive mutex
    mutable FRecursiveMutex Mutex;
 
#if UE_ENABLE_VIRTUALIZATION_TOGGLE
    bool bSkipVirtualization = false;
#endif //UE_ENABLE_VIRTUALIZATION_TOGGLE
 
    Private::FCompressionSettings CompressionSettings;
};
 
ENUM_CLASS_FLAGS(FEditorBulkData::EFlags);
 
COREUOBJECT_API FGuid IoHashToGuid(const FIoHash& Hash);
 
} // namespace UE::Serialization
 
//#endif //WITH_EDITORONLY_DATA