VirtualizationSystem.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "Compression/CompressedBuffer.h"
#include "Containers/Array.h"
#include "Containers/ArrayView.h"
#include "Containers/StringFwd.h"
#include "Containers/StringView.h"
#include "Containers/UnrealString.h"
#include "CoreTypes.h"
#include "Delegates/Delegate.h"
#include "Features/IModularFeature.h"
#include "Features/IModularFeatures.h"
#include "IO/IoHash.h"
#include "Misc/ConfigCacheIni.h"
#include "Templates/Function.h"
#include "Templates/UniquePtr.h"
#include "Templates/UnrealTemplate.h"
#include "UObject/NameTypes.h"
#include "Virtualization/VirtualizationTypes.h"
 
class FPackagePath;
class FText;
class UObject;
struct FAnalyticsEventAttribute;
 
namespace UE::Virtualization
{
 
struct FPayloadActivityInfo
{
    struct FActivity
    {
        int64 PayloadCount = 0;
        int64 TotalBytes = 0;
        int64 CyclesSpent = 0;
    };
 
    FActivity Pull;
    FActivity Push;
    FActivity Cache;
};
 
struct FBackendStats
{
    FString ConfigName;
    FString DebugName;
    
    EStorageType Type;
 
    FPayloadActivityInfo PayloadActivity;
};
 
struct FRehydrationInfo
{
    uint64 OriginalSize = 0;
    uint64 RehydratedSize = 0;
    int32 NumPayloadsRehydrated = 0;
};
 
class IPayloadProvider
{
public:
    IPayloadProvider() = default;
    virtual ~IPayloadProvider() = default;
 
    virtual FCompressedBuffer RequestPayload(const FIoHash& Identifier) = 0;
 
    virtual uint64 GetPayloadSize(const FIoHash& Identifier) = 0;
};
 
struct FPushResult
{
    enum class EStatus : int8
    {
        Error           = -5,
        ProcessDisabled = -4,
        NoBackend       = -3,
        Invalid         = -2,
        Filtered        = -1,
        Pending         = 0,
        AlreadyExisted  = 1,
        Pushed          = 2
    };
 
    FPushResult() = default;
    ~FPushResult() = default;
 
    static FPushResult GetAsError()
    {
        return FPushResult(EStatus::Error);
    }
 
    static FPushResult GetAsProcessDisabled()
    {
        return FPushResult(EStatus::ProcessDisabled);
    }
 
    static FPushResult GetAsNoBackend()
    {
        return FPushResult(EStatus::NoBackend);
    }
 
    static FPushResult GetAsInvalid()
    {
        return FPushResult(EStatus::Invalid);
    }
 
    static FPushResult GetAsFiltered(EPayloadFilterReason Reason)
    {
        return FPushResult(EStatus::Filtered, Reason);
    }
 
    static FPushResult GetAsAlreadyExists()
    {
        return FPushResult(EStatus::AlreadyExisted);
    }
 
    static FPushResult GetAsPushed()
    {
        return FPushResult(EStatus::Pushed);
    }
 
    bool WasPushed() const
    {
        return Status == EStatus::Pushed;
    }
 
    bool WasError() const
    {
        return Status < EStatus::Pending;
    }
 
    bool IsVirtualized() const
    {
        return Status > EStatus::Pending;
    }
 
    bool IsFiltered() const
    {
        if (Status == EStatus::Filtered)
        {
            // Quick sanity check, if the payload was filtered there must be a reason
            check(FilterReason != EPayloadFilterReason::None);
            return true;
        }
        else
        {
            return false;
        }
    }
 
    EPayloadFilterReason GetFilterReason() const
    {
        return FilterReason;
    }
 
private:
 
    FPushResult(EStatus InStatus)
        : Status(InStatus)
    {
        check(Status != EStatus::Pending);
    }
 
    FPushResult(EStatus InStatus, EPayloadFilterReason InReason)
        : Status(InStatus)
        , FilterReason(InReason)
    {
        check(Status != EStatus::Pending);
        check(FilterReason != EPayloadFilterReason::None);
    }
 
    EStatus Status = EStatus::Pending;
    EPayloadFilterReason FilterReason = EPayloadFilterReason::None;
};
 
struct FPushRequest
{
    FPushRequest() = delete;
    ~FPushRequest() = default;
 
    FPushRequest(const FIoHash& InIdentifier, const FCompressedBuffer& InPayload, const FString& InContext)
        : Identifier(InIdentifier)
        , Payload(InPayload)
        , Context(InContext)
    {
 
    }
 
    FPushRequest(const FIoHash& InIdentifier, FCompressedBuffer&& InPayload, FString&& InContext)
        : Identifier(InIdentifier)
        , Payload(InPayload)
        , Context(InContext)
    {
 
    }
 
    FPushRequest(const FIoHash& InIdentifier, IPayloadProvider& InProvider, FString&& InContext)
        : Identifier(InIdentifier)
        , Provider(&InProvider)
        , Context(InContext)
    {
 
    }
 
    const FIoHash& GetIdentifier() const
    {
        return Identifier;
    }
 
    uint64 GetPayloadSize() const
    {
        if (Provider != nullptr)
        {
            return Provider->GetPayloadSize(Identifier);
        }
        else
        {
            return Payload.GetCompressedSize();
        }
    }
    
    FCompressedBuffer GetPayload() const
    {
        if (Provider != nullptr)
        {
            return Provider->RequestPayload(Identifier);
        }
        else
        {
            return Payload;
        }
    }
 
    const FString& GetContext() const
    {
        return Context;
    }
 
    void ResetResult()
    {
        Result = FPushResult();
    }
 
    void SetResult(FPushResult InResult)
    {
        Result = InResult;
    }
 
    const FPushResult& GetResult() const
    {
        return Result;
    }
 
private:
    FIoHash Identifier;
 
    FCompressedBuffer Payload;
 
    IPayloadProvider* Provider = nullptr;
 
    FString Context;
 
    FPushResult Result;
};
 
struct FPullRequest
{
private:
    enum class EStatus : int8
    {
        Error = -1,
        Pending = 0,
        Success = 1
    };
 
public:
    FPullRequest(const FIoHash& InIdentifier)
        : Identifier(InIdentifier)
    {
 
    }
 
    const FIoHash& GetIdentifier() const
    {
        return Identifier;
    }
 
    const FCompressedBuffer& GetPayload() const
    {
        return Payload;
    }
 
    bool IsSuccess() const
    {
        return Status == EStatus::Success;
    }
 
    void SetError()
    {
        Status = EStatus::Error;
        Payload.Reset();
    }
 
    void SetPayload(const FCompressedBuffer& InPayload)
    {
        Payload = InPayload;
        Status = Payload ? EStatus::Success : EStatus::Error;
    }
 
    void SetPayload(FCompressedBuffer&& InPayload)
    {
        Payload = MoveTemp(InPayload);
        Status = Payload ? EStatus::Success : EStatus::Error;
    }
 
private:
    FIoHash Identifier;
    FCompressedBuffer Payload;
    EStatus Status = EStatus::Pending;
};
 
struct FVirtualizationResult
{
    TArray<FText> Errors;
 
    TArray<FText> DescriptionTags;
 
    TArray<FString> VirtualizedPackages;
    TArray<FString> CheckedOutPackages;
 
    double TimeTaken = 0.0;
 
    uint64 PreOperationSize = 0;
    uint64 PostOperationSize = 0;
 
    /* Returns if the virtualization process succeeded or not */
    bool WasSuccessful() const
    {
        return Errors.IsEmpty();
    }
 
    void AddError(const FText& ErrorMsg)
    {
        Errors.Add(ErrorMsg);
    }
 
    void AddError(FText&& ErrorMsg)
    {
        Errors.Add(MoveTemp(ErrorMsg));
    }
 
    int32 GetNumErrors() const
    {
        return Errors.Num();
    }
 
};
 
struct FRehydrationResult
{
    TArray<FText> Errors;
 
    TArray<FString> RehydratedPackages;
    TArray<FString> CheckedOutPackages;
 
    double TimeTaken = 0.0;
 
    uint64 PreOperationSize = 0;
    uint64 PostOperationSize = 0;
 
    /* Returns if the rehydration process succeeded or not */
    bool WasSuccessful() const
    {
        return Errors.IsEmpty();
    }
 
    void AddError(const FText& ErrorMsg)
    {
        Errors.Add(ErrorMsg);
    }
 
    void AddError(FText&& ErrorMsg)
    {
        Errors.Add(MoveTemp(ErrorMsg));
    }
 
    int32 GetNumErrors() const
    {
        return Errors.Num();
    }
};
 
struct FInitParams
{
    FInitParams(FStringView InProjectName, const FConfigFile& InConfigFile)
        : ProjectName(InProjectName)
        , ConfigFile(InConfigFile)
    {
 
    }
 
    FStringView ProjectName;
 
    const FConfigFile& ConfigFile;
};
 
CORE_API void Initialize(EInitializationFlags Flags);
 
CORE_API void Initialize(const FInitParams& InitParams, EInitializationFlags Flags);
 
CORE_API bool ShouldInitializePreSlate();
 
CORE_API void Shutdown();
 
enum class EAnalyticsFlags : uint32
{
    None = 0,
    Flush = 1 << 0
};
 
using FAnalyticsRecordEvent = TTSMulticastDelegate<void(const FString&, const TArray<FAnalyticsEventAttribute>&, EAnalyticsFlags)>;
 
CORE_API FAnalyticsRecordEvent& GetAnalyticsRecordEvent();
 
class IVirtualizationSystem
{
public:
    IVirtualizationSystem() = default;
    virtual ~IVirtualizationSystem() = default;
 
    virtual bool Initialize(const FInitParams& InitParams) = 0;
 
    CORE_API static bool IsInitialized();
 
    CORE_API static FName GetSystemName();
 
    CORE_API static IVirtualizationSystem& Get();
 
    virtual bool IsEnabled() const = 0;
 
    virtual bool IsPushingEnabled(EStorageType StorageType) const = 0;
 
    virtual EPayloadFilterReason FilterPayload(const UObject* Owner) const = 0;
 
    virtual bool AllowSubmitIfVirtualizationFailed() const = 0;
    
    virtual bool PushData(TArrayView<FPushRequest> Requests, EStorageType StorageType) = 0;
 
    bool PushData(FPushRequest Request, EStorageType StorageType)
    {
        return PushData(MakeArrayView(&Request, 1), StorageType);
    }
 
    bool PushData(const FIoHash& Id, const FCompressedBuffer& Payload, const FString& Context, EStorageType StorageType)
    {
        FPushRequest Request(Id, Payload, Context);
        return PushData(MakeArrayView(&Request, 1), StorageType);
    }
 
    virtual bool PullData(TArrayView<FPullRequest> Requests) = 0;
 
    FCompressedBuffer PullData(const FIoHash& Id)
    {
        FPullRequest Request(Id);
 
        if (PullData(MakeArrayView(&Request, 1)))
        {
            return Request.GetPayload();
        }
        else
        {
            return FCompressedBuffer();
        }
    }
 
    bool PullData(FPullRequest Request)
    {
        return PullData(MakeArrayView(&Request, 1));
    }
 
    virtual EQueryResult QueryPayloadStatuses(TArrayView<const FIoHash> Ids, EStorageType StorageType, TArray<EPayloadStatus>& OutStatuses) = 0;
    
    virtual FVirtualizationResult TryVirtualizePackages(TConstArrayView<FString> PackagePaths, EVirtualizationOptions Options) = 0;
 
    virtual FRehydrationResult TryRehydratePackages(TConstArrayView<FString> PackagePaths, ERehydrationOptions Options) = 0;
 
    virtual ERehydrationResult TryRehydratePackages(TConstArrayView<FString> PackagePaths, uint64 PaddingAlignment, TArray<FText>& OutErrors, TArray<FSharedBuffer>& OutPackages, TArray<FRehydrationInfo>* OutInfo) = 0;
 
    virtual void DumpStats() const = 0;
 
    using GetPayloadActivityInfoFuncRef = TFunctionRef<void(const FString& DebugName, const FString& ConfigName, const FPayloadActivityInfo& PayloadInfo)>;
    
    virtual TArray<FBackendStats> GetBackendStatistics() const = 0;
    
    virtual FPayloadActivityInfo GetSystemStatistics() const = 0;
 
    virtual void GatherAnalytics(TArray<FAnalyticsEventAttribute>& Attributes) const =0;
 
    //* Notification messages
    enum ENotification
    {
        PushBegunNotification,
        PushEndedNotification,
        PushFailedNotification,
 
        PullBegunNotification,
        PullEndedNotification,
        PullFailedNotification,
    };
 
    using FOnNotification = TTSMulticastDelegate<void(ENotification, const FIoHash&)>;
 
    virtual FOnNotification& GetNotificationEvent() = 0;
};
 
namespace Private
{
 
class IVirtualizationSystemFactory : public IModularFeature
{
public:
    virtual TUniquePtr<IVirtualizationSystem> Create() = 0;
 
    virtual FName GetName() = 0;
};
 
} // namespace Private
 
#define UE_REGISTER_VIRTUALIZATION_SYSTEM(SystemClass, SystemName) \
    class FVirtualizationSystem##Factory : public Private::IVirtualizationSystemFactory \
    { \
    public: \
        FVirtualizationSystem##Factory() { IModularFeatures::Get().RegisterModularFeature(FName("VirtualizationSystem"), this); }\
        virtual ~FVirtualizationSystem##Factory() { IModularFeatures::Get().UnregisterModularFeature(FName("VirtualizationSystem"), this); } \
    private: \
        virtual TUniquePtr<IVirtualizationSystem> Create() override { return MakeUnique<SystemClass>(); } \
        virtual FName GetName() override { return FName(#SystemName); } \
    }; \
    static FVirtualizationSystem##Factory FVirtualizationSystem##Factory##Instance;
 
namespace Experimental
{
 
class IVirtualizationSourceControlUtilities : public IModularFeature
{
public:
    virtual bool SyncPayloadSidecarFile(const FPackagePath& PackagePath) = 0;
};
 
} // namespace Experimental
 
} // namespace UE::Virtualization