BuildPatchFileConstructor.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "CoreMinimal.h"
#include "Async/Mutex.h"
#include "Misc/Guid.h"
#include "HAL/Runnable.h"
#include "BuildPatchProgress.h"
#include "BuildPatchManifest.h"
#include "Installer/ChunkSource.h"
#include "Installer/Controllable.h"
#include "Common/SpeedRecorder.h"
#include "BuildPatchInstall.h"
 
 
// Forward declarations
struct FBatchState;
struct FResumeData;
struct FFileConstructionState;
class FBuildPatchAppManifest;
enum class EConstructionError : uint8;
 
class IBuildInstallerSharedContext;
class FChunkBackingStore;
 
namespace BuildPatchServices
{
    enum EConstructorChunkLocation : uint8
    {
        Install,
        ChunkDb,
        Memory,
        DiskOverflow,
        Cloud,
        Retired,
        COUNT
    };
 
    struct FChunkPart;
    class IFileSystem;
    class IChunkSource;
    class IChunkReferenceTracker;
    class IInstallerError;
    class IInstallerAnalytics;
    class IFileConstructorStat;
    class IMessagePump;
    class IBuildManifestSet;
    class IBuildInstallerThread;
    class IConstructorChunkDbChunkSource;
    class IConstructorInstallChunkSource;
    class IConstructorCloudChunkSource;
    
 
    struct FFileConstructorConfig
    {
        // The manifest set class for details on the installation files.
        IBuildManifestSet* ManifestSet;
 
        // The location for the installation.
        FString InstallDirectory;
 
        // The location where new installation files will be constructed.
        FString StagingDirectory;
 
        // The location where temporary files for tracking can be stored.
        FString MetaDirectory;
 
        // The list of files to be constructed, filename paths should match those contained in manifest.
        TArray<FString> ConstructList;
 
        // The install mode used for this installation.
        EInstallMode InstallMode;
 
        // The location where memory overflow will get written to.
        FString BackingStoreDirectory;
 
        IBuildInstallerSharedContext* SharedContext;
 
        // See comments in installer config
        bool bInstallToMemory = false;
        bool bConstructInMemory = false;
        bool bSkipInitialDiskSizeCheck = false;
 
        bool bDeleteChunkDBFilesAfterUse = false;
 
        static const bool bDefaultSpawnAdditionalIOThreads = true;
        static const int32 DefaultIOBatchSizeMB = 10;
        static const int32 DefaultIOBufferSizeMB = 64;
 
        TOptional<bool> SpawnAdditionalIOThreads;
        TOptional<int32> IOBatchSizeMB;
        TOptional<int32> IOBufferSizeMB;
        TOptional<bool> StallWhenFileSystemThrottled;
        TOptional<int32> DisableResumeBelowMB;
    };
 
    class FBuildPatchFileConstructor : public IControllable
    {
        friend FChunkBackingStore;
    public:
 
        FBuildPatchFileConstructor(
            FFileConstructorConfig Configuration, IFileSystem* FileSystem, 
            IConstructorChunkDbChunkSource* ChunkDbChunkSource, IConstructorCloudChunkSource* CloudChunkSource, IConstructorInstallChunkSource* InstallChunkSource, 
            IChunkReferenceTracker* ChunkReferenceTracker, IInstallerError* InstallerError, IInstallerAnalytics* InstallerAnalytics, IMessagePump* MessagePump,
            IFileConstructorStat* FileConstructorStat, TMap<FGuid, EConstructorChunkLocation>&& ChunkLocations);
 
        ~FBuildPatchFileConstructor();
 
        void Run();
 
        // IControllable interface begin.
        virtual void SetPaused(bool bInIsPaused) override;
        virtual void Abort() override;
        // IControllable interface end.
 
        void WakeUpDispatch();
 
        uint64 GetRequiredDiskSpace();
 
        uint64 GetAvailableDiskSpace();
 
        DECLARE_EVENT_OneParam(FBuildPatchFileConstructor, FOnBeforeDeleteFile, const FString& /*BuildFile*/);
        FOnBeforeDeleteFile& OnBeforeDeleteFile();
 
        void GrabFilesInstalledToMemory(TMap<FString, TArray64<uint8>>& OutFilesInstalledToMemory)
        {
            OutFilesInstalledToMemory = MoveTemp(MemoryOutputFiles);
        }
 
    private:
 
        void SetChunkLocation(const FGuid& InGuid, EConstructorChunkLocation InNewLocation);
 
        void CountBytesProcessed(const int64& ByteCount);
 
        int64 GetRemainingBytes();
 
        uint64 CalculateInProgressDiskSpaceRequired(const FFileManifest& InProgressFileManifest, uint64 InProgressFileSize);
 
        // Calculates the amount of disk space we need to finish the install, needs to be called on file boundaries on the construct thread.
        uint64 CalculateDiskSpaceRequirementsWithDeleteDuringInstall();
 
        void ConstructFiles(const FResumeData& ResumeData);
 
    private:
        // The configuration for the constructor.
        const FFileConstructorConfig Configuration;
 
        // A flag marking that we told the chunk cache to queue required downloads.
        bool bIsDownloadStarted;
 
        // A flag marking that we have made the initial disk space check following resume logic complete.
        bool bInitialDiskSizeCheck;
 
        // If true, the chunkdb source has chunks to provide
        bool bHasChunkDbSource = false;
 
        // Our local resolved copy of the cvar with overrides applied.
        bool bStallWhenFileSystemThrottled = false;
 
        // A flag marking whether we should be paused.
        FThreadSafeBool bIsPaused;
 
        // A flag marking whether we should abort operations and exit. Always call Abort() to set this.
        FThreadSafeBool bShouldAbort;
 
        // Indexes in to ConstructionList and associated parallel arrays. This is the next file that will
        // start construction when dependencies are met.
        std::atomic_int32_t NextIndexToConstruct = 0;
 
        struct FFileToConstruct
        {
            const FFileManifest* FileManifest = nullptr;
 
            // When using an install source with multiple files in flight, we can't start this 
            // file until all of the install sources it needs have been harvested. Since files are
            // constructed in order, we only track the file that will be harvested last. If this
            // file has no dependencies this is -1
            int32 LatestDependentInstallSource = -1;
        };
 
        // The in-oder list of files to construct. The array is parallel with Configuration.ConstructList.
        TArray<FFileToConstruct> ConstructionList;
 
        // Pointer to the file system.
        IFileSystem* FileSystem;
 
        IConstructorChunkDbChunkSource* ChunkDbSource; // can be null if not using.
        IConstructorInstallChunkSource* InstallSource;
        IConstructorCloudChunkSource* CloudSource;
 
        // Keyed off of the filename relative to the install directory.
        TMap<FString, TArray64<uint8>> MemoryOutputFiles;
 
        // We always want to know exactly where we think a chunk should be. If it's not there,
        // we update this list to where it can be found (i.e. cloud)
        // This is almost always read only after initialization, but in rare situations can be updated
        // (chunk failures, file resume) and is multi threaded access.
        FRWLock ChunkLocationsLock;
        TMap<FGuid, EConstructorChunkLocation> ChunkLocations;
        // Track how much data we expect to have to download. This is protected by the ChunkLocationsLock since they are in sync.
        uint64 DownloadRequirement = 0;
 
        // Pointer to the chunk reference tracker.
        IChunkReferenceTracker* ChunkReferenceTracker;
 
        // Pointer to the installer error class.
        IInstallerError* InstallerError;
 
        // Pointer to the installer analytics handler.
        IInstallerAnalytics* InstallerAnalytics;
 
        IMessagePump* MessagePump = nullptr;
 
        // Pointer to the stat class.
        IFileConstructorStat* FileConstructorStat;
 
        bool bAllowMultipleFilesInFlight = true;
 
        bool bShouldLogNextDependencyWait = true;
 
        // The size we expect for chunks. This should be used for estimation purposes, not anything requiring hard limits.
        uint32 ExpectedChunkSize = 0;
 
        // Total job size for tracking progress.
        int64 TotalJobSize;
 
        // Byte processed so far for tracking progress.
        int64 ByteProcessed;
 
        uint32 MaxWriteBatchSize = 0;
        uint32 IOBufferSize = 0;
 
        int32 WriteCount = 0;
 
        // The amount of disk space requirement that was calculated when beginning the process. 0 if the install process was not started, or no additional space was needed.
        std::atomic_uint64_t RequiredDiskSpace;
 
        // The amount of disk space available when beginning the process. 0 if the install process was not started.
        std::atomic_uint64_t AvailableDiskSpace;
 
        // Event executed before deleting an old installation file.
        FOnBeforeDeleteFile BeforeDeleteFileEvent;
 
 
        TArray<FEvent*> ThreadWakeups;
        TArray<TArray<IConstructorChunkSource::FRequestProcessFn>> ThreadJobPostings;
        TArray<UE::FMutex> ThreadJobPostingLocks;
        TArray<FEvent*> ThreadCompleteEvents;
        TArray<IBuildInstallerThread*> Threads;
 
        void QueueGenericThreadTask(int32 ThreadIndex, TUniqueFunction<void(bool)>&& Task);
        void GenericThreadFn(int32 ThreadIndex);
        int8 ThreadAssignments[EConstructorChunkLocation::COUNT];
        int8 WriteThreadIndex = -1;
 
        TUniquePtr<FChunkBackingStore> BackingStore;
 
        // Fire this to wake up the main thread to process completed tasks.
        FEvent* WakeUpDispatchThreadEvent = nullptr;
 
        IConstructorChunkSource::FRequestProcessFn CreateWriteRequest(FArchive* File, FBatchState& Batch);
 
        // Where we are in the chunk consumption list after each file.
        TArray<int32> FileCompletionPositions;
 
        void StartReadBatch(FFileConstructionState& CurrentFile, FBatchState& Buffer);
        void CompleteReadBatch(const FFileManifest& FileManifest, FBatchState& Buffer);
        void RequestCompletedFn(const FGuid& Guid, bool bAborted, bool bFailedToRead, void* UserPtr);
 
        std::atomic_int32_t PendingHarvestRequests = 0;
        void ChunkHarvestCompletedFn(const FGuid& Guid, bool bAborted, bool bFailedToRead, void* UserPtr);
 
        void StartFile(FFileConstructionState& CurrentFile, const FResumeData& ResumeData);
        void ResumeFile(FFileConstructionState& FileToResume);
        void OpenFileToConstruct(FFileConstructionState& CurrentFile);
        bool HandleInitialDiskSizeCheck(const FFileManifest& CurrentFileManifest, int64 BytesInToCurrentFile);
        bool HarvestChunksForCompletedFile(const FString& CompletedBuildFileName);
        void CompleteConstructedFile(FFileConstructionState& CurrentFile);
        void InitFile(FFileConstructionState& CurrentFile, const FResumeData& ResumeData);
 
 
 
    public:
        struct FBackingStoreStats
        {
            uint64 DiskPeakUsageBytes=0;
            uint64 MemoryPeakUsageBytes=0;
            uint64 MemoryLimitBytes=0;
            uint32 DiskLoadFailureCount=0;
            uint32 DiskLostChunkCount=0;
            uint32 DiskChunkLoadCount=0;
        };
 
        // This isn't safe to call during operations as values are changing on other threads.
        FBackingStoreStats GetBackingStoreStats()
        {
            return BackingStoreStats;
        }
    private:
 
        FBackingStoreStats BackingStoreStats;
    };
 
    class IFileConstructorStat
    {
    public:
        virtual ~IFileConstructorStat() {}
 
        virtual void OnResumeStarted() = 0;
 
        virtual void OnResumeCompleted() = 0;
 
        virtual void OnChunkGet(const FGuid& ChunkId) = 0;
 
        virtual void OnFileStarted(const FString& Filename, int64 FileSize) = 0;
 
        virtual void OnFileProgress(const FString& Filename, int64 TotalBytes) = 0;
 
        virtual void OnFileCompleted(const FString& Filename, bool bSuccess) = 0;
 
        virtual void OnConstructionCompleted() = 0;
 
        virtual void OnProcessedDataUpdated(int64 TotalBytes) = 0;
 
        virtual void OnTotalRequiredUpdated(int64 TotalBytes) = 0;
 
        virtual void OnBeforeAdminister() = 0;
 
        virtual void OnAfterAdminister(const ISpeedRecorder::FRecord& Record) = 0;
 
        virtual void OnBeforeRead() = 0;
 
        virtual void OnAfterRead(const ISpeedRecorder::FRecord& Record) = 0;
 
        virtual void OnBeforeWrite() = 0;
 
        virtual void OnAfterWrite(const ISpeedRecorder::FRecord& Record) = 0;
    };
}
 
namespace FileConstructorHelpers
{
    uint64 CalculateRequiredDiskSpace(const FBuildPatchAppManifestPtr& CurrentManifest, const FBuildPatchAppManifestRef& BuildManifest, const BuildPatchServices::EInstallMode& InstallMode, const TSet<FString>& InstallTags);
}