BuildPatchSettings.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
 
#include "BuildPatchDelta.h"
#include "BuildPatchFeatureLevel.h"
#include "BuildPatchInstall.h"
#include "BuildPatchVerify.h"
#include "Containers/Array.h"
#include "Containers/Map.h"
#include "Containers/Set.h"
#include "Containers/SparseArray.h"
#include "Containers/UnrealString.h"
#include "CoreMinimal.h"
#include "HAL/Platform.h"
#include "Interfaces/IBuildManifest.h"
#include "Interfaces/IBuildInstallerSharedContext.h"
#include "Misc/Variant.h"
#include "Templates/UnrealTemplate.h"
 
class FVariant;
 
namespace BuildPatchServices
{
    enum class EInstallActionIntent : int32;
 
    struct FBuildPatchServicesInitSettings
    {
    public:
        BUILDPATCHSERVICES_API FBuildPatchServicesInitSettings();
 
    public:
        // The application settings directory.
        FString ApplicationSettingsDir;
        // The application project name.
        FString ProjectName;
        // The local machine config file name.
        FString LocalMachineConfigFileName;
    };
 
    struct FInstallerAction
    {
    public:
 
        static BUILDPATCHSERVICES_API FInstallerAction MakeInstall(const IBuildManifestRef& Manifest, TSet<FString> InstallTags = TSet<FString>(), FString InstallSubdirectory = FString(), FString CloudSubdirectory = FString());
 
        static BUILDPATCHSERVICES_API FInstallerAction MakeUpdate(const IBuildManifestRef& CurrentManifest, const IBuildManifestRef& InstallManifest, TSet<FString> InstallTags = TSet<FString>(), FString InstallSubdirectory = FString(), FString CloudSubdirectory = FString());
 
        static BUILDPATCHSERVICES_API FInstallerAction MakeRepair(const IBuildManifestRef& Manifest, TSet<FString> InstallTags = TSet<FString>(), FString InstallSubdirectory = FString(), FString CloudSubdirectory = FString());
 
 
        static BUILDPATCHSERVICES_API FInstallerAction MakeUninstall(const IBuildManifestRef& Manifest, FString InstallSubdirectory = FString(), FString CloudSubdirectory = FString());
 
        static FInstallerAction MakeInstallOrUpdate(const IBuildManifestPtr& CurrentManifest, const IBuildManifestRef& InstallManifest, TSet<FString> InstallTags = TSet<FString>(), FString InstallSubdirectory = FString(), FString CloudSubdirectory = FString())
        {
            if (!CurrentManifest.IsValid())
            {
                return MakeInstall(InstallManifest, MoveTemp(InstallTags), MoveTemp(InstallSubdirectory), MoveTemp(CloudSubdirectory));
            }
            else
            {
                return MakeUpdate(CurrentManifest.ToSharedRef(), InstallManifest, MoveTemp(InstallTags), MoveTemp(InstallSubdirectory), MoveTemp(CloudSubdirectory));
            }
        }
 
        BUILDPATCHSERVICES_API FInstallerAction(const FInstallerAction& CopyFrom);
 
        BUILDPATCHSERVICES_API FInstallerAction(FInstallerAction&& MoveFrom);
 
    public:
        BUILDPATCHSERVICES_API bool IsInstall() const;
 
        BUILDPATCHSERVICES_API bool IsUpdate() const;
 
        BUILDPATCHSERVICES_API bool IsRepair() const;
 
        BUILDPATCHSERVICES_API bool IsUninstall() const;
 
        BUILDPATCHSERVICES_API const TSet<FString>& GetInstallTags() const;
 
        BUILDPATCHSERVICES_API const FString& GetInstallSubdirectory() const;
 
        BUILDPATCHSERVICES_API const FString& GetCloudSubdirectory() const;
 
        BUILDPATCHSERVICES_API IBuildManifestRef GetCurrentManifest() const;
 
        BUILDPATCHSERVICES_API IBuildManifestRef GetInstallManifest() const;
 
    public:
        IBuildManifestPtr TryGetCurrentManifest() const
        {
            if (IsUpdate() || IsRepair() || IsUninstall())
            {
                return GetCurrentManifest();
            }
            return nullptr;
        }
 
        IBuildManifestPtr TryGetInstallManifest() const
        {
            if (IsInstall() || IsUpdate() || IsRepair())
            {
                return GetInstallManifest();
            }
            return nullptr;
        }
 
        IBuildManifestRef GetCurrentOrInstallManifest() const
        {
            return (CurrentManifest.IsValid() ? CurrentManifest : InstallManifest).ToSharedRef();
        }
 
        IBuildManifestRef GetInstallOrCurrentManifest() const
        {
            return (InstallManifest.IsValid() ? InstallManifest : CurrentManifest).ToSharedRef();
        }
 
    private:
        BUILDPATCHSERVICES_API FInstallerAction();
        IBuildManifestPtr CurrentManifest;
        IBuildManifestPtr InstallManifest;
        TSet<FString> InstallTags;
        FString InstallSubdirectory;
        FString CloudSubdirectory;
        EInstallActionIntent ActionIntent;
    };
 
    struct FInstallerConfiguration
    {
        BUILDPATCHSERVICES_API FInstallerConfiguration(const IBuildManifestRef& InInstallManifest);
        BUILDPATCHSERVICES_API FInstallerConfiguration(const FInstallerConfiguration& CopyFrom);
        BUILDPATCHSERVICES_API FInstallerConfiguration(FInstallerConfiguration&& MoveFrom);
 
    public:
        IBuildManifestPtr CurrentManifest;
        IBuildManifestRef InstallManifest;
        FString InstallDirectory;
        FString StagingDirectory;
        FString BackupDirectory;
        TArray<FString> ChunkDatabaseFiles;
        TArray<FString> CloudDirectories;
        TSet<FString> InstallTags;
        EInstallMode InstallMode;
        EVerifyMode VerifyMode;
        EDeltaPolicy DeltaPolicy;
        bool bIsRepair;
        bool bRunRequiredPrereqs;
        bool bAllowConcurrentExecution;
    };
 
    struct FBuildInstallerConfiguration
    {
        BUILDPATCHSERVICES_API FBuildInstallerConfiguration(TArray<FInstallerAction> InstallerActions);
 
    public:
        // The array of intended actions to perform.
        TArray<FInstallerAction> InstallerActions;
        // The context for allocating shared resources.
        IBuildInstallerSharedContextPtr SharedContext;
        // The directory to install to.
        FString InstallDirectory;
        // The directory for storing the intermediate files. This would usually be inside the InstallDirectory. Empty string will use module's global setting.
        FString StagingDirectory;
        // The directory for placing files that are believed to have local changes, before we overwrite them. Empty string will use module's global setting. If both empty, the feature disables.
        FString BackupDirectory;
        // The list of chunk database filenames that will be used to pull patch data from. These must be local (i.e. no URLs)
        TArray<FString> ChunkDatabaseFiles;
        // If set, when we've read everything we're going to read from a chunkdb file, we delete it to minimize required disk space for install/patch.
        bool bDeleteChunkDbFilesAfterUse = false;
 
        // If set, don't actually do an installation, just figure out how much disk space is required if we are using entirely
        // chunkdbs and we delete them as we go.
        bool bCalculateDeleteChunkDbMaxDiskSpaceAndExit = false;
 
        // The list of cloud directory roots that will be used to pull patch data from. Empty array will use module's global setting. This is only hit if a chunk can not be satisfied by
        // other sources (i.e. chunkdbs or install directory for patches). If the chunk can not be serviced from here, it's a failure.
        TArray<FString> CloudDirectories;
 
        // The mode for installation.
        EInstallMode InstallMode;
        // The mode for verification. This will be overridden for any files referenced by a repair action.
        EVerifyMode VerifyMode;
        // The policy to follow for requesting an optimised delta.
        EDeltaPolicy DeltaPolicy;
        // Whether to run the prerequisite installer provided if it hasn't been ran before on this machine.
        bool bRunRequiredPrereqs;
        // Whether to allow this installation to run concurrently with any existing installations.
        bool bAllowConcurrentExecution;
        // Whether to gather individual file operation statistics during install
        bool bTrackFileOperations;
 
        // Whether the file constructor is allowed to spawn multiple threads for IO read/write servicing.
        // Unset uses Engine.ini value [Portal.BuildPatch]ConstructorSpawnAdditionalIOThreads if present, FFileConstructorConfig::bDefaultSpawnAdditionalIOThreads otherwise.
        TOptional<bool> ConstructorSpawnAdditionalIOThreads;
        // File constructor batch size, in MB. File will attempt to fill/write this chunk size to disk. 
        // Unset uses Engine.ini value [Portal.BuildPatch]ConstructorIOBatchSizeMB if present, FFileConstructorConfig::DefaultIOBatchSizeMB otherwise.
        TOptional<int32> ConstructorIOBatchSizeMB;
 
        // File constructor buffer size, in MB. This affects constructor performance heavily and must be at least big enough to fit a batch. 
        // For purely disk->disk installs, this can be relatively small - even 2x batch size will facilitate multiple GB/s installations. However
        // for cloud/download installs this needs to be fairly large to mitigate the effect of out of order download completion. 
        // When bConstructFilesInMemory or bInstallToMemory is set, this is used to determine how many active files can be in flight.
        // Unset uses Engine.ini value [Portal.BuildPatch]ConstructorIOBufferSizeMB if present, FFileConstructorConfig::DefaultIOBufferSizeMB otherwise.
        TOptional<int32> ConstructorIOBufferSizeMB;
 
        // Whether the file construction should adhere to file system write limits. If a write limit is hit, writing will stall until
        // sufficient quota exists for the write. Inheritance for this property is:
        // CVar BuildPatchFileConstructor.bStallWhenFileSystemThrottled, which defaults to true.
        // if set, this member.
        TOptional<bool> ConstructorStallWhenFileSystemThrottled;
 
        // Whether to disable resume on clean installs below a given threshold, in megabytes. 0 means "never disable".
        // Inheritance:
        // CVar BuildPatchFileConstructor.DisableResumeBelowMB, which defaults to 0.
        // if set, this member
        TOptional<int32> ConstructorDisableResumeBelowMB;
 
        // If set, files will be built entirely in memory - requiring at least the largest file's size worth of memory.
        // IO batch and buffer sizes are ignored as operations operate directly on the output memory buffer. If allowing
        // multiple files in flight, this causes the size of the entire installation to be allocated as there's no IO buffer
        // size limiting the number of in flight requests.
        // Once the file completes construction it is written to disk in the normal place defined by InstallMode. This occurs
        // after the old file is deleted for Destructive installations in order to manage peak disk space usage.
        //
        // This mode does not support resuming mid-file.
        bool bConstructFilesInMemory = false;
 
        // As bConstructFilesInMemory, except the file is never written to disk and is instead made available after via 
        // GetFilesInstalledToMemory 
        bool bInstallToMemory = false;
 
        // If set, no initial size check will be performed prior to starting the installation. Running out of disk space during installation is unchanged.
        bool bSkipInitialDiskSizeCheck = false;
    };
 
    struct FChunkBuildConfiguration
    {
    public:
        BUILDPATCHSERVICES_API FChunkBuildConfiguration();
 
    public:
        // The client feature level to output data for.
        EFeatureLevel FeatureLevel;
        // The directory to analyze.
        FString RootDirectory;
        // The ID of the app of this build.
        uint32 AppId;
        // The name of the app of this build.
        FString AppName;
        // The version string for this build.
        FString BuildVersion;
        // The local exe path that would launch this build.
        FString LaunchExe;
        // The command line that would launch this build.
        FString LaunchCommand;
        // The path to a file containing a \r\n separated list of RootDirectory relative files to read.
        FString InputListFile;
        // The path to a file containing a \r\n separated list of RootDirectory relative files to ignore.
        FString IgnoreListFile;
        // The path to a file containing a \r\n separated list of RootDirectory relative files followed by attribute keywords.
        FString AttributeListFile;
        // The set of identifiers which the prerequisites satisfy.
        TSet<FString> PrereqIds;
        // The display name of the prerequisites installer.
        FString PrereqName;
        // The path to the prerequisites installer.
        FString PrereqPath;
        // The command line arguments for the prerequisites installer.
        FString PrereqArgs;
        // The maximum age (in days) of existing data files which can be reused in this build.
        float DataAgeThreshold;
        // Indicates whether data age threshold should be honored. If false, ALL data files can be reused.
        bool bShouldHonorReuseThreshold;
        // The chunk window size to be used when saving out new data.
        uint32 OutputChunkWindowSize;
        // Indicates whether any window size chunks should be matched, rather than just out output window size.
        bool bShouldMatchAnyWindowSize;
        // Map of custom fields to add to the manifest.
        TMap<FString, FVariant> CustomFields;
        // The cloud directory that all patch data will be saved to. An empty value will use module's global setting.
        FString CloudDirectory;
        // The output manifest filename.
        FString OutputFilename;
        // Allow Manifest Creation for builds with no data
        bool bAllowEmptyBuild;
    };
 
    // Temporary for use with deprecated module function.
    typedef FChunkBuildConfiguration FGenerationConfiguration;
 
    struct FChunkDeltaOptimiserConfiguration
    {
    public:
        BUILDPATCHSERVICES_API FChunkDeltaOptimiserConfiguration();
 
    public:
        // A full file or http path for the manifest to be used as the source build.
        FString ManifestAUri;
        // A full file or http path for the manifest to be used as the destination build.
        FString ManifestBUri;
        // The cloud directory that all patch data will be saved to. An empty value will use ManifestB's directory.
        FString CloudDirectory;
        // The window size to use for find new matches.
        uint32 ScanWindowSize;
        // The chunk size to use for saving new diff data.
        uint32 OutputChunkSize;
        // A threshold for the original delta size, for which we would abort and not process.
        uint64 DiffAbortThreshold;
    };
 
    struct FPatchDataEnumerationConfiguration
    {
    public:
        BUILDPATCHSERVICES_API FPatchDataEnumerationConfiguration();
 
    public:
        // A full file path for the manifest or chunkdb to enumerate referenced data for.
        FString InputFile;
        // A full file path to a file where the list will be saved out to.
        FString OutputFile;
        // Whether to include files sizes.
        bool bIncludeSizes;
    };
 
    struct FDiffManifestsConfiguration
    {
    public:
        BUILDPATCHSERVICES_API FDiffManifestsConfiguration();
 
    public:
        // A full file or http path for the manifest to be used as the source build.
        FString ManifestAUri;
        // A full file or http path for the manifest to be used as the destination build.
        FString ManifestBUri;
        // The tag set to use to filter desired files from ManifestA.
        TSet<FString> TagSetA;
        // The tag set to use to filter desired files from ManifestB.
        TSet<FString> TagSetB;
        // Tag sets that will be used to calculate additional differential size statistics between manifests.
        // They must all be a subset of anything used in TagSetB.
        TArray<TSet<FString>> CompareTagSets;
        // A full file path where a JSON object will be saved for the diff details.Empty string if not desired.
        FString OutputFilePath;
 
        // A destination base directory to use for writing out patch description files for the patch between the two
        // manifest.
        FString OutputPatchDescriptorPath;
 
        // If true, the tool will throw an error if the optimized delta file is missing.
        bool bRequireOptimizedDelta=false;
 
        // If true, only emit patch descriptors and not the entirety of the diff manifests output.
        bool bOnlyPatchDescriptors=false;
        
        // If true, emit simulated installation time estimations
        bool bEmitInstallTime=true;
    };
 
    struct FCompactifyConfiguration
    {
    public:
        BUILDPATCHSERVICES_API FCompactifyConfiguration();
 
    public:
        // The path to the directory to compactify.
        FString CloudDirectory;
        // Chunks which are not referenced by a valid manifest, and which are older than this age(in days), will be deleted.
        float DataAgeThreshold;
        // The full path to a file to which a list of all chunk files deleted by compactify will be written.The output filenames will be relative to the cloud directory.
        FString DeletedChunkLogFile;
        // If ran in preview mode, then the process will run in logging mode only - no files will be deleted.
        bool bRunPreview;
    };
 
    struct FPackageChunksConfiguration
    {
    public:
        BUILDPATCHSERVICES_API FPackageChunksConfiguration();
 
    public:
        // The client feature level to output data for.
        EFeatureLevel FeatureLevel;
        // A full file path to the manifest to enumerate chunks from.
        FString ManifestFilePath;
        // A full file path to a manifest describing a previous build, which will filter out saved chunks for patch only chunkdbs.
        FString PrevManifestFilePath;
        // Optional list of tagsets to split chunkdb files on. Empty array will include all data as normal.
        TArray<TSet<FString>> TagSetArray;
        // Optional tagset to filter the files used from PrevManifestFilePath, potentially increasing the number of chunks saved.
        TSet<FString> PrevTagSet;
        // A full file path to the chunkdb file to save. Extension of .chunkdb will be added if not present.
        FString OutputFile;
        // Cloud directory where chunks to be packaged can be found.
        FString CloudDir;
        // The maximum desired size for each chunkdb file.
        uint64 MaxOutputFileSize;
        // A full file path to use when saving the json output data.
        FString ResultDataFilePath;
    };
}