AssetDataGatherer.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "AssetRegistry/PackageReader.h"
#include "Containers/Array.h"
#include "Containers/ArrayView.h"
#include "Containers/DirectoryTree.h"
#include "Containers/LockFreeList.h"
#include "Containers/Map.h"
#include "Containers/MpscQueue.h"
#include "Containers/RingBuffer.h"
#include "Containers/StringFwd.h"
#include "Containers/StringView.h"
#include "Containers/UnrealString.h"
#include "CriticalSectionQueryable.h"
#include "DiskCachedAssetData.h"
#include "HAL/CriticalSection.h"
#include "HAL/FileManager.h"
#include "HAL/Platform.h"
#include "HAL/Runnable.h"
#include "Misc/DateTime.h"
#include "Misc/Optional.h"
#include "NonBufferingReadOnlyArchive.h"
#include "PackageDependencyData.h"
#include "Tasks/TaskConcurrencyLimiter.h"
#include "Templates/UniquePtr.h"
#include "UObject/NameTypes.h"
 
#include <atomic>
 
class FArchive;
class FAssetRegistryReader;
class FAssetRegistryWriter; // Not defined if !ALLOW_NAME_BATCH_SAVING
class FDiskCachedAssetData;
struct FAssetData;
namespace UE::AssetDataGather { struct FResults; }
namespace UE::AssetDataGather::Private { class FAssetDataDiscovery; }
namespace UE::AssetDataGather::Private { class FFileReadScheduler; }
namespace UE::AssetDataGather::Private { class FFilesToSearch; }
namespace UE::AssetDataGather::Private { struct FCachePayload; }
namespace UE::AssetDataGather::Private { struct FDirectoryReadTaskData; }
namespace UE::AssetDataGather::Private { struct FGatheredPathData; }
namespace UE::AssetDataGather::Private { struct FPathExistence; }
namespace UE::AssetDataGather::Private { struct FReadContext; }
namespace UE::AssetDataGather::Private { struct FSetPathProperties; }
namespace UE::AssetDataGather::Private { enum class EPriority : uint8; }
namespace UE::AssetDataGather::Private 
{ 
struct FWaitBatchDirectory
{
    FWaitBatchDirectory(const FStringView InPath, const bool InIsRecursive)
        : Path(InPath)
        , bIsRecursive(InIsRecursive)
    {
    }
 
    friend int32 GetTypeHash(const FWaitBatchDirectory& WaitBatchDirectory)
    {
        return GetTypeHash(WaitBatchDirectory.Path);
    }
 
    FString Path;
    bool bIsRecursive;
};
 
struct FWaitBatchDirectorySetFuncs : BaseKeyFuncs<FWaitBatchDirectory, FString>
{
    static FORCEINLINE const FString& GetSetKey(const FWaitBatchDirectory& Element)
    {
        return Element.Path;
    }
    static FORCEINLINE bool Matches(const FString& A, const FString& B)
    {
        return A.Equals(B, ESearchCase::CaseSensitive);
    }
    static FORCEINLINE uint32 GetKeyHash(const FString& Key)
    {
        return GetTypeHash(Key);
    }
};
 
typedef TSet<FWaitBatchDirectory, FWaitBatchDirectorySetFuncs> FWaitBatchDirectorySet;
}
namespace UE::AssetRegistry { class FAssetRegistryImpl; }
namespace UE::AssetRegistry::Impl { enum class EGatherStatus : uint8; }
 
#if DO_CHECK
typedef FCriticalSectionQueryable FGathererCriticalSection;
typedef FScopeLockQueryable FGathererScopeLock;
#define CHECK_IS_LOCKED_CURRENT_THREAD(CritSec) check(CritSec.IsLockedOnCurrentThread())
#define CHECK_IS_NOT_LOCKED_CURRENT_THREAD(CritSec) check(!CritSec.IsLockedOnCurrentThread())
#else
typedef FCriticalSection FGathererCriticalSection;
typedef FScopeLock FGathererScopeLock;
#define CHECK_IS_LOCKED_CURRENT_THREAD(CritSec) do {} while (false)
#define CHECK_IS_NOT_LOCKED_CURRENT_THREAD(CritSec) do {} while (false)
#endif
 
DECLARE_TS_MULTICAST_DELEGATE_OneParam(FGatheredResultsEvent, const UE::AssetDataGather::FResults&);
 
struct FAssetGatherDiagnostics
{
    float DiscoveryTimeSeconds;
    float GatherTimeSeconds;
    float WallTimeSeconds;
    int32 NumCachedDirectories;
    int32 NumUncachedDirectories;
    int32 NumCachedAssetFiles;
    int32 NumUncachedAssetFiles;
};
 
namespace UE::AssetDataGather
{
 
struct FResults
{
    TMultiMap<FName, TUniquePtr<FAssetData>> Assets;
    TMultiMap<FName, TUniquePtr<FAssetData>> AssetsForGameThread;
    TRingBuffer<FString> Paths;
    TMultiMap<FName, FPackageDependencyData> Dependencies;
    TMultiMap<FName, FPackageDependencyData> DependenciesForGameThread;
    TRingBuffer<FString> CookedPackageNamesWithoutAssetData;
    TRingBuffer<FName> VerseFiles;
    TArray<FString> BlockedFiles;
 
    SIZE_T GetAllocatedSize() const
    {
        return Assets.GetAllocatedSize() + AssetsForGameThread.GetAllocatedSize() + Paths.GetAllocatedSize() + Dependencies.GetAllocatedSize() +
            DependenciesForGameThread.GetAllocatedSize() + CookedPackageNamesWithoutAssetData.GetAllocatedSize() + VerseFiles.GetAllocatedSize() +
            BlockedFiles.GetAllocatedSize();
    }
    void Shrink()
    {
        Assets.Shrink();
        AssetsForGameThread.Shrink();
        Paths.Trim();
        Dependencies.Shrink();
        DependenciesForGameThread.Shrink();
        CookedPackageNamesWithoutAssetData.Trim();
        VerseFiles.Trim();
        BlockedFiles.Shrink();
    }
};
 
struct FResultContext
{
    bool bIsSearching = false;
    bool bAbleToProgress = false;
    TArray<double> SearchTimes;
    int32 NumFilesToSearch = 0;
    int32 NumPathsToSearch = 0;
    bool bIsDiscoveringFiles = false;
};
 
} // namespace UE::AssetDataGather
 
 
class FAssetDataGatherer : public FRunnable
{
public:
    FAssetDataGatherer(UAssetRegistryImpl& InRegistryImpl);
    virtual ~FAssetDataGatherer();
 
    void UpdateCacheForSaving();
    void OnInitialSearchCompleted();
    void OnAdditionalMountSearchCompleted();
    void RequestAsyncCacheSave();
 
    // Controlling Async behavior
 
    void StartAsync();
 
    // FRunnable implementation
    virtual bool Init() override;
    virtual uint32 Run() override;
    virtual void Stop() override;
    virtual void Exit() override;
 
    bool IsAsyncEnabled() const;
    bool IsSynchronous() const;
    void EnsureCompletion();
 
    FGatheredResultsEvent& GetGatheredResultsEvent();
 
    void GetAndTrimSearchResults(UE::AssetDataGather::FResults& InOutResults,
        UE::AssetDataGather::FResultContext& OutContext);
    FAssetGatherDiagnostics GetDiagnostics();
    void GetPackageResults(UE::AssetDataGather::FResults& InOutResults);
    void WaitOnPath(FStringView LocalPath);
    void ClearCache();
 
    void ScanPathsSynchronous(const TArray<FString>& InPaths, bool bForceRescan, bool bIgnoreDenyListScanFilters);
    void WaitForIdle(float TimeoutSeconds = -1.0f);
    bool IsComplete() const;
    bool HasSerializedDiscoveryCache() const;
 
    // Reading/writing/triggering depot-wide properties and events (possibly while tick is running)
 
    void SetInitialPluginsLoaded();
    bool IsGatheringDependencies() const;
    bool IsCacheReadEnabled() const;
    bool IsCacheWriteEnabled() const;
    SIZE_T GetAllocatedSize() const;
 
 
    // Configuring mount points (possibly while tick is running)
 
    void AddMountPoint(FStringView LocalPath, FStringView LongPackageName);
    void RemoveMountPoint(FStringView LocalPath);
    void AddRequiredMountPoints(TArrayView<FString> LocalPaths);
 
 
    // Reading/Writing properties of files and directories (possibly while tick is running)
 
    void OnDirectoryCreated(FStringView LocalPath);
    void OnFilesCreated(TConstArrayView<FString> LocalPaths);
    void PrioritizeSearchPath(const FString& PathToPrioritize);
    void SetIsOnAllowList(FStringView LocalPath, bool bIsAllowed);
    bool IsOnAllowList(FStringView LocalPath) const;
    bool IsOnDenyList(FStringView LocalPath) const;
    bool IsMonitored(FStringView LocalPath) const;
 
    static bool IsVerseFile(FStringView FilePath);
    static TConstArrayView<const TCHAR*> GetVerseFileExtensions();
 
    static bool ReadAssetFile(FPackageReader& PackageReader, TArray<FAssetData*>& AssetDataList,
        FPackageDependencyData& DependencyData, TArray<FString>& CookedPackagesToLoadUponDiscovery,
        FPackageReader::EReadOptions Options);
 
    void PauseProcessing() { IsProcessingPaused.fetch_add(1, std::memory_order_relaxed); }
    void ResumeProcessing() { IsProcessingPaused.fetch_sub(1, std::memory_order_relaxed); }
    bool IsProcessingPauseRequested() const { return IsProcessingPaused.load(std::memory_order_relaxed) != 0; }
 
    void SetGatherOnGameThreadOnly(bool bValue);
    bool IsGatherOnGameThreadOnly() const;
 
    /*
    * Mark that the gatherer is in the process of handling an additional search.
    */
    void SetIsAdditionalMountSearchInProgress(bool bIsInProgress);
 
private:
    enum class ETickResult
    {
        KeepTicking,
        PollDiscovery,
        Idle,
        Interrupt,
    };
    ETickResult InnerTickLoop(bool bInSynchronousTick, bool bContributeToCacheSave, double EndTimeSeconds);
    ETickResult TickInternal(double& TickStartTime, bool bPollDiscovery);
    UE::AssetRegistry::Impl::EGatherStatus TryTickOnBackgroundThread();
 
    bool ShouldBackgroundGatherThreadPauseGathering(bool& bOutGathererIsIdle);
 
    void IngestDiscoveryResults();
 
    void OnFileCreated(FStringView LocalPath);
 
    void SetDirectoryProperties(FStringView LocalPath, const UE::AssetDataGather::Private::FSetPathProperties& Properties);
 
    void WaitOnPathsInternal(TArrayView<UE::AssetDataGather::Private::FPathExistence> QueryPaths);
 
    void SortPathsByPriority(TArrayView<UE::AssetDataGather::Private::FPathExistence> QueryPaths,
        UE::AssetDataGather::Private::EPriority Priority, int32& OutNumPaths);
    bool ReadAssetFile(const FString& AssetLongPackageName, const FString& AssetFilename,
        TArray<FAssetData*>& AssetDataList, FPackageDependencyData& DependencyData,
        TArray<FString>& CookedPackagesToLoadUponDiscovery, bool& OutCanRetry) const;
 
    void ConsumeCacheFiles(TArray<UE::AssetDataGather::Private::FCachePayload> Payloads);
    void TryReserveSaveCache(bool& bOutShouldSave, TArray<TPair<FName,FDiskCachedAssetData*>>& AssetsToSave);
    void SaveCacheFile(const TArray<TPair<FName,FDiskCachedAssetData*>>& AssetsToSave);
    int64 SaveCacheFileInternal(const FString& CacheFilename,
        const TArray<TPair<FName,FDiskCachedAssetData*>>& AssetsToSave);
    void GetAssetsToSave(TArrayView<const FString> SaveCacheLongPackageNameDirs,
        TArray<TPair<FName,FDiskCachedAssetData*>>& OutAssetsToSave);
    void GetCacheAssetsToSave(TArray<TPair<FName,FDiskCachedAssetData*>>& OutAssetsToSave);
 
    /* Adds the given pair into NewCachedAssetDataMap. Detects collisions for multiple files with the same PackageName */
    void AddToCache(FName PackageName, FDiskCachedAssetData* DiskCachedAssetData);
 
    void SetIsIdle(bool IsIdle);
    void SetIsIdle(bool IsIdle, double& TickStartTime);
 
    void Shrink();
 
    struct FScopedGatheringPause
    {
        FScopedGatheringPause(const FAssetDataGatherer& InOwner);
        ~FScopedGatheringPause();
        const FAssetDataGatherer& Owner;
    };
 
    static FString NormalizeLocalPath(FStringView LocalPath);
    static FStringView NormalizeLongPackageName(FStringView LongPackageName);
 
private:
 
    mutable FGathererCriticalSection TickLock;
    mutable FGathererCriticalSection ResultsLock;
 
    FGatheredResultsEvent GatheredResultsEvent;
 
    // Variable section for variables that are constant during threading.
    UAssetRegistryImpl& AssetRegistry;
 
    FRunnableThread* Thread;
 
    bool bAsyncEnabled;
    bool bGatherAssetPackageData;
    bool bGatherDependsData;
 
    FDateTime GatherStartTime;
 
    // Variable section for variables that are atomics read/writable from outside critical sections.
 
    std::atomic<uint32> IsStopped;
    mutable std::atomic<uint32> IsGatheringPaused;
    
    mutable std::atomic<uint32> IsProcessingPaused;
 
    TUniquePtr<UE::AssetDataGather::Private::FAssetDataDiscovery> Discovery;
    std::atomic<bool> bSaveAsyncCacheTriggered;
    std::atomic<bool> bCacheReadEnabled;
    std::atomic<bool> bCacheWriteEnabled;
 
    // Variable section for variables that are read/writable only within ResultsLock.
 
    TUniquePtr<UE::AssetDataGather::Private::FFilesToSearch> FilesToSearch;
 
    TArray<TUniquePtr<FAssetData>> AssetResults;
    TArray<TUniquePtr<FAssetData>> AssetResultsForGameThread;
    TArray<FPackageDependencyData> DependencyResults;
    TArray<FPackageDependencyData> DependencyResultsForGameThread;
    TArray<FString> CookedPackageNamesWithoutAssetDataResults;
    TArray<FName> VerseResults;
    TArray<FString> BlockedResults;
 
    TArray<double> SearchTimes;
    float CumulativeGatherTime = 0.f;
 
    TArray<FString> DiscoveredPaths;
 
    double CurrentSearchTime = 0.;
    double LastCacheWriteTime;
    int32 NumPathsToSearchAtLastSyncPoint;
    int32 NumCachedAssetFiles = 0;
    int32 NumUncachedAssetFiles = 0;
    int32 NumUncachedAssetFilesOutstanding = 0;
    bool bHasLoadedCache;
    bool bDiscoveryIsComplete;
    bool bIsComplete;
    bool bRequestAssetRegistryTick;
    bool bIsIdle;
    bool bFirstTickAfterIdle;
    bool bFinishedInitialDiscovery;
    std::atomic<bool> bIsInitialSearchCompleted;
    std::atomic<bool> bIsAdditionalMountSearchInProgress;
    std::atomic<bool> bGatherOnGameThreadOnly;
 
    // Variable section for variables that are read/writable only within TickLock.
 
    TArray<FDiskCachedAssetData*> NewCachedAssetData;
    TArray<TPair<int32, FDiskCachedAssetData*>> DiskCachedAssetBlocks;
    TMap<FName, FDiskCachedAssetData*> DiskCachedAssetDataMap;
    TMap<FName, FDiskCachedAssetData*> NewCachedAssetDataMap;
    int32 LastCacheSaveNumUncachedAssetFiles;
    int32 CacheInUseCount;
    bool bSynchronousTick;
    bool bIsSavingAsyncCache;
    bool bFlushedRetryFiles;
    TOptional<UE::AssetDataGather::Private::FWaitBatchDirectorySet> WaitBatchDirectories;
    TUniquePtr<UE::AssetDataGather::Private::FFileReadScheduler> FileReadScheduler;
};
 
 
// Inline implementations
 
inline void FAssetDataGatherer::RequestAsyncCacheSave()
{
    bSaveAsyncCacheTriggered = true;
}
 
inline FGatheredResultsEvent& FAssetDataGatherer::GetGatheredResultsEvent()
{
    return GatheredResultsEvent;
}
 
inline void FAssetDataGatherer::SetGatherOnGameThreadOnly(bool bValue)
{
    bGatherOnGameThreadOnly.store(bValue, std::memory_order_relaxed);
}
 
inline bool FAssetDataGatherer::IsGatherOnGameThreadOnly() const
{
    return bGatherOnGameThreadOnly.load(std::memory_order_relaxed);
}
 
inline void FAssetDataGatherer::SetIsAdditionalMountSearchInProgress(bool bIsInProgress)
{
    bIsAdditionalMountSearchInProgress.store(bIsInProgress, std::memory_order_relaxed);
}