StallDetector.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "Containers/Array.h"
#include "Containers/Set.h"
#include "Containers/UnrealString.h"
#include "CoreTypes.h"
#include "Delegates/Delegate.h"
#include "GenericPlatform/GenericPlatformCrashContext.h"
#include "HAL/CriticalSection.h"
#include "HAL/PreprocessorHelpers.h"
#include "Logging/LogMacros.h"
#include "Misc/Build.h"
#include "Misc/ScopeLock.h"
#include "ProfilingDebugging/CountersTrace.h"
 
#include <atomic>
 
#ifndef STALL_DETECTOR
 #if WITH_EDITOR && ( PLATFORM_WINDOWS || PLATFORM_UNIX ) && !UE_BUILD_SHIPPING && COUNTERSTRACE_ENABLED && !USING_THREAD_SANITISER
  #define STALL_DETECTOR 1
 #else
  #define STALL_DETECTOR 0
 #endif
#endif // STALL_DETECTOR
 
#if STALL_DETECTOR
 
#define SCOPE_STALL_COUNTER(InName, InBudgetSeconds)                                                                                                                                           \
    static_assert(UE_ARRAY_COUNT(PREPROCESSOR_TO_STRING(InName)) <= UE::StallDetectorScopeMaxNameLength, "Scope name must be less than UE::StallDetectorScopeMaxNameLength characters long "); \
    static UE::FStallDetectorStats PREPROCESSOR_JOIN(StallDetectorStats, __LINE__)(TEXT(#InName), InBudgetSeconds, UE::EStallDetectorReportingMode::Disabled);                                 \
    UE::FStallDetector PREPROCESSOR_JOIN(ScopeStallDetector, __LINE__)(PREPROCESSOR_JOIN(StallDetectorStats, __LINE__));
 
#define SCOPE_STALL_REPORTER(InName, InBudgetSeconds)                                                                                                                                          \
    static_assert(UE_ARRAY_COUNT(PREPROCESSOR_TO_STRING(InName)) <= UE::StallDetectorScopeMaxNameLength, "Scope name must be less than UE::StallDetectorScopeMaxNameLength characters long "); \
    static UE::FStallDetectorStats PREPROCESSOR_JOIN(StallDetectorStats, __LINE__)(TEXT(#InName), InBudgetSeconds, UE::EStallDetectorReportingMode::First);                                    \
    UE::FStallDetector PREPROCESSOR_JOIN(ScopeStallDetector, __LINE__)(PREPROCESSOR_JOIN(StallDetectorStats, __LINE__));
 
#define SCOPE_STALL_REPORTER_ALWAYS(InName, InBudgetSeconds)                                                                                                                                   \
    static_assert(UE_ARRAY_COUNT(PREPROCESSOR_TO_STRING(InName)) <= UE::StallDetectorScopeMaxNameLength, "Scope name must be less than UE::StallDetectorScopeMaxNameLength characters long "); \
    static UE::FStallDetectorStats PREPROCESSOR_JOIN(StallDetectorStats, __LINE__)(TEXT(#InName), InBudgetSeconds, UE::EStallDetectorReportingMode::Always);                                   \
    UE::FStallDetector PREPROCESSOR_JOIN(ScopeStallDetector, __LINE__)(PREPROCESSOR_JOIN(StallDetectorStats, __LINE__));
 
#define SCOPE_STALL_DETECTOR_PAUSE() \
    UE::FStallDetectorPause PREPROCESSOR_JOIN(ScopeStallDetectorPause, __LINE__);
 
#else // STALL_DETECTOR
 
#define SCOPE_STALL_COUNTER(InName, InBudgetSeconds)
#define SCOPE_STALL_REPORTER(InName, InBudgetSeconds)
#define SCOPE_STALL_REPORTER_ALWAYS(InName, InBudgetSeconds)
#define SCOPE_STALL_DETECTOR_PAUSE()
 
#endif // STALL_DETECTOR
 
#if STALL_DETECTOR
 
CORE_API DECLARE_LOG_CATEGORY_EXTERN(LogStall, Log, All);
 
namespace UE
{
    class FStallDetector;
 
    // Stall detected event is fired on a worker thread when it is detected that a stall scope has been active for too long.
    // Fired from a worker thread.
    struct FStallDetectedParams
    {
        FStringView StatName;
        uint32 ThreadId;
        uint64 UniqueID;
        TArray<uint64> Backtrace;
    };
    DECLARE_TS_MULTICAST_DELEGATE_OneParam(FOnStallDetected, const FStallDetectedParams&);
 
    // Stall completed event is fired synchronously on the stalling thread after it recovers from the stall.
    // It may be fired without a matching StallDetected event in the case that the background detector's resolution
    // is too low or that thread is blocked for some reason.
    struct FStallCompletedParams
    {
        uint32 ThreadId;
        FStringView StatName;
        double BudgetSeconds;
        double OverbudgetSeconds;
        uint64 UniqueID;
        bool bWasTriggered;
    };
    DECLARE_TS_MULTICAST_DELEGATE_OneParam(FOnStallCompleted, const FStallCompletedParams&);
 
    enum EStallDetectorReportingMode
    {
        Disabled,
        First,
        Always
    };
 
    static constexpr SIZE_T StallDetectorStatNameBufferSize = 256;
    static constexpr int32 StallDetectorMaxBacktraceEntries = 32;
 
    class FStallDetectorRunnable;
 
    class FStallDetectorStats
    {
        friend class FStallDetectorRunnable;
 
    public:
        CORE_API FStallDetectorStats(const TCHAR* InName, const double InBudgetSeconds, const EStallDetectorReportingMode InReportingMode);
 
        CORE_API ~FStallDetectorStats();
 
        CORE_API void OnStallCompleted(double InOverageSeconds);
 
        struct TabulatedResult
        {
            const TCHAR* Name = TEXT("");
            double BudgetSeconds = 0.0;
            int64 TriggerCount = 0;
            double OverageSeconds = 0.0;
            double OverageRatio = 0.0;
        };
        static CORE_API void TabulateStats(TArray<TabulatedResult>& TabulatedResults);
 
        // The name to refer to this callsite in the codebase
        const TCHAR* Name;
 
        // The budgeted time we tolerate without sending a report
        const double BudgetSeconds;
 
        // Drives the behavior about what to do when a budget is violated (triggered)
        const EStallDetectorReportingMode ReportingMode;
 
        // Has this been reported yet, this is different than the trigger count for coherency reasons
        bool bReported;
 
        // The total number of times all callsites have been triggered
        static CORE_API FCountersTrace::FCounterAtomicInt TotalTriggeredCount;
 
        // The total number of reports that have been sent
        static CORE_API FCountersTrace::FCounterAtomicInt TotalReportedCount;
 
    private:
        // The number of times this callsite has been triggered
        TCHAR TriggerCountCounterName[StallDetectorStatNameBufferSize];
        FCountersTrace::TCounter<int64, TraceCounterType_Int> TriggerCount;
 
        // The cumulative overage time for this callsite
        TCHAR OverageSecondsCounterName[StallDetectorStatNameBufferSize];
        FCountersTrace::TCounter<double, TraceCounterType_Float> OverageSeconds;
 
        // Guards access to the stats from multiple threads, for coherency
        mutable FCriticalSection StatsSection;
    };
 
    class FStallTimer
    {
    public:
        CORE_API FStallTimer();
 
        CORE_API void Reset(const double InSeconds, const double InRemainingSeconds);
 
        CORE_API void Check(const double InSeconds, double& OutDeltaSeconds, double& OutOverageSeconds);
 
        CORE_API void Pause(const double InSeconds);
 
        CORE_API void Resume(const double InSeconds);
 
    private:
        // To track pause/unpause calls
        uint32 PauseCount;
 
        // The timestamp of when we last called Reset() or Check()
        double LastCheckSeconds;
 
        // The time remaining before we we go beyond the remaining time
        double RemainingSeconds;
 
        // Guards access to internal state from multiple threads
        FCriticalSection Section;
    };
 
    class FStallDetector
    {
        friend class FStallDetectorRunnable;
 
    public:
        CORE_API FStallDetector(FStallDetectorStats& InStats);
 
        CORE_API ~FStallDetector();
 
        CORE_API void CheckAndReset();
 
        CORE_API void Pause(const double InSeconds);
 
        CORE_API void Resume(const double InSeconds);
 
        FStallDetector* GetParent()
        {
            return Parent;
        }
 
        static CORE_API double Seconds();
 
        static CORE_API void Startup();
 
        static CORE_API void Shutdown();
 
        static CORE_API bool IsRunning();
 
        static CORE_API FOnStallDetected StallDetected;
 
        static CORE_API FOnStallCompleted StallCompleted;
 
    private:
        static CORE_API TAtomic<uint64> UIDGenerator;
        
        // The time provided, the deadline
        FStallDetectorStats& Stats;
 
        // A stall detector on the same thread with a longer lifetime if any
        FStallDetector* Parent;
 
        // Thread it was constructed on, and to stack trace when budgeted time is elapsed
        uint32 ThreadId;
 
        // Did we actually start the detector (was the thread started at construction time)
        bool bStarted;
 
        // Persistent usage mode
        bool bPersistent;
 
        // Track whether a stall was detected from the background thread.
        // This is protected by the same mutex which protects the registered stall detector instances.
        // Only set from background thread, can be reset from another thread with CheckAndReset
        bool bTriggered;
 
        // The timer data for this detector
        FStallTimer Timer;
 
        // Unique ID given to a StallDetector
        uint64 UniqueID;
 
        CORE_API double Check(bool bFinalCheck, double InCheckSeconds);
    };
 
    class FStallDetectorPause
    {
    public:
        CORE_API FStallDetectorPause();
        CORE_API ~FStallDetectorPause();
        bool IsPaused() const;
 
    private:
        bool bPaused;
    };
 
    static constexpr TCHAR StallDetectorCounterPrefix[] = TEXT("StallDetector/");
    static constexpr TCHAR StallDetectorTriggerCountSuffix[] = TEXT(" TriggerCount");
    static constexpr TCHAR StallDetectorOverageTimeSuffix[] = TEXT(" OverageSeconds");
    static constexpr SIZE_T StallDetectorScopeMaxNameLength = StallDetectorStatNameBufferSize - 1 - UE_ARRAY_COUNT(StallDetectorCounterPrefix)
                                                            + FMath::Max(UE_ARRAY_COUNT(StallDetectorTriggerCountSuffix), UE_ARRAY_COUNT(StallDetectorOverageTimeSuffix));
} // namespace UE
#endif // STALL_DETECTOR