ThreadHeartBeat.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
 
#include "Containers/Map.h"
#include "CoreTypes.h"
#include "Delegates/Delegate.h"
#include "Delegates/DelegateCombinations.h" // for DECLARE_DELEGATE_OneParam
#include "HAL/CriticalSection.h"
#include "HAL/Runnable.h"
#include "HAL/ThreadSafeBool.h"
#include "HAL/ThreadSafeCounter.h"
#include "Misc/AssertionMacros.h"
#include "UObject/NameTypes.h"
 
#if PLATFORM_UNIX
#include "Unix/UnixSignalHeartBeat.h"
#endif
 
DECLARE_DELEGATE_OneParam(FOnThreadStuck, uint32);
DECLARE_DELEGATE_OneParam(FOnThreadUnstuck, uint32);
DECLARE_DELEGATE_OneParam(FOnHangDelegate, uint32);
 
class FThreadHeartBeatClock
{
    uint64 CurrentCycles;
    uint64 LastRealTickCycles;
    const uint64 MaxTimeStepCycles;
 
public:
    CORE_API FThreadHeartBeatClock(double InMaxTimeStep);
 
    CORE_API void Tick();
    CORE_API double Seconds();
};
 
class FThreadHeartBeat : public FRunnable
{
    static CORE_API FThreadHeartBeat* Singleton;
 
    struct FHeartBeatInfo
    {
        FHeartBeatInfo()
            : LastHeartBeatTime(0.0)
            , LastHangTime(0.0)
            , SuspendedCount(0)
            , HangDuration(0)
            , LastStuckTime(0.0)
            , StuckDuration(0.0)
            , HeartBeatName()
        {}
 
        double LastHeartBeatTime;
        double LastHangTime;
        int32 SuspendedCount;
        double HangDuration;
 
        double LastStuckTime;
        double StuckDuration;
        FName HeartBeatName;
 
        void Suspend()
        {
            SuspendedCount++;
        }
        void Resume(double CurrentTime)
        {
            check(SuspendedCount > 0);
            if (--SuspendedCount == 0)
            {
                LastHeartBeatTime = CurrentTime;
            }
        }
    };
    FRunnableThread* Thread;
    FThreadSafeCounter StopTaskCounter;
    FCriticalSection HeartBeatCritical;
    TMap<uint32, FHeartBeatInfo> ThreadHeartBeat;
    FHeartBeatInfo PresentHeartBeat;
 
    FCriticalSection FunctionHeartBeatCritical;
    TMap<uint32, FHeartBeatInfo> FunctionHeartBeat;
 
    FCriticalSection CheckpointHeartBeatCritical;
    TMap<FName, FHeartBeatInfo> CheckpointHeartBeat;
 
    FThreadSafeBool bReadyToCheckHeartbeat;
    double ConfigHangDuration;
    double CurrentHangDuration;
    double ConfigPresentDuration;
    double CurrentPresentDuration;
    double ConfigStuckDuration;
    double CurrentStuckDuration;
 
    double HangDurationMultiplier;
 
    uint32 LastHangCallstackCRC;
    uint32 LastHungThreadId;
 
    uint32 LastStuckThreadId;
 
    bool bHangsAreFatal;
 
    FThreadSafeCounter GlobalSuspendCount;
    int32 CheckpointSuspendCount;
 
    FThreadHeartBeatClock Clock;
 
    FOnThreadStuck OnStuck;
    FOnThreadUnstuck OnUnstuck;
    FOnHangDelegate OnHangDelegate;
 
    CORE_API FThreadHeartBeat();
    CORE_API virtual ~FThreadHeartBeat();
 
    CORE_API void FORCENOINLINE OnHang(double HangDuration, uint32 ThreadThatHung);
    CORE_API void FORCENOINLINE OnPresentHang(double HangDuration);
 
    CORE_API bool IsEnabled();
 
    void InitSettingsInternal();
 
public:
 
    enum EConstants
    {
        InvalidThreadId = (uint32)-1,
 
        PresentThreadId = (uint32)-2
    };
 
    static CORE_API FThreadHeartBeat& Get();
    static CORE_API FThreadHeartBeat* GetNoInit();
 
    CORE_API void Start();
    CORE_API void InitSettings();
    UE_DEPRECATED(5.7, "HeartBeat(bool bReadConfig) is deprecated, please use HeartBeat() instead.")
    CORE_API void HeartBeat(bool bReadConfig);
    CORE_API void HeartBeat();
    CORE_API void PresentFrame();
    CORE_API uint32 CheckHeartBeat(double& OutHangDuration);
    CORE_API void KillHeartBeat();
 
    CORE_API void MonitorFunctionStart();
    CORE_API void MonitorFunctionEnd();
    CORE_API uint32 CheckFunctionHeartBeat(double& OutHangDuration);
 
    /* 
        Called from a thread to register a checkpoint to be monitored 
        @param EndCheckPoint name of the checkpoint that needs to be reached. TimeToReachCheckPoint the time duration we have to reach the specified checkpoint.
    */
    CORE_API void MonitorCheckpointStart(FName EndCheckPoint, double TimeToReachCheckpoint);
    /* Called from a thread when a checkpoint has ended */
    CORE_API void MonitorCheckpointEnd(FName CheckPoint);
    /* Called by a supervising thread to check all checkpoints forward progress */
    CORE_API uint32 CheckCheckpointHeartBeat(double& OutHangDuration);
 
    CORE_API void SuspendHeartBeat(bool bAllThreads = false);
    CORE_API void ResumeHeartBeat(bool bAllThreads = false);
 
    CORE_API bool IsBeating();
 
    CORE_API void SetDurationMultiplier(double NewMultiplier);
 
    /*
    * Get the Id of the last thread to trigger the hang detector.
    * Returns InvalidThreadId if hang detector has not been triggered.
    */
    uint32 GetLastHungThreadId() const { return LastHungThreadId; }
 
    /*
    * Get the Id of the last thread to pass the stuck thread time.
    * Returns InvalidThreadId if hang detector has not been triggered.
    */
    uint32 GetLastStuckThreadId() const { return LastStuckThreadId; }
 
    /*
    * Get delegate for callback on stuck or unstuck thread.
    */
    FOnThreadStuck& GetOnThreadStuck() { return OnStuck; }
    FOnThreadUnstuck& GetOnThreadUnstuck() { return OnUnstuck; }
 
    /*
    * Get delegate for callback on hang.
    * Delegate implementation will be called from the hang detector thread and not from the hung thread
    */
    FOnHangDelegate& GetOnHangDelegate() { return OnHangDelegate; }
 
    /*
    *  Get hang duration threshold.
    */
    double GetHangDuration() const { return ConfigHangDuration; };
 
    //~ Begin FRunnable Interface.
    CORE_API virtual bool Init();
    CORE_API virtual uint32 Run();
    CORE_API virtual void Stop();
    //~ End FRunnable Interface
};
 
struct FSlowHeartBeatScope
{
private:
    bool bSuspendedAllThreads;
    bool bSuspended;
public:
    inline FSlowHeartBeatScope(bool bAllThreads = false)
        : bSuspendedAllThreads(bAllThreads)
        , bSuspended(false)
    {
        if (FThreadHeartBeat* HB = FThreadHeartBeat::GetNoInit())
        {
            bSuspended = true;
            HB->SuspendHeartBeat(bSuspendedAllThreads);
        }
    }
    inline ~FSlowHeartBeatScope()
    {
        if (bSuspended)
        {
            if (FThreadHeartBeat* HB = FThreadHeartBeat::GetNoInit())
            {
                HB->ResumeHeartBeat(bSuspendedAllThreads);
            }
        }
    }
};
 
struct FFunctionHeartBeatScope
{
private:
    bool bStartedMonitor;
public:
    inline FFunctionHeartBeatScope()
        : bStartedMonitor(false)
    {
        if (FThreadHeartBeat* HB = FThreadHeartBeat::GetNoInit())
        {
            bStartedMonitor = true;
            HB->MonitorFunctionStart();
        }
    }
    inline ~FFunctionHeartBeatScope()
    {
        if (bStartedMonitor)
        {
            if (FThreadHeartBeat* HB = FThreadHeartBeat::GetNoInit())
            {
                HB->MonitorFunctionEnd();
            }
        }
    }
};
 
 
// When 1, performs a full symbol lookup in hitch call stacks, otherwise only
// a backtrace is performed and the raw addresses are written to the log.
#ifndef LOOKUP_SYMBOLS_IN_HITCH_STACK_WALK
#define LOOKUP_SYMBOLS_IN_HITCH_STACK_WALK 0
#endif
 
#ifndef WALK_STACK_ON_HITCH_DETECTED
#define WALK_STACK_ON_HITCH_DETECTED 0
#endif
 
class FGameThreadHitchHeartBeatThreaded : public FRunnable
{
    static CORE_API FGameThreadHitchHeartBeatThreaded* Singleton;
 
    FRunnableThread* Thread;
    FThreadSafeCounter StopTaskCounter;
    FCriticalSection HeartBeatCritical;
 
#if USE_HITCH_DETECTION
    float HangDuration = -1.0f;
    bool bWalkStackOnHitch;
    double FirstStartTime;
    double FrameStartTime;
    int32 SuspendedCount;
    bool bStartSuspended = false;
 
#if WALK_STACK_ON_HITCH_DETECTED
#if LOOKUP_SYMBOLS_IN_HITCH_STACK_WALK
    static constexpr SIZE_T StackTraceSize = 65535;
    ANSICHAR StackTrace[StackTraceSize];
#else
    static constexpr uint32 MaxStackDepth = 128;
    uint64 StackTrace[MaxStackDepth]; 
#endif // LOOKUP_SYMBOLS_IN_HITCH_STACK_WALK
#endif // WALK_STACK_ON_HITCH_DETECTED
#endif // USE_HITCH_DETECTION
 
    FThreadHeartBeatClock Clock;
 
    CORE_API FGameThreadHitchHeartBeatThreaded();
    CORE_API virtual ~FGameThreadHitchHeartBeatThreaded();
 
    void InitSettingsInternal();
 
public:
 
    enum EConstants
    {
        InvalidThreadId = (uint32)-1
    };
 
    static CORE_API FGameThreadHitchHeartBeatThreaded& Get();
    static CORE_API FGameThreadHitchHeartBeatThreaded* GetNoInit();
 
    CORE_API void InitSettings();
 
    CORE_API void FrameStart(bool bSkipThisFrame = false);
 
    CORE_API double GetFrameStartTime();
    CORE_API double GetCurrentTime();
 
    CORE_API void SuspendHeartBeat();
 
    CORE_API void ResumeHeartBeat();
    
    CORE_API bool IsStartedSuspended();
 
    // No-op, used in FUnixSignalGameHitchHeartBeat
    void Restart() {}
    void PostFork() {}
 
    //~ Begin FRunnable Interface.
    CORE_API virtual bool Init();
    CORE_API virtual uint32 Run();
    CORE_API virtual void Stop();
    //~ End FRunnable Interface
};
 
#if PLATFORM_UNIX
typedef FUnixSignalGameHitchHeartBeat FGameThreadHitchHeartBeat;
#else
typedef FGameThreadHitchHeartBeatThreaded FGameThreadHitchHeartBeat;
#endif
 
struct FDisableHitchDetectorScope
{
    UE_FORCEINLINE_HINT FDisableHitchDetectorScope()
    {
        FGameThreadHitchHeartBeat::Get().SuspendHeartBeat();
    }
    UE_FORCEINLINE_HINT ~FDisableHitchDetectorScope()
    {
        FGameThreadHitchHeartBeat::Get().ResumeHeartBeat();
    }
};