AutomationTestExcludelist.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "AutomationTestPlatform.h"
#include "CoreMinimal.h"
#include "Misc/SecureHash.h"
 
#include "AutomationTestExcludelist.generated.h"
 
UENUM()
enum class ETEST_RHI_Options
{
    DirectX11,
    DirectX12,
    Vulkan,
    Metal,
    Null
};
 
inline FString LexToString(ETEST_RHI_Options Option)
{
    switch (Option)
    {
    case ETEST_RHI_Options::DirectX11:    return TEXT("DirectX 11");
    case ETEST_RHI_Options::DirectX12:    return TEXT("DirectX 12");
    case ETEST_RHI_Options::Vulkan:       return TEXT("Vulkan");
    case ETEST_RHI_Options::Metal:        return TEXT("Metal");
    case ETEST_RHI_Options::Null:         return TEXT("Null");
    default:                              return TEXT("Unknown");
    }
}
 
UENUM()
enum class ETEST_RHI_FeatureLevel_Options
{
    SM5,
    SM6
};
 
inline FString LexToString(ETEST_RHI_FeatureLevel_Options Option)
{
    switch (Option)
    {
    case ETEST_RHI_FeatureLevel_Options::SM5:   return TEXT("SM5");
    case ETEST_RHI_FeatureLevel_Options::SM6:   return TEXT("SM6");
    default:                                    return TEXT("Unknown");
    }
}
 
inline FString SetToString(const TSet<FName>& Set)
{
    if (Set.IsEmpty())
    {
        return TEXT("");
    }
 
    TArray<FString> List;
    for (auto& Item : Set)
    {
        List.Add(Item.ToString());
    }
    List.Sort();
 
    return FString::Join(List, TEXT(", "));
}
 
inline FString SetToShortString(const TSet<FName>& Set)
{
    if (Set.IsEmpty())
    {
        return TEXT("");
    }
 
    TArray<FString> List;
    for (auto& Item : Set)
    {
        List.Add(Item.ToString());
    }
    if (List.Num() == 1)
    {
        return List[0];
    }       
    List.Sort();
 
    return List[0] + TEXT("+");
}
 
UCLASS(config = Engine, defaultconfig, MinimalAPI)
class UAutomationTestExcludelistSettings : public UAutomationTestPlatformSettings
{
    GENERATED_BODY()
 
public:
 
    UPROPERTY(Config)
    TArray<FName> SupportedRHIs;
 
protected:
    virtual void InitializeSettingsDefault() { }
    virtual FString GetSectionName() { return TEXT("AutomationTestExcludelistSettings"); }
 
};
 
USTRUCT()
struct FAutomationTestExcludeOptions
{
    GENERATED_BODY()
 
    template<typename EnumType>
    static const TSet<FName>& GetAllRHIOptionNames()
    {
        LLM_SCOPE_BYNAME(TEXT("AutomationTest/Settings"));
        static TSet<FName> NameSet;
        if (NameSet.IsEmpty())
        {
            if constexpr (std::is_same_v<EnumType, ETEST_RHI_Options> || std::is_same_v<EnumType, ETEST_RHI_FeatureLevel_Options>)
            {
                UEnum* Enum = StaticEnum<EnumType>();
                int32 Num_Flags = Enum->NumEnums() - 1;
                for (int32 i = 0; i < Num_Flags; i++)
                {
                    NameSet.Add(*LexToString((EnumType)Enum->GetValueByIndex(i)));
                }
            }
        }
 
        return NameSet;
    }
 
    static const TSet<FName>& GetAllRHIOptionNamesFromSettings()
    {
        LLM_SCOPE_BYNAME(TEXT("AutomationTest/Settings"));
        static TSet<FName> NameSet;
        if (NameSet.IsEmpty())
        {
            for (auto Settings : AutomationTestPlatform::GetAllPlatformsSettings(UAutomationTestExcludelistSettings::StaticClass()))
            {
                NameSet.Append(CastChecked<UAutomationTestExcludelistSettings>(Settings)->SupportedRHIs);
            }
            NameSet.Sort([](const FName& A, const FName& B) { return A.ToString() < B.ToString(); });
        }
 
        return NameSet;
    }
 
    static const TSet<FName>& GetPlatformRHIOptionNamesFromSettings(const FName& Platform)
    {
        LLM_SCOPE_BYNAME(TEXT("AutomationTest/Settings"));
        static TMap<FName, TSet<FName>> PlatformSettings;
        if (PlatformSettings.IsEmpty())
        {
            for (auto Settings : AutomationTestPlatform::GetAllPlatformsSettings(UAutomationTestExcludelistSettings::StaticClass()))
            {
                PlatformSettings.Emplace(Settings->GetPlatformName()).Append(CastChecked<UAutomationTestExcludelistSettings>(Settings)->SupportedRHIs);
            }
        }
 
        return PlatformSettings.FindOrAdd(Platform);
    }
 
#if WITH_EDITOR
    AUTOMATIONTEST_API void UpdateReason(const FString& BeautifiedReason, const FString& TaskTrackerTicketId);
#endif //WITH_EDITOR
 
    /* Name of the target test */
    UPROPERTY(VisibleAnywhere, Category = ExcludeTestOptions)
    FName Test;
 
    /* Reason to why the test is excluded */
    UPROPERTY(EditAnywhere, Category = ExcludeTestOptions)
    FName Reason;
 
    /* Options to target specific RHI. No option means it should be applied to all RHIs */
    UPROPERTY(EditAnywhere, Category = ExcludeTestOptions)
    TSet<FName> RHIs;
 
    /* Options to target specific platform. No option means it should be applied to all platforms */
    UPROPERTY(EditAnywhere, Category = ExcludeTestOptions)
    TSet<FName> Platforms;
 
    /* Should the Reason be reported as a warning in the log */
    UPROPERTY(EditAnywhere, Category = ExcludeTestOptions)
    bool Warn = false;
};
 
USTRUCT()
struct FAutomationTestExcludelistEntry
{
    GENERATED_BODY()
 
    FAutomationTestExcludelistEntry() { }
 
    FAutomationTestExcludelistEntry(const FAutomationTestExcludeOptions& Options)
        : Platforms(Options.Platforms)
        , Test(Options.Test)
        , Reason(Options.Reason)
        , RHIs(Options.RHIs)
        , Warn(Options.Warn)
    { }
 
    TSharedPtr<FAutomationTestExcludeOptions> GetOptions() const
    {
        TSharedPtr<FAutomationTestExcludeOptions> Options = MakeShareable(new FAutomationTestExcludeOptions());
        Options->Test = Test;
        Options->Reason = Reason;
        Options->Warn = Warn;
        Options->RHIs = RHIs;
        Options->Platforms = Platforms;
        
        return Options;
    }
 
    /* Return the number of RHI types that needs to be matched for exclusion */
    int8 NumRHIType() const
    {
        int8 Num = 0;
        // Test mentions of each RHI option types
        static const TSet<FName> AllRHI_OptionNames = FAutomationTestExcludeOptions::GetAllRHIOptionNamesFromSettings();
        if (RHIs.Difference(AllRHI_OptionNames).Num() < RHIs.Num())
        {
            Num++;
        }
        static const TSet<FName> AllRHI_FeatureLevel_OptionNames = FAutomationTestExcludeOptions::GetAllRHIOptionNames<ETEST_RHI_FeatureLevel_Options>();
        if (RHIs.Difference(AllRHI_FeatureLevel_OptionNames).Num() < RHIs.Num())
        {
            Num++;
        }
 
        return Num;
    }
 
    /* Determine if exclusion entry is propagated based on test name - used for management in test automation window */
    void SetPropagation(const FString& ForTestName)
    {
        bIsPropagated = FullTestName != ForTestName.TrimStartAndEnd().ToLower();
    }
 
    /* Return true if the entry is not specific */
    bool IsEmpty() const
    {
        return FullTestName.IsEmpty();
    }
 
    /* Reset entry to be un-specific */
    void Reset()
    {
        FullTestName.Empty();
        bIsPropagated = false;
    }
 
    /* Make the entry specific */
    void Finalize();
 
    /* Output string used to generate hash */
    FString GetStringForHash() const;
 
    /* Has conditional exclusion */
    bool HasConditions() const
    {
        return !RHIs.IsEmpty() || !Platforms.IsEmpty();
    }
 
    /* Remove overlapping exclusion conditions, return true if at least one condition was removed */
    bool RemoveConditions(const FAutomationTestExcludelistEntry& Entry)
    {
        if (!Entry.HasConditions())
        {
            return false;
        }
 
        bool GotRemoved = false;
        int Length;
        // Check RHIs
        Length = RHIs.Num();
        if (Length > 0)
        {
            RHIs = RHIs.Difference(Entry.RHIs);
            GotRemoved = GotRemoved || Length != RHIs.Num();
        }
        // Check Platforms
        Length = Platforms.Num();
        if (Length > 0)
        {
            Platforms = Platforms.Difference(Entry.Platforms);
            GotRemoved = GotRemoved || Length != Platforms.Num();
        }
 
        return GotRemoved;
    }
 
    /* Hold full test name/path */
    FString FullTestName;
    /* Is the entry comes from propagation */
    bool bIsPropagated = false;
    /* Platforms to which the entry applies */
    TSet<FName> Platforms;
 
    // Use FName instead of FString to read params from config that aren't wrapped with quotes
 
    /* Hold the name of the target functional test map */
    UPROPERTY(Config)
    FName Map;
 
    /* Hold the name of the target test - full test name is require here unless for functional tests */
    UPROPERTY(Config)
    FName Test;
 
    /* Reason to why the test is excluded */
    UPROPERTY(Config)
    FName Reason;
 
    /* Option to target specific RHI. Empty array means it should be applied to all RHI */
    UPROPERTY(Config)
    TSet<FName> RHIs;
 
    /* Should the Reason be reported as a warning in the log */
    UPROPERTY(Config)
    bool Warn = false;
};
 
UCLASS(config = Engine, defaultconfig, MinimalAPI)
class UAutomationTestExcludelistConfig : public UAutomationTestPlatformSettings
{
    GENERATED_BODY()
 
public:
    void Reset();
    void AddEntry(const FAutomationTestExcludelistEntry& Entry);
    const TArray<FAutomationTestExcludelistEntry>& GetEntries() const;
    FString GetTaskTrackerURLHashtag() const { return TaskTrackerURLHashtag; }
    FString GetTaskTrackerURLBase() const { return TaskTrackerURLBase; }
 
    void SaveConfig();
 
    void LoadTaskTrackerProperties();
 
protected:
    virtual void PostInitProperties() override;
    virtual void InitializeSettingsDefault() override;
    virtual FString GetSectionName() override { return TEXT("AutomationTestExcludelist"); }
 
private:
    void UpdateHash(const FAutomationTestExcludelistEntry& Entry);
 
    UPROPERTY(Transient)
    FString TaskTrackerURLHashtag;
 
    UPROPERTY(Transient)
    FString TaskTrackerURLBase;
 
    UPROPERTY(Config)
    TArray<FAutomationTestExcludelistEntry> ExcludeTest;
 
    FString PlatformName;
    FSHAHash EntriesHash;
    FSHAHash SavedEntriesHash;
};
 
UCLASS(MinimalAPI)
class UAutomationTestExcludelist : public UObject
{
    GENERATED_BODY()
 
public:
    UAutomationTestExcludelist() { };
 
    static AUTOMATIONTEST_API UAutomationTestExcludelist* Get();
 
    AUTOMATIONTEST_API void AddToExcludeTest(const FString& TestName, const FAutomationTestExcludelistEntry& ExcludelistEntry);
 
    AUTOMATIONTEST_API void RemoveFromExcludeTest(const FString& TestName);
 
    AUTOMATIONTEST_API bool IsTestExcluded(const FString& TestName) const;
 
    AUTOMATIONTEST_API bool IsTestExcluded(const FString& TestName, const TSet<FName>& RHI, FName* OutReason, bool* OutWarn) const;
 
    AUTOMATIONTEST_API bool IsTestExcluded(const FString& TestName, const FName& Platform, const TSet<FName>& RHI, FName* OutReason, bool* OutWarn) const;
 
    AUTOMATIONTEST_API const FAutomationTestExcludelistEntry* GetExcludeTestEntry(const FString& TestName) const;
 
    AUTOMATIONTEST_API const FAutomationTestExcludelistEntry* GetExcludeTestEntry(const FString& TestName, const TSet<FName>& RHI) const;
 
    AUTOMATIONTEST_API const FAutomationTestExcludelistEntry* GetExcludeTestEntry(const FString& TestName, const FName& Platform, const TSet<FName>& RHI) const;
 
    UE_DEPRECATED(5.4, "Please use GetConfigFilenameForEntry instead.")
    AUTOMATIONTEST_API FString GetConfigFilename() const;
 
    AUTOMATIONTEST_API FString GetConfigFilenameForEntry(const FAutomationTestExcludelistEntry& Entry) const;
 
    AUTOMATIONTEST_API FString GetConfigFilenameForEntry(const FAutomationTestExcludelistEntry& Entry, const FName& Platform) const;
 
    AUTOMATIONTEST_API FString GetTaskTrackerURLBase() const;
 
    AUTOMATIONTEST_API FString GetConfigTaskTrackerHashtag() const;
 
    AUTOMATIONTEST_API FString GetTaskTrackerName() const;
 
    AUTOMATIONTEST_API FString GetTaskTrackerTicketTag() const;
 
    AUTOMATIONTEST_API void SaveToConfigs();
 
    UE_DEPRECATED(5.4, "Please use SaveToConfigs instead.")
    void SaveConfig() { SaveToConfigs(); }
 
private:
    void Initialize();
 
    void LoadPlatformConfigs();
 
    void PopulateEntries();
 
    FString GetBeautifiedTaskTrackerTicketTagSuffix() const;
 
    TMap<FString, FAutomationTestExcludelistEntry> Entries;
 
    UPROPERTY(Transient)
    TObjectPtr<UAutomationTestExcludelistConfig> DefaultConfig;
 
    UPROPERTY(Transient)
    TMap<FName, TObjectPtr<UAutomationTestExcludelistConfig>> PlatformConfigs;
};