IMediaTextureSample.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "CoreTypes.h"
#include "IMediaTimeSource.h"
#include "Math/Color.h"
#include "Math/IntPoint.h"
#include "Math/MathFwd.h"
#include "Math/Matrix.h"
#include "Math/Plane.h"
#include "Misc/Optional.h"
#include "Misc/Timecode.h"
#include "Misc/Timespan.h"
#include "Misc/FrameRate.h"
#include "Templates/SharedPointer.h"
 
#include "HDRHelper.h"
#include "ColorManagement/ColorManagementDefines.h"
#include "ColorManagement/ColorSpace.h"
#include "MediaShaders.h"
 
#if WITH_ENGINE
    class FRHITexture;
    class IMediaTextureSampleConverter;
    class IMediaTextureSampleColorConverter;
#endif
 
 
enum class EMediaTextureSampleFormat
{
    Undefined,
 
    CharAYUV,
 
    CharBGRA,
 
    CharRGBA,
 
    CharBGR10A2,
 
    CharBMP,
 
    CharNV12,
 
    CharNV21,
 
    CharUYVY,
 
    CharYUY2,
 
    Char2VUY,
 
    CharYVYU,
 
    FloatRGB,
 
    FloatRGBA,
 
    YUVv210,
 
    YUVv216,
 
    ShortYUY2,
 
    Y416,
 
    R4FL,
 
    P010,
 
    DXT1,
 
    DXT5,
 
    BC4,
 
    YCoCg_DXT5,
 
    YCoCg_DXT5_Alpha_BC4,
 
    P010_RGB1010102,
 
    RGBA16,
 
    ABGR16,
 
    ARGB16_BIG,
 
    ExternalVYU,
};
 
namespace MediaTextureSampleFormat
{
     MEDIA_API const TCHAR* EnumToString(const EMediaTextureSampleFormat InSampleFormat);
 
     static inline bool IsBlockCompressedFormat(EMediaTextureSampleFormat InSampleFormat)
     {
        PRAGMA_DISABLE_SWITCH_UNHANDLED_ENUM_CASE_WARNINGS;
        switch (InSampleFormat)
        {
        case EMediaTextureSampleFormat::DXT1:
        case EMediaTextureSampleFormat::DXT5:
        case EMediaTextureSampleFormat::BC4:
        case EMediaTextureSampleFormat::YCoCg_DXT5:
        case EMediaTextureSampleFormat::YCoCg_DXT5_Alpha_BC4:
         return true;
        }
        PRAGMA_RESTORE_SWITCH_UNHANDLED_ENUM_CASE_WARNINGS;
        return false;
     }
};
 
struct FMediaTextureTilingDescription
{
    FIntPoint TileNum = FIntPoint::ZeroValue;
    FIntPoint TileSize = FIntPoint::ZeroValue;
    int32 TileBorderSize = 0;
 
    inline bool IsValid() const
    {
        return TileNum.X > 0 && TileNum.Y > 0 && TileSize.X > 0 && TileSize.Y > 0;
    }
};
 
enum class EMediaOrientation
{
    Original = 0,
    CW90,
    CW180,
    CW270
};
 
namespace MediaTextureSample
{
    /* This the reference white level for mapping UE scene-referred colors to nits (see TonemapCommon.ush). */
    static constexpr float kLinearToNitsScale_UE = 100.0f;
 
    /* This the reference white level for mapping SDR 1.0 to nits, as defined by ITU-R Report BT.2408. */
    static constexpr float kLinearToNitsScale_BT2408 = 203.0f;
}
 
static constexpr float kMediaSample_HDR_NitsNormalizationFactor = 1.0f / MediaTextureSample::kLinearToNitsScale_UE;
 
class IMediaTextureSample
{
public:
    virtual const void* GetBuffer() = 0;
 
    virtual FIntPoint GetDim() const = 0;
 
    virtual uint8 GetNumMips() const
    {
        return 1;
    }
 
    virtual FMediaTextureTilingDescription GetTilingDescription() const
    {
        return FMediaTextureTilingDescription();
    }
 
    virtual FTimespan GetDuration() const = 0;
 
    virtual EMediaTextureSampleFormat GetFormat() const = 0;
 
    virtual FIntPoint GetOutputDim() const = 0;
 
    virtual uint32 GetStride() const = 0;
 
    virtual bool ShouldApplyColorConversion() const { return true; }
 
#if WITH_ENGINE
 
    virtual FRHITexture* GetTexture() const = 0;
 
    virtual IMediaTextureSampleConverter* GetMediaTextureSampleConverter()
    {
        return nullptr;
    }
 
    virtual IMediaTextureSampleColorConverter* GetMediaTextureSampleColorConverter()
    {
        return nullptr;
    }
 
#endif //WITH_ENGINE
 
    virtual FMediaTimeStamp GetTime() const = 0;
 
    virtual TOptional<FTimecode> GetTimecode() const { return TOptional<FTimecode>(); }
 
    virtual TOptional<FFrameRate> GetFramerate() const { return TOptional<FFrameRate>(); }
 
    virtual bool IsCacheable() const = 0;
 
    virtual bool IsOutputSrgb() const = 0;
 
    virtual EMediaOrientation GetOrientation() const
    {
        return EMediaOrientation::Original;
    }
 
    virtual double GetAspectRatio() const
    {
        FIntPoint OutputDim = GetOutputDim();
        return (double)OutputDim.X / (double)OutputDim.Y;
    }
 
    virtual bool IsExternalImage() const
    {
        return false;
    }
    
    virtual FLinearColor GetScaleRotation() const
    {
        return FLinearColor(1.0f, 0.0f, 0.0f, 1.0f);
    }
 
    virtual FLinearColor GetOffset() const
    {
        return FLinearColor(0.0f, 0.0f, 0.0f, 0.0f);
    }
 
    virtual const FMatrix& GetYUVToRGBMatrix() const
    {
        return MediaShaders::YuvToRgbRec709Scaled;
    }
 
    /*
    * Get full range color flag
    */
    virtual bool GetFullRange() const
    {
        return false;
    }
 
    virtual FMatrix44f GetSampleToRGBMatrix() const
    {
        FMatrix Pre = FMatrix::Identity;
        FVector Off;
        switch (GetFormat())
        {
            case EMediaTextureSampleFormat::R4FL:       Off = MediaShaders::YUVOffsetFloat; break;
            case EMediaTextureSampleFormat::Y416:
            case EMediaTextureSampleFormat::P010:
            case EMediaTextureSampleFormat::ShortYUY2:
            case EMediaTextureSampleFormat::YUVv216:    Off = MediaShaders::YUVOffset16bits; break;
            case EMediaTextureSampleFormat::YUVv210:    Off = MediaShaders::YUVOffset10bits; break;
            default:                                    Off = MediaShaders::YUVOffset8bits; break;
        }
        Pre.M[0][3] = -Off.X;
        Pre.M[1][3] = -Off.Y;
        Pre.M[2][3] = -Off.Z;
        return FMatrix44f(MediaShaders::YuvToRgbRec709Scaled * Pre);    // assumes sRGB & video range
    }
 
    /*
    * Get sample source color space (defaults to the sRGB/Rec709 gamut)
    */
    virtual const UE::Color::FColorSpace& GetSourceColorSpace() const
    {
        return UE::Color::FColorSpace::GetSRGB();
    }
 
    UE_DEPRECATED(5.5, "GetGamutToXYZMatrix is deprecated, please use GetSourceColorSpace instead.")
    virtual FMatrix44d GetGamutToXYZMatrix() const
    {
        return FMatrix44d(GamutToXYZMatrix(EDisplayColorGamut::sRGB_D65));
    }
 
    UE_DEPRECATED(5.5, "GetWhitePoint is deprecated, please use GetSourceColorSpace instead.")
    virtual FVector2d GetWhitePoint() const
    {
        return UE::Color::GetWhitePoint(UE::Color::EWhitePoint::CIE1931_D65);
    }
 
    UE_DEPRECATED(5.5, "GetDisplayPrimaryRed is deprecated, please use GetMasteringDisplayColorSpace instead.")
    virtual FVector2d GetDisplayPrimaryRed() const
    {
        return FVector2d(0.64, 0.33);
    }
 
    UE_DEPRECATED(5.5, "GetDisplayPrimaryGreen is deprecated, please use GetMasteringDisplayColorSpace instead.")
    virtual FVector2d GetDisplayPrimaryGreen() const
    {
        return FVector2d(0.30, 0.60);
    }
 
    UE_DEPRECATED(5.5, "GetDisplayPrimaryBlue is deprecated, please use GetMasteringDisplayColorSpace instead.")
    virtual FVector2d GetDisplayPrimaryBlue() const
    {
        return FVector2d(0.15, 0.06);
    }
 
    virtual UE::Color::EChromaticAdaptationMethod GetChromaticAdapationMethod() const
    {
        return UE::Color::DEFAULT_CHROMATIC_ADAPTATION_METHOD;
    }
 
    virtual UE::Color::EEncoding GetEncodingType() const
    {
        return IsOutputSrgb() ? UE::Color::EEncoding::sRGB : UE::Color::EEncoding::Linear;
    }
 
    virtual float GetHDRNitsNormalizationFactor() const
    {
        return (GetEncodingType() == UE::Color::EEncoding::sRGB || GetEncodingType() == UE::Color::EEncoding::Linear) ? 1.0f : kMediaSample_HDR_NitsNormalizationFactor;
    }
 
    virtual bool GetDisplayMasteringLuminance(float& OutMin, float& OutMax) const
    {
        return false;
    }
 
    virtual TOptional<UE::Color::FColorSpace> GetDisplayMasteringColorSpace() const
    {
        return TOptional<UE::Color::FColorSpace>();
    }
 
    virtual bool GetMaxLuminanceLevels(uint16& OutCLL, uint16& OutFALL) const
    {
        return false;
    }
 
    virtual MediaShaders::EToneMapMethod GetToneMapMethod() const
    {
        return MediaShaders::EToneMapMethod::None;
    }
 
    virtual void Reset() { }
 
public:
 
    virtual ~IMediaTextureSample() { }
};