MediaSoundComponent.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "Components/SynthComponent.h"
 
#include "Containers/Array.h"
#include "MediaSampleQueue.h"
#include "Misc/Timespan.h"
#include "Templates/Atomic.h"
#include "Templates/SharedPointer.h"
#include "UObject/ObjectMacros.h"
#include "UObject/ScriptMacros.h"
#include "DSP/SpectrumAnalyzer.h"
#include "DSP/BufferVectorOperations.h"
#include "DSP/EnvelopeFollower.h"
#include "Sound/SoundClass.h"
#include "Sound/SoundGenerator.h"
#include "MediaAudioResampler.h"
#include "Misc/TransactionallySafeCriticalSection.h"
 
#include "MediaSoundComponent.generated.h"
 
class FMediaAudioResampler;
class FMediaPlayerFacade;
class FMediaSoundComponentClockSink;
class IMediaAudioSample;
class IMediaPlayer;
class UMediaPlayer;
 
 
UENUM()
enum class EMediaSoundChannels
{
    Mono,
 
    Stereo,
 
    Surround
};
 
UENUM(BlueprintType)
enum class EMediaSoundComponentFFTSize : uint8
{
    Min_64,
    Small_256,
    Medium_512,
    Large_1024,
};
 
USTRUCT(BlueprintType)
struct FMediaSoundComponentSpectralData
{
    GENERATED_USTRUCT_BODY()
 
    // The frequency hz of the spectrum value
    UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "SpectralData")
    float FrequencyHz = 0.0f;
 
    // The magnitude of the spectrum at this frequency
    UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "SpectralData")
    float Magnitude = 0.0f;
};
 
// Class implements an ISoundGenerator to feed decoded audio to audio renderering async tasks
class FMediaSoundGenerator : public ISoundGenerator
{
public:
    struct FSoundGeneratorParams
    {
        int32 SampleRate = 0;
        int32 NumChannels = 0;
 
        TSharedPtr<FMediaAudioSampleQueue, ESPMode::ThreadSafe> SampleQueue;
        uint32 PreviousSampleQueueFlushCount = 0;
 
        bool bSpectralAnalysisEnabled = false;
        bool bEnvelopeFollowingEnabled = false;
 
        int32 EnvelopeFollowerAttackTime = 0;
        int32 EnvelopeFollowerReleaseTime = 0;
 
        Audio::FSpectrumAnalyzerSettings SpectrumAnalyzerSettings;
        TArray<float> FrequenciesToAnalyze;
 
        float CachedRate = 0.0f;
        FTimespan CachedTime;
        FTimespan LastPlaySampleTime;
    };
 
    FMediaSoundGenerator(FSoundGeneratorParams& InParams);
 
    virtual ~FMediaSoundGenerator();
 
    virtual void OnEndGenerate() override;
 
    virtual int32 OnGenerateAudio(float* OutAudio, int32 NumSamples) override;
 
    void SetCachedData(float InCachedRate, const FTimespan& InCachedTime);
    void SetLastPlaySampleTime(const FTimespan& InLastPlaySampleTime);
 
    void SetEnableSpectralAnalysis(bool bInSpectralAnlaysisEnabled);
    void SetEnableEnvelopeFollowing(bool bInEnvelopeFollowingEnabled);
 
    void SetSpectrumAnalyzerSettings(Audio::FSpectrumAnalyzerSettings::EFFTSize InFFTSize, const TArray<float>& InFrequenciesToAnalyze);
    void SetEnvelopeFollowingSettings(int32 InAttackTimeMsec, int32 InReleaseTimeMsec);
 
    void SetSampleQueue(TSharedPtr<FMediaAudioSampleQueue, ESPMode::ThreadSafe>& InSampleQueue);
 
    TArray<FMediaSoundComponentSpectralData> GetSpectralData() const;
    TArray<FMediaSoundComponentSpectralData> GetNormalizedSpectralData() const;
    float GetCurrentEnvelopeValue() const { return CurrentEnvelopeValue; }
 
    FTimespan GetLastPlayTime() const { return LastPlaySampleTime.Load(); }
 
private:
 
    FSoundGeneratorParams Params;
 
    FMediaAudioResampler Resampler;
 
    Audio::AlignedFloatBuffer AudioScratchBuffer;
 
    mutable Audio::FAsyncSpectrumAnalyzer SpectrumAnalyzer;
 
    Audio::FEnvelopeFollower EnvelopeFollower;
 
    TAtomic<float> CachedRate;
    TAtomic<FTimespan> CachedTime;
    TAtomic<FTimespan> LastPlaySampleTime;
 
    float CurrentEnvelopeValue = 0.0f;
    bool bEnvelopeFollowerSettingsChanged = false;
 
    mutable FCriticalSection AnalysisCritSect;
    mutable FCriticalSection SampleQueueCritSect;
};
 
UCLASS(ClassGroup=Media, editinlinenew, meta=(BlueprintSpawnableComponent), MinimalAPI)
class UMediaSoundComponent
    : public USynthComponent
{
    GENERATED_BODY()
 
public:
 
    UPROPERTY(EditAnywhere, Category="Media")
    EMediaSoundChannels Channels;
 
    UPROPERTY(EditAnywhere, Category="Media", AdvancedDisplay)
    bool DynamicRateAdjustment;
 
    UPROPERTY(EditAnywhere, Category="Media", AdvancedDisplay)
    float RateAdjustmentFactor;
 
    UPROPERTY(EditAnywhere, Category="Media", AdvancedDisplay)
    FFloatRange RateAdjustmentRange;
 
public:
 
    MEDIAASSETS_API UMediaSoundComponent(const FObjectInitializer& ObjectInitializer);
 
    MEDIAASSETS_API ~UMediaSoundComponent();
 
public:
 
    UFUNCTION(BlueprintCallable, Category="Media|MediaSoundComponent", meta=(DisplayName="Get Attenuation Settings To Apply", ScriptName="GetAttenuationSettingsToApply"))
    MEDIAASSETS_API bool BP_GetAttenuationSettingsToApply(FSoundAttenuationSettings& OutAttenuationSettings);
 
    UFUNCTION(BlueprintCallable, Category="Media|MediaSoundComponent")
    MEDIAASSETS_API UMediaPlayer* GetMediaPlayer() const;
 
    virtual USoundClass* GetSoundClass() override
    {
        if (SoundClass)
        {
            return SoundClass;
        }
 
        if (const UAudioSettings* AudioSettings = GetDefault<UAudioSettings>())
        {
            if (USoundClass* DefaultSoundClass = AudioSettings->GetDefaultMediaSoundClass())
            {
                return DefaultSoundClass;
            }
 
            if (USoundClass* DefaultSoundClass = AudioSettings->GetDefaultSoundClass())
            {
                return DefaultSoundClass;
            }
        }
 
        return nullptr;
    }
 
    UFUNCTION(BlueprintCallable, Category="Media|MediaSoundComponent")
    MEDIAASSETS_API void SetMediaPlayer(UMediaPlayer* NewMediaPlayer);
 
    UFUNCTION(BlueprintCallable, Category = "Media|MediaSoundComponent")
    MEDIAASSETS_API void SetEnableSpectralAnalysis(bool bInSpectralAnalysisEnabled);
 
    UFUNCTION(BlueprintCallable, Category = "Media|MediaSoundComponent")
    MEDIAASSETS_API void SetSpectralAnalysisSettings(TArray<float> InFrequenciesToAnalyze, EMediaSoundComponentFFTSize InFFTSize = EMediaSoundComponentFFTSize::Medium_512);
 
    UFUNCTION(BlueprintCallable, Category = "TimeSynth")
    MEDIAASSETS_API TArray<FMediaSoundComponentSpectralData> GetSpectralData();
 
    UFUNCTION(BlueprintCallable, Category = "TimeSynth")
    MEDIAASSETS_API TArray<FMediaSoundComponentSpectralData> GetNormalizedSpectralData();
 
    UFUNCTION(BlueprintCallable, Category = "Media|MediaSoundComponent")
    MEDIAASSETS_API void SetEnableEnvelopeFollowing(bool bInEnvelopeFollowing);
 
    UFUNCTION(BlueprintCallable, Category = "Media|MediaSoundComponent")
    MEDIAASSETS_API void SetEnvelopeFollowingsettings(int32 AttackTimeMsec, int32 ReleaseTimeMsec);
 
    UFUNCTION(BlueprintCallable, Category = "TimeSynth")
    MEDIAASSETS_API float GetEnvelopeValue() const;
 
public:
 
    MEDIAASSETS_API void AddClockSink();
 
    MEDIAASSETS_API void RemoveClockSink();
 
    MEDIAASSETS_API void UpdatePlayer();
 
#if WITH_EDITOR
    MEDIAASSETS_API void SetDefaultMediaPlayer(UMediaPlayer* NewMediaPlayer);
#endif
 
public:
 
    //~ TAttenuatedComponentVisualizer interface
 
    MEDIAASSETS_API void CollectAttenuationShapesForVisualization(TMultiMap<EAttenuationShape::Type, FBaseAttenuationSettings::AttenuationShapeDetails>& ShapeDetailsMap) const;
 
public:
 
    //~ UActorComponent interface
 
    MEDIAASSETS_API virtual void OnRegister() override;
    MEDIAASSETS_API virtual void OnUnregister() override;
    MEDIAASSETS_API virtual void TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction *ThisTickFunction) override;
 
public:
 
    //~ USceneComponent interface
 
    MEDIAASSETS_API virtual void Activate(bool bReset = false) override;
    MEDIAASSETS_API virtual void Deactivate() override;
 
public:
 
    //~ UObject interface
    MEDIAASSETS_API virtual void PostLoad() override;
 
#if WITH_EDITOR
    MEDIAASSETS_API virtual void PostEditChangeProperty(FPropertyChangedEvent& PropertyChangedEvent) override;
#endif
 
protected:
 
    MEDIAASSETS_API const FSoundAttenuationSettings* GetSelectedAttenuationSettings() const;
 
protected:
 
    //~ USynthComponent interface
 
    MEDIAASSETS_API virtual bool Init(int32& SampleRate) override;
 
    MEDIAASSETS_API virtual ISoundGeneratorPtr CreateSoundGenerator(const FSoundGeneratorInitParams& InParams) override;
 
protected:
 
    UPROPERTY(EditAnywhere, Category="Media")
    TObjectPtr<UMediaPlayer> MediaPlayer;
 
private:
 
    float CachedRate;
 
    FTimespan CachedTime;
 
    FTransactionallySafeCriticalSection CriticalSection;
 
    TWeakObjectPtr<UMediaPlayer> CurrentPlayer;
 
    TWeakPtr<FMediaPlayerFacade, ESPMode::ThreadSafe> CurrentPlayerFacade;
 
    float RateAdjustment;
 
    TSharedPtr<FMediaAudioSampleQueue, ESPMode::ThreadSafe> SampleQueue;
 
    /* Time of last sample played. */
    FTimespan LastPlaySampleTime;
 
    TArray<float> FrequenciesToAnalyze;
    Audio::FSpectrumAnalyzerSettings SpectrumAnalyzerSettings;
    int32 EnvelopeFollowerAttackTime;
    int32 EnvelopeFollowerReleaseTime;
 
    bool bSpectralAnalysisEnabled;
 
    bool bEnvelopeFollowingEnabled;
 
    TSharedPtr<FMediaSoundComponentClockSink, ESPMode::ThreadSafe> ClockSink;
 
    ISoundGeneratorPtr MediaSoundGenerator;
};