SoundWaveScrubber.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "DSP/AlignedBuffer.h"
#include "DSP/GrainEnvelope.h"
#include "DSP/SampleRateConverter.h"
#include "Sound/SoundBase.h"
#include "Sound/SoundGenerator.h"
#include "Sound/SoundWave.h"
#include "Sound/SoundWaveProxyReader.h"
#include "AudioDynamicParameter.h"
 
#include "SoundWaveScrubber.generated.h"
 
#define UE_API AUDIOMIXER_API
 
namespace Audio
{
 
// Class manages the runtime generation of scrubbed audio from a reference sound wave
// It does this using a simple granulation technique of overlapping grains using a Hanning window.
class FSoundWaveScrubber
{
public:
    UE_API FSoundWaveScrubber();
    UE_API ~FSoundWaveScrubber();
 
    // Initialize the sound wave scrubber
    UE_API void Init(FSoundWaveProxyPtr InSoundWaveProxyPtr, float InSampleRate, int32 InNumChannels, float InPlayheadTimeSeconds = 0.0f);
 
    // Sets whether or not to scrub the audio file. If false, then the output of the scrubber will be silence.
    UE_API void SetIsScrubbing(bool bInIsScrubbing);
 
    // Sets if the scrubber should scrub while playhead is stationary (after it hits it's target playhead)
    UE_API void SetIsScrubbingWhileStationary(bool bInIsScrubWhileStationary);
 
    // Sets the scrubbing location in seconds
    UE_API void SetPlayheadTime(float InPlayheadTimeSeconds);
 
    // Sets the grain duration range in seconds (clamped in range of of 0.01 to 0.5 seconds)
    // The grain duration used during grain spawn is based the scrubbing speed. Sloweer the speed, the longer the grain.
    UE_API void SetGrainDurationRange(const FVector2D& InGrainDurationRange);
 
    // Renders the audio from the sound wave scrubber into the output audio buffer.
    UE_API int32 RenderAudio(TArrayView<float>& OutAudio);
 
private:
 
    // ----------------
    // General state and/or settings
    float AudioMixerSampleRate = 0.0f;
    float SourceFileSampleRate = 0.0f;
    float SourceFileDurationSeconds = 0.0f;
    int32 NumChannels = 0;
 
    // How much audio to decode per decode block
    static constexpr float DecodedAudioSizeInSeconds = 1.0f;
 
    // The current target playhead time in seconds
    // The scrubber will interpolate to this if the current playhead time is less than a given threshold,
    // otherwise the scrubber will "jump" to the playhead time.
    float TargetPlayheadTimeSeconds = 0.0f;
 
    // How long we've been stationary
    float TimeSincePlayheadHasNotChanged = 0.0f;
 
    // Interpolated playhead time
    FDynamicParameter CurrentPlayheadTimeSeconds;
 
    // ----------------
    // Data for managing the decoded audio
    struct FDecodedDataChunk
    {
        // The start frame of the decoded audio chunk
        int32 FrameStart = INDEX_NONE;
 
        // The actual decoded audio. 
        // Size is NumFrames * NumChannels * AudioMixerSampleRate (after SRC)
        Audio::FAlignedFloatBuffer PCMAudio;
 
        // Count of the number of grains actively using this chunk
        int32 NumGrainsUsingChunk = 0;
    };
    // Place to store decoded chunks. We need 2 chunks for actively playing grains plus a free chunk.
    TArray<FDecodedDataChunk> DecodedChunks;
 
    // Sound wave proxy to safely query and use the referenced sound wave asset
    FSoundWaveProxyPtr SoundWaveProxyPtr;
    // Sound wave proxy reader is a decoder we can use to decode audio into chunks
    TUniquePtr<FSoundWaveProxyReader> SoundWaveProxyReaderPtr;
    // Simple SRC interface. Decoded audio is SRC'd to match the audio mixer sample rate
    TUniquePtr<Audio::ISampleRateConverter> SRC;
 
    // Function utility to decode audio from the current sate to the given decoded data chunk
    void DecodeToDataChunk(FDecodedDataChunk& InOutDataChunk, float InDecoderSeekTimeSeconds);
 
    // This retrieves decoded data for the given read frame index. 
    // Returns INDEX_NONE if no decoded data was available for the given read index
    int32 GetDecodedDataChunkIndexForCurrentReadIndex(int32 InReadFrameIndex);
 
    // This function will decode the audio and cache it.
    // Returns the index for the decoded chunk
    int32 DecodeDataChunkIndexForCurrentReadIndex(int32 InReadFrameIndex);
 
    // ----------------
    // Data for granulation
    // Envelope shared across grains
    Audio::Grain::FEnvelope GrainEnvelope;
 
    // The grain duration in frames
    FVector2D TargetGrainDurationRange;
    FVector2D GrainDurationRange;
    float GrainDurationSeconds = 0.0f;
 
    // Grain duration as computed by scrub velocity based on the grain duration ranges
    int32 CurrentGrainDurationFrames = 0;
    int32 CurrentHalfGrainDurationFrames = 0;
 
    // Utility data useful for debugging the scrubber.
    // The grain count is a running tally of the numebr of grains rendered
    int32 GrainCount = 0;
    // This should never go above 2
    int32 NumActiveGrains = 0;
 
    // Active grain data for a single grain
    struct FGrain
    {
        // The number of frames this grain has rendered
        // If this is larger than GrainDurationFrames it's an inactive grain
        // this is used to look up the grain envelope to find the amplitude of the grain per frame
        int32 CurrentRenderedFramesCount = INDEX_NONE;
 
        // Index into the decoded data array
        int32 DecodedDataChunkIndex = 0;
 
        // The current read frame of the grain
        int32 CurrentReadFrame = INDEX_NONE;
 
        // The duration of this grain. Set when the grain spawns. 
        int32 GrainDurationFrames = 0;
    };
    // The grain data used to render the granular audio
    TArray<FGrain> ActiveGrains;
 
    // This is a frame count until we need to spawn another grain
    int32 NumFramesTillNextGrainSpawn = 0;
 
    // Spawns a new grain at the value of the current playhead
    FGrain SpawnGrain();
 
    // Makes sure the grain is referencing valid decoded data
    void UpdateGrainDecodeData(FGrain& InGrain);
 
    // Renders the grain, mixing it into the output buffer. Returns the frame index that is half
    void RenderActiveGrains(TArrayView<float>& OutAudio, int32 InStartFrame, int32 InNumFramesToRender);
 
    // Used for setting user parameters
    FCriticalSection CritSect;
 
    // Whether or not we're actively scrubbing audio playback. 
    bool bIsScrubbing = false;
    bool bIsScrubbingDueToBeingStationary = true;
 
    // Whether or not we scrub while the playhead doesn't move
    bool bIsScrubbingWhileStationary = true;
 
    // Whether or not we have an error with the decoder
    bool bHasErrorWithDecoder = false;
};
 
 
class FSoundWaveScrubberGenerator : public ISoundGenerator
{
public:
    UE_API void Init(FSoundWaveProxyPtr InSoundWaveProxyPtr, float InSampleRate, int32 InNumChannels, float InPlayheadTimeSeconds = 0.0f);
 
    // ~ISoundGenerator Begin
    UE_API virtual int32 OnGenerateAudio(float* OutAudio, int32 NumSamples) override;
    UE_API virtual int32 GetDesiredNumSamplesToRenderPerCallback() const override;
    UE_API virtual bool IsFinished() const override;
    // ~ISoundGenerator End
 
    // Sets whether or not to scrub the audio file. If false, then the output of the scrubber will be silence.
    UE_API void SetIsScrubbing(bool bInIsScrubbing);
 
    // Sets if the scrubber should scrub while playhead is stationary (after it hits it's target playhead)
    UE_API void SetIsScrubbingWhileStationary(bool bInScrubWhileStationary);
 
    // Sets the scrubbing location in seconds
    UE_API void SetPlayheadTime(float InPlayheadTimeSeconds);
 
    // Sets the grain duration range in seconds (clamped in range of of 0.01 to 0.5 seconds)
    // The grain duration used during grain spawn is based the scrubbing speed. Sloweer the speed, the longer the grain.
    UE_API void SetGrainDurationRange(const FVector2D& InGrainDurationRange);
 
private:
    int32 NumChannels = 0;
    FSoundWaveScrubber SoundWaveScrubber;
 
};
 
}
 
 
UCLASS(MinimalAPI)
class UScrubbedSound : public USoundWave
{
    GENERATED_UCLASS_BODY()
public:
 
    // ~USoundBase
    virtual ISoundGeneratorPtr CreateSoundGenerator(const FSoundGeneratorInitParams& InParams) override;
    // ~USoundBase
 
    ISoundGeneratorPtr GetSoundGenerator() { return SoundWaveScrubber; }
    
    // Sets the sound wave to scrub
    UFUNCTION(BlueprintCallable, Category = "Scrubbing")
    AUDIOMIXER_API void SetSoundWave(USoundWave* InSoundWave);
    
    // Sets the scrub time in seconds
    UFUNCTION(BlueprintCallable, Category = "Scrubbing")
    AUDIOMIXER_API void SetPlayheadTime(float InPlayheadTimeSeconds);
 
    // Returns the current playhead time
    UFUNCTION(BlueprintCallable, Category = "Scrubbing")
    float GetPlayheadTime() const { return PlayheadTimeSeconds; }
 
    // Sets the scrub grain duration range.
    UFUNCTION(BlueprintCallable, Category = "Scrubbing")
    AUDIOMIXER_API void SetGrainDurationRange(const FVector2D& InGrainDurationRangeSeconds);
 
    // Sets if the scrubber is actively scrubbing or not
    UFUNCTION(BlueprintCallable, Category = "Scrubbing")
    AUDIOMIXER_API void SetIsScrubbing(bool bInIsScrubbing);
 
    // Sets if the scrubber should scrub while playhead is stationary (after it hits it's target playhead)
    UFUNCTION(BlueprintCallable, Category = "Scrubbing")
    AUDIOMIXER_API void SetIsScrubbingWhileStationary(bool bInScrubWhileStationary);
 
private:
    float PlayheadTimeSeconds = 0.0f;
    FVector2D GrainDurationRange = { 0.4f, 0.05f };
 
    bool bIsScrubbing = false;
    bool bScrubWhileStationary = true;
    float StationaryTimeSeconds = 0.1f;
 
    ISoundGeneratorPtr SoundWaveScrubber;
 
    UPROPERTY(Transient)
    TObjectPtr<USoundWave> SoundWaveToScrub;
    
};
 
#undef UE_API