audio_renderer.hpp

Go to the documentation of this file.

#pragma once
 
#include <pulp/audio/audio_file.hpp>
#include <pulp/audio/buffer.hpp>
#include <pulp/audio/rt_safety_contract.hpp>
#include <pulp/audio/wav_decoder.hpp>
#include <pulp/playback/audio_renderer_limits.hpp>
#include <pulp/playback/realtime_stretch_state_bank.hpp>
#include <pulp/playback/stable_renderer_shell.hpp>
#include <pulp/playback/transport.hpp>
#include <pulp/runtime/result.hpp>
#include <pulp/timebase/compiled_tempo_map.hpp>
#include <pulp/timeline/model.hpp>
 
#include <cstdint>
#include <memory>
#include <span>
#include <vector>
 
namespace pulp::playback {
 
class PlaybackProgram;
class ProgramCompilerTask;
class AudioClipConversionArtifact;
class RealtimeStretchProgramRuntime;
struct OfflineStretchArtifact;
struct OfflineStretchProvenance;
 
enum class AudioRendererErrorCode : std::uint8_t {
    InvalidIdentity,
    DuplicateAsset,
    MissingDecodedAsset,
    InvalidAsset,
    AssetMetadataMismatch,
    UnsupportedSampleRate,
    InvalidClipRange,
    InvalidFade,
    CapacityExceeded,
    InvalidTakeComp,
    OfflineStretchRequired,
    OfflineStretchFailed,
};
 
struct AudioRendererError {
    AudioRendererErrorCode code = AudioRendererErrorCode::InvalidAsset;
    timeline::ItemId item;
    timeline::ItemId related_item;
    std::uint64_t actual = 0;
    std::uint64_t limit = 0;
};
 
struct DecodedAudioAsset {
    timeline::ItemId id;
    std::shared_ptr<const audio::AudioFileData> audio;
    timeline::ContentHash content_hash;
    timeline::ContentHash decoded_content_hash;
};

class DecodedAudioAssetPool {
  public:
    static runtime::Result<std::shared_ptr<const DecodedAudioAssetPool>, AudioRendererError>
    create(std::vector<DecodedAudioAsset> assets, AudioRendererLimits limits = {});
 
    static runtime::Result<DecodedAudioAsset, AudioRendererError>
    decode_wav(timeline::ItemId id, std::span<const std::uint8_t> bytes,
               audio::WavDecodeLimits decode_limits = {});
 
    const DecodedAudioAsset* find(timeline::ItemId id) const noexcept;
    std::span<const DecodedAudioAsset> assets() const noexcept {
        return assets_;
    }
 
  private:
    explicit DecodedAudioAssetPool(std::vector<DecodedAudioAsset> assets) noexcept
        : assets_(std::move(assets)) {}
    std::vector<DecodedAudioAsset> assets_;
};

struct ClipPlacementFadeSegment {
    double silent_frame = 0.0;
    double open_frame = 0.0;
    timeline::ClipFadeShape shape = timeline::ClipFadeShape::Linear;
};

struct ClipPlacementFadeProgram {
    std::vector<ClipPlacementFadeSegment> segments;
};
 
struct AudioClipRendererProgram {
    enum class SourceKind : std::uint8_t { ArrangementClip, TakeCompSegment, FrozenTrack };
    enum class TimeDomain : std::uint8_t { Musical, Absolute };
    enum class SourceTimeMapping : std::uint8_t {
        NativeRate,
        MusicalPhaseResample,
        OfflineStretchArtifact,
    };
 
    timeline::ItemId id;
    timeline::ItemId asset_id;
    std::shared_ptr<const audio::AudioFileData> audio;
    std::int64_t timeline_start = 0;
    std::uint64_t timeline_frame_count = 0;
    std::uint64_t source_start = 0;
    std::uint64_t source_frame_count = 0;
    double source_frame_offset = 0.0;
    std::uint64_t renderable_timeline_frames = 0;
    double source_frames_per_timeline_frame = 1.0;
    std::shared_ptr<const AudioClipConversionArtifact> conversion_artifact;
    float gain_linear = 1.0f;
    std::uint64_t fade_in_frames = 0;
    std::uint64_t fade_out_frames = 0;
    timeline::ClipFadeShape fade_shape = timeline::ClipFadeShape::Linear;
    SourceKind source_kind = SourceKind::ArrangementClip;
    std::uint32_t source_ordinal = 0;
    TimeDomain time_domain = TimeDomain::Absolute;
    timebase::TickPosition musical_tick_start{};
    timebase::TickPosition musical_tick_end{};
    SourceTimeMapping source_time_mapping = SourceTimeMapping::NativeRate;
    std::shared_ptr<const OfflineStretchArtifact> offline_stretch_artifact;
    std::shared_ptr<const OfflineStretchProvenance> offline_stretch_provenance;
    // Null unless flattening placed this clip under a faded `SequenceRef`, which
    // is what keeps an ordinary clip's per-sample gain exactly the arithmetic it
    // always was.
    std::shared_ptr<const ClipPlacementFadeProgram> placement_fade;
    // Exclusive source-frame bound of the musical phase map, paired with
    // `source_frame_offset` as its inclusive start. Zero selects
    // `source_frame_count`, which is every clip whose placement carries its
    // whole authored source span. The two differ only when flattening cut a
    // conforming clip down to a sub-span of that span, which no clip in the
    // document model can express: a conforming clip maps all of its source
    // onto all of its placement, so a retained window needs the matching
    // sub-range named explicitly. Read only under `MusicalPhaseResample`.
    double source_frame_phase_end = 0.0;
 
    std::int64_t timeline_end() const noexcept;
    bool uses_sample_rate_conversion() const noexcept;
    bool uses_host_rate_conversion() const noexcept;
    bool shares_sample_rate_conversion_with(const AudioClipRendererProgram& other) const noexcept;
    bool shares_host_rate_conversion_with(const AudioClipRendererProgram& other) const noexcept;
};
 
class AudioTrackRendererProgram {
  public:
    timeline::ItemId id() const noexcept {
        return id_;
    }
    std::span<const AudioClipRendererProgram> clips() const noexcept {
        return clips_;
    }
 
  private:
    friend class ProgramCompilerTask;
    friend runtime::Result<std::shared_ptr<const AudioTrackRendererProgram>, AudioRendererError>
    link_audio_track_program(timeline::ItemId, std::vector<AudioClipRendererProgram>,
                             const AudioRendererLimits&);
    AudioTrackRendererProgram(timeline::ItemId id,
                              std::vector<AudioClipRendererProgram> clips) noexcept
        : id_(id), clips_(std::move(clips)) {}
    timeline::ItemId id_;
    std::vector<AudioClipRendererProgram> clips_;
};
 
runtime::Result<AudioClipRendererProgram, AudioRendererError>
compile_audio_clip_program(const timeline::Clip& clip, const timeline::Project& project,
                           const timebase::CompiledTempoMap& tempo_map,
                           const DecodedAudioAssetPool& assets, const AudioRendererLimits& limits);

runtime::Result<AudioClipRendererProgram, AudioRendererError> compile_take_comp_segment_program(
    const timeline::TakeLane& lane, std::size_t segment_index, const timeline::Project& project,
    const timebase::CompiledTempoMap& tempo_map, const DecodedAudioAssetPool& assets,
    const AudioRendererLimits& limits);

runtime::Result<AudioClipRendererProgram, AudioRendererError>
compile_track_freeze_program(const timeline::Track& track, const timeline::Project& project,
                             const timebase::CompiledTempoMap& tempo_map,
                             const DecodedAudioAssetPool& assets,
                             const AudioRendererLimits& limits);
 
runtime::Result<std::shared_ptr<const AudioTrackRendererProgram>, AudioRendererError>
link_audio_track_program(timeline::ItemId track_id, std::vector<AudioClipRendererProgram> clips,
                         const AudioRendererLimits& limits);
 
enum class AudioRenderStatus : std::uint8_t {
    Rendered,
    Silent,
    InvalidProgram,
    InvalidTransport,
    InvalidOutput,
    CapacityExceeded,
    RealtimeStretchStateRequired,
    RealtimeStretchGap,
    RealtimeStretchStalePublication,
    RealtimeStretchImpossibleRatio,
    RealtimeStretchBackpressure,
    RealtimeStretchUnderflow,
    RealtimeStretchUnsupportedScrubbing,
};

class ArrangementAudioRenderer {
  public:
    static constexpr bool carries_mutable_state = false;
    static constexpr audio::RtSafetyClass process_rt_safety_class =
        audio::RtSafetyClass::AudioCallbackSafeWithImmutableInputs;
 
    static AudioRenderStatus process(const PlaybackProgram& program,
                                     const TransportSnapshot& transport,
                                     audio::BufferView<float> output,
                                     AudioRendererLimits limits = {}) noexcept;
};

class ArrangementAudioTrackRenderer {
  public:
    static constexpr audio::RtSafetyClass process_rt_safety_class =
        audio::RtSafetyClass::AudioCallbackSafeAfterPrepare;
 
    explicit ArrangementAudioTrackRenderer(timeline::ItemId track_id,
                                           bool apply_track_mixer = true) noexcept
        : shell_(track_id), apply_track_mixer_(apply_track_mixer) {}
 
    AudioRenderStatus process(const PlaybackProgramBlock& block, const TransportSnapshot& transport,
                              audio::BufferView<float> output, AudioRendererLimits limits = {},
                              RealtimeStretchProgramRuntime* realtime_stretch = nullptr) noexcept;
 
    RendererProgramKey active_key() const noexcept {
        return shell_.active_key();
    }
    RendererCarryState state_snapshot() const noexcept {
        return shell_.state_snapshot();
    }
    void reset() noexcept {
        shell_.reset();
    }
 
  private:
    StableRendererShell shell_;
    bool apply_track_mixer_ = true;
};

class TrackMixerTrackRenderer {
  public:
    static constexpr audio::RtSafetyClass process_rt_safety_class =
        audio::RtSafetyClass::AudioCallbackSafeAfterPrepare;
 
    explicit TrackMixerTrackRenderer(timeline::ItemId track_id) noexcept : shell_(track_id) {}
 
    AudioRenderStatus process(const PlaybackProgramBlock& block, const TransportSnapshot& transport,
                              audio::BufferView<float> output,
                              const audio::BufferView<const float>& input,
                              AudioRendererLimits limits = {}) noexcept;
 
    void reset() noexcept {
        shell_.reset();
    }
 
  private:
    StableRendererShell shell_;
};
 
} // namespace pulp::playback