validation_harness.hpp

Go to the documentation of this file.

#pragma once

 
#include <pulp/format/headless.hpp>
#include <pulp/audio/load_measurer.hpp>
#include <chrono>
#include <filesystem>
#include <functional>
#include <string>
#include <vector>
 
namespace pulp::format {

enum class ValidationStatus { pass, fail, skip, error };

inline const char* to_string(ValidationStatus s) {
    switch (s) {
        case ValidationStatus::pass:  return "pass";
        case ValidationStatus::fail:  return "fail";
        case ValidationStatus::skip:  return "skip";
        case ValidationStatus::error: return "error";
    }
    return "error";
}

struct ReportEntry {
    std::string type;               // screenshot, screenshot_diff, inspector, validator, sanitizer, test_suite
    ValidationStatus status = ValidationStatus::pass;
    std::string target;             // plugin name, test name, view ID
    double duration_ms = 0;
    std::string error_message;
 
    // Type-specific payloads stored as pre-formatted JSON fragments.
    // The harness populates these when generating entries.
    std::string payload_json;
};

struct ValidationRunOptions {
    std::filesystem::path output_dir;       // Where to write artifacts
    double sample_rate = 48000.0;
    int buffer_size = 512;
    int input_channels = 2;
    int output_channels = 2;
    uint32_t screenshot_width = 800;
    uint32_t screenshot_height = 600;
    float screenshot_scale = 2.0f;
    std::string screenshot_backend = "default";
 
    // Diff settings
    uint8_t diff_tolerance = 32;
    float diff_threshold = 0.85f;
 
    // Git metadata (optional)
    std::string git_ref;
    std::string run_id;
};

class ValidationHarness {
public:
    explicit ValidationHarness(ProcessorFactory factory);

    void configure(const ValidationRunOptions& opts);
 
    // ── Audio control surface ───────────────────────────────────────────

    void prepare();

    void set_param(state::ParamID id, float value);

    float get_param(state::ParamID id) const;

    void send_midi_note_on(int channel, int note, int velocity);

    void send_midi_note_off(int channel, int note);

    void send_midi_cc(int channel, int controller, int value);

    std::vector<float> process_blocks(int num_blocks);

    std::vector<float> process_buffer(const std::vector<float>& interleaved_input,
                                       int num_channels, int num_samples);

    std::vector<double> process_buffer_f64(
        const std::vector<double>& interleaved_input,
        int num_channels, int num_samples);

    std::vector<uint8_t> save_state() const;

    bool load_state(std::span<const uint8_t> data);
 
    // ── Capture ─────────────────────────────────────────────────────────
    //
    // Screenshot and inspector capture are delegated to an external
    // provider callback so the harness stays free of a direct view-
    // library dependency. Register a provider before calling
    // capture_screenshot()/capture_inspector() — without one, the
    // entry is reported as `skip` with a clear "no provider attached"
    // message. Callers must never confuse skip with pass.

    using CaptureScreenshotProvider = std::function<bool(
        const std::filesystem::path& output_path,
        uint32_t width, uint32_t height, float scale,
        const std::string& backend)>;

    using CaptureInspectorProvider = std::function<std::string()>;

    void set_capture_screenshot_provider(CaptureScreenshotProvider p);

    void set_capture_inspector_provider(CaptureInspectorProvider p);

    ReportEntry capture_screenshot(const std::filesystem::path& output_path);

    ReportEntry compare_screenshots(const std::filesystem::path& reference,
                                     const std::filesystem::path& rendered);

    ReportEntry capture_inspector();
 
    // ── External validators ─────────────────────────────────────────────

    ReportEntry run_validator(const std::string& tool,
                               const std::filesystem::path& plugin_path,
                               const std::string& format = "");

    ReportEntry validate_plugin_runtime_manifest(
        const std::filesystem::path& plugin_path,
        const std::string& expected_plugin_id = "");

    ReportEntry record_runtime_overload(
        const audio::AudioProcessLoadSnapshot& load,
        std::uint64_t xrun_count,
        const audio::AudioRuntimeOverloadPolicy& policy = {});
 
    // ── Report generation ───────────────────────────────────────────────

    void add_entry(ReportEntry entry);

    std::string generate_report() const;

    bool write_report(const std::filesystem::path& output_path) const;

    const std::vector<ReportEntry>& entries() const { return entries_; }

    void clear_entries() { entries_.clear(); }

    HeadlessHost& host() { return host_; }
    const HeadlessHost& host() const { return host_; }

    const PluginDescriptor& descriptor() const { return host_.descriptor(); }
 
private:
    HeadlessHost host_;
    ValidationRunOptions opts_;
    std::vector<ReportEntry> entries_;
    midi::MidiBuffer pending_midi_in_;
    bool prepared_ = false;
 
    // Capture delegation is unset by default; callers register a provider
    // backed by the view layer (or a test fake) before invoking
    // capture_screenshot / capture_inspector.
    CaptureScreenshotProvider screenshot_provider_;
    CaptureInspectorProvider  inspector_provider_;
 
    std::string platform_string() const;
    std::string iso_timestamp() const;
    std::string escape_json(const std::string& s) const;
    std::string json_string_array(const std::vector<std::string>& values) const;
};
 
} // namespace pulp::format