code_editor_tokenizer.hpp

Go to the documentation of this file.

#pragma once

 
#include <cstddef>
#include <string_view>
#include <vector>
 
#include <pulp/view/code_editor.hpp>
 
namespace pulp::view {

enum class TokenClass {
    text,        
    keyword,     
    type,        
    identifier,  
    number,      
    string,      
    comment,     
    preprocessor,
    punctuation, 
    heading,     
    link,        
};

struct Token {
    std::size_t start = 0;
    std::size_t length = 0;
    TokenClass cls = TokenClass::text;
};

std::vector<Token> tokenize_line(std::string_view line, CodeLanguage language);
 
// ── Per-language entry points (callable independently) ────────────────
 
std::vector<Token> tokenize_cpp_line(std::string_view line);
std::vector<Token> tokenize_javascript_line(std::string_view line);
std::vector<Token> tokenize_python_line(std::string_view line);
std::vector<Token> tokenize_json_line(std::string_view line);
std::vector<Token> tokenize_lua_line(std::string_view line);
std::vector<Token> tokenize_markdown_line(std::string_view line);

bool is_keyword(std::string_view name, CodeLanguage language);
 
} // namespace pulp::view