Function leo_parser::tokenizer::tokenize

pub(crate) fn tokenize(
    input: &str,
    start_pos: BytePos,
) -> Result<Vec<SpannedToken>>

Creates a new vector of spanned tokens from a given file path and source code text.