Rust Tokenizer

A rust tokenizer from rust standard library.

extern crate syntax;

use syntax::parse;
use syntax::ast;

fn main() {
    let sess = parse::new_parse_sess();
    let cfg = Vec::new();
    let mut p = parse::new_parser_from_file(&sess, cfg, &Path::new("./mytest.rs"));
    while p.token != parse::token::EOF {
        p.bump();
        println!("debug => {}", parse::token::to_str(&p.token));
    }
}

DONE!

p.parse_token_tree() will return a TokenTree, which is a nested token list.