#ifndef TOKENIZER_HPP
#define TOKENIZER_HPP

#include "code_point.h"
#include "html_token.hpp"

enum TokenizerFlag {
    TokenizerFlag_None = 0,
    TokenizerFlag_Emit = 1,
    TokenizerFlag_NoIncrement = 2
};

enum TokenizerState {
    TokenizerState_None,
    TokenizerState_Data,
    TokenizerState_CharacterReferenceInData,
    TokenizerState_TagOpen,
    TokenizerState_MarkupDeclarationOpen,
    TokenizerState_EndTagOpen,
    TokenizerState_TagName,
    TokenizerState_BogusComment,
    TokenizerState_CommentState,
    TokenizerState_BeforeAttribute,
    TokenizerState_SelfClosingStartTag
};

struct Tokenizer {
    code_point_t* ptr = nullptr;
    size_t length = 0;

    TokenizerState state = TokenizerState_Data;
    HtmlToken last;
    int flag = TokenizerFlag_None;
};


Tokenizer create(code_point_t*);
HtmlToken read_next(Tokenizer*);


#endif