1 files changed, 12 insertions, 1 deletions
diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp
index 8b844cd..e2c17f9 100644
--- a/src/tokenizer.hpp
+++ b/src/tokenizer.hpp
@@ -6,7 +6,11 @@
 
 enum TokenizerFlag {
     TokenizerFlag_None = 0,
+
+    /// When set, the tokenizer will emit the HtmlToken stored in "last".
     TokenizerFlag_Emit = 1,
+
+    /// When set, the tokenizer will not increment the pointer when it attempts its next read.
     TokenizerFlag_NoIncrement = 2
 };
 
@@ -20,7 +24,14 @@ enum TokenizerState {
     TokenizerState_TagName,
     TokenizerState_BogusComment,
     TokenizerState_CommentState,
-    TokenizerState_BeforeAttribute,
+    TokenizerState_BeforeAttributeName,
+    TokenizerState_AttributeNameState,
+    TokenizerState_AfterAttributeNameState,
+    TokenizerState_BeforeAttributeValueState,
+    TokenizerState_AttributeValueUnquoted,
+    TokenizerState_AttributeValueDoubleQuoted,
+    TokenizerState_AttributeValueSingleQuoted,
+    TokenizerState_AfterAttributeValueQuoted,
     TokenizerState_SelfClosingStartTag
 };