diff --git a/parser/tokenizer.go b/parser/tokenizer.go index 5d3e23c..7481f7a 100644 --- a/parser/tokenizer.go +++ b/parser/tokenizer.go @@ -24,7 +24,7 @@ type TokenizerState struct { func newTokenizerState() *TokenizerState { return &TokenizerState{ - curColumn: 1, + curColumn: 0, curIndent: 0, curLevel: 0, curLine: 1, @@ -53,6 +53,7 @@ func (tokenizer *TokenizerState) readNext() rune { next = EOF } tokenizer.offset += 1 + tokenizer.curColumn += 1 if next != EOF { tokenizer.curLiteral += string(next) } @@ -62,6 +63,7 @@ func (tokenizer *TokenizerState) readNext() rune { func (tokenizer *TokenizerState) unread() error { err := tokenizer.buffer.UnreadRune() tokenizer.offset -= 1 + tokenizer.curColumn -= 1 if len(tokenizer.curLiteral) > 0 { tokenizer.curLiteral = tokenizer.curLiteral[0 : len(tokenizer.curLiteral)-1] } @@ -70,7 +72,8 @@ func (tokenizer *TokenizerState) unread() error { func (tokenizer *TokenizerState) finalizeToken(tok *token.Token, tokId token.TokenID) *token.Token { tok.ID = tokId - tok.End = tokenizer.offset + tok.LineEnd = tokenizer.curLine + tok.ColumnEnd = tokenizer.curColumn tok.Literal = tokenizer.curLiteral return tok } @@ -78,10 +81,12 @@ func (tokenizer *TokenizerState) finalizeToken(tok *token.Token, tokId token.Tok func (tokenizer *TokenizerState) newToken() *token.Token { tokenizer.curLiteral = "" return &token.Token{ - ID: token.ERRORTOKEN, - Start: tokenizer.offset, - End: tokenizer.offset - 1, - Literal: tokenizer.curLiteral, + ID: token.ERRORTOKEN, + LineStart: tokenizer.curLine, + ColumnStart: tokenizer.curColumn, + LineEnd: tokenizer.curLine, + ColumnEnd: tokenizer.curColumn, + Literal: tokenizer.curLiteral, } } @@ -262,7 +267,8 @@ again: break } } - curTok.Start = tokenizer.offset - 1 + curTok.LineStart = tokenizer.curLine + curTok.ColumnStart = tokenizer.curColumn - 1 tokenizer.curLiteral = string(nextChar) // Skip comments @@ -306,6 +312,8 @@ again: // Newline if nextChar == '\n' { + tokenizer.curLine += 1 + tokenizer.curColumn = 0 return tokenizer.finalizeToken(curTok, token.NEWLINE) } diff --git a/token/token.go b/token/token.go index bf1e051..0f4e083 100644 --- a/token/token.go +++ b/token/token.go @@ -3,22 +3,34 @@ package token import "fmt" type Token struct { - End int - ID TokenID - Literal string - Start int + ColumnEnd int + LineEnd int + ID TokenID + Literal string + ColumnStart int + LineStart int } func (token *Token) String() string { return TokenNames[token.ID] } +func (token *Token) Start() []int { + return []int{token.LineStart, token.ColumnStart} +} + +func (token *Token) End() []int { + return []int{token.LineEnd, token.ColumnEnd} +} + func (token *Token) Repr() string { return fmt.Sprintf( - "Token{ID: %#v, Literal: %#v, Start: %#v, End: %#v}", + "Token{ID: %#v, Literal: %#v, LineStart: %#v, ColumnStart: %#v, LineEnd: %#v, ColumnEnd: %#v}", token.ID, token.Literal, - token.Start, - token.End, + token.LineStart, + token.ColumnStart, + token.LineEnd, + token.ColumnEnd, ) }