Browse Source

give main similar output as python -m tokenize

master
Brett Langdon 10 years ago
parent
commit
8ac42a22ef
1 changed files with 3 additions and 1 deletions
  1. +3
    -1
      main.go

+ 3
- 1
main.go View File

@ -15,9 +15,11 @@ func main() {
} }
for { for {
tok := tokenizer.Next() tok := tokenizer.Next()
tokenRange := fmt.Sprintf("%d,%d-%d,%d:", tok.LineStart, tok.ColumnStart, tok.LineEnd, tok.ColumnEnd)
literalRep := fmt.Sprintf("%#v", tok.Literal)
fmt.Printf("%-20s%-15s%15s\n", tokenRange, tok.String(), literalRep)
if tok.ID == token.ENDMARKER || tok.ID == token.ERRORTOKEN { if tok.ID == token.ENDMARKER || tok.ID == token.ERRORTOKEN {
break break
} }
fmt.Println(fmt.Sprintf("<%s> %s", tok, tok.Repr()))
} }
} }

Loading…
Cancel
Save