85 lines
1.6 KiB
Python
85 lines
1.6 KiB
Python
from dataclasses import dataclass
|
|
from enum import Enum, auto
|
|
from typing import Any
|
|
|
|
from src.position import Position
|
|
|
|
|
|
class TokenType(Enum):
|
|
# Punctuation
|
|
LEFT_PAREN = auto()
|
|
RIGHT_PAREN = auto()
|
|
LEFT_BRACE = auto()
|
|
RIGHT_BRACE = auto()
|
|
COMMA = auto()
|
|
DOT = auto()
|
|
SEMICOLON = auto()
|
|
COLON = auto()
|
|
|
|
# Operators
|
|
PLUS = auto()
|
|
PLUS_EQUAL = auto()
|
|
MINUS = auto()
|
|
MINUS_EQUAL = auto()
|
|
SLASH = auto()
|
|
SLASH_EQUAL = auto()
|
|
STAR = auto()
|
|
STAR_EQUAL = auto()
|
|
EQUAL = auto()
|
|
EQUAL_EQUAL = auto()
|
|
BANG = auto()
|
|
BANG_EQUAL = auto()
|
|
GREATER = auto()
|
|
GREATER_EQUAL = auto()
|
|
LESS = auto()
|
|
LESS_EQUAL = auto()
|
|
|
|
# Literals
|
|
IDENTIFIER = auto()
|
|
STRING = auto()
|
|
NUMBER = auto()
|
|
TRUE = auto()
|
|
FALSE = auto()
|
|
NULL = auto()
|
|
|
|
# Keywords
|
|
LET = auto()
|
|
FUN = auto()
|
|
AND = auto()
|
|
OR = auto()
|
|
IF = auto()
|
|
ELSE = auto()
|
|
FOR = auto()
|
|
WHILE = auto()
|
|
FROM = auto()
|
|
TO = auto()
|
|
UNTIL = auto()
|
|
BY = auto()
|
|
RETURN = auto()
|
|
BREAK = auto()
|
|
CONTINUE = auto()
|
|
|
|
# Misc
|
|
PRINT = auto()
|
|
COMMENT = auto()
|
|
WHITESPACE = auto()
|
|
EOF = auto()
|
|
NEWLINE = auto()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Token:
|
|
type: TokenType
|
|
lexeme: str
|
|
value: Any
|
|
position: Position
|
|
|
|
def __repr__(self) -> str:
|
|
res: str = f"[{self.type.name}"
|
|
if self.value is not None:
|
|
res += f" ({self.value!r})"
|
|
elif self.type == TokenType.IDENTIFIER:
|
|
res += f" ({self.lexeme})"
|
|
res += "]"
|
|
return res
|