refactor: add token submodule

src/ast/expr.py

@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, TypeVar, Generic
 
-from src.token import Token
+from src.token.token import Token
 
 
 T = TypeVar("T")

src/ast/stmt.py

@@ -5,7 +5,7 @@ from dataclasses import dataclass
 from typing import TypeVar, Generic, Optional
 
 from src.ast.expr import Expr, VariableExpr
-from src.token import Token
+from src.token.token import Token
 
 T = TypeVar("T")
 

src/core/instance.py

@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Optional
 
 from src.core.function import PebbleFunction
 from src.interpreter.error import PebbleRuntimeError
-from src.token import Token
+from src.token.token import Token
 
 if TYPE_CHECKING:
     from src.core.klass import PebbleClass

src/interpreter/environment.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from typing import Any, Optional
 
 from src.interpreter.error import PebbleRuntimeError
-from src.token import Token
+from src.token.token import Token
 
 
 class Environment:

src/interpreter/error.py

@@ -1,4 +1,4 @@
-from src.token import Token
+from src.token.token import Token
 
 
 class PebbleRuntimeError(RuntimeError):

src/interpreter/interpreter.py

@@ -1,7 +1,7 @@
 from typing import Any, Optional
 
 from src.ast.expr import LiteralExpr, GroupingExpr, UnaryExpr, BinaryExpr, Expr, VariableExpr, AssignExpr, LogicalExpr, \
-    CallExpr, GetExpr, T, SetExpr, ThisExpr, SuperExpr
+    CallExpr, GetExpr, SetExpr, ThisExpr, SuperExpr
 from src.ast.stmt import Stmt, ExpressionStmt, LetStmt, BlockStmt, IfStmt, WhileStmt, ForStmt, FunctionStmt, \
     ReturnStmt, BreakStmt, ContinueStmt, ClassStmt
 from src.consts import CONSTRUCTOR_NAME
@@ -14,7 +14,7 @@ from src.interpreter.error import PebbleRuntimeError
 from src.interpreter.exceptions import ReturnException, BreakException, ContinueException
 from src.interpreter.globals import GlobalEnvironment
 from src.pebble import Pebble
-from src.token import TokenType, Token
+from src.token.token import TokenType, Token
 
 
 class Interpreter(Expr.Visitor[Any], Stmt.Visitor[None]):

src/interpreter/resolver.py

@@ -4,12 +4,12 @@ from enum import Enum, auto
 from typing import TYPE_CHECKING
 
 from src.ast.expr import Expr, LogicalExpr, VariableExpr, LiteralExpr, GroupingExpr, CallExpr, UnaryExpr, BinaryExpr, \
-    AssignExpr, GetExpr, SetExpr, ThisExpr, SuperExpr, T
+    AssignExpr, GetExpr, SetExpr, ThisExpr, SuperExpr
 from src.ast.stmt import Stmt, ForStmt, WhileStmt, LetStmt, ReturnStmt, IfStmt, FunctionStmt, \
     ExpressionStmt, BlockStmt, BreakStmt, ContinueStmt, ClassStmt
 from src.consts import CONSTRUCTOR_NAME
 from src.pebble import Pebble
-from src.token import Token
+from src.token.token import Token
 
 if TYPE_CHECKING:
     from src.interpreter.interpreter import Interpreter

src/parser/parser.py

@@ -7,7 +7,7 @@ from src.ast.stmt import Stmt, ExpressionStmt, LetStmt, BlockStmt, IfStmt, While
 from src.consts import MAX_FUNCTION_ARGS
 from src.parser.error import ParsingError
 from src.pebble import Pebble
-from src.token import Token, TokenType
+from src.token.token import Token, TokenType
 
 
 class Parser:

src/pebble.py

@@ -4,8 +4,8 @@ from typing import Optional
 
 from src.consts import VERSION
 from src.interpreter.error import PebbleRuntimeError
-from src.position import Position
+from src.token.position import Position
-from src.token import Token, TokenType
+from src.token.token import Token, TokenType
 
 
 class Pebble:

src/runner.py

@@ -5,10 +5,10 @@ from src.ast.stmt import Stmt
 from src.formatter import Formatter
 from src.interpreter.interpreter import Interpreter
 from src.interpreter.resolver import Resolver
-from src.lexer import Lexer
+from src.token.lexer import Lexer
 from src.parser.parser import Parser
 from src.pebble import Pebble
-from src.token import Token, TokenType
+from src.token.token import Token, TokenType
 
 
 class Runner:

src/token/keyword.py

@@ -1,4 +1,4 @@
-from src.token import TokenType
+from src.token.token import TokenType
 
 KEYWORDS: dict[str, TokenType] = {
     "let": TokenType.LET,

src/token/lexer.py

@@ -1,8 +1,8 @@
 from typing import Optional, Any
 
-from src.keyword import KEYWORDS
+from src.token.keyword import KEYWORDS
-from src.position import Position
+from src.token.position import Position
-from src.token import Token, TokenType
+from src.token.token import Token, TokenType
 
 
 class Lexer:

src/token/token.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from enum import Enum, auto
 from typing import Any
 
-from src.position import Position
+from src.token.position import Position
 
 
 class TokenType(Enum):