From 3578b6e49d3c6edb6c0432c65f833c5a01dda819 Mon Sep 17 00:00:00 2001
From: LordBaryhobal <lordbaryhobal@gmail.com>
Date: Sat, 7 Feb 2026 02:17:06 +0100
Subject: [PATCH] refactor: add token submodule

---
 src/ast/expr.py                | 2 +-
 src/ast/stmt.py                | 2 +-
 src/core/instance.py           | 2 +-
 src/interpreter/environment.py | 2 +-
 src/interpreter/error.py       | 2 +-
 src/interpreter/interpreter.py | 4 ++--
 src/interpreter/resolver.py    | 4 ++--
 src/parser/parser.py           | 2 +-
 src/pebble.py                  | 4 ++--
 src/runner.py                  | 4 ++--
 src/token/__init__.py          | 0
 src/{ => token}/keyword.py     | 2 +-
 src/{ => token}/lexer.py       | 6 +++---
 src/{ => token}/position.py    | 0
 src/{ => token}/token.py       | 2 +-
 15 files changed, 19 insertions(+), 19 deletions(-)
 create mode 100644 src/token/__init__.py
 rename src/{ => token}/keyword.py (94%)
 rename src/{ => token}/lexer.py (97%)
 rename src/{ => token}/position.py (100%)
 rename src/{ => token}/token.py (97%)

diff --git a/src/ast/expr.py b/src/ast/expr.py
index 002353c..1446001 100644
--- a/src/ast/expr.py
+++ b/src/ast/expr.py
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, TypeVar, Generic
 
-from src.token import Token
+from src.token.token import Token
 
 
 T = TypeVar("T")
diff --git a/src/ast/stmt.py b/src/ast/stmt.py
index 6e64837..3796b3a 100644
--- a/src/ast/stmt.py
+++ b/src/ast/stmt.py
@@ -5,7 +5,7 @@ from dataclasses import dataclass
 from typing import TypeVar, Generic, Optional
 
 from src.ast.expr import Expr, VariableExpr
-from src.token import Token
+from src.token.token import Token
 
 T = TypeVar("T")
 
diff --git a/src/core/instance.py b/src/core/instance.py
index 36e4780..877a56c 100644
--- a/src/core/instance.py
+++ b/src/core/instance.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Optional
 
 from src.core.function import PebbleFunction
 from src.interpreter.error import PebbleRuntimeError
-from src.token import Token
+from src.token.token import Token
 
 if TYPE_CHECKING:
     from src.core.klass import PebbleClass
diff --git a/src/interpreter/environment.py b/src/interpreter/environment.py
index e99fc49..867f83a 100644
--- a/src/interpreter/environment.py
+++ b/src/interpreter/environment.py
@@ -3,7 +3,7 @@ from __future__ import annotations
 from typing import Any, Optional
 
 from src.interpreter.error import PebbleRuntimeError
-from src.token import Token
+from src.token.token import Token
 
 
 class Environment:
diff --git a/src/interpreter/error.py b/src/interpreter/error.py
index ac5761b..8be4b61 100644
--- a/src/interpreter/error.py
+++ b/src/interpreter/error.py
@@ -1,4 +1,4 @@
-from src.token import Token
+from src.token.token import Token
 
 
 class PebbleRuntimeError(RuntimeError):
diff --git a/src/interpreter/interpreter.py b/src/interpreter/interpreter.py
index 3585868..4473166 100644
--- a/src/interpreter/interpreter.py
+++ b/src/interpreter/interpreter.py
@@ -1,7 +1,7 @@
 from typing import Any, Optional
 
 from src.ast.expr import LiteralExpr, GroupingExpr, UnaryExpr, BinaryExpr, Expr, VariableExpr, AssignExpr, LogicalExpr, \
-    CallExpr, GetExpr, T, SetExpr, ThisExpr, SuperExpr
+    CallExpr, GetExpr, SetExpr, ThisExpr, SuperExpr
 from src.ast.stmt import Stmt, ExpressionStmt, LetStmt, BlockStmt, IfStmt, WhileStmt, ForStmt, FunctionStmt, \
     ReturnStmt, BreakStmt, ContinueStmt, ClassStmt
 from src.consts import CONSTRUCTOR_NAME
@@ -14,7 +14,7 @@ from src.interpreter.error import PebbleRuntimeError
 from src.interpreter.exceptions import ReturnException, BreakException, ContinueException
 from src.interpreter.globals import GlobalEnvironment
 from src.pebble import Pebble
-from src.token import TokenType, Token
+from src.token.token import TokenType, Token
 
 
 class Interpreter(Expr.Visitor[Any], Stmt.Visitor[None]):
diff --git a/src/interpreter/resolver.py b/src/interpreter/resolver.py
index 26821f8..84be0a0 100644
--- a/src/interpreter/resolver.py
+++ b/src/interpreter/resolver.py
@@ -4,12 +4,12 @@ from enum import Enum, auto
 from typing import TYPE_CHECKING
 
 from src.ast.expr import Expr, LogicalExpr, VariableExpr, LiteralExpr, GroupingExpr, CallExpr, UnaryExpr, BinaryExpr, \
-    AssignExpr, GetExpr, SetExpr, ThisExpr, SuperExpr, T
+    AssignExpr, GetExpr, SetExpr, ThisExpr, SuperExpr
 from src.ast.stmt import Stmt, ForStmt, WhileStmt, LetStmt, ReturnStmt, IfStmt, FunctionStmt, \
     ExpressionStmt, BlockStmt, BreakStmt, ContinueStmt, ClassStmt
 from src.consts import CONSTRUCTOR_NAME
 from src.pebble import Pebble
-from src.token import Token
+from src.token.token import Token
 
 if TYPE_CHECKING:
     from src.interpreter.interpreter import Interpreter
diff --git a/src/parser/parser.py b/src/parser/parser.py
index ceb692c..14c6d83 100644
--- a/src/parser/parser.py
+++ b/src/parser/parser.py
@@ -7,7 +7,7 @@ from src.ast.stmt import Stmt, ExpressionStmt, LetStmt, BlockStmt, IfStmt, While
 from src.consts import MAX_FUNCTION_ARGS
 from src.parser.error import ParsingError
 from src.pebble import Pebble
-from src.token import Token, TokenType
+from src.token.token import Token, TokenType
 
 
 class Parser:
diff --git a/src/pebble.py b/src/pebble.py
index 1f56b73..a7823c7 100644
--- a/src/pebble.py
+++ b/src/pebble.py
@@ -4,8 +4,8 @@ from typing import Optional
 
 from src.consts import VERSION
 from src.interpreter.error import PebbleRuntimeError
-from src.position import Position
-from src.token import Token, TokenType
+from src.token.position import Position
+from src.token.token import Token, TokenType
 
 
 class Pebble:
diff --git a/src/runner.py b/src/runner.py
index 46993cf..b42b775 100644
--- a/src/runner.py
+++ b/src/runner.py
@@ -5,10 +5,10 @@ from src.ast.stmt import Stmt
 from src.formatter import Formatter
 from src.interpreter.interpreter import Interpreter
 from src.interpreter.resolver import Resolver
-from src.lexer import Lexer
+from src.token.lexer import Lexer
 from src.parser.parser import Parser
 from src.pebble import Pebble
-from src.token import Token, TokenType
+from src.token.token import Token, TokenType
 
 
 class Runner:
diff --git a/src/token/__init__.py b/src/token/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/keyword.py b/src/token/keyword.py
similarity index 94%
rename from src/keyword.py
rename to src/token/keyword.py
index cec6276..2f3c2f7 100644
--- a/src/keyword.py
+++ b/src/token/keyword.py
@@ -1,4 +1,4 @@
-from src.token import TokenType
+from src.token.token import TokenType
 
 KEYWORDS: dict[str, TokenType] = {
     "let": TokenType.LET,
diff --git a/src/lexer.py b/src/token/lexer.py
similarity index 97%
rename from src/lexer.py
rename to src/token/lexer.py
index 5fe9491..ce822d2 100644
--- a/src/lexer.py
+++ b/src/token/lexer.py
@@ -1,8 +1,8 @@
 from typing import Optional, Any
 
-from src.keyword import KEYWORDS
-from src.position import Position
-from src.token import Token, TokenType
+from src.token.keyword import KEYWORDS
+from src.token.position import Position
+from src.token.token import Token, TokenType
 
 
 class Lexer:
diff --git a/src/position.py b/src/token/position.py
similarity index 100%
rename from src/position.py
rename to src/token/position.py
diff --git a/src/token.py b/src/token/token.py
similarity index 97%
rename from src/token.py
rename to src/token/token.py
index e83ec10..8224106 100644
--- a/src/token.py
+++ b/src/token/token.py
@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from enum import Enum, auto
 from typing import Any
 
-from src.position import Position
+from src.token.position import Position
 
 
 class TokenType(Enum):