HEL/pebble
1
0
Fork 0
Code Issues Pull Requests Releases Activity

fix(lexer): correct token positions

Browse Source
This commit is contained in:
Louis Heredero
2026-02-05 03:43:51 +01:00
parent 0195a84bfd
commit 1362a071cf
1 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/lexer.py
View File

@@ -13,8 +13,8 @@ class Lexer:
self.start: int = 0 self.start: int = 0
self.idx: int = 0 self.idx: int = 0
self.length: int = 0 self.length: int = 0
self.line: int = 0 self.line: int = 1
self.column: int = 0 self.column: int = 1
self.start_pos: Position = self.get_position() self.start_pos: Position = self.get_position()
def error(self, msg: str): def error(self, msg: str):
@@ -27,8 +27,8 @@ class Lexer:
self.start = 0 self.start = 0
self.idx = 0 self.idx = 0
self.length = len(self.source) self.length = len(self.source)
self.line = 0 self.line = 1
self.column = 0 self.column = 1
while not self.is_at_end(): while not self.is_at_end():
self.start_pos = self.get_position() self.start_pos = self.get_position()
@@ -59,8 +59,14 @@ class Lexer:
char: str = self.peek() char: str = self.peek()
self.idx += 1 self.idx += 1
self.column += 1 self.column += 1
if char == "\n":
self.newline()
return char return char
def newline(self):
self.line += 1
self.column = 1
def match(self, expected: str) -> bool: def match(self, expected: str) -> bool:
if self.peek() == expected: if self.peek() == expected:
self.advance() self.advance()
@@ -119,9 +125,6 @@ class Lexer:
else: else:
self.add_token(TokenType.SLASH_EQUAL if self.match("=") else TokenType.SLASH) self.add_token(TokenType.SLASH_EQUAL if self.match("=") else TokenType.SLASH)
case " " | "\r" | "\t" | "\n": case " " | "\r" | "\t" | "\n":
if char == "\n":
self.line += 1
self.column = 0
while self.peek().isspace() and not self.is_at_end(): while self.peek().isspace() and not self.is_at_end():
self.advance() self.advance()
self.add_token(TokenType.WHITESPACE) self.add_token(TokenType.WHITESPACE)
@@ -138,9 +141,6 @@ class Lexer:
def scan_string(self): def scan_string(self):
while self.peek() != '"' and not self.is_at_end(): while self.peek() != '"' and not self.is_at_end():
if self.peek() == "\n":
self.line += 1
self.column = 0
self.advance() self.advance()
if self.is_at_end(): if self.is_at_end():