diff --git a/src/lexer.py b/src/lexer.py index 852623c..5077f89 100644 --- a/src/lexer.py +++ b/src/lexer.py @@ -124,8 +124,10 @@ class Lexer: self.scan_comment_multiline() else: self.add_token(TokenType.SLASH_EQUAL if self.match("=") else TokenType.SLASH) - case " " | "\r" | "\t" | "\n": - while self.peek().isspace() and not self.is_at_end(): + case "\n": + self.add_token(TokenType.NEWLINE) + case " " | "\r" | "\t": + while self.peek().isspace() and self.peek() != "\n" and not self.is_at_end(): self.advance() self.add_token(TokenType.WHITESPACE) case '"': diff --git a/src/parser/parser.py b/src/parser/parser.py index 59df0b4..55c4b96 100644 --- a/src/parser/parser.py +++ b/src/parser/parser.py @@ -69,7 +69,8 @@ class Parser: def synchronize(self): self.advance() while not self.is_at_end(): - # TODO: if self.previous().type == TokenType.NEWLINE: return + if self.previous().type == TokenType.NEWLINE: + return if self.peek().type in self.STATEMENT_BOUNDARY: return self.advance() diff --git a/src/token.py b/src/token.py index 420a9cf..1176d19 100644 --- a/src/token.py +++ b/src/token.py @@ -59,6 +59,7 @@ class TokenType(Enum): COMMENT = auto() WHITESPACE = auto() EOF = auto() + NEWLINE = auto() @dataclass