feat(ast): add newline token

src/lexer.py

@@ -124,8 +124,10 @@ class Lexer:
                     self.scan_comment_multiline()
                 else:
                     self.add_token(TokenType.SLASH_EQUAL if self.match("=") else TokenType.SLASH)
-            case " " | "\r" | "\t" | "\n":
-                while self.peek().isspace() and not self.is_at_end():
+            case "\n":
+                self.add_token(TokenType.NEWLINE)
+            case " " | "\r" | "\t":
+                while self.peek().isspace() and self.peek() != "\n" and not self.is_at_end():
                     self.advance()
                 self.add_token(TokenType.WHITESPACE)
             case '"':

src/parser/parser.py

@@ -69,7 +69,8 @@ class Parser:
     def synchronize(self):
         self.advance()
         while not self.is_at_end():
-            # TODO: if self.previous().type == TokenType.NEWLINE: return
+            if self.previous().type == TokenType.NEWLINE:
+                return
             if self.peek().type in self.STATEMENT_BOUNDARY:
                 return
             self.advance()

src/token.py

@@ -59,6 +59,7 @@ class TokenType(Enum):
     COMMENT = auto()
     WHITESPACE = auto()
     EOF = auto()
+    NEWLINE = auto()
 
 
 @dataclass