Merge pull request 'Syntax prototype and basic parser' (#1) from feat/syntax-prototype into main

Reviewed-on: #1

core/ast/midas.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Generic, Optional, TypeVar
+from typing import Any, Generic, Optional, TypeVar
 
 from lexer.token import Token
 
@@ -79,6 +79,12 @@ class Expr(ABC):
     def accept(self, visitor: Visitor[T]) -> T: ...
 
     class Visitor(ABC, Generic[T]):
+        @abstractmethod
+        def visit_wildcard_expr(self, expr: WildcardExpr) -> T: ...
+
+        @abstractmethod
+        def visit_literal_expr(self, expr: LiteralExpr) -> T: ...
+
         @abstractmethod
         def visit_type_expr(self, expr: TypeExpr) -> T: ...
 
@@ -89,6 +95,22 @@ class Expr(ABC):
         def visit_type_body_expr(self, expr: TypeBodyExpr) -> T: ...
 
 
+@dataclass(frozen=True)
+class WildcardExpr(Expr):
+    token: Token
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_wildcard_expr(self)
+
+
+@dataclass(frozen=True)
+class LiteralExpr(Expr):
+    value: Any
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_literal_expr(self)
+
+
 @dataclass(frozen=True)
 class TypeExpr(Expr):
     name: Token
@@ -100,6 +122,10 @@ class TypeExpr(Expr):
 
 @dataclass(frozen=True)
 class ConstraintExpr(Expr):
+    left: Expr
+    op: Token
+    right: Expr
+
     def accept(self, visitor: Expr.Visitor[T]) -> T:
         return visitor.visit_constraint_expr(self)
 

core/ast/printer.py

@@ -102,7 +102,7 @@ class AnnotationAstPrinter(AstPrinter, a.Expr.Visitor[None], a.Stmt.Visitor[None
                     if i == len(expr.constraints) - 1:
                         self._mark_last()
                     constraint.accept(self)
-    
+
     def visit_constraint_expr(self, expr: a.ConstraintExpr) -> None:
         self._write_line("ConstraintExpr")
         with self._child_level():
@@ -110,9 +110,9 @@ class AnnotationAstPrinter(AstPrinter, a.Expr.Visitor[None], a.Stmt.Visitor[None
             with self._child_level():
                 self._mark_last()
                 expr.left.accept(self)
-            
+
             self._write_line(f"operator: {expr.op.lexeme}")
-            
+
             self._write_line("right", last=True)
             with self._child_level():
                 self._mark_last()
@@ -133,14 +133,14 @@ class AnnotationAstPrinter(AstPrinter, a.Expr.Visitor[None], a.Stmt.Visitor[None
             name_text: str = "None" if expr.name is None else f'"{expr.name.lexeme}"'
             self._write_line(f"name: {name_text}")
             self._write_optional_child("type", expr.type, last=True)
-    
+
     def visit_wildcard_expr(self, expr: a.WildcardExpr) -> None:
         self._write_line("WildcardExpr")
-    
+
     def visit_literal_expr(self, expr: a.LiteralExpr) -> None:
         self._write_line("LiteralExpr")
         with self._child_level():
-            self._write_line(f'value: {expr.value}', last=True)
+            self._write_line(f"value: {expr.value}", last=True)
 
 
 class AnnotationPrinter(a.Expr.Visitor[str], a.Stmt.Visitor[str]):
@@ -158,12 +158,12 @@ class AnnotationPrinter(a.Expr.Visitor[str], a.Stmt.Visitor[str]):
         for constraint in expr.constraints:
             parts.append("(" + constraint.accept(self) + ")")
         return " + ".join(parts)
-    
+
     def visit_constraint_expr(self, expr: a.ConstraintExpr) -> str:
         parts: list[str] = [
             expr.left.accept(self),
             expr.op.lexeme,
-            expr.right.accept(self)
+            expr.right.accept(self),
         ]
         return " ".join(parts)
 
@@ -257,6 +257,18 @@ class MidasAstPrinter(AstPrinter, m.Expr.Visitor[None], m.Stmt.Visitor[None]):
 
     def visit_constraint_expr(self, expr: m.ConstraintExpr):
         self._write_line("ConstraintExpr")
+        with self._child_level():
+            self._write_line("left")
+            with self._child_level():
+                self._mark_last()
+                expr.left.accept(self)
+
+            self._write_line(f"operator: {expr.op.lexeme}")
+
+            self._write_line("right", last=True)
+            with self._child_level():
+                self._mark_last()
+                expr.right.accept(self)
 
     def visit_type_body_expr(self, expr: m.TypeBodyExpr):
         self._write_line("TypeBodyExpr")
@@ -268,3 +280,81 @@ class MidasAstPrinter(AstPrinter, m.Expr.Visitor[None], m.Stmt.Visitor[None]):
                     if i == len(expr.properties) - 1:
                         self._mark_last()
                     property.accept(self)
+
+    def visit_wildcard_expr(self, expr: m.WildcardExpr) -> None:
+        self._write_line("WildcardExpr")
+
+    def visit_literal_expr(self, expr: m.LiteralExpr) -> None:
+        self._write_line("LiteralExpr")
+        with self._child_level():
+            self._write_line(f"value: {expr.value}", last=True)
+
+class MidasPrinter(m.Expr.Visitor[str], m.Stmt.Visitor[str]):
+    def __init__(self, indent: int = 4):
+        self.indent: int = indent
+        self.level: int = 0
+
+    def indented(self, text: str) -> str:
+        return " " * (self.level * self.indent) + text
+
+    def print(self, expr: m.Expr | m.Stmt):
+        self.level = 0
+        return expr.accept(self)
+
+    def visit_type_stmt(self, stmt: m.TypeStmt):
+        bases: list[str] = [
+            b.accept(self)
+            for b in stmt.bases
+        ]
+        
+        res: str = self.indented(f"type {stmt.name.lexeme}<{', '.join(bases)}>")
+        if stmt.body is not None:
+            res += " {\n"
+            self.level += 1
+            res += stmt.body.accept(self)
+            self.level -= 1
+            res += "\n" + self.indented("}")
+
+        return res
+
+    def visit_property_stmt(self, stmt: m.PropertyStmt):
+        return f"{stmt.name.lexeme}: {stmt.type.accept(self)}"
+
+    def visit_op_stmt(self, stmt: m.OpStmt):
+        left: str = stmt.left.accept(self)
+        op: str = stmt.op.lexeme
+        right: str = stmt.right.accept(self)
+        result: str = stmt.result.accept(self)
+        return self.indented(f"op <{left}> {op} <{right}> = <{result}>")
+
+    def visit_constraint_stmt(self, stmt: m.ConstraintStmt):
+        name: str = stmt.name.lexeme
+        constraint: str = stmt.constraint.accept(self)
+        return self.indented(f"constraint {name} = {constraint}")
+
+    def visit_type_expr(self, expr: m.TypeExpr):
+        parts: list[str] = [expr.name.lexeme]
+        for constraint in expr.constraints:
+            parts.append("(" + constraint.accept(self) + ")")
+        return " + ".join(parts)
+
+    def visit_constraint_expr(self, expr: m.ConstraintExpr):
+        parts: list[str] = [
+            expr.left.accept(self),
+            expr.op.lexeme,
+            expr.right.accept(self),
+        ]
+        return " ".join(parts)
+
+    def visit_type_body_expr(self, expr: m.TypeBodyExpr):
+        properties: list[str] = [
+            self.indented(prop.accept(self))
+            for prop in expr.properties
+        ]
+        return "\n".join(properties)
+
+    def visit_wildcard_expr(self, expr: m.WildcardExpr):
+        return "_"
+
+    def visit_literal_expr(self, expr: m.LiteralExpr):
+        return str(expr.value)

examples/00_syntax_prototype/01_simple_types.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 df: Frame[
     verified: bool,
     birth_year: int,
-    height: float,
+    height: float + ( _ > 0 ) + ( _ < 250 ),
     name: str,
     date: datetime,
     float,  # unnamed

examples/00_syntax_prototype/02_custom_types.midas

@@ -16,7 +16,7 @@ op <Latitude> - <Latitude> = <LatitudeDiff>
 op <Longitude> - <Longitude> = <LongitudeDiff>
 
 // Simple custom type with a constraint
-type Age<int + (0 <= _ < 150)>
+type Age<int + (0 <= _) + (_ < 150)>
 
 // Predefined custom constraints that can be referenced in other definitions
 constraint Positive = _ >= 0

lexer/annotations.py

@@ -1,4 +1,5 @@
 from lexer.base import Lexer
+from lexer.keyword import ANNOTATION_KEYWORDS
 from lexer.token import TokenType
 
 
@@ -27,7 +28,7 @@ class AnnotationLexer(Lexer):
                     TokenType.EQUAL_EQUAL if self.match("=") else TokenType.EQUAL
                 )
             case "!":
-                if self.peek() == "=":
+                if self.match("="):
                     self.add_token(TokenType.BANG_EQUAL)
                 else:
                     self.error("Unexpected single bang. Did you mean '!=' ?")
@@ -86,7 +87,10 @@ class AnnotationLexer(Lexer):
         """
         while self.peek().isalnum() or self.peek() == "_":
             self.advance()
-        self.add_token(TokenType.IDENTIFIER)
+        
+        lexeme: str = self.source[self.start : self.idx]
+        token_type: TokenType = ANNOTATION_KEYWORDS.get(lexeme, TokenType.IDENTIFIER)
+        self.add_token(token_type)
 
     def scan_comment(self):
         """Scan the rest of a comment and add it as a token

lexer/keyword.py

@@ -1,9 +1,16 @@
 from lexer.token import TokenType
 
-KEYWORDS: dict[str, TokenType] = {
+ANNOTATION_KEYWORDS: dict[str, TokenType] = {
+    "True": TokenType.TRUE,
+    "False": TokenType.FALSE,
+    "None": TokenType.NONE,
+}
+
+MIDAS_KEYWORDS: dict[str, TokenType] = {
     "type": TokenType.TYPE,
     "op": TokenType.OP,
     "constraint": TokenType.CONSTRAINT,
     "true": TokenType.TRUE,
     "false": TokenType.FALSE,
+    "none": TokenType.NONE,
 }

lexer/midas.py

@@ -1,5 +1,5 @@
 from lexer.base import Lexer
-from lexer.keyword import KEYWORDS
+from lexer.keyword import MIDAS_KEYWORDS
 from lexer.token import TokenType
 
 
@@ -32,7 +32,7 @@ class MidasLexer(Lexer):
                     TokenType.EQUAL_EQUAL if self.match("=") else TokenType.EQUAL
                 )
             case "!":
-                if self.peek() == "=":
+                if self.match("="):
                     self.add_token(TokenType.BANG_EQUAL)
                 else:
                     self.error("Unexpected single bang. Did you mean '!=' ?")
@@ -102,7 +102,7 @@ class MidasLexer(Lexer):
             self.advance()
 
         lexeme: str = self.source[self.start : self.idx]
-        token_type: TokenType = KEYWORDS.get(lexeme, TokenType.IDENTIFIER)
+        token_type: TokenType = MIDAS_KEYWORDS.get(lexeme, TokenType.IDENTIFIER)
         self.add_token(token_type)
 
     def scan_comment(self):

parser/annotations.py

@@ -68,8 +68,6 @@ class AnnotationParser(Parser):
         constraints: list[ConstraintExpr] = []
 
         while not self.is_at_end() and self.match(TokenType.PLUS):
-            print(self.peek())
-            print(self.tokens)
             self.consume(TokenType.LEFT_PAREN, "Expected '(' before type constraint")
             constraints.append(self.constraint_expr())
             self.consume(TokenType.RIGHT_PAREN, "Expected ')' after type constraint")

parser/midas.py

@@ -3,12 +3,15 @@ from typing import Optional
 from core.ast.midas import (
     ConstraintExpr,
     ConstraintStmt,
+    Expr,
+    LiteralExpr,
     OpStmt,
     PropertyStmt,
     Stmt,
     TypeBodyExpr,
     TypeExpr,
     TypeStmt,
+    WildcardExpr,
 )
 from lexer.token import Token, TokenType
 from parser.base import Parser
@@ -59,6 +62,7 @@ class MidasParser(Parser):
                 return self.op_declaration()
             if self.match(TokenType.CONSTRAINT):
                 return self.constraint_declaration()
+            raise self.error(self.peek(), "Unexpected token")
         except ParsingError:
             self.synchronize()
             return None
@@ -96,7 +100,9 @@ class MidasParser(Parser):
         constraints: list[ConstraintExpr] = []
 
         while not self.is_at_end() and self.match(TokenType.PLUS):
+            self.consume(TokenType.LEFT_PAREN, "Expected '(' before type constraint")
             constraints.append(self.constraint_expr())
+            self.consume(TokenType.RIGHT_PAREN, "Expected ')' after type constraint")
 
         return TypeExpr(name=name, constraints=constraints)
 
@@ -106,8 +112,41 @@ class MidasParser(Parser):
         Returns:
             ConstraintExpr: the parsed type constraint expression
         """
-        # TODO
-        return ConstraintExpr()
+
+        left: Expr = self.constraint_value()
+        op: Token = self.constraint_operator()
+        right: Expr = self.constraint_value()
+        return ConstraintExpr(left=left, op=op, right=right)
+
+    def constraint_value(self) -> Expr:
+        if self.match(TokenType.UNDERSCORE):
+            return WildcardExpr(self.previous())
+        return self.literal()
+
+    def literal(self) -> LiteralExpr:
+        if self.match(TokenType.FALSE):
+            return LiteralExpr(False)
+        if self.match(TokenType.TRUE):
+            return LiteralExpr(True)
+        if self.match(TokenType.NONE):
+            return LiteralExpr(None)
+
+        if self.match(TokenType.NUMBER):
+            return LiteralExpr(self.previous().value)
+
+        raise self.error(self.peek(), "Expected literal")
+
+    def constraint_operator(self) -> Token:
+        if self.match(
+            TokenType.LESS,
+            TokenType.LESS_EQUAL,
+            TokenType.GREATER,
+            TokenType.GREATER_EQUAL,
+            TokenType.EQUAL_EQUAL,
+            TokenType.BANG_EQUAL,
+        ):
+            return self.previous()
+        raise self.error(self.peek(), "Expected constraint operator")
 
     def type_body_expr(self) -> TypeBodyExpr:
         """Parse a type definition body

tests/lexer/test_annotation_lexer.py

@@ -0,0 +1,129 @@
+from typing import Any
+
+import pytest
+
+from lexer.annotations import AnnotationLexer
+from lexer.token import Token, TokenType
+
+
+def scan(source: str) -> list[Token]:
+    return AnnotationLexer(source).process()
+
+
+def assert_n_tokens(tokens: list[Token], n: int):
+    assert len(tokens) == n + 1
+    assert tokens[-1].type == TokenType.EOF
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("(", TokenType.LEFT_PAREN),
+        (")", TokenType.RIGHT_PAREN),
+        ("[", TokenType.LEFT_BRACKET),
+        ("]", TokenType.RIGHT_BRACKET),
+        (":", TokenType.COLON),
+        (",", TokenType.COMMA),
+        ("_", TokenType.UNDERSCORE),
+    ],
+)
+def test_punctuation(src: str, expected: TokenType):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("+", TokenType.PLUS),
+        (">", TokenType.GREATER),
+        (">=", TokenType.GREATER_EQUAL),
+        ("<", TokenType.LESS),
+        ("<=", TokenType.LESS_EQUAL),
+        ("=", TokenType.EQUAL),
+        ("==", TokenType.EQUAL_EQUAL),
+        ("!=", TokenType.BANG_EQUAL),
+    ],
+)
+def test_operators(src: str, expected: TokenType):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("a", TokenType.IDENTIFIER),
+        ("foo", TokenType.IDENTIFIER),
+        ("foo1", TokenType.IDENTIFIER),
+        ("foo_", TokenType.IDENTIFIER),
+        ("foo_bar1_baz2", TokenType.IDENTIFIER),
+        ("FOO_BAR1_BAZ2", TokenType.IDENTIFIER),
+        ("True", TokenType.TRUE),
+        ("False", TokenType.FALSE),
+        ("None", TokenType.NONE),
+    ],
+)
+def test_identifiers_keywords(src: str, expected: TokenType):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("#", TokenType.COMMENT),
+        ("# This is a comment", TokenType.COMMENT),
+        (" ", TokenType.WHITESPACE),
+        ("\t", TokenType.WHITESPACE),
+        ("\r", TokenType.WHITESPACE),
+        ("  \t  \t", TokenType.WHITESPACE),
+        ("\n", TokenType.NEWLINE),
+    ],
+)
+def test_misc(src: str, expected: TokenType):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected
+
+
+@pytest.mark.parametrize(
+    "src,expected_type,expected_value",
+    [
+        ("0", TokenType.NUMBER, 0),
+        ("0.0", TokenType.NUMBER, 0),
+        ("1234.56", TokenType.NUMBER, 1234.56),
+    ],
+)
+def test_literals(src: str, expected_type: TokenType, expected_value: Any):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected_type
+    assert tokens[0].value == expected_value
+
+
+def test_single_bang_error():
+    with pytest.raises(SyntaxError):
+        scan("!")
+
+
+@pytest.mark.parametrize(
+    "src",
+    [
+        "-",
+        "*",
+        "/",
+        "{",
+        "}",
+        "@",
+        '"',
+        "'",
+        ".",
+    ],
+)
+def test_unexpected_character(src: str):
+    with pytest.raises(SyntaxError):
+        scan(src)

tests/lexer/test_midas_lexer.py

@@ -0,0 +1,129 @@
+from typing import Any
+
+import pytest
+
+from lexer.midas import MidasLexer
+from lexer.token import Token, TokenType
+
+
+def scan(source: str) -> list[Token]:
+    return MidasLexer(source).process()
+
+
+def assert_n_tokens(tokens: list[Token], n: int):
+    assert len(tokens) == n + 1
+    assert tokens[-1].type == TokenType.EOF
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("(", TokenType.LEFT_PAREN),
+        (")", TokenType.RIGHT_PAREN),
+        ("[", TokenType.LEFT_BRACKET),
+        ("]", TokenType.RIGHT_BRACKET),
+        ("{", TokenType.LEFT_BRACE),
+        ("}", TokenType.RIGHT_BRACE),
+        (":", TokenType.COLON),
+        (",", TokenType.COMMA),
+        ("_", TokenType.UNDERSCORE),
+    ],
+)
+def test_punctuation(src: str, expected: TokenType):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("+", TokenType.PLUS),
+        ("-", TokenType.MINUS),
+        ("*", TokenType.STAR),
+        ("/", TokenType.SLASH),
+        (">", TokenType.GREATER),
+        (">=", TokenType.GREATER_EQUAL),
+        ("<", TokenType.LESS),
+        ("<=", TokenType.LESS_EQUAL),
+        ("=", TokenType.EQUAL),
+        ("==", TokenType.EQUAL_EQUAL),
+        ("!=", TokenType.BANG_EQUAL),
+    ],
+)
+def test_operators(src: str, expected: TokenType):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("a", TokenType.IDENTIFIER),
+        ("foo", TokenType.IDENTIFIER),
+        ("foo1", TokenType.IDENTIFIER),
+        ("foo_", TokenType.IDENTIFIER),
+        ("foo_bar1_baz2", TokenType.IDENTIFIER),
+        ("FOO_BAR1_BAZ2", TokenType.IDENTIFIER),
+        ("true", TokenType.TRUE),
+        ("false", TokenType.FALSE),
+        ("none", TokenType.NONE),
+    ],
+)
+def test_identifiers_keywords(src: str, expected: TokenType):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("// This is a comment", TokenType.COMMENT),
+        ("/* This is a comment */", TokenType.COMMENT),
+        (" ", TokenType.WHITESPACE),
+        ("\t", TokenType.WHITESPACE),
+        ("\r", TokenType.WHITESPACE),
+        ("  \t  \t", TokenType.WHITESPACE),
+        ("\n", TokenType.NEWLINE),
+    ],
+)
+def test_misc(src: str, expected: TokenType):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected
+
+
+@pytest.mark.parametrize(
+    "src,expected_type,expected_value",
+    [
+        ("0", TokenType.NUMBER, 0),
+        ("0.0", TokenType.NUMBER, 0),
+        ("1234.56", TokenType.NUMBER, 1234.56),
+    ],
+)
+def test_literals(src: str, expected_type: TokenType, expected_value: Any):
+    tokens: list[Token] = scan(src)
+    assert_n_tokens(tokens, 1)
+    assert tokens[0].type == expected_type
+    assert tokens[0].value == expected_value
+
+
+def test_single_bang_error():
+    with pytest.raises(SyntaxError):
+        scan("!")
+
+
+@pytest.mark.parametrize(
+    "src",
+    [
+        "@",
+        '"',
+        "'",
+        ".",
+    ],
+)
+def test_unexpected_character(src: str):
+    with pytest.raises(SyntaxError):
+        scan(src)

tests/parser/test_annotation_parser.py

@@ -0,0 +1,130 @@
+from typing import Optional
+
+import pytest
+
+from core.ast.annotations import (
+    AnnotationStmt,
+    ConstraintExpr,
+    Expr,
+    LiteralExpr,
+    SchemaElementExpr,
+    SchemaExpr,
+    Stmt,
+    TypeExpr,
+    WildcardExpr,
+)
+from lexer.annotations import AnnotationLexer
+from lexer.position import Position
+from lexer.token import Token
+from parser.annotations import AnnotationParser
+
+
+class AstSerializer(Stmt.Visitor[str], Expr.Visitor[str]):
+    def serialize(self, stmt: Stmt):
+        return stmt.accept(self)
+
+    def visit_annotation_stmt(self, stmt: AnnotationStmt) -> str:
+        schema: str = ""
+        if stmt.schema is not None:
+            schema = " " + stmt.schema.accept(self)
+        return f"(annotation {stmt.name.lexeme}{schema})"
+
+    def visit_schema_expr(self, expr: SchemaExpr) -> str:
+        elements: list[str] = [elmt.accept(self) for elmt in expr.elements]
+        return f"(schema {' '.join(elements)})"
+
+    def visit_schema_element_expr(self, expr: SchemaElementExpr) -> str:
+        name: str = expr.name.lexeme if expr.name is not None else "_"
+        type: str = expr.type.accept(self) if expr.type is not None else "_"
+        return f"({name} {type})"
+
+    def visit_type_expr(self, expr: TypeExpr) -> str:
+        res: str = f"({expr.name.lexeme}"
+        for constraint in expr.constraints:
+            res += " " + constraint.accept(self)
+        res += ")"
+        return res
+
+    def visit_constraint_expr(self, expr: ConstraintExpr) -> str:
+        return f"(constraint {expr.left.accept(self)} {expr.op.lexeme} {expr.right.accept(self)})"
+
+    def visit_wildcard_expr(self, expr: WildcardExpr) -> str:
+        return "(_)"
+
+    def visit_literal_expr(self, expr: LiteralExpr) -> str:
+        return f"({expr.value})"
+
+
+def parse(source: str) -> Optional[Stmt]:
+    tokens: list[Token] = AnnotationLexer(source).process()
+    return AnnotationParser(tokens).parse()
+
+
+def must_parse(source: str) -> Stmt:
+    stmt: Optional[Stmt] = parse(source)
+    assert stmt is not None
+    return stmt
+
+
+def ast_str(source: str) -> str:
+    stmt: Stmt = must_parse(source)
+    return AstSerializer().serialize(stmt)
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("Type", "(annotation Type)"),
+        ("Type[]", "(annotation Type (schema ))"),
+        (
+            """
+            Frame[
+                verified: bool,
+                birth_year: int,
+                height: float + ( _ > 0 ) + ( _ < 250 ),
+                name: str,
+                date: datetime,
+                float,  # unnamed
+                unknown: _,  # untyped
+                _  # unnamed and untyped
+            ]
+            """,
+            "(annotation Frame (schema (verified (bool)) (birth_year (int)) (height (float (constraint (_) > (0.0)) (constraint (_) < (250.0)))) (name (str)) (date (datetime)) (_ (float)) (unknown _) (_ _)))",
+        ),
+    ],
+)
+def test_expressions(src: str, expected: str):
+    assert ast_str(src) == expected
+
+
+@pytest.mark.parametrize(
+    "src,pos,should_fail",
+    [
+        ("", (1, 1), True),
+        ("42", (1, 1), True),
+        ("True", (1, 1), True),
+        ("Type[", (1, 6), True),
+        ("Type[] Type2", (1, 8), False),
+        ("Type[bool:]", (1, 11), True),
+        ("Type[3]", (1, 6), True),
+        ("Type[bool float]", (1, 11), True),
+        ("Type[bool (_ < 2)]", (1, 11), True),
+        ("Type[bool + _ < 2)]", (1, 13), True),
+        ("Type[bool + (_ < 2]", (1, 19), True),
+        ("Type[bool + (< 2)]", (1, 14), True),
+        ("Type[bool + (_ + 2)]", (1, 16), True),
+        ("Type[bool + (Foo + Bar)]", (1, 14), True),
+        # ("Type[bool,]", (1, 11), True),  # trailing comma is accepted, TODO: update parser or EBNF
+        ("Type[bool, Type[]]", (1, 16), True),
+        ("Type[foo: 3]", (1, 11), True),
+    ],
+)
+def test_parsing_error(src: str, pos: tuple[int, int], should_fail: bool):
+    tokens: list[Token] = AnnotationLexer(src).process()
+    parser: AnnotationParser = AnnotationParser(tokens)
+    stmt: Optional[Stmt] = parser.parse()
+    if should_fail:
+        assert stmt is None
+    assert len(parser.errors) != 0
+    error_pos: Position = parser.errors[0].token.position
+    assert (error_pos.line, error_pos.column) == pos

tests/parser/test_midas_parser.py

@@ -0,0 +1,202 @@
+import textwrap
+
+import pytest
+
+from core.ast.midas import (
+    ConstraintExpr,
+    ConstraintStmt,
+    Expr,
+    LiteralExpr,
+    OpStmt,
+    PropertyStmt,
+    Stmt,
+    TypeBodyExpr,
+    TypeExpr,
+    TypeStmt,
+    WildcardExpr,
+)
+from lexer.midas import MidasLexer
+from lexer.position import Position
+from lexer.token import Token
+from parser.midas import MidasParser
+
+
+class AstSerializer(Stmt.Visitor[str], Expr.Visitor[str]):
+    def serialize(self, stmt: Stmt):
+        return stmt.accept(self)
+
+    def visit_type_stmt(self, stmt: TypeStmt) -> str:
+        res: str = f"(type_def {stmt.name.lexeme}"
+        for base in stmt.bases:
+            res += " " + base.accept(self)
+        if stmt.body is not None:
+            res += " " + stmt.body.accept(self)
+        res += ")"
+        return res
+
+    def visit_type_expr(self, expr: TypeExpr) -> str:
+        res: str = f"({expr.name.lexeme}"
+        for constraint in expr.constraints:
+            res += " " + constraint.accept(self)
+        res += ")"
+        return res
+
+    def visit_constraint_expr(self, expr: ConstraintExpr) -> str:
+        return f"(constraint {expr.left.accept(self)} {expr.op.lexeme} {expr.right.accept(self)})"
+
+    def visit_wildcard_expr(self, expr: WildcardExpr) -> str:
+        return "(_)"
+
+    def visit_literal_expr(self, expr: LiteralExpr) -> str:
+        return f"({expr.value})"
+
+    def visit_type_body_expr(self, expr: TypeBodyExpr) -> str:
+        res: str = "(body"
+        for prop in expr.properties:
+            res += " " + prop.accept(self)
+        res += ")"
+        return res
+
+    def visit_property_stmt(self, stmt: PropertyStmt) -> str:
+        return f"(property {stmt.name.lexeme} {stmt.type.accept(self)})"
+
+    def visit_op_stmt(self, stmt: OpStmt) -> str:
+        left: str = stmt.left.accept(self)
+        right: str = stmt.right.accept(self)
+        result: str = stmt.result.accept(self)
+        return f"(op_def {left} {stmt.op.lexeme} {right} {result})"
+
+    def visit_constraint_stmt(self, stmt: ConstraintStmt) -> str:
+        return f"(constraint_def {stmt.name.lexeme} {stmt.constraint.accept(self)})"
+
+
+def parse(source: str) -> list[Stmt]:
+    tokens: list[Token] = MidasLexer(source).process()
+    return MidasParser(tokens).parse()
+
+
+def ast_str(source: str) -> list[str]:
+    stmts: list[Stmt] = parse(source)
+    return [AstSerializer().serialize(stmt) for stmt in stmts]
+
+
+@pytest.mark.parametrize(
+    "src,expected",
+    [
+        ("type Foo<>", "(type_def Foo)"),
+        ("type Foo<Bar>", "(type_def Foo (Bar))"),
+        ("type Foo<Bar, Baz>", "(type_def Foo (Bar) (Baz))"),
+        (
+            "type Foo<Bar + (_ < 2), Baz>",
+            "(type_def Foo (Bar (constraint (_) < (2.0))) (Baz))",
+        ),
+        (
+            """
+            type Foo<> {
+                foo: Bar
+            }
+            """,
+            "(type_def Foo (body (property foo (Bar))))",
+        ),
+        (
+            """
+            type Foo<> {
+                foo: Bar + (_ != none)
+                foo2: Bar2 + (0 <= _) + (_ <= 100)
+            }
+            """,
+            "(type_def Foo (body (property foo (Bar (constraint (_) != (None)))) (property foo2 (Bar2 (constraint (0.0) <= (_)) (constraint (_) <= (100.0))))))",
+        ),
+        ("op <A> + <B> = <C>", "(op_def (A) + (B) (C))"),
+        (
+            "op <A + (_ < 100)> + <B + (_ < 100)> = <C + (_ < 200)>",
+            "(op_def (A (constraint (_) < (100.0))) + (B (constraint (_) < (100.0))) (C (constraint (_) < (200.0))))",
+        ),
+        (
+            "constraint Positive = _ >= 0",
+            "(constraint_def Positive (constraint (_) >= (0.0)))",
+        ),
+    ],
+)
+def test_expressions(src: str, expected: str | list[str]):
+    if isinstance(expected, str):
+        expected = [expected]
+    assert ast_str(src) == expected
+
+
+@pytest.mark.parametrize(
+    "src,pos",
+    [
+        ###
+        # Misc
+        ###
+        ("42", (1, 1)),
+        ("true", (1, 1)),
+        ("foo", (1, 1)),
+        ###
+        # Type statements
+        ###
+        ("type", (1, 5)),
+        ("type true", (1, 6)),
+        ("type Foo", (1, 9)),
+        ("type Foo<1>", (1, 10)),
+        # ("type Foo<float,>", (1, 16)),  # trailing comma is accepted, TODO: update parser or EBNF
+        ("type Foo<float, 1>", (1, 17)),
+        ("type Foo<float", (1, 15)),
+        ("type Foo<float> { 3 }", (1, 19)),
+        (
+            """
+            type Foo<float> {
+                foo
+            }
+            """,
+            (4, 1),
+        ),
+        (
+            """
+            type Foo<float> {
+                foo: 3
+            }
+            """,
+            (3, 10),
+        ),
+        ###
+        # Operation statements
+        ###
+        ("op", (1, 3)),
+        ("op float", (1, 4)),
+        ("op <", (1, 5)),
+        ("op <float", (1, 10)),
+        ("op <float>", (1, 11)),
+        ("op <float> +", (1, 13)),
+        ("op <float> + float", (1, 14)),
+        ("op <float> + <", (1, 15)),
+        ("op <float> + <float", (1, 20)),
+        ("op <float> + <float>", (1, 21)),
+        ("op <float> + <float> =", (1, 23)),
+        ("op <float> + <float> = float", (1, 24)),
+        ("op <float> + <float> = <", (1, 25)),
+        ("op <float> + <float> = <float", (1, 30)),
+        ("op <float + 3> + <float> = <float>", (1, 13)),
+        ("op <float> + <float + 3> = <float>", (1, 23)),
+        ("op <float> + <float> = <float + 3>", (1, 33)),
+        ###
+        # Constraint statements
+        ###
+        ("constraint", (1, 11)),
+        ("constraint 3", (1, 12)),
+        ("constraint Foo", (1, 15)),
+        ("constraint Foo =", (1, 17)),
+        ("constraint Foo = 3", (1, 19)),
+        ("constraint Foo = 3 <", (1, 21)),
+    ],
+)
+def test_parsing_error(src: str, pos: tuple[int, int]):
+    src = textwrap.dedent(src)
+    tokens: list[Token] = MidasLexer(src).process()
+    parser: MidasParser = MidasParser(tokens)
+    stmt: list[Stmt] = parser.parse()
+    assert len(stmt) == 0
+    assert len(parser.errors) != 0
+    error_pos: Position = parser.errors[0].token.position
+    assert (error_pos.line, error_pos.column) == pos