test(parser): remove pytest tests

modify the lexer to allow underscores in an identifier, but keep scanning single underscores as a specific underscore token

core/ast/json_serializer.py

@@ -1,14 +1,24 @@
+from typing import Optional, Sequence
+
 from core.ast.midas import (
-    ConstraintExpr,
+    BinaryExpr,
-    ConstraintStmt,
+    ComplexTypeStmt,
     Expr,
+    ExtendStmt,
+    GetExpr,
+    GroupingExpr,
     LiteralExpr,
+    LogicalExpr,
     OpStmt,
+    PredicateStmt,
     PropertyStmt,
+    SimpleTypeExpr,
+    SimpleTypeStmt,
     Stmt,
-    TypeBodyExpr,
+    TemplateExpr,
     TypeExpr,
-    TypeStmt,
+    UnaryExpr,
+    VariableExpr,
     WildcardExpr,
 )
 
@@ -19,42 +29,29 @@ class AstJsonSerializer(Stmt.Visitor[dict], Expr.Visitor[dict]):
     def serialize(self, stmts: list[Stmt]) -> list[dict]:
         return [stmt.accept(self) for stmt in stmts]
 
-    def visit_type_stmt(self, stmt: TypeStmt) -> dict:
+    def _serialize_optional(self, element: Optional[Stmt | Expr]) -> Optional[dict]:
+        if element is None:
+            return None
+        return element.accept(self)
+
+    def _serialize_list(self, elements: Sequence[Stmt | Expr]) -> list[dict]:
+        return [element.accept(self) for element in elements]
+
+    def visit_simple_type_stmt(self, stmt: SimpleTypeStmt) -> dict:
         return {
-            "_type": "TypeStmt",
+            "_type": "SimpleTypeStmt",
+            "template": self._serialize_optional(stmt.template),
             "name": stmt.name.lexeme,
-            "bases": [base.accept(self) for base in stmt.bases],
+            "base": stmt.base.accept(self),
-            "body": stmt.body.accept(self) if stmt.body is not None else None,
+            "constraint": self._serialize_optional(stmt.constraint),
         }
 
-    def visit_type_expr(self, expr: TypeExpr) -> dict:
+    def visit_complex_type_stmt(self, stmt: ComplexTypeStmt) -> dict:
         return {
-            "_type": "TypeExpr",
+            "_type": "ComplexTypeStmt",
-            "name": expr.name.lexeme,
+            "name": stmt.name.lexeme,
-            "constraints": [constraint.accept(self) for constraint in expr.constraints],
+            "template": self._serialize_optional(stmt.template),
-        }
+            "properties": self._serialize_list(stmt.properties),
-
-    def visit_constraint_expr(self, expr: ConstraintExpr) -> dict:
-        return {
-            "_type": "ConstraintExpr",
-            "left": expr.left.accept(self),
-            "op": expr.op.lexeme,
-            "right": expr.right.accept(self),
-        }
-
-    def visit_wildcard_expr(self, expr: WildcardExpr) -> dict:
-        return {"_type": "WildcardExpr"}
-
-    def visit_literal_expr(self, expr: LiteralExpr) -> dict:
-        return {
-            "_type": "LiteralExpr",
-            "value": expr.value,
-        }
-
-    def visit_type_body_expr(self, expr: TypeBodyExpr) -> dict:
-        return {
-            "_type": "TypeBodyExpr",
-            "properties": [prop.accept(self) for prop in expr.properties],
         }
 
     def visit_property_stmt(self, stmt: PropertyStmt) -> dict:
@@ -62,20 +59,101 @@ class AstJsonSerializer(Stmt.Visitor[dict], Expr.Visitor[dict]):
             "_type": "PropertyStmt",
             "name": stmt.name.lexeme,
             "type": stmt.type.accept(self),
+            "constraint": self._serialize_optional(stmt.constraint),
+        }
+
+    def visit_extend_stmt(self, stmt: ExtendStmt) -> dict:
+        return {
+            "_type": "ExtendStmt",
+            "type": stmt.type.accept(self),
+            "operations": self._serialize_list(stmt.operations),
         }
 
     def visit_op_stmt(self, stmt: OpStmt) -> dict:
         return {
             "_type": "OpStmt",
-            "left": stmt.left.accept(self),
+            "name": stmt.name.lexeme,
-            "op": stmt.op.lexeme,
+            "operand": stmt.operand.accept(self),
-            "right": stmt.right.accept(self),
             "result": stmt.result.accept(self),
         }
 
-    def visit_constraint_stmt(self, stmt: ConstraintStmt) -> dict:
+    def visit_predicate_stmt(self, stmt: PredicateStmt) -> dict:
         return {
-            "_type": "ConstraintStmt",
+            "_type": "PredicateStmt",
             "name": stmt.name.lexeme,
-            "constraint": stmt.constraint.accept(self),
+            "subject": stmt.subject.lexeme,
+            "type": stmt.type.accept(self),
+            "condition": stmt.condition.accept(self),
+        }
+
+    def visit_simple_type_expr(self, expr: SimpleTypeExpr) -> dict:
+        return {
+            "_type": "SimpleTypeExpr",
+            "name": expr.name.lexeme,
+            "optional": expr.optional,
+        }
+
+    def visit_logical_expr(self, expr: LogicalExpr) -> dict:
+        return {
+            "_type": "LogicalExpr",
+            "left": expr.left.accept(self),
+            "operator": expr.operator.lexeme,
+            "right": expr.right.accept(self),
+        }
+
+    def visit_binary_expr(self, expr: BinaryExpr) -> dict:
+        return {
+            "_type": "BinaryExpr",
+            "left": expr.left.accept(self),
+            "operator": expr.operator.lexeme,
+            "right": expr.right.accept(self),
+        }
+
+    def visit_unary_expr(self, expr: UnaryExpr) -> dict:
+        return {
+            "_type": "UnaryExpr",
+            "operator": expr.operator.lexeme,
+            "right": expr.right.accept(self),
+        }
+
+    def visit_get_expr(self, expr: GetExpr) -> dict:
+        return {
+            "_type": "GetExpr",
+            "expr": expr.expr.accept(self),
+            "name": expr.name.lexeme,
+        }
+
+    def visit_variable_expr(self, expr: VariableExpr) -> dict:
+        return {
+            "_type": "VariableExpr",
+            "name": expr.name.lexeme,
+        }
+
+    def visit_grouping_expr(self, expr: GroupingExpr) -> dict:
+        return {
+            "_type": "GroupingExpr",
+            "expr": expr.expr.accept(self),
+        }
+
+    def visit_literal_expr(self, expr: LiteralExpr) -> dict:
+        return {
+            "_type": "LiteralExpr",
+            "value": expr.value,
+        }
+
+    def visit_wildcard_expr(self, expr: WildcardExpr) -> dict:
+        return {"_type": "WildcardExpr"}
+
+    def visit_template_expr(self, expr: TemplateExpr) -> dict:
+        return {
+            "_type": "TemplateExpr",
+            "type": expr.type.accept(self),
+        }
+
+    def visit_type_expr(self, expr: TypeExpr) -> dict:
+        return {
+            "_type": "TypeExpr",
+            "name": expr.name.lexeme,
+            "template": self._serialize_optional(expr.template),
+            "optional": expr.optional,
         }

core/ast/midas.py

@@ -1,3 +1,8 @@
+"""
+This file was generated by a script. Any manual changes might be overwritten.
+Please modify gen/ast.py instead and run gen/gen.py
+"""
+
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
@@ -8,8 +13,9 @@ from lexer.token import Token
 
 T = TypeVar("T")
 
-
+##############
-# Statements
+# Statements #
+##############
 
 
 @dataclass(frozen=True)
@@ -19,42 +25,68 @@ class Stmt(ABC):
 
     class Visitor(ABC, Generic[T]):
         @abstractmethod
-        def visit_type_stmt(self, stmt: TypeStmt) -> T: ...
+        def visit_simple_type_stmt(self, stmt: SimpleTypeStmt) -> T: ...
+
+        @abstractmethod
+        def visit_complex_type_stmt(self, stmt: ComplexTypeStmt) -> T: ...
 
         @abstractmethod
         def visit_property_stmt(self, stmt: PropertyStmt) -> T: ...
 
+        @abstractmethod
+        def visit_extend_stmt(self, stmt: ExtendStmt) -> T: ...
+
         @abstractmethod
         def visit_op_stmt(self, stmt: OpStmt) -> T: ...
 
         @abstractmethod
-        def visit_constraint_stmt(self, stmt: ConstraintStmt) -> T: ...
+        def visit_predicate_stmt(self, stmt: PredicateStmt) -> T: ...
 
 
 @dataclass(frozen=True)
-class TypeStmt(Stmt):
+class SimpleTypeStmt(Stmt):
     name: Token
-    bases: list[TypeExpr]
+    template: Optional[TemplateExpr]
-    body: Optional[TypeBodyExpr]
+    base: TypeExpr
+    constraint: Optional[Expr]
 
     def accept(self, visitor: Stmt.Visitor[T]) -> T:
-        return visitor.visit_type_stmt(self)
+        return visitor.visit_simple_type_stmt(self)
+
+
+@dataclass(frozen=True)
+class ComplexTypeStmt(Stmt):
+    name: Token
+    template: Optional[TemplateExpr]
+    properties: list[PropertyStmt]
+
+    def accept(self, visitor: Stmt.Visitor[T]) -> T:
+        return visitor.visit_complex_type_stmt(self)
 
 
 @dataclass(frozen=True)
 class PropertyStmt(Stmt):
     name: Token
     type: TypeExpr
+    constraint: Optional[Expr]
 
     def accept(self, visitor: Stmt.Visitor[T]) -> T:
         return visitor.visit_property_stmt(self)
 
 
+@dataclass(frozen=True)
+class ExtendStmt(Stmt):
+    type: TypeExpr
+    operations: list[OpStmt]
+
+    def accept(self, visitor: Stmt.Visitor[T]) -> T:
+        return visitor.visit_extend_stmt(self)
+
+
 @dataclass(frozen=True)
 class OpStmt(Stmt):
-    left: TypeExpr
+    name: Token
-    op: Token
+    operand: TypeExpr
-    right: TypeExpr
     result: TypeExpr
 
     def accept(self, visitor: Stmt.Visitor[T]) -> T:
@@ -62,15 +94,19 @@ class OpStmt(Stmt):
 
 
 @dataclass(frozen=True)
-class ConstraintStmt(Stmt):
+class PredicateStmt(Stmt):
     name: Token
-    constraint: ConstraintExpr
+    subject: Token
+    type: TypeExpr
+    condition: Expr
 
     def accept(self, visitor: Stmt.Visitor[T]) -> T:
-        return visitor.visit_constraint_stmt(self)
+        return visitor.visit_predicate_stmt(self)
 
 
-# Expressions
+###############
+# Expressions #
+###############
 
 
 @dataclass(frozen=True)
@@ -80,27 +116,100 @@ class Expr(ABC):
 
     class Visitor(ABC, Generic[T]):
         @abstractmethod
-        def visit_wildcard_expr(self, expr: WildcardExpr) -> T: ...
+        def visit_simple_type_expr(self, expr: SimpleTypeExpr) -> T: ...
+
+        @abstractmethod
+        def visit_logical_expr(self, expr: LogicalExpr) -> T: ...
+
+        @abstractmethod
+        def visit_binary_expr(self, expr: BinaryExpr) -> T: ...
+
+        @abstractmethod
+        def visit_unary_expr(self, expr: UnaryExpr) -> T: ...
+
+        @abstractmethod
+        def visit_get_expr(self, expr: GetExpr) -> T: ...
+
+        @abstractmethod
+        def visit_variable_expr(self, expr: VariableExpr) -> T: ...
+
+        @abstractmethod
+        def visit_grouping_expr(self, expr: GroupingExpr) -> T: ...
 
         @abstractmethod
         def visit_literal_expr(self, expr: LiteralExpr) -> T: ...
 
+        @abstractmethod
+        def visit_wildcard_expr(self, expr: WildcardExpr) -> T: ...
+
+        @abstractmethod
+        def visit_template_expr(self, expr: TemplateExpr) -> T: ...
+
         @abstractmethod
         def visit_type_expr(self, expr: TypeExpr) -> T: ...
 
-        @abstractmethod
-        def visit_constraint_expr(self, expr: ConstraintExpr) -> T: ...
-
-        @abstractmethod
-        def visit_type_body_expr(self, expr: TypeBodyExpr) -> T: ...
-
 
 @dataclass(frozen=True)
-class WildcardExpr(Expr):
+class SimpleTypeExpr(Expr):
-    token: Token
+    name: Token
+    optional: bool
 
     def accept(self, visitor: Expr.Visitor[T]) -> T:
-        return visitor.visit_wildcard_expr(self)
+        return visitor.visit_simple_type_expr(self)
+
+
+@dataclass(frozen=True)
+class LogicalExpr(Expr):
+    left: Expr
+    operator: Token
+    right: Expr
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_logical_expr(self)
+
+
+@dataclass(frozen=True)
+class BinaryExpr(Expr):
+    left: Expr
+    operator: Token
+    right: Expr
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_binary_expr(self)
+
+
+@dataclass(frozen=True)
+class UnaryExpr(Expr):
+    operator: Token
+    right: Expr
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_unary_expr(self)
+
+
+@dataclass(frozen=True)
+class GetExpr(Expr):
+    expr: Expr
+    name: Token
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_get_expr(self)
+
+
+@dataclass(frozen=True)
+class VariableExpr(Expr):
+    name: Token
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_variable_expr(self)
+
+
+@dataclass(frozen=True)
+class GroupingExpr(Expr):
+    expr: Expr
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_grouping_expr(self)
 
 
 @dataclass(frozen=True)
@@ -111,28 +220,27 @@ class LiteralExpr(Expr):
         return visitor.visit_literal_expr(self)
 
 
+@dataclass(frozen=True)
+class WildcardExpr(Expr):
+    token: Token
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_wildcard_expr(self)
+
+
+@dataclass(frozen=True)
+class TemplateExpr(Expr):
+    type: TypeExpr
+
+    def accept(self, visitor: Expr.Visitor[T]) -> T:
+        return visitor.visit_template_expr(self)
+
+
 @dataclass(frozen=True)
 class TypeExpr(Expr):
     name: Token
-    constraints: list[ConstraintExpr]
+    template: Optional[TemplateExpr]
+    optional: bool
 
     def accept(self, visitor: Expr.Visitor[T]) -> T:
         return visitor.visit_type_expr(self)
-
-
-@dataclass(frozen=True)
-class ConstraintExpr(Expr):
-    left: Expr
-    op: Token
-    right: Expr
-
-    def accept(self, visitor: Expr.Visitor[T]) -> T:
-        return visitor.visit_constraint_expr(self)
-
-
-@dataclass(frozen=True)
-class TypeBodyExpr(Expr):
-    properties: list[PropertyStmt]
-
-    def accept(self, visitor: Expr.Visitor[T]) -> T:
-        return visitor.visit_type_body_expr(self)

core/ast/printer.py

@@ -38,8 +38,8 @@ class AstPrinter(Generic[T]):
         return self._buf.getvalue()
 
     @contextmanager
-    def _child_level(self, last: bool = False) -> Generator[None, None, None]:
+    def _child_level(self, single: bool = False) -> Generator[None, None, None]:
-        self._levels.append(_Level.LAST if last else _Level.ACTIVE)
+        self._levels.append(_Level.LAST if single else _Level.ACTIVE)
         try:
             yield
         finally:
@@ -79,108 +79,169 @@ class AstPrinter(Generic[T]):
             self._write_line(f"{label}: None")
         else:
             self._write_line(label)
-            with self._child_level(last=True):
+            with self._child_level(single=True):
                 child.accept(self)
 
 
 class MidasAstPrinter(AstPrinter, m.Expr.Visitor[None], m.Stmt.Visitor[None]):
-    def visit_type_stmt(self, stmt: m.TypeStmt):
+    #Statements
-        self._write_line("TypeStmt")
+
+    def visit_simple_type_stmt(self, stmt: m.SimpleTypeStmt):
+        self._write_line("SimpleTypeStmt")
         with self._child_level():
             self._write_line(f'name: "{stmt.name.lexeme}"')
-            self._write_line("bases")
+            self._write_optional_child("template", stmt.template)
+            self._write_line("base")
+            with self._child_level(single=True):
+                stmt.base.accept(self)
+            self._write_optional_child("constraint", stmt.constraint, last=True)
+
+    def visit_complex_type_stmt(self, stmt: m.ComplexTypeStmt):
+        self._write_line("ComplexTypeStmt")
+        with self._child_level():
+            self._write_line(f'name: "{stmt.name.lexeme}"')
+            self._write_optional_child("template", stmt.template)
+            self._write_line("properties", last=True)
             with self._child_level():
-                for i, base in enumerate(stmt.bases):
+                for i, prop in enumerate(stmt.properties):
                     self._idx = i
-                    if i == len(stmt.bases) - 1:
+                    if i == len(stmt.properties) - 1:
                         self._mark_last()
-                    base.accept(self)
+                    prop.accept(self)
-            self._write_optional_child("body", stmt.body, last=True)
 
     def visit_property_stmt(self, stmt: m.PropertyStmt):
         self._write_line("PropertyStmt")
         with self._child_level():
             self._write_line(f'name: "{stmt.name.lexeme}"')
-            self._write_line("type", last=True)
+            self._write_line("type")
-            with self._child_level():
+            with self._child_level(single=True):
-                self._mark_last()
                 stmt.type.accept(self)
+            self._write_optional_child("constraint", stmt.constraint, last=True)
+
+    def visit_extend_stmt(self, stmt: m.ExtendStmt) -> None:
+        self._write_line("ExtendStmt")
+        with self._child_level():
+            self._write_line("type")
+            with self._child_level(single=True):
+                stmt.type.accept(self)
+            self._write_line("operations", last=True)
+            with self._child_level():
+                for i, op in enumerate(stmt.operations):
+                    self._idx = i
+                    if i == len(stmt.operations) - 1:
+                        self._mark_last()
+                    op.accept(self)
 
     def visit_op_stmt(self, stmt: m.OpStmt) -> None:
         self._write_line("OpStmt")
         with self._child_level():
-            self._write_line("left")
+            self._write_line(f'name: "{stmt.name.lexeme}"')
-            with self._child_level():
-                self._mark_last()
-                stmt.left.accept(self)
 
-            self._write_line(f'op: "{stmt.op.lexeme}"')
+            self._write_line("operand")
-
+            with self._child_level(single=True):
-            self._write_line("right")
+                stmt.operand.accept(self)
-            with self._child_level():
-                self._mark_last()
-                stmt.right.accept(self)
 
             self._write_line("result", last=True)
-            with self._child_level():
+            with self._child_level(single=True):
-                self._mark_last()
                 stmt.result.accept(self)
 
-    def visit_constraint_stmt(self, stmt: m.ConstraintStmt):
+    def visit_predicate_stmt(self, stmt: m.PredicateStmt):
-        self._write_line("ConstraintStmt")
+        self._write_line("PredicateStmt")
         with self._child_level():
             self._write_line(f'name: "{stmt.name.lexeme}"')
-            self._write_line("constraint", last=True)
+            self._write_line(f'subject: "{stmt.subject.lexeme}"')
-            with self._child_level():
+            self._write_line("type")
-                self._mark_last()
+            with self._child_level(single=True):
-                stmt.constraint.accept(self)
+                stmt.type.accept(self)
+            self._write_line("condition", last=True)
+            with self._child_level(single=True):
+                stmt.condition.accept(self)
 
-    def visit_type_expr(self, expr: m.TypeExpr):
+    # Expressions
-        self._write_line("TypeExpr")
+
+    def visit_simple_type_expr(self, expr: m.SimpleTypeExpr):
+        self._write_line("SimpleTypeExpr")
         with self._child_level():
             self._write_line(f'name: "{expr.name.lexeme}"')
-            self._write_line("constraints", last=True)
+            self._write_line(f"optional: {expr.optional}", last=True)
-            with self._child_level():
-                for i, constraint in enumerate(expr.constraints):
-                    self._idx = i
-                    if i == len(expr.constraints) - 1:
-                        self._mark_last()
-                    constraint.accept(self)
 
-    def visit_constraint_expr(self, expr: m.ConstraintExpr):
+    def visit_logical_expr(self, expr: m.LogicalExpr):
-        self._write_line("ConstraintExpr")
+        self._write_line("LogicalExpr")
         with self._child_level():
             self._write_line("left")
-            with self._child_level():
+            with self._child_level(single=True):
-                self._mark_last()
                 expr.left.accept(self)
 
-            self._write_line(f"operator: {expr.op.lexeme}")
+            self._write_line(f"operator: {expr.operator.lexeme}")
 
             self._write_line("right", last=True)
-            with self._child_level():
+            with self._child_level(single=True):
-                self._mark_last()
                 expr.right.accept(self)
 
-    def visit_type_body_expr(self, expr: m.TypeBodyExpr):
+    def visit_binary_expr(self, expr: m.BinaryExpr):
-        self._write_line("TypeBodyExpr")
+        self._write_line("BinaryExpr")
         with self._child_level():
-            self._write_line("properties", last=True)
+            self._write_line("left")
-            with self._child_level():
+            with self._child_level(single=True):
-                for i, property in enumerate(expr.properties):
+                expr.left.accept(self)
-                    self._idx = i
-                    if i == len(expr.properties) - 1:
-                        self._mark_last()
-                    property.accept(self)
 
-    def visit_wildcard_expr(self, expr: m.WildcardExpr) -> None:
+            self._write_line(f"operator: {expr.operator.lexeme}")
-        self._write_line("WildcardExpr")
+
+            self._write_line("right", last=True)
+            with self._child_level(single=True):
+                expr.right.accept(self)
+
+    def visit_unary_expr(self, expr: m.UnaryExpr):
+        self._write_line("UnaryExpr")
+        with self._child_level():
+            self._write_line(f"operator: {expr.operator.lexeme}")
+
+            self._write_line("right", last=True)
+            with self._child_level(single=True):
+                expr.right.accept(self)
+
+    def visit_get_expr(self, expr: m.GetExpr):
+        self._write_line("GetExpr")
+        with self._child_level():
+            self._write_line("expr")
+            with self._child_level(single=True):
+                expr.expr.accept(self)
+            self._write_line(f'name: "{expr.name.lexeme}"', last=True)
+
+    def visit_variable_expr(self, expr: m.VariableExpr):
+        self._write_line("VariableExpr")
+        with self._child_level():
+            self._write_line(f'name: "{expr.name.lexeme}"', last=True)
+
+    def visit_grouping_expr(self, expr: m.GroupingExpr):
+        self._write_line("GroupingExpr")
+        with self._child_level():
+            self._write_line("expr", last=True)
+            with self._child_level(single=True):
+                expr.expr.accept(self)
 
     def visit_literal_expr(self, expr: m.LiteralExpr) -> None:
         self._write_line("LiteralExpr")
         with self._child_level():
             self._write_line(f"value: {expr.value}", last=True)
 
+    def visit_wildcard_expr(self, expr: m.WildcardExpr) -> None:
+        self._write_line("WildcardExpr")
+
+    def visit_template_expr(self, expr: m.TemplateExpr) -> None:
+        self._write_line("TemplateExpr")
+        with self._child_level(single=True):
+            self._write_line("type")
+            with self._child_level(single=True):
+                expr.type.accept(self)
+
+    def visit_type_expr(self, expr: m.TypeExpr):
+        self._write_line("TypeExpr")
+        with self._child_level():
+            self._write_line(f'name: "{expr.name.lexeme}"')
+            self._write_optional_child("template", expr.template)
+            self._write_line(f"optional: {expr.optional}", last=True)
+
 
 class MidasPrinter(m.Expr.Visitor[str], m.Stmt.Visitor[str]):
     def __init__(self, indent: int = 4):
@@ -194,56 +255,94 @@ class MidasPrinter(m.Expr.Visitor[str], m.Stmt.Visitor[str]):
         self.level = 0
         return expr.accept(self)
 
-    def visit_type_stmt(self, stmt: m.TypeStmt):
+    def visit_simple_type_stmt(self, stmt: m.SimpleTypeStmt):
-        bases: list[str] = [b.accept(self) for b in stmt.bases]
+        template: str = stmt.template.accept(self) if stmt.template is not None else ""
-
+        res: str = f"type {stmt.name.lexeme}{template}({stmt.base.accept(self)})"
-        res: str = self.indented(f"type {stmt.name.lexeme}<{', '.join(bases)}>")
+        if stmt.constraint is not None:
-        if stmt.body is not None:
+            res += " where " + stmt.constraint.accept(self)
-            res += " {\n"
+        return self.indented(res)
-            self.level += 1
-            res += stmt.body.accept(self)
-            self.level -= 1
-            res += "\n" + self.indented("}")
 
+    def visit_complex_type_stmt(self, stmt: m.ComplexTypeStmt):
+        template: str = stmt.template.accept(self) if stmt.template is not None else ""
+        res: str = self.indented(f"type {stmt.name.lexeme}{template}")
+        res += " {\n"
+        self.level += 1
+        for prop in stmt.properties:
+            res += prop.accept(self)
+            res += "\n"
+        self.level -= 1
+        res += self.indented("}")
         return res
 
     def visit_property_stmt(self, stmt: m.PropertyStmt):
-        return f"{stmt.name.lexeme}: {stmt.type.accept(self)}"
+        res: str = f"{stmt.name.lexeme}: {stmt.type.accept(self)}"
+        if stmt.constraint is not None:
+            res += " where " + stmt.constraint.accept(self)
+        return self.indented(res)
+
+    def visit_extend_stmt(self, stmt: m.ExtendStmt):
+        res: str = self.indented(f"extend {stmt.type.accept(self)}")
+        res += " {\n"
+        self.level += 1
+        for op in stmt.operations:
+            res += op.accept(self)
+        self.level -= 1
+        res += "\n" + self.indented("}")
+        return res
 
     def visit_op_stmt(self, stmt: m.OpStmt):
-        left: str = stmt.left.accept(self)
+        operand: str = stmt.operand.accept(self)
-        op: str = stmt.op.lexeme
-        right: str = stmt.right.accept(self)
         result: str = stmt.result.accept(self)
-        return self.indented(f"op <{left}> {op} <{right}> = <{result}>")
+        return self.indented(f"op {stmt.name.lexeme}({operand}) -> {result}")
 
-    def visit_constraint_stmt(self, stmt: m.ConstraintStmt):
+    def visit_predicate_stmt(self, stmt: m.PredicateStmt):
         name: str = stmt.name.lexeme
-        constraint: str = stmt.constraint.accept(self)
+        subject: str = stmt.subject.lexeme
-        return self.indented(f"constraint {name} = {constraint}")
+        type: str = stmt.type.accept(self)
+        condition: str = stmt.condition.accept(self)
+        return self.indented(f"predicate {name}({subject}: {type}) = {condition}")
 
-    def visit_type_expr(self, expr: m.TypeExpr):
+    def visit_simple_type_expr(self, expr: m.SimpleTypeExpr):
-        parts: list[str] = [expr.name.lexeme]
+        return f"{expr.name.lexeme}{'?' if expr.optional else ''}"
-        for constraint in expr.constraints:
-            parts.append("(" + constraint.accept(self) + ")")
-        return " + ".join(parts)
 
-    def visit_constraint_expr(self, expr: m.ConstraintExpr):
+    def visit_logical_expr(self, expr: m.LogicalExpr):
-        parts: list[str] = [
+        left: str = expr.left.accept(self)
-            expr.left.accept(self),
+        operator: str = expr.operator.lexeme
-            expr.op.lexeme,
+        right: str = expr.right.accept(self)
-            expr.right.accept(self),
+        return f"{left} {operator} {right}"
-        ]
-        return " ".join(parts)
 
-    def visit_type_body_expr(self, expr: m.TypeBodyExpr):
+    def visit_binary_expr(self, expr: m.BinaryExpr):
-        properties: list[str] = [
+        left: str = expr.left.accept(self)
-            self.indented(prop.accept(self)) for prop in expr.properties
+        operator: str = expr.operator.lexeme
-        ]
+        right: str = expr.right.accept(self)
-        return "\n".join(properties)
+        return f"{left} {operator} {right}"
+
+    def visit_unary_expr(self, expr: m.UnaryExpr):
+        operator: str = expr.operator.lexeme
+        right: str = expr.right.accept(self)
+        return f"{operator}{right}"
+
+    def visit_get_expr(self, expr: m.GetExpr):
+        expr_: str = expr.expr.accept(self)
+        name: str = expr.name.lexeme
+        return f"{expr_}.{name}"
+
+    def visit_variable_expr(self, expr: m.VariableExpr):
+        return expr.name.lexeme
+
+    def visit_grouping_expr(self, expr: m.GroupingExpr):
+        expr_: str = expr.expr.accept(self)
+        return f"({expr_})"
+
+    def visit_literal_expr(self, expr: m.LiteralExpr):
+        return str(expr.value)
 
     def visit_wildcard_expr(self, expr: m.WildcardExpr):
         return "_"
 
-    def visit_literal_expr(self, expr: m.LiteralExpr):
+    def visit_template_expr(self, expr: m.TemplateExpr):
-        return str(expr.value)
+        return f"[{expr.type.accept(self)}]"
+
+    def visit_type_expr(self, expr: m.TypeExpr):
+        template: str = expr.template.accept(self) if expr.template is not None else ""
+        return f"{expr.name.lexeme}{template}{'?' if expr.optional else ''}"

gen/ast.py

@@ -0,0 +1,72 @@
+class SimpleTypeStmt:
+    name: Token
+    template: Optional[TemplateExpr]
+    base: TypeExpr
+    constraint: Optional[Expr]
+
+class SimpleTypeExpr:
+    name: Token
+    optional: bool
+
+class LogicalExpr:
+    left: Expr
+    operator: Token
+    right: Expr
+
+class BinaryExpr:
+    left: Expr
+    operator: Token
+    right: Expr
+
+class UnaryExpr:
+    operator: Token
+    right: Expr
+
+class GetExpr:
+    expr: Expr
+    name: Token
+
+class VariableExpr:
+    name: Token
+
+class GroupingExpr:
+    expr: Expr
+
+class LiteralExpr:
+    value: Any
+
+class WildcardExpr:
+    token: Token
+
+class TemplateExpr:
+    type: TypeExpr
+
+class TypeExpr:
+    name: Token
+    template: Optional[TemplateExpr]
+    optional: bool
+
+class ComplexTypeStmt:
+    name: Token
+    template: Optional[TemplateExpr]
+    properties: list[PropertyStmt]
+
+class PropertyStmt:
+    name: Token
+    type: TypeExpr
+    constraint: Optional[Expr]
+
+class ExtendStmt:
+    type: TypeExpr
+    operations: list[OpStmt]
+
+class OpStmt:
+    name: Token
+    operand: TypeExpr
+    result: TypeExpr
+
+class PredicateStmt:
+    name: Token
+    subject: Token
+    type: TypeExpr
+    condition: Expr

gen/gen.py

@@ -0,0 +1,128 @@
+from pathlib import Path
+import re
+
+HEADER = '''"""
+This file was generated by a script. Any manual changes might be overwritten.
+Please modify gen/ast.py instead and run gen/gen.py
+"""'''
+
+TEMPLATE = """{header}
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Generic, Optional, TypeVar
+
+from lexer.token import Token
+
+T = TypeVar("T")
+
+##############
+# Statements #
+##############
+
+
+@dataclass(frozen=True)
+class Stmt(ABC):
+    @abstractmethod
+    def accept(self, visitor: Visitor[T]) -> T: ...
+
+    class Visitor(ABC, Generic[T]):
+{stmt_visitor_methods}
+
+
+{statements}
+
+
+###############
+# Expressions #
+###############
+
+
+@dataclass(frozen=True)
+class Expr(ABC):
+    @abstractmethod
+    def accept(self, visitor: Visitor[T]) -> T: ...
+
+    class Visitor(ABC, Generic[T]):
+{expr_visitor_methods}
+
+
+{expressions}
+"""
+
+VISITOR_METHOD_TEMPLATE = """
+        @abstractmethod
+        def visit_{func_name}(self, {param}: {cls}) -> T: ...
+"""
+
+CLASS_TEMPLATE = """
+@dataclass(frozen=True)
+class {cls}({base}):
+{body}
+
+    def accept(self, visitor: {base}.Visitor[T]) -> T:
+        return visitor.visit_{func_name}(self)
+"""
+
+def snake_case(text: str) -> str:
+    return re.sub(r"[A-Z]", lambda c: "_" + c.group().lower(), text).lower().strip("_")
+
+def make_visitor_method(cls: str, param: str):
+    method: str = VISITOR_METHOD_TEMPLATE.format(
+        func_name=snake_case(cls),
+        param=param,
+        cls=cls
+    )
+    return method.strip("\n")
+
+def make_class(name: str, cls: str, base: str):
+    body: str = cls.split("\n", 1)[1]
+    func_name: str = snake_case(name)
+    cls_def: str = CLASS_TEMPLATE.format(
+        cls=name,
+        base=base,
+        body=body,
+        func_name=func_name,
+    )
+    return cls_def.strip("\n")
+
+def generate(src: str):
+    classes: list[str] = src.split("\n\n")
+    stmt_visitor_methods: list[str] = []
+    expr_visitor_methods: list[str] = []
+    statements: list[str] = []
+    expressions: list[str] = []
+
+    for cls in classes:
+        cls = cls.strip("\n")
+        name: str = re.match("class (.*?):", cls).group(1)  # type: ignore
+        print(f"Processing {name}")
+        if name.endswith("Stmt"):
+            stmt_visitor_methods.append(make_visitor_method(name, "stmt"))
+            statements.append(make_class(name, cls, "Stmt"))
+        elif name.endswith("Expr"):
+            expr_visitor_methods.append(make_visitor_method(name, "expr"))
+            expressions.append(make_class(name, cls, "Expr"))
+
+    return TEMPLATE.format(
+        header=HEADER,
+        stmt_visitor_methods="\n\n".join(stmt_visitor_methods),
+        expr_visitor_methods="\n\n".join(expr_visitor_methods),
+        statements="\n\n\n".join(statements),
+        expressions="\n\n\n".join(expressions),
+    )
+
+def main():
+    root: Path = Path(__file__).parent.parent
+    in_path: Path = root / "gen" / "ast.py"
+    out_path: Path = root / "core" / "ast" / "midas.py"
+
+    src: str = in_path.read_text()
+    generated: str = generate(src)
+    out_path.write_text(generated)
+
+
+if __name__ == "__main__":
+    main()

lexer/keyword.py

@@ -3,7 +3,9 @@ from lexer.token import TokenType
 KEYWORDS: dict[str, TokenType] = {
     "type": TokenType.TYPE,
     "op": TokenType.OP,
-    "constraint": TokenType.CONSTRAINT,
+    "predicate": TokenType.PREDICATE,
+    "extend": TokenType.EXTEND,
+    "where": TokenType.WHERE,
     "true": TokenType.TRUE,
     "false": TokenType.FALSE,
     "none": TokenType.NONE,

lexer/midas.py

@@ -31,30 +31,32 @@ class MidasLexer(Lexer):
                 self.add_token(
                     TokenType.EQUAL_EQUAL if self.match("=") else TokenType.EQUAL
                 )
-            case "!":
+            case "!" if self.match("="):
-                if self.match("="):
+                self.add_token(TokenType.BANG_EQUAL)
-                    self.add_token(TokenType.BANG_EQUAL)
-                else:
-                    self.error("Unexpected single bang. Did you mean '!=' ?")
             case ":":
                 self.add_token(TokenType.COLON)
-            case ",":
+            case ".":
-                self.add_token(TokenType.COMMA)
+                self.add_token(TokenType.DOT)
-            case "_":
+            case "&":
+                self.add_token(TokenType.AND)
+            case "?":
+                self.add_token(TokenType.QMARK)
+            # case ",":
+            #     self.add_token(TokenType.COMMA)
+            case "_" if not self.is_identifier_char(self.peek_next(), start=False):
                 self.add_token(TokenType.UNDERSCORE)
-            case "+":
+            case "-" if self.match(">"):
-                self.add_token(TokenType.PLUS)
+                self.add_token(TokenType.ARROW)
+            # case "+":
+            #     self.add_token(TokenType.PLUS)
             case "-":
                 self.add_token(TokenType.MINUS)
-            case "*":
+            # case "*":
-                self.add_token(TokenType.STAR)
+            #     self.add_token(TokenType.STAR)
-            case "/":
+            case "/" if self.match("/"):
-                if self.match("/"):
+                self.scan_comment()
-                    self.scan_comment()
+            case "/" if self.match("*"):
-                elif self.match("*"):
+                self.scan_comment_multiline()
-                    self.scan_comment_multiline()
-                else:
-                    self.add_token(TokenType.SLASH)
             case "\n":
                 self.add_token(TokenType.NEWLINE)
             case " " | "\r" | "\t":
@@ -69,7 +71,7 @@ class MidasLexer(Lexer):
             case _:
                 if char.isdigit():
                     self.scan_number()
-                elif char.isalpha():
+                elif self.is_identifier_char(char, start=True):
                     self.scan_identifier()
                 else:
                     self.error("Unexpected character")
@@ -98,7 +100,7 @@ class MidasLexer(Lexer):
         An identifier starts with a letter, followed by any number of
         alphanumerical characters or underscores
         """
-        while self.peek().isalnum() or self.peek() == "_":
+        while self.is_identifier_char(self.peek(), start=False):
             self.advance()
 
         lexeme: str = self.source[self.start : self.idx]
@@ -129,3 +131,12 @@ class MidasLexer(Lexer):
         if not self.is_at_end():
             self.advance()
         self.add_token(TokenType.COMMENT)
+
+    def is_identifier_char(self, char: str, *, start: bool) -> bool:
+        if char == "_":
+            return True
+        if char.isalpha():
+            return True
+        if not start and char.isdigit():
+            return True
+        return False

lexer/token.py

@@ -14,14 +14,18 @@ class TokenType(Enum):
     LEFT_BRACE = auto()
     RIGHT_BRACE = auto()
     COLON = auto()
-    COMMA = auto()
+    # COMMA = auto()
     UNDERSCORE = auto()
+    ARROW = auto()
+    AND = auto()
+    QMARK = auto()
+    DOT = auto()
 
     # Operators
-    PLUS = auto()
+    # PLUS = auto()
     MINUS = auto()
-    STAR = auto()
+    # STAR = auto()
-    SLASH = auto()
+    # SLASH = auto()
     GREATER = auto()
     GREATER_EQUAL = auto()
     LESS = auto()
@@ -40,7 +44,9 @@ class TokenType(Enum):
     # Keywords
     TYPE = auto()
     OP = auto()
-    CONSTRAINT = auto()
+    PREDICATE = auto()
+    EXTEND = auto()
+    WHERE = auto()
 
     # Misc
     COMMENT = auto()

parser/midas.py

@@ -1,16 +1,24 @@
 from typing import Optional
 
 from core.ast.midas import (
-    ConstraintExpr,
+    BinaryExpr,
-    ConstraintStmt,
+    ComplexTypeStmt,
     Expr,
+    ExtendStmt,
+    GetExpr,
+    GroupingExpr,
     LiteralExpr,
+    LogicalExpr,
     OpStmt,
+    PredicateStmt,
     PropertyStmt,
+    SimpleTypeExpr,
+    SimpleTypeStmt,
     Stmt,
-    TypeBodyExpr,
+    TemplateExpr,
     TypeExpr,
-    TypeStmt,
+    UnaryExpr,
+    VariableExpr,
     WildcardExpr,
 )
 from lexer.token import Token, TokenType
@@ -21,7 +29,12 @@ from parser.errors import ParsingError
 class MidasParser(Parser):
     """A simple parser for midas type definitions"""
 
-    SYNC_BOUNDARY: set[TokenType] = {TokenType.TYPE, TokenType.OP, TokenType.CONSTRAINT}
+    SYNC_BOUNDARY: set[TokenType] = {
+        TokenType.TYPE,
+        TokenType.OP,
+        TokenType.EXTEND,
+        TokenType.PREDICATE,
+    }
 
     def parse(self) -> list[Stmt]:
         statements: list[Stmt] = []
@@ -58,16 +71,16 @@ class MidasParser(Parser):
         try:
             if self.match(TokenType.TYPE):
                 return self.type_declaration()
-            if self.match(TokenType.OP):
+            if self.match(TokenType.EXTEND):
-                return self.op_declaration()
+                return self.extend_declaration()
-            if self.match(TokenType.CONSTRAINT):
+            if self.match(TokenType.PREDICATE):
-                return self.constraint_declaration()
+                return self.predicate_declaration()
             raise self.error(self.peek(), "Unexpected token")
         except ParsingError:
             self.synchronize()
             return None
 
-    def type_declaration(self) -> TypeStmt:
+    def type_declaration(self) -> SimpleTypeStmt | ComplexTypeStmt:
         """Parse a type declaration
 
         A type declaration is written `type Name<TypeExpr, ...>` optionally followed by a brace-wrapped body
@@ -76,19 +89,28 @@ class MidasParser(Parser):
             TypeStmt: the parsed type declaration statement
         """
         name: Token = self.consume(TokenType.IDENTIFIER, "Expected type name")
-        self.consume(TokenType.LESS, "Expected '<' after type name")
+        template: Optional[TemplateExpr] = None
-        bases: list[TypeExpr] = []
+        if self.check(TokenType.LEFT_BRACKET):
-        while not self.check(TokenType.GREATER) and not self.is_at_end():
+            template = self.template_expr()
-            bases.append(self.type_expr())
-            if not self.check(TokenType.GREATER):
-                self.consume(TokenType.COMMA, "Expected ',' between type bases")
-        self.consume(TokenType.GREATER, "Expected '>' after base type")
 
-        body: Optional[TypeBodyExpr] = None
+        if self.match(TokenType.LEFT_PAREN):
+            base: TypeExpr = self.type_expr()
+            self.consume(TokenType.RIGHT_PAREN, "Unclosed base type parenthesis")
+            constraint: Optional[Expr] = None
+            if self.match(TokenType.WHERE):
+                constraint = self.constraint()
+            return SimpleTypeStmt(
+                name=name, template=template, base=base, constraint=constraint
+            )
+        else:
+            properties: list[PropertyStmt] = self.type_properties()
+            return ComplexTypeStmt(name=name, template=template, properties=properties)
 
-        if self.check(TokenType.LEFT_BRACE):
+    def template_expr(self) -> TemplateExpr:
-            body = self.type_body_expr()
+        self.consume(TokenType.LEFT_BRACKET, "Missing '[' before template expression")
-        return TypeStmt(name=name, bases=bases, body=body)
+        type: TypeExpr = self.type_expr()
+        self.consume(TokenType.RIGHT_BRACKET, "Missing ']' after template expression")
+        return TemplateExpr(type=type)
 
     def type_expr(self) -> TypeExpr:
         """Parse a type expression
@@ -97,33 +119,66 @@ class MidasParser(Parser):
             TypeExpr: the parsed type expression
         """
         name: Token = self.consume(TokenType.IDENTIFIER, "Expected type name")
-        constraints: list[ConstraintExpr] = []
+        template: Optional[TemplateExpr] = None
+        if self.check(TokenType.LEFT_BRACKET):
+            template = self.template_expr()
+        optional: bool = self.match(TokenType.QMARK)
+        return TypeExpr(name=name, template=template, optional=optional)
 
-        while not self.is_at_end() and self.match(TokenType.PLUS):
+    def simple_type_expr(self) -> SimpleTypeExpr:
-            self.consume(TokenType.LEFT_PAREN, "Expected '(' before type constraint")
+        name: Token = self.consume(TokenType.IDENTIFIER, "Expected type name")
-            constraints.append(self.constraint_expr())
+        optional: bool = self.match(TokenType.QMARK)
-            self.consume(TokenType.RIGHT_PAREN, "Expected ')' after type constraint")
+        return SimpleTypeExpr(name=name, optional=optional)
 
-        return TypeExpr(name=name, constraints=constraints)
+    def constraint(self) -> Expr:
+        return self.and_()
 
-    def constraint_expr(self) -> ConstraintExpr:
+    def and_(self) -> Expr:
-        """Parse a type constraint
+        expr: Expr = self.equality()
+        while self.match(TokenType.AND):
+            operator: Token = self.previous()
+            right: Expr = self.equality()
+            expr = LogicalExpr(left=expr, operator=operator, right=right)
+        return expr
 
-        Returns:
+    def equality(self) -> Expr:
-            ConstraintExpr: the parsed type constraint expression
+        expr: Expr = self.comparison()
-        """
+        while self.match(TokenType.BANG_EQUAL, TokenType.EQUAL_EQUAL):
+            operator: Token = self.previous()
+            right: Expr = self.comparison()
+            expr = BinaryExpr(left=expr, operator=operator, right=right)
+        return expr
 
-        left: Expr = self.constraint_value()
+    def comparison(self) -> Expr:
-        op: Token = self.constraint_operator()
+        expr: Expr = self.unary()
-        right: Expr = self.constraint_value()
+        while self.match(
-        return ConstraintExpr(left=left, op=op, right=right)
+            TokenType.LESS,
+            TokenType.LESS_EQUAL,
+            TokenType.GREATER,
+            TokenType.GREATER_EQUAL,
+        ):
+            operator: Token = self.previous()
+            right: Expr = self.unary()
+            expr = BinaryExpr(left=expr, operator=operator, right=right)
+        return expr
 
-    def constraint_value(self) -> Expr:
+    def unary(self) -> Expr:
-        if self.match(TokenType.UNDERSCORE):
+        if self.match(TokenType.MINUS):
-            return WildcardExpr(self.previous())
+            operator: Token = self.previous()
-        return self.literal()
+            right: Expr = self.unary()
+            return UnaryExpr(operator=operator, right=right)
+        return self.reference()
 
-    def literal(self) -> LiteralExpr:
+    def reference(self) -> Expr:
+        expr: Expr = self.primary()
+        while self.match(TokenType.DOT):
+            name: Token = self.consume(
+                TokenType.IDENTIFIER, "Expected property name after '.'"
+            )
+            expr = GetExpr(expr=expr, name=name)
+        return expr
+
+    def primary(self) -> Expr:
         if self.match(TokenType.FALSE):
             return LiteralExpr(False)
         if self.match(TokenType.TRUE):
@@ -134,35 +189,34 @@ class MidasParser(Parser):
         if self.match(TokenType.NUMBER):
             return LiteralExpr(self.previous().value)
 
-        raise self.error(self.peek(), "Expected literal")
+        if self.match(TokenType.IDENTIFIER):
+            return VariableExpr(self.previous())
 
-    def constraint_operator(self) -> Token:
+        if self.match(TokenType.UNDERSCORE):
-        if self.match(
+            return WildcardExpr(self.previous())
-            TokenType.LESS,
-            TokenType.LESS_EQUAL,
-            TokenType.GREATER,
-            TokenType.GREATER_EQUAL,
-            TokenType.EQUAL_EQUAL,
-            TokenType.BANG_EQUAL,
-        ):
-            return self.previous()
-        raise self.error(self.peek(), "Expected constraint operator")
 
-    def type_body_expr(self) -> TypeBodyExpr:
+        if self.match(TokenType.LEFT_PAREN):
+            expr: Expr = self.constraint()
+            self.consume(TokenType.RIGHT_PAREN, "Unclosed parenthesis")
+            return GroupingExpr(expr)
+
+        raise self.error(self.peek(), "Expected expression")
+
+    def type_properties(self) -> list[PropertyStmt]:
         """Parse a type definition body
 
         A type definition body is a set of whitespace-separated
         property statements enclosed in curly braces
 
         Returns:
-            TypeBodyExpr: the parsed type body expression
+            TypeBodyStmt: the parsed type body expression
         """
         self.consume(TokenType.LEFT_BRACE, "Expected '{' to start type body")
         properties: list[PropertyStmt] = []
         while not self.check(TokenType.RIGHT_BRACE) and not self.is_at_end():
             properties.append(self.property_stmt())
         self.consume(TokenType.RIGHT_BRACE, "Unclosed type body")
-        return TypeBodyExpr(properties=properties)
+        return properties
 
     def property_stmt(self) -> PropertyStmt:
         """Parse a property statement
@@ -175,7 +229,19 @@ class MidasParser(Parser):
         name: Token = self.consume(TokenType.IDENTIFIER, "Expected property name")
         self.consume(TokenType.COLON, "Expected ':' after property name")
         type: TypeExpr = self.type_expr()
-        return PropertyStmt(name=name, type=type)
+        constraint: Optional[Expr] = None
+        if self.match(TokenType.WHERE):
+            constraint = self.constraint()
+        return PropertyStmt(name=name, type=type, constraint=constraint)
+
+    def extend_declaration(self) -> ExtendStmt:
+        type: TypeExpr = self.type_expr()
+        self.consume(TokenType.LEFT_BRACE, "Expected '{' to start extend body")
+        operations: list[OpStmt] = []
+        while not self.is_at_end() and not self.check(TokenType.RIGHT_BRACE):
+            operations.append(self.op_declaration())
+        self.consume(TokenType.RIGHT_BRACE, "Unclosed extend body")
+        return ExtendStmt(type=type, operations=operations)
 
     def op_declaration(self) -> OpStmt:
         """Parse an operation definition
@@ -185,25 +251,19 @@ class MidasParser(Parser):
         Returns:
             OpStmt: the parsed operation statement
         """
-        self.consume(TokenType.LESS, "Expected '<' before first type")
+        self.consume(TokenType.OP, "Expected 'op' keyword")
-        left: TypeExpr = self.type_expr()
-        self.consume(TokenType.GREATER, "Expected '>' after first type")
 
-        op: Token = self.advance()
+        name: Token = self.consume(TokenType.IDENTIFIER, "Expected operation name")
+        self.consume(TokenType.LEFT_PAREN, "Expected '(' before operand type")
+        operand: TypeExpr = self.type_expr()
+        self.consume(TokenType.RIGHT_PAREN, "Expected ')' after operand type")
 
-        self.consume(TokenType.LESS, "Expected '<' before second type")
+        self.consume(TokenType.ARROW, "Expected '->' before result type")
-        right: TypeExpr = self.type_expr()
-        self.consume(TokenType.GREATER, "Expected '>' after second type")
-
-        self.consume(TokenType.EQUAL, "Expected '=' after second type")
-
-        self.consume(TokenType.LESS, "Expected '<' before result type")
         result: TypeExpr = self.type_expr()
-        self.consume(TokenType.GREATER, "Expected '>' after result type")
 
-        return OpStmt(left=left, op=op, right=right, result=result)
+        return OpStmt(name=name, operand=operand, result=result)
 
-    def constraint_declaration(self) -> ConstraintStmt:
+    def predicate_declaration(self) -> PredicateStmt:
         """Parse a type constraint declaration
 
         A constraint is written `constraint Name = constraint_expression`
@@ -211,7 +271,12 @@ class MidasParser(Parser):
         Returns:
             ConstraintStmt: the parsed constraint declaration statement
         """
-        name: Token = self.consume(TokenType.IDENTIFIER, "Expected constraint name")
+        name: Token = self.consume(TokenType.IDENTIFIER, "Expected predicate name")
-        self.consume(TokenType.EQUAL, "Expected '=' after constraint name")
+        self.consume(TokenType.LEFT_PAREN, "Expected '(' before predicate subject")
-        constraint: ConstraintExpr = self.constraint_expr()
+        subject: Token = self.consume(TokenType.IDENTIFIER, "Expected subject name")
-        return ConstraintStmt(name=name, constraint=constraint)
+        self.consume(TokenType.COLON, "Expected ':' after subject name")
+        type: TypeExpr = self.type_expr()
+        self.consume(TokenType.RIGHT_PAREN, "Expected ')' after predicate subject")
+        self.consume(TokenType.EQUAL, "Expected '=' after predicate subject")
+        condition: Expr = self.constraint()
+        return PredicateStmt(name=name, subject=subject, type=type, condition=condition)

test.py

@@ -1,40 +1,21 @@
-import importlib
+import json
 from pathlib import Path
 
-from core.ast.printer import AnnotationAstPrinter, MidasAstPrinter
+from core.ast.printer import MidasAstPrinter
-from lexer.annotations import AnnotationLexer
 from lexer.midas import MidasLexer
 from lexer.token import Token
-from parser.annotations import AnnotationParser
 from parser.midas import MidasParser
 
 
-def test_annotation():
-    # Frame annotation
-    mod = importlib.import_module("examples.00_syntax_prototype.01_simple_types")
-
-    annotation: str = mod.__annotations__["df"]
-    lexer: AnnotationLexer = AnnotationLexer(annotation, "01_simple_types.py")
-    tokens: list[Token] = lexer.process()
-    # print([f"{t.type.name}('{t.lexeme}')" for t in tokens])
-
-    parser = AnnotationParser(tokens)
-    parsed = parser.parse()
-    print(parsed)
-    for err in parser.errors:
-        print(err.get_report())
-    printer = AnnotationAstPrinter()
-    if parsed is not None:
-        print(printer.print(parsed))
-
-
 def test_midas():
     # Midas type definitions
-    path: Path = Path("examples") / "00_syntax_prototype" / "02_custom_types.midas"
+    path: Path = Path("examples") / "00_syntax_prototype" / "03_custom_types_v2.midas"
     definitions: str = path.read_text()
     midas_lexer: MidasLexer = MidasLexer(definitions, path.name)
     tokens: list[Token] = midas_lexer.process()
     # print([f"{t.type.name}('{t.lexeme}')" for t in tokens])
+    with open("tokens.json", "w") as f:
+        json.dump([f"{t.type.name}('{t.lexeme}')" for t in tokens], f, indent=4)
 
     parser = MidasParser(tokens)
     parsed = parser.parse()

tests/cases/parser/01_simple_types.midas

@@ -1,24 +1,57 @@
-// Simple custom type derived from floats
+// Simple custom type derived from float
-type Latitude<float>
+type Custom(float)
-type Longitude<float>
+
+// Simple custom types with constraints
+type Latitude(float) where (-90 <= _ <= 90)
+type Longitude(float) where (-180 <= _ <= 180)
+
+// Generic custom type (a Difference of T is derived from T, e.g. a difference of floats is a float
+type Difference[T](T)
 
 // Complex custom type, containing two values accessible through properties
-type GeoLocation<Latitude, Longitude> {
+type GeoLocation {
     lat: Latitude
     lon: Longitude
 }
 
-type LatitudeDiff<float>
+// Define operations on our custom type
-type LongitudeDiff<float>
+extend GeoLocation {
+    // This type is compatible with the `-` operation with another GeoLocation
+    // i.e. you can subtract a GeoLocation from another GeoLocation, resulting
+    // in a Difference of GeoLocations
+    op __sub__(GeoLocation) -> Difference[GeoLocation]
+}
+
+// For complex generics, you need to specify how the genericity the properties
+// are handled
+type Difference[GeoLocation] {
+    lat: Difference[Latitude]
+    lon: Difference[Longitude]
+}
 
 // Simple operation defined on our custom types
-op <Latitude> - <Latitude> = <LatitudeDiff>
+extend Latitude {
-op <Longitude> - <Longitude> = <LongitudeDiff>
+    op __sub__(Latitude) -> Difference[Latitude]
+}
 
-// Simple custom type with a constraint
+extend Longitude {
-type Age<int + (0 <= _) + (_ < 150)>
+    op __sub__(Longitude) -> Difference[Longitude]
+}
 
-// Predefined custom constraints that can be referenced in other definitions
+// Predefined custom predicates that can be referenced in other definitions
-constraint Positive = _ >= 0
+predicate Positive(v: float) = v >= 0
-constraint StrictlyPositive = _ > 0
+predicate StrictlyPositive(v: float) = v > 0
-//constraint Even = _ % 2 == 0
+predicate Equatorial(loc: GeoLocation) = (-10 <= loc.lat <= 10)
+predicate Arctic(loc: GeoLocation) = (loc.lat >= 66)
+
+type Person {
+    name: str
+
+    // Property with an inline constraint
+    age: int? where (0 <= _ < 150)
+
+    // Property referencing a predicate
+    height: float where StrictlyPositive
+
+    home: GeoLocation
+}

tests/lexer/test_midas_lexer.py

@@ -1,130 +0,0 @@
-from typing import Any
-
-import pytest
-
-from lexer.base import MidasSyntaxError
-from lexer.midas import MidasLexer
-from lexer.token import Token, TokenType
-
-
-def scan(source: str) -> list[Token]:
-    return MidasLexer(source).process()
-
-
-def assert_n_tokens(tokens: list[Token], n: int):
-    assert len(tokens) == n + 1
-    assert tokens[-1].type == TokenType.EOF
-
-
-@pytest.mark.parametrize(
-    "src,expected",
-    [
-        ("(", TokenType.LEFT_PAREN),
-        (")", TokenType.RIGHT_PAREN),
-        ("[", TokenType.LEFT_BRACKET),
-        ("]", TokenType.RIGHT_BRACKET),
-        ("{", TokenType.LEFT_BRACE),
-        ("}", TokenType.RIGHT_BRACE),
-        (":", TokenType.COLON),
-        (",", TokenType.COMMA),
-        ("_", TokenType.UNDERSCORE),
-    ],
-)
-def test_punctuation(src: str, expected: TokenType):
-    tokens: list[Token] = scan(src)
-    assert_n_tokens(tokens, 1)
-    assert tokens[0].type == expected
-
-
-@pytest.mark.parametrize(
-    "src,expected",
-    [
-        ("+", TokenType.PLUS),
-        ("-", TokenType.MINUS),
-        ("*", TokenType.STAR),
-        ("/", TokenType.SLASH),
-        (">", TokenType.GREATER),
-        (">=", TokenType.GREATER_EQUAL),
-        ("<", TokenType.LESS),
-        ("<=", TokenType.LESS_EQUAL),
-        ("=", TokenType.EQUAL),
-        ("==", TokenType.EQUAL_EQUAL),
-        ("!=", TokenType.BANG_EQUAL),
-    ],
-)
-def test_operators(src: str, expected: TokenType):
-    tokens: list[Token] = scan(src)
-    assert_n_tokens(tokens, 1)
-    assert tokens[0].type == expected
-
-
-@pytest.mark.parametrize(
-    "src,expected",
-    [
-        ("a", TokenType.IDENTIFIER),
-        ("foo", TokenType.IDENTIFIER),
-        ("foo1", TokenType.IDENTIFIER),
-        ("foo_", TokenType.IDENTIFIER),
-        ("foo_bar1_baz2", TokenType.IDENTIFIER),
-        ("FOO_BAR1_BAZ2", TokenType.IDENTIFIER),
-        ("true", TokenType.TRUE),
-        ("false", TokenType.FALSE),
-        ("none", TokenType.NONE),
-    ],
-)
-def test_identifiers_keywords(src: str, expected: TokenType):
-    tokens: list[Token] = scan(src)
-    assert_n_tokens(tokens, 1)
-    assert tokens[0].type == expected
-
-
-@pytest.mark.parametrize(
-    "src,expected",
-    [
-        ("// This is a comment", TokenType.COMMENT),
-        ("/* This is a comment */", TokenType.COMMENT),
-        (" ", TokenType.WHITESPACE),
-        ("\t", TokenType.WHITESPACE),
-        ("\r", TokenType.WHITESPACE),
-        ("  \t  \t", TokenType.WHITESPACE),
-        ("\n", TokenType.NEWLINE),
-    ],
-)
-def test_misc(src: str, expected: TokenType):
-    tokens: list[Token] = scan(src)
-    assert_n_tokens(tokens, 1)
-    assert tokens[0].type == expected
-
-
-@pytest.mark.parametrize(
-    "src,expected_type,expected_value",
-    [
-        ("0", TokenType.NUMBER, 0),
-        ("0.0", TokenType.NUMBER, 0),
-        ("1234.56", TokenType.NUMBER, 1234.56),
-    ],
-)
-def test_literals(src: str, expected_type: TokenType, expected_value: Any):
-    tokens: list[Token] = scan(src)
-    assert_n_tokens(tokens, 1)
-    assert tokens[0].type == expected_type
-    assert tokens[0].value == expected_value
-
-
-def test_single_bang_error():
-    with pytest.raises(MidasSyntaxError):
-        scan("!")
-
-
-@pytest.mark.parametrize(
-    "src",
-    [
-        "@",
-        '"',
-        "'",
-        ".",
-    ],
-)
-def test_unexpected_character(src: str):
-    with pytest.raises(MidasSyntaxError):
-        scan(src)

tests/parser/test_midas_parser.py

@@ -1,202 +0,0 @@
-import textwrap
-
-import pytest
-
-from core.ast.midas import (
-    ConstraintExpr,
-    ConstraintStmt,
-    Expr,
-    LiteralExpr,
-    OpStmt,
-    PropertyStmt,
-    Stmt,
-    TypeBodyExpr,
-    TypeExpr,
-    TypeStmt,
-    WildcardExpr,
-)
-from lexer.midas import MidasLexer
-from lexer.position import Position
-from lexer.token import Token
-from parser.midas import MidasParser
-
-
-class AstSerializer(Stmt.Visitor[str], Expr.Visitor[str]):
-    def serialize(self, stmt: Stmt):
-        return stmt.accept(self)
-
-    def visit_type_stmt(self, stmt: TypeStmt) -> str:
-        res: str = f"(type_def {stmt.name.lexeme}"
-        for base in stmt.bases:
-            res += " " + base.accept(self)
-        if stmt.body is not None:
-            res += " " + stmt.body.accept(self)
-        res += ")"
-        return res
-
-    def visit_type_expr(self, expr: TypeExpr) -> str:
-        res: str = f"({expr.name.lexeme}"
-        for constraint in expr.constraints:
-            res += " " + constraint.accept(self)
-        res += ")"
-        return res
-
-    def visit_constraint_expr(self, expr: ConstraintExpr) -> str:
-        return f"(constraint {expr.left.accept(self)} {expr.op.lexeme} {expr.right.accept(self)})"
-
-    def visit_wildcard_expr(self, expr: WildcardExpr) -> str:
-        return "(_)"
-
-    def visit_literal_expr(self, expr: LiteralExpr) -> str:
-        return f"({expr.value})"
-
-    def visit_type_body_expr(self, expr: TypeBodyExpr) -> str:
-        res: str = "(body"
-        for prop in expr.properties:
-            res += " " + prop.accept(self)
-        res += ")"
-        return res
-
-    def visit_property_stmt(self, stmt: PropertyStmt) -> str:
-        return f"(property {stmt.name.lexeme} {stmt.type.accept(self)})"
-
-    def visit_op_stmt(self, stmt: OpStmt) -> str:
-        left: str = stmt.left.accept(self)
-        right: str = stmt.right.accept(self)
-        result: str = stmt.result.accept(self)
-        return f"(op_def {left} {stmt.op.lexeme} {right} {result})"
-
-    def visit_constraint_stmt(self, stmt: ConstraintStmt) -> str:
-        return f"(constraint_def {stmt.name.lexeme} {stmt.constraint.accept(self)})"
-
-
-def parse(source: str) -> list[Stmt]:
-    tokens: list[Token] = MidasLexer(source).process()
-    return MidasParser(tokens).parse()
-
-
-def ast_str(source: str) -> list[str]:
-    stmts: list[Stmt] = parse(source)
-    return [AstSerializer().serialize(stmt) for stmt in stmts]
-
-
-@pytest.mark.parametrize(
-    "src,expected",
-    [
-        ("type Foo<>", "(type_def Foo)"),
-        ("type Foo<Bar>", "(type_def Foo (Bar))"),
-        ("type Foo<Bar, Baz>", "(type_def Foo (Bar) (Baz))"),
-        (
-            "type Foo<Bar + (_ < 2), Baz>",
-            "(type_def Foo (Bar (constraint (_) < (2.0))) (Baz))",
-        ),
-        (
-            """
-            type Foo<> {
-                foo: Bar
-            }
-            """,
-            "(type_def Foo (body (property foo (Bar))))",
-        ),
-        (
-            """
-            type Foo<> {
-                foo: Bar + (_ != none)
-                foo2: Bar2 + (0 <= _) + (_ <= 100)
-            }
-            """,
-            "(type_def Foo (body (property foo (Bar (constraint (_) != (None)))) (property foo2 (Bar2 (constraint (0.0) <= (_)) (constraint (_) <= (100.0))))))",
-        ),
-        ("op <A> + <B> = <C>", "(op_def (A) + (B) (C))"),
-        (
-            "op <A + (_ < 100)> + <B + (_ < 100)> = <C + (_ < 200)>",
-            "(op_def (A (constraint (_) < (100.0))) + (B (constraint (_) < (100.0))) (C (constraint (_) < (200.0))))",
-        ),
-        (
-            "constraint Positive = _ >= 0",
-            "(constraint_def Positive (constraint (_) >= (0.0)))",
-        ),
-    ],
-)
-def test_expressions(src: str, expected: str | list[str]):
-    if isinstance(expected, str):
-        expected = [expected]
-    assert ast_str(src) == expected
-
-
-@pytest.mark.parametrize(
-    "src,pos",
-    [
-        ###
-        # Misc
-        ###
-        ("42", (1, 1)),
-        ("true", (1, 1)),
-        ("foo", (1, 1)),
-        ###
-        # Type statements
-        ###
-        ("type", (1, 5)),
-        ("type true", (1, 6)),
-        ("type Foo", (1, 9)),
-        ("type Foo<1>", (1, 10)),
-        # ("type Foo<float,>", (1, 16)),  # trailing comma is accepted, TODO: update parser or EBNF
-        ("type Foo<float, 1>", (1, 17)),
-        ("type Foo<float", (1, 15)),
-        ("type Foo<float> { 3 }", (1, 19)),
-        (
-            """
-            type Foo<float> {
-                foo
-            }
-            """,
-            (4, 1),
-        ),
-        (
-            """
-            type Foo<float> {
-                foo: 3
-            }
-            """,
-            (3, 10),
-        ),
-        ###
-        # Operation statements
-        ###
-        ("op", (1, 3)),
-        ("op float", (1, 4)),
-        ("op <", (1, 5)),
-        ("op <float", (1, 10)),
-        ("op <float>", (1, 11)),
-        ("op <float> +", (1, 13)),
-        ("op <float> + float", (1, 14)),
-        ("op <float> + <", (1, 15)),
-        ("op <float> + <float", (1, 20)),
-        ("op <float> + <float>", (1, 21)),
-        ("op <float> + <float> =", (1, 23)),
-        ("op <float> + <float> = float", (1, 24)),
-        ("op <float> + <float> = <", (1, 25)),
-        ("op <float> + <float> = <float", (1, 30)),
-        ("op <float + 3> + <float> = <float>", (1, 13)),
-        ("op <float> + <float + 3> = <float>", (1, 23)),
-        ("op <float> + <float> = <float + 3>", (1, 33)),
-        ###
-        # Constraint statements
-        ###
-        ("constraint", (1, 11)),
-        ("constraint 3", (1, 12)),
-        ("constraint Foo", (1, 15)),
-        ("constraint Foo =", (1, 17)),
-        ("constraint Foo = 3", (1, 19)),
-        ("constraint Foo = 3 <", (1, 21)),
-    ],
-)
-def test_parsing_error(src: str, pos: tuple[int, int]):
-    src = textwrap.dedent(src)
-    tokens: list[Token] = MidasLexer(src).process()
-    parser: MidasParser = MidasParser(tokens)
-    stmt: list[Stmt] = parser.parse()
-    assert len(stmt) == 0
-    assert len(parser.errors) != 0
-    error_pos: Position = parser.errors[0].token.position
-    assert (error_pos.line, error_pos.column) == pos