Compare commits

...
10 Commits
11 changed files with 523 additions and 24 deletions

No files matched your search

+56 -2
View File
@@ -2,22 +2,50 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Generic, Optional, TypeVar
from typing import Any, Generic, Optional, TypeVar
from lexer.token import Token
T = TypeVar("T")
@dataclass(frozen=True)
class Stmt(ABC):
@abstractmethod
def accept(self, visitor: Visitor[T]) -> T: ...
class Visitor(ABC, Generic[T]):
@abstractmethod
def visit_annotation_stmt(self, stmt: AnnotationStmt) -> T: ...
@dataclass(frozen=True)
class AnnotationStmt(Stmt):
name: Token
schema: Optional[SchemaExpr]
def accept(self, visitor: Stmt.Visitor[T]) -> T:
return visitor.visit_annotation_stmt(self)
@dataclass(frozen=True)
class Expr(ABC):
@abstractmethod
def accept(self, visitor: Visitor[T]) -> T: ...
class Visitor(ABC, Generic[T]):
@abstractmethod
def visit_wildcard_expr(self, expr: WildcardExpr) -> T: ...
@abstractmethod
def visit_literal_expr(self, expr: LiteralExpr) -> T: ...
@abstractmethod
def visit_type_expr(self, expr: TypeExpr) -> T: ...
@abstractmethod
def visit_constraint_expr(self, expr: ConstraintExpr) -> T: ...
@abstractmethod
def visit_schema_expr(self, expr: SchemaExpr) -> T: ...
@@ -25,15 +53,41 @@ class Expr(ABC):
def visit_schema_element_expr(self, expr: SchemaElementExpr) -> T: ...
@dataclass(frozen=True)
class WildcardExpr(Expr):
token: Token
def accept(self, visitor: Expr.Visitor[T]) -> T:
return visitor.visit_wildcard_expr(self)
@dataclass(frozen=True)
class LiteralExpr(Expr):
value: Any
def accept(self, visitor: Expr.Visitor[T]) -> T:
return visitor.visit_literal_expr(self)
@dataclass(frozen=True)
class TypeExpr(Expr):
name: Token
schema: Optional[SchemaExpr]
constraints: list[ConstraintExpr]
def accept(self, visitor: Expr.Visitor[T]) -> T:
return visitor.visit_type_expr(self)
@dataclass(frozen=True)
class ConstraintExpr(Expr):
left: Expr
op: Token
right: Expr
def accept(self, visitor: Expr.Visitor[T]) -> T:
return visitor.visit_constraint_expr(self)
@dataclass(frozen=True)
class SchemaExpr(Expr):
left: Token
+63 -8
View File
@@ -84,12 +84,39 @@ class AstPrinter(Generic[T]):
child.accept(self)
class AnnotationAstPrinter(AstPrinter, a.Expr.Visitor[None]):
class AnnotationAstPrinter(AstPrinter, a.Expr.Visitor[None], a.Stmt.Visitor[None]):
def visit_annotation_stmt(self, stmt: a.AnnotationStmt) -> None:
self._write_line("AnnotationStmt")
with self._child_level():
self._write_line(f'name: "{stmt.name.lexeme}"')
self._write_optional_child("schema", stmt.schema, last=True)
def visit_type_expr(self, expr: a.TypeExpr):
self._write_line("TypeExpr")
with self._child_level():
self._write_line(f'name: "{expr.name.lexeme}"')
self._write_optional_child("schema", expr.schema, last=True)
self._write_line("constraints", last=True)
with self._child_level():
for i, constraint in enumerate(expr.constraints):
self._idx = i
if i == len(expr.constraints) - 1:
self._mark_last()
constraint.accept(self)
def visit_constraint_expr(self, expr: a.ConstraintExpr) -> None:
self._write_line("ConstraintExpr")
with self._child_level():
self._write_line("left")
with self._child_level():
self._mark_last()
expr.left.accept(self)
self._write_line(f"operator: {expr.op.lexeme}")
self._write_line("right", last=True)
with self._child_level():
self._mark_last()
expr.right.accept(self)
def visit_schema_expr(self, expr: a.SchemaExpr):
self._write_line("SchemaExpr")
@@ -106,17 +133,39 @@ class AnnotationAstPrinter(AstPrinter, a.Expr.Visitor[None]):
name_text: str = "None" if expr.name is None else f'"{expr.name.lexeme}"'
self._write_line(f"name: {name_text}")
self._write_optional_child("type", expr.type, last=True)
def visit_wildcard_expr(self, expr: a.WildcardExpr) -> None:
self._write_line("WildcardExpr")
def visit_literal_expr(self, expr: a.LiteralExpr) -> None:
self._write_line("LiteralExpr")
with self._child_level():
self._write_line(f'value: {expr.value}', last=True)
class AnnotationPrinter(a.Expr.Visitor[str]):
def print(self, expr: a.Expr):
class AnnotationPrinter(a.Expr.Visitor[str], a.Stmt.Visitor[str]):
def print(self, expr: a.Expr | a.Stmt):
return expr.accept(self)
def visit_type_expr(self, expr: a.TypeExpr) -> str:
def visit_annotation_stmt(self, stmt: a.AnnotationStmt) -> str:
schema: str = ""
if expr.schema is not None:
schema = expr.schema.accept(self)
return f"{expr.name.lexeme}{schema}"
if stmt.schema is not None:
schema = stmt.schema.accept(self)
return f"{stmt.name.lexeme}{schema}"
def visit_type_expr(self, expr: a.TypeExpr) -> str:
parts: list[str] = [expr.name.lexeme]
for constraint in expr.constraints:
parts.append("(" + constraint.accept(self) + ")")
return " + ".join(parts)
def visit_constraint_expr(self, expr: a.ConstraintExpr) -> str:
parts: list[str] = [
expr.left.accept(self),
expr.op.lexeme,
expr.right.accept(self)
]
return " ".join(parts)
def visit_schema_expr(self, expr: a.SchemaExpr) -> str:
res: str = expr.left.lexeme
@@ -135,6 +184,12 @@ class AnnotationPrinter(a.Expr.Visitor[str]):
parts.append(expr.type.accept(self))
return ": ".join(parts)
def visit_wildcard_expr(self, expr: a.WildcardExpr) -> str:
return "_"
def visit_literal_expr(self, expr: a.LiteralExpr) -> str:
return str(expr.value)
class MidasAstPrinter(AstPrinter, m.Expr.Visitor[None], m.Stmt.Visitor[None]):
def visit_type_stmt(self, stmt: m.TypeStmt):
+17
View File
@@ -14,6 +14,23 @@ class AnnotationLexer(Lexer):
self.add_token(TokenType.LEFT_BRACKET)
case "]":
self.add_token(TokenType.RIGHT_BRACKET)
case "<":
self.add_token(
TokenType.LESS_EQUAL if self.match("=") else TokenType.LESS
)
case ">":
self.add_token(
TokenType.GREATER_EQUAL if self.match("=") else TokenType.GREATER
)
case "=":
self.add_token(
TokenType.EQUAL_EQUAL if self.match("=") else TokenType.EQUAL
)
case "!":
if self.peek() == "=":
self.add_token(TokenType.BANG_EQUAL)
else:
self.error("Unexpected single bang. Did you mean '!=' ?")
case ":":
self.add_token(TokenType.COLON)
case ",":
+5
View File
@@ -31,6 +31,11 @@ class MidasLexer(Lexer):
self.add_token(
TokenType.EQUAL_EQUAL if self.match("=") else TokenType.EQUAL
)
case "!":
if self.peek() == "=":
self.add_token(TokenType.BANG_EQUAL)
else:
self.error("Unexpected single bang. Did you mean '!=' ?")
case ":":
self.add_token(TokenType.COLON)
case ",":
+1
View File
@@ -28,6 +28,7 @@ class TokenType(Enum):
LESS_EQUAL = auto()
EQUAL = auto()
EQUAL_EQUAL = auto()
BANG_EQUAL = auto()
# Literals
IDENTIFIER = auto()
+104 -14
View File
@@ -1,42 +1,124 @@
from typing import Optional
from core.ast.annotations import Expr, SchemaElementExpr, SchemaExpr, TypeExpr
from core.ast.annotations import (
AnnotationStmt,
ConstraintExpr,
Expr,
LiteralExpr,
SchemaElementExpr,
SchemaExpr,
Stmt,
TypeExpr,
WildcardExpr,
)
from lexer.token import Token, TokenType
from parser.base import Parser
from parser.errors import ParsingError
class AnnotationParser(Parser):
"""A simple parser for custom type annotations"""
SYNC_BOUNDARY: set[TokenType] = set()
def parse(self) -> Optional[Expr]:
expression: Optional[Expr] = self.annotation()
def parse(self) -> Optional[Stmt]:
stmt: Optional[Stmt] = None
try:
stmt = self.annotation()
except ParsingError:
self.synchronize()
if not self.is_at_end():
self.error(self.peek(), "Extra tokens")
return expression
return stmt
def synchronize(self):
"""Skip tokens until a synchronization boundary is found
This method allows gracefully recovering from a parse error
to a safe place and continue parsing
"""
self.advance()
while not self.is_at_end():
if self.peek().type in self.SYNC_BOUNDARY:
return
self.advance()
def annotation(self) -> Optional[Expr]:
try:
return self.type()
except ParsingError:
self.synchronize()
return None
def annotation(self) -> AnnotationStmt:
"""Parse an annotation
An annotation is written as `Type` or `Type[Schema]`
Returns:
AnnotationStmt: the parsed annotation statement
"""
def type(self) -> TypeExpr:
name: Token = self.consume(TokenType.IDENTIFIER, "Expected type identifier")
schema: Optional[SchemaExpr] = None
if self.match(TokenType.LEFT_BRACKET):
schema = self.schema()
return TypeExpr(name=name, schema=schema)
return AnnotationStmt(name=name, schema=schema)
def type_expr(self) -> TypeExpr:
"""Parse a type expression
Returns:
TypeExpr: the parsed type expression
"""
name: Token = self.consume(TokenType.IDENTIFIER, "Expected type name")
constraints: list[ConstraintExpr] = []
while not self.is_at_end() and self.match(TokenType.PLUS):
print(self.peek())
print(self.tokens)
self.consume(TokenType.LEFT_PAREN, "Expected '(' before type constraint")
constraints.append(self.constraint_expr())
self.consume(TokenType.RIGHT_PAREN, "Expected ')' after type constraint")
return TypeExpr(name=name, constraints=constraints)
def constraint_expr(self) -> ConstraintExpr:
"""Parse a type constraint
Returns:
ConstraintExpr: the parsed type constraint expression
"""
left: Expr = self.constraint_value()
op: Token = self.constraint_operator()
right: Expr = self.constraint_value()
return ConstraintExpr(left=left, op=op, right=right)
def constraint_value(self) -> Expr:
if self.match(TokenType.UNDERSCORE):
return WildcardExpr(self.previous())
return self.literal()
def literal(self) -> LiteralExpr:
if self.match(TokenType.FALSE):
return LiteralExpr(False)
if self.match(TokenType.TRUE):
return LiteralExpr(True)
if self.match(TokenType.NONE):
return LiteralExpr(None)
if self.match(TokenType.NUMBER):
return LiteralExpr(self.previous().value)
raise self.error(self.peek(), "Expected literal")
def constraint_operator(self) -> Token:
if self.match(TokenType.LESS, TokenType.LESS_EQUAL, TokenType.GREATER, TokenType.GREATER_EQUAL, TokenType.EQUAL_EQUAL, TokenType.BANG_EQUAL):
return self.previous()
raise self.error(self.peek(), "Expected constraint operator")
def schema(self) -> SchemaExpr:
"""Parse a schema definition
A comma separated list of schema elements
Returns:
SchemaExpr: the parsed schema expression
"""
left: Token = self.previous()
elements: list[Expr] = []
while not self.check(TokenType.RIGHT_BRACKET) and not self.is_at_end():
@@ -47,7 +129,15 @@ class AnnotationParser(Parser):
right: Token = self.consume(TokenType.RIGHT_BRACKET, "Unclosed schema")
return SchemaExpr(left=left, elements=elements, right=right)
def schema_element(self) -> Expr:
def schema_element(self) -> SchemaElementExpr:
"""Parse a schema element
An anonymous element (`_`), a type, an untyped named column (`name: _`),
or a named column (`name: Type`)
Returns:
SchemaElementExpr: the parsed schema element expression
"""
if self.match(TokenType.UNDERSCORE):
return SchemaElementExpr(name=None, type=None)
@@ -60,5 +150,5 @@ class AnnotationParser(Parser):
name = self.advance()
self.advance()
if not self.match(TokenType.UNDERSCORE):
type = self.type()
type = self.type_expr()
return SchemaElementExpr(name=name, type=type)
+60
View File
@@ -16,6 +16,8 @@ from parser.errors import ParsingError
class MidasParser(Parser):
"""A simple parser for midas type definitions"""
SYNC_BOUNDARY: set[TokenType] = {TokenType.TYPE, TokenType.OP, TokenType.CONSTRAINT}
def parse(self) -> list[Stmt]:
@@ -29,6 +31,11 @@ class MidasParser(Parser):
return statements
def synchronize(self):
"""Skip tokens until a synchronization boundary is found
This method allows gracefully recovering from a parse error
to a safe place and continue parsing
"""
self.advance()
while not self.is_at_end():
if self.previous().type == TokenType.NEWLINE:
@@ -38,6 +45,13 @@ class MidasParser(Parser):
self.advance()
def declaration(self) -> Optional[Stmt]:
"""Try and parse a declaration
Any parsing error is caught and None is returned
Returns:
Optional[Stmt]: the parsed Midas statement, or None if a ParsingError was raised
"""
try:
if self.match(TokenType.TYPE):
return self.type_declaration()
@@ -50,6 +64,13 @@ class MidasParser(Parser):
return None
def type_declaration(self) -> TypeStmt:
"""Parse a type declaration
A type declaration is written `type Name<TypeExpr, ...>` optionally followed by a brace-wrapped body
Returns:
TypeStmt: the parsed type declaration statement
"""
name: Token = self.consume(TokenType.IDENTIFIER, "Expected type name")
self.consume(TokenType.LESS, "Expected '<' after type name")
bases: list[TypeExpr] = []
@@ -66,6 +87,11 @@ class MidasParser(Parser):
return TypeStmt(name=name, bases=bases, body=body)
def type_expr(self) -> TypeExpr:
"""Parse a type expression
Returns:
TypeExpr: the parsed type expression
"""
name: Token = self.consume(TokenType.IDENTIFIER, "Expected type name")
constraints: list[ConstraintExpr] = []
@@ -75,10 +101,23 @@ class MidasParser(Parser):
return TypeExpr(name=name, constraints=constraints)
def constraint_expr(self) -> ConstraintExpr:
"""Parse a type constraint
Returns:
ConstraintExpr: the parsed type constraint expression
"""
# TODO
return ConstraintExpr()
def type_body_expr(self) -> TypeBodyExpr:
"""Parse a type definition body
A type definition body is a set of whitespace-separated
property statements enclosed in curly braces
Returns:
TypeBodyExpr: the parsed type body expression
"""
self.consume(TokenType.LEFT_BRACE, "Expected '{' to start type body")
properties: list[PropertyStmt] = []
while not self.check(TokenType.RIGHT_BRACE) and not self.is_at_end():
@@ -87,12 +126,26 @@ class MidasParser(Parser):
return TypeBodyExpr(properties=properties)
def property_stmt(self) -> PropertyStmt:
"""Parse a property statement
A type property statement is written `name: Type`
Returns:
PropertyStmt: the parsed property statement
"""
name: Token = self.consume(TokenType.IDENTIFIER, "Expected property name")
self.consume(TokenType.COLON, "Expected ':' after property name")
type: TypeExpr = self.type_expr()
return PropertyStmt(name=name, type=type)
def op_declaration(self) -> OpStmt:
"""Parse an operation definition
An operation is written `op <Type1> operator <Type2> = <Type3>` where `operator` can be any single token
Returns:
OpStmt: the parsed operation statement
"""
self.consume(TokenType.LESS, "Expected '<' before first type")
left: TypeExpr = self.type_expr()
self.consume(TokenType.GREATER, "Expected '>' after first type")
@@ -112,6 +165,13 @@ class MidasParser(Parser):
return OpStmt(left=left, op=op, right=right, result=result)
def constraint_declaration(self) -> ConstraintStmt:
"""Parse a type constraint declaration
A constraint is written `constraint Name = constraint_expression`
Returns:
ConstraintStmt: the parsed constraint declaration statement
"""
name: Token = self.consume(TokenType.IDENTIFIER, "Expected constraint name")
self.consume(TokenType.EQUAL, "Expected '=' after constraint name")
constraint: ConstraintExpr = self.constraint_expr()
+20
View File
@@ -0,0 +1,20 @@
identifier ::= '[a-zA-Z][a-zA-Z_]*'
integer ::= '\d+'
number ::= integer ["." integer]
boolean ::= "False" | "True"
none ::= "None"
value ::= number | boolean | none
lambda-value ::= "_" | value
lambda-operator ::= ">" | "<" | ">=" | "<=" | "==" | "!="
lambda ::= lambda-value lambda-operator lambda-value
constraint ::= identifier | "(" lambda ")"
base-type ::= identifier
type ::= base-type { "+" constraint }
column-type ::= type | "_"
column-def ::= [ identifier ":" ] column-type
frame-def ::= column-def { "," column-def }
+74
View File
@@ -0,0 +1,74 @@
#import "@preview/fervojo:0.1.1": render
#let value = ```
{[`value` <
[`number` 'digit' * ! <!, ["." 'digit' * !]>],
[`boolean` <"False", "True">],
[`none` "None"]
>]}
```
#let constraint = ```
{[`constraint` <"_", 'value'> <">", "<", ">=", "<=", "==", "!="> <"_", 'value'>]}
```
#let type-with-constraints = ```
{[`type-with-constraints` 'identifier' <!, ["+" "(" 'constraint' ")"] * !>]}
```
#let column-def = ```
{[`column-def` <!, ['identifier' ":"]> <"_", 'type-with-constraints'>]}
```
#let frame-def = ```
{[`frame-def` 'column-def' * ","]}
```
#let annotation = ```
{[`annotation` 'identifier' <!, ["[" 'frame-def' "]"]>]}
```
#let rules = (
value,
constraint,
type-with-constraints,
column-def,
frame-def,
annotation,
)
#set text(font: "Source Sans 3")
= Type annotation syntax
#for rule in rules {
render(rule)
}
/*
#let by-name = (
annotation: annotation,
frame-def: frame-def,
column-def: column-def,
type-with-constraints: type-with-constraints,
constraint: constraint,
value: value,
)
#let substitute(base-rule) = {
let new-rule = base-rule
for (key, rule) in by-name.pairs() {
new-rule = new-rule.replace("'" + key + "'", rule.text.slice(1, -1))
}
if new-rule != base-rule {
new-rule = substitute(new-rule)
}
return new-rule
}
#let combined = raw(substitute(annotation.text))
#set page(flipped: true)
#render(combined)
*/
+26
View File
@@ -0,0 +1,26 @@
identifier ::= '[a-zA-Z][a-zA-Z_]*'
integer ::= '\d+'
number ::= integer ["." integer]
boolean ::= "False" | "True"
none ::= "None"
value ::= number | boolean | none
lambda-value ::= "_" | value
lambda-operator ::= ">" | "<" | ">=" | "<=" | "==" | "!="
lambda ::= lambda-value lambda-operator lambda-value
constraint ::= identifier | "(" lambda ")"
base-type ::= identifier
type ::= base-type { "+" constraint }
type-property ::= 'identifier' ":" 'type'
type-body ::= "{" { 'type-property' } "}"
operation-type ::= "<" 'type' ">"
type-statement ::= "type" 'identifier' "<" 'type' {"," 'type'} ">" ['type-body']
operation-statement ::= "op" 'operation-type' 'operator' 'operation-type' "=" 'operation-type'
constraint-statement ::= "constraint" 'identifier' "=" 'lambda'
statement ::= type-statement | operation-statement | constraint-statement
+97
View File
@@ -0,0 +1,97 @@
#import "@preview/fervojo:0.1.1": render
#let value = ```
{[`value` <
[`number` 'digit' * ! <!, ["." 'digit' * !]>],
[`boolean` <"False", "True">],
[`none` "None"]
>]}
```
#let constraint = ```
{[`constraint` <"_", 'value'> <">", "<", ">=", "<=", "==", "!="> <"_", 'value'>]}
```
#let type-with-constraints = ```
{[`type-with-constraints` 'identifier' <!, ["+" "(" 'constraint' ")"] * !>]}
```
#let type-property = ```
{[`type-property` 'identifier' ":" 'type-with-constraints']}
```
#let type-body = ```
{[`type-body` "{" <!, 'type-property'*!> "}"]}
```
#let operation-type = ```
{[`operation-type` "<" 'type-with-constraints' ">"]}
```
#let type-statement = ```
{[`type-statement` "type" 'identifier' "<" 'type-with-constraints'*"," ">" <!, 'type-body'>]}
```
#let operation-statement = ```
{[`operation-statement` "op" 'operation-type' "operator" 'operation-type' "=" 'operation-type']}
```
#let constraint-statement = ```
{[`constraint-statement` "constraint" 'identifier' "=" 'constraint']}
```
#let statement = ```
{[`statement` <'type-statement', 'operation-statement', 'constraint-statement'>]}
```
#let rules = (
value,
constraint,
type-with-constraints,
type-property,
type-body,
operation-type,
type-statement,
operation-statement,
constraint-statement,
statement,
)
#set text(font: "Source Sans 3")
= Midas type definition syntax
#for rule in rules {
render(rule)
}
/*
#let by-name = (
value: value,
constraint: constraint,
type-with-constraints: type-with-constraints,
type-property: type-property,
type-body: type-body,
operation-type: operation-type,
type-statement: type-statement,
operation-statement: operation-statement,
constraint-statement: constraint-statement,
)
#let substitute(base-rule) = {
let new-rule = base-rule
for (key, rule) in by-name.pairs() {
new-rule = new-rule.replace("'" + key + "'", rule.text.slice(1, -1))
}
if new-rule != base-rule {
new-rule = substitute(new-rule)
}
return new-rule.replace(regex("`.*?`"), "")
}
#let combined = raw(substitute(statement.text))
#set page(flipped: true)
#render(combined)
*/