HEL/midas
1
0
Fork 0
Code Issues Pull Requests Releases Activity

feat(parser): update lexer with new tokens

Browse Source
This commit is contained in:
Louis Heredero
2026-05-21 09:15:14 +02:00
parent 429d0d98fe
commit 98c3510bd4
3 changed files with 34 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lexer/keyword.py
View File

@@ -3,7 +3,9 @@ from lexer.token import TokenType
KEYWORDS: dict[str, TokenType] = { KEYWORDS: dict[str, TokenType] = {
"type": TokenType.TYPE, "type": TokenType.TYPE,
"op": TokenType.OP, "op": TokenType.OP,
"constraint": TokenType.CONSTRAINT, "predicate": TokenType.PREDICATE,
"extend": TokenType.EXTEND,
"where": TokenType.WHERE,
"true": TokenType.TRUE, "true": TokenType.TRUE,
"false": TokenType.FALSE, "false": TokenType.FALSE,
"none": TokenType.NONE, "none": TokenType.NONE,

32
lexer/midas.py
View File

@@ -31,30 +31,32 @@ class MidasLexer(Lexer):
self.add_token( self.add_token(
TokenType.EQUAL_EQUAL if self.match("=") else TokenType.EQUAL TokenType.EQUAL_EQUAL if self.match("=") else TokenType.EQUAL
) )
case "!": case "!" if self.match("="):
if self.match("="):
self.add_token(TokenType.BANG_EQUAL) self.add_token(TokenType.BANG_EQUAL)
else:
self.error("Unexpected single bang. Did you mean '!=' ?")
case ":": case ":":
self.add_token(TokenType.COLON) self.add_token(TokenType.COLON)
case ",": case ".":
self.add_token(TokenType.COMMA) self.add_token(TokenType.DOT)
case "&":
self.add_token(TokenType.AND)
case "?":
self.add_token(TokenType.QMARK)
# case ",":
# self.add_token(TokenType.COMMA)
case "_": case "_":
self.add_token(TokenType.UNDERSCORE) self.add_token(TokenType.UNDERSCORE)
case "+": case "-" if self.match(">"):
self.add_token(TokenType.PLUS) self.add_token(TokenType.ARROW)
# case "+":
# self.add_token(TokenType.PLUS)
case "-": case "-":
self.add_token(TokenType.MINUS) self.add_token(TokenType.MINUS)
case "*": # case "*":
self.add_token(TokenType.STAR) # self.add_token(TokenType.STAR)
case "/": case "/" if self.match("/"):
if self.match("/"):
self.scan_comment() self.scan_comment()
elif self.match("*"): case "/" if self.match("*"):
self.scan_comment_multiline() self.scan_comment_multiline()
else:
self.add_token(TokenType.SLASH)
case "\n": case "\n":
self.add_token(TokenType.NEWLINE) self.add_token(TokenType.NEWLINE)
case " " | "\r" | "\t": case " " | "\r" | "\t":

16
lexer/token.py
View File

@@ -14,14 +14,18 @@ class TokenType(Enum):
LEFT_BRACE = auto() LEFT_BRACE = auto()
RIGHT_BRACE = auto() RIGHT_BRACE = auto()
COLON = auto() COLON = auto()
COMMA = auto() # COMMA = auto()
UNDERSCORE = auto() UNDERSCORE = auto()
ARROW = auto()
AND = auto()
QMARK = auto()
DOT = auto()
# Operators # Operators
PLUS = auto() # PLUS = auto()
MINUS = auto() MINUS = auto()
STAR = auto() # STAR = auto()
SLASH = auto() # SLASH = auto()
GREATER = auto() GREATER = auto()
GREATER_EQUAL = auto() GREATER_EQUAL = auto()
LESS = auto() LESS = auto()
@@ -40,7 +44,9 @@ class TokenType(Enum):
# Keywords # Keywords
TYPE = auto() TYPE = auto()
OP = auto() OP = auto()
CONSTRAINT = auto() PREDICATE = auto()
EXTEND = auto()
WHERE = auto()
# Misc # Misc
COMMENT = auto() COMMENT = auto()