diff --git a/midas/lexer/base.py b/midas/lexer/base.py index c4f4d82..f21205b 100644 --- a/midas/lexer/base.py +++ b/midas/lexer/base.py @@ -16,9 +16,10 @@ class Lexer(ABC): """An abstract lexer which provides methods to easily extend it into a concrete one This implementation is based on the [_Crafting Interpreters_][1] book by Robert Nystrom, - more specifically on my [previous Python implementation](https://git.kb28.ch/HEL/pebble) + more specifically on my [previous Python implementation][2] [1]: https://craftinginterpreters.com/ + [2]: https://git.kb28.ch/HEL/pebble """ def __init__(self, source: str, file: Optional[str] = None) -> None: @@ -168,6 +169,6 @@ class Lexer(ABC): def scan_token(self) -> None: """Scan a token - This function should (at least) consume the current character and produce the appropriate token(s), using `add_token` + This function should (at least) consume the current character and produce the appropriate token(s), using :func:`add_token` """ pass diff --git a/midas/lexer/midas.py b/midas/lexer/midas.py index a530b47..e1f6758 100644 --- a/midas/lexer/midas.py +++ b/midas/lexer/midas.py @@ -81,6 +81,12 @@ class MidasLexer(Lexer): return None def scan_string(self, opening: str): + """Scan the rest of a string and add it as a token + + Args: + opening (str): the opening quote or double quote, to be matched + at the end of the string + """ while self.peek() != opening and not self.is_at_end(): self.advance() @@ -147,6 +153,18 @@ class MidasLexer(Lexer): self.add_token(TokenType.COMMENT) def is_identifier_char(self, char: str, *, start: bool) -> bool: + """Check whether a character is a valid as part of an identifier + + Identifiers can contain any alphanumerical character or underscore. + They cannot start with a digit. + + Args: + char (str): the character to check + start (bool): whether this is the first character of the identifier + + Returns: + bool: `True` if the character is valid, `False` otherwise + """ if char == "_": return True if char.isalpha(): diff --git a/midas/lexer/token.py b/midas/lexer/token.py index f06c21e..d06052c 100644 --- a/midas/lexer/token.py +++ b/midas/lexer/token.py @@ -104,6 +104,15 @@ class Token: ) def location_to(self, to: Token) -> Location: + """Create a new :class:`Location` spanning from this token to another + + Args: + to (Token): the end token + + Returns: + Location: a new :class:`Location` starting at this token and ending + at `to`, both included + """ return Location.span(self.get_location(), to.get_location()) @property