From 8c036d9582efe00e4588075d2e98ead4b6354130 Mon Sep 17 00:00:00 2001 From: LeonardoBizzoni Date: Thu, 5 Sep 2024 14:16:08 +0200 Subject: [PATCH] Replaced `Lexer` instances with fn calls --- src/LBPLFunction.cpp | 2 +- src/LBPLInstance.cpp | 12 +- src/environment.cpp | 19 +- src/interpreter.cpp | 38 ++-- src/lexer.cpp | 523 ++++++++++++++++++++++--------------------- src/lexer.hpp | 101 ++++++--- src/main.cpp | 21 +- src/parser.cpp | 290 ++++++++++++------------ src/parser.hpp | 54 +++-- src/resolver.cpp | 21 +- src/token.cpp | 24 ++ src/token.hpp | 28 ++- src/token_type.hpp | 77 ++++++- 13 files changed, 691 insertions(+), 519 deletions(-) create mode 100644 src/token.cpp diff --git a/src/LBPLFunction.cpp b/src/LBPLFunction.cpp index ac9c602..b516285 100644 --- a/src/LBPLFunction.cpp +++ b/src/LBPLFunction.cpp @@ -17,7 +17,7 @@ Value LBPLFunc::call(Interpreter *interpreter, std::vector &args) { auto env = std::make_shared(closureEnv); for (int i = 0; i < stmt->args.size(); i++) { - env->define(stmt->args[i]->lexeme, args[i]); + env->define(std::get(stmt->args[i]->lexeme), args[i]); } try { diff --git a/src/LBPLInstance.cpp b/src/LBPLInstance.cpp index 72951b9..c6771f7 100644 --- a/src/LBPLInstance.cpp +++ b/src/LBPLInstance.cpp @@ -1,20 +1,22 @@ #include "LBPLInstance.hpp" #include "runtime_error.hpp" +#include Value LBPLInstance::get(const Token *name) { - if (fields.contains(name->lexeme)) { - return fields.find(name->lexeme)->second; + const char *lexeme = std::get(name->lexeme); + if (fields.contains(lexeme)) { + return fields.find(lexeme)->second; } - LBPLFunc *method = lbplClass->findMethod(name->lexeme); + LBPLFunc *method = lbplClass->findMethod(lexeme); if (method) { method->bind(std::make_shared(this)); return std::make_shared(*method); } - throw RuntimeError(name, "Undefined field '" + name->lexeme + "'."); + throw RuntimeError(name, "Undefined field '" + std::string(lexeme) + "'."); } void LBPLInstance::set(const Token *name, Value &value) { - fields.insert_or_assign(name->lexeme, value); + fields.insert_or_assign(std::get(name->lexeme), value); } diff --git a/src/environment.cpp b/src/environment.cpp index 4ef6a96..20f2656 100755 --- a/src/environment.cpp +++ b/src/environment.cpp @@ -2,11 +2,11 @@ #include "LBPLTypes.hpp" #include "runtime_error.hpp" +#include #include #include #include #include -#include void Environment::define(const std::string &name, Value &value) { env.insert(std::make_pair(name, value)); @@ -16,7 +16,8 @@ void Environment::define(const std::string &name, Value &&value) { } Value Environment::get(std::shared_ptr &name) { - auto it = env.find(name->lexeme); + const char *namestr = std::get(name->lexeme); + auto it = env.find(namestr); if (it != env.end()) { return it->second; @@ -25,11 +26,12 @@ Value Environment::get(std::shared_ptr &name) { return enclosing->get(name); } - throw RuntimeError(name.get(), "Undefined name '" + name->lexeme + "'."); + throw RuntimeError(name.get(), + "Undefined name '" + std::string(namestr) + "'."); } Value Environment::getAt(int depth, std::shared_ptr &name) { - return getAt(depth, name->lexeme); + return getAt(depth, std::get(name->lexeme)); } Value Environment::getAt(int depth, const std::string &name) { @@ -42,13 +44,14 @@ Value Environment::getAt(int depth, const std::string &name) { } void Environment::assign(std::shared_ptr &name, Value &value) { - if (env.contains(name->lexeme)) { - env.insert_or_assign(name->lexeme, value); + if (auto namestr = std::get(name->lexeme); + env.contains(namestr)) { + env.insert_or_assign(namestr, value); } else if (enclosing) { enclosing->assign(name, value); } else { throw RuntimeError(name.get(), - "Undefined variable '" + name->lexeme + "'."); + "Undefined variable '" + std::string(namestr) + "'."); } } @@ -62,7 +65,7 @@ void Environment::assignAt(int depth, std::shared_ptr &name, enclosing->assignAt(depth - 1, name, value); } - env.insert_or_assign(name->lexeme, value); + env.insert_or_assign(std::get(name->lexeme), value); } void Environment::printEnv(const std::string &&msg) { diff --git a/src/interpreter.cpp b/src/interpreter.cpp index a2f9f6a..de80ef8 100644 --- a/src/interpreter.cpp +++ b/src/interpreter.cpp @@ -6,6 +6,7 @@ #include "runtime_error.hpp" #include +#include #include #include #include @@ -26,7 +27,7 @@ void Interpreter::resolve(Expr *expr, int depth) { } void Interpreter::visitFnStmt(FnStmt *stmt) { - currentEnv->define(stmt->name->lexeme, + currentEnv->define(std::get(stmt->name->lexeme), std::make_shared(stmt, currentEnv, false)); } @@ -36,12 +37,12 @@ void Interpreter::visitVarStmt(VarStmt *stmt) { value = stmt->value->accept(this); } - currentEnv->define(stmt->name->lexeme, value); + currentEnv->define(std::get(stmt->name->lexeme), value); } void Interpreter::visitClassStmt(ClassStmt *stmt) { Value superclass; - currentEnv->define(stmt->name->lexeme, nullptr); + currentEnv->define(std::get(stmt->name->lexeme), nullptr); if (stmt->superclass) { superclass = stmt->superclass->accept(this); @@ -62,20 +63,22 @@ void Interpreter::visitClassStmt(ClassStmt *stmt) { throw RuntimeError(methodStmt.get(), "Excepted class method."); } - auto fn = new LBPLFunc(method, currentEnv, method->name->lexeme == "init"); - methods.insert(std::make_pair(method->name->lexeme, fn)); + auto namestr = std::get(method->name->lexeme); + auto fn = new LBPLFunc(method, currentEnv, std::string(namestr) == "init"); + methods.insert(std::make_pair(namestr, fn)); } if (stmt->superclass) { currentEnv = currentEnv->enclosing; currentEnv->assign(stmt->name, std::make_shared( - stmt->name->lexeme, + std::get(stmt->name->lexeme), std::get>(superclass), methods)); } else { currentEnv->assign( - stmt->name, std::make_shared(stmt->name->lexeme, methods)); + stmt->name, std::make_shared( + std::get(stmt->name->lexeme), methods)); } } @@ -165,14 +168,15 @@ Value Interpreter::visitLiteralExpr(LiteralExpr *expr) { } else if (expr->token->type == TokenType::Nil) { return nullptr; } else if (expr->token->type == TokenType::Char) { - return expr->token->lexeme[0]; + return std::get(expr->token->lexeme); } else if (expr->token->type == TokenType::String) { - return expr->token->lexeme; + return std::get(expr->token->lexeme); } else if (expr->token->type == TokenType::Number) { - if (expr->token->lexeme.find('.') == std::string::npos) { - return std::stoi(expr->token->lexeme); + if (std::holds_alternative(expr->token->lexeme)) { + return std::get(expr->token->lexeme); + } else { + return std::get(expr->token->lexeme); } - return std::stod(expr->token->lexeme); } else if (expr->token->type == TokenType::Identifier) { return lookupVariable(expr->token, expr); } @@ -185,7 +189,9 @@ Value Interpreter::visitGroupExpr(GroupingExpr *expr) { } Value Interpreter::visitSuperExpr(SuperExpr *) { return nullptr; } -Value Interpreter::visitThisExpr(ThisExpr *expr) { return lookupVariable(expr->keyword, expr); } +Value Interpreter::visitThisExpr(ThisExpr *expr) { + return lookupVariable(expr->keyword, expr); +} Value Interpreter::visitCallExpr(FnCallExpr *expr) { Value callee = expr->callee->accept(this); @@ -274,8 +280,8 @@ Value Interpreter::evaluate(std::unique_ptr &expr) { } Value Interpreter::performBinaryOperation(std::shared_ptr &op, - const Value &left, - const Value &right) { + const Value &left, + const Value &right) { auto performIntOp = [](int l, int r, std::shared_ptr &op) -> Value { switch (op->type) { @@ -443,7 +449,7 @@ bool Interpreter::isTruthy(const Value &value) { bool Interpreter::isTruthy(Value &&value) { return isTruthy(value); } Value Interpreter::lookupVariable(std::shared_ptr &name, - Expr *expr) { + Expr *expr) { auto it = locals.find(expr); if (it == locals.end()) { diff --git a/src/lexer.cpp b/src/lexer.cpp index 4fc6691..8c0914d 100755 --- a/src/lexer.cpp +++ b/src/lexer.cpp @@ -1,321 +1,340 @@ #include "lexer.hpp" +#include "token.hpp" +#include "token_type.hpp" + +#include +#include +#include +#include +#include + +#define MAKE_TOKEN(TYPE, LEXEME) \ + std::make_shared(TYPE, LEXEME, file.line, file.column, \ + file.filepath) + +namespace Lexer { +std::shared_ptr getNextToken(Source &file) { + if (file.stream.peek() == EOF) { + return MAKE_TOKEN(TokenType::Eof, 0); + } -Lexer::Lexer(const char *stream, const std::string &filename) - : line(1), hadError(false), filename(filename), current(stream), start(stream) {} + skipWhitespace(file); + char ch = file.stream.peek(); -bool Lexer::isAtEnd() const { return *current == '\0'; } -bool Lexer::isDigit(char ch) const { return ch >= '0' && ch <= '9'; } -bool Lexer::isAlpha(char ch) const { - return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch == '_'; -} + if (isDigit(ch)) { + return makeNumberToken(file); + } else if (isAlpha(ch)) { + return makeIdentifierToken(file); + } -bool Lexer::match(char ch) { - if (isAtEnd() || ch != *current) { - return false; + switch ((ch = file.advance())) { + case '(': + return MAKE_TOKEN(TokenType::LeftParen, 0); + case ')': + return MAKE_TOKEN(TokenType::RightParen, 0); + case '{': + return MAKE_TOKEN(TokenType::LeftBrace, 0); + case '}': + return MAKE_TOKEN(TokenType::RightBrace, 0); + case '?': + return MAKE_TOKEN(TokenType::Question, 0); + case ',': + return MAKE_TOKEN(TokenType::Comma, 0); + case '.': + return MAKE_TOKEN(TokenType::Dot, 0); + case ':': + return MAKE_TOKEN(TokenType::Colon, 0); + case ';': + return MAKE_TOKEN(TokenType::Semicolon, 0); + case '%': + return MAKE_TOKEN(TokenType::ModOp, 0); + case '&': + if (char _ch = file.stream.peek(); _ch == '&') { + file.advance(); + return MAKE_TOKEN(TokenType::And, 0); + } else { + std::string str_msg = "invalid token '" + std::string(1, _ch) + "'"; + char *msg = (char *)malloc(str_msg.size() + 1); + std::strncpy(msg, str_msg.c_str(), str_msg.size() + 1); + return MAKE_TOKEN(TokenType::Error, msg); + } + case '|': + if (char _ch = file.stream.peek(); _ch == '|') { + file.advance(); + return MAKE_TOKEN(TokenType::Or, 0); + } else { + std::string str_msg = "invalid token '" + std::string(1, _ch) + "'"; + char *msg = (char *)malloc(str_msg.size() + 1); + std::strncpy(msg, str_msg.c_str(), str_msg.size() + 1); + return MAKE_TOKEN(TokenType::Error, msg); + } + case '-': + return MAKE_TOKEN(TokenType::Minus, 0); + case '+': + return MAKE_TOKEN(TokenType::Plus, 0); + case '/': + return MAKE_TOKEN(TokenType::Slash, 0); + case '*': + return MAKE_TOKEN(TokenType::Star, 0); + case '!': + if (file.stream.peek() == '=') { + file.advance(); + return MAKE_TOKEN(TokenType::BangEqual, 0); + } else { + return MAKE_TOKEN(TokenType::Bang, 0); + } + case '=': + if (file.stream.peek() == '=') { + file.advance(); + return MAKE_TOKEN(TokenType::EqualEqual, 0); + } else { + return MAKE_TOKEN(TokenType::Equal, 0); + } + case '>': + if (file.stream.peek() == '>') { + file.advance(); + return MAKE_TOKEN(TokenType::ShiftRight, 0); + } else if (file.stream.peek() == '=') { + file.advance(); + return MAKE_TOKEN(TokenType::GreaterEqual, 0); + } else { + return MAKE_TOKEN(TokenType::Greater, 0); + } + case '<': + if (file.stream.peek() == '<') { + file.advance(); + return MAKE_TOKEN(TokenType::ShiftLeft, 0); + } else if (file.stream.peek() == '=') { + file.advance(); + return MAKE_TOKEN(TokenType::LessEqual, 0); + } else { + return MAKE_TOKEN(TokenType::Less, 0); + } + case '\'': { + char lexeme = file.advance(); + if (file.stream.peek() != '\'') { + return MAKE_TOKEN(TokenType::Error, "A char must be one character long."); + } else { + return MAKE_TOKEN(TokenType::Char, lexeme); + } } + case '"': { + size_t len = 0; + char *lexeme = 0; + char new_ch = 0; - advance(); - return true; -} + while (file.stream.eof() && file.stream.peek() != '"') { + if (file.stream.peek() == '\\') { + file.advance(); -char Lexer::peek() const { return *current; } -char Lexer::peekNext() const { return *(current + 1); } + switch (file.stream.peek()) { + case 'n': + new_ch = '\n'; + break; + case 't': + new_ch = '\t'; + break; + case 'r': + new_ch = '\r'; + break; + case '\'': + new_ch = '\''; + break; + case '\"': + new_ch = '"'; + break; + case '\\': + new_ch = '\\'; + break; + default: + return MAKE_TOKEN(TokenType::Error, "invalid escape sequence"); + } -char Lexer::advance() { - if (isAtEnd()) { - return '\0'; + file.advance(); + } else { + new_ch = file.advance(); + } + + lexeme = (char *)realloc(lexeme, ++len); + lexeme[len - 1] = new_ch; + } + + if (file.stream.eof()) { + return MAKE_TOKEN(TokenType::Error, "Unterminated string."); + } else { + file.advance(); + return MAKE_TOKEN(TokenType::String, lexeme); + } + } + } + + if (file.stream.eof()) { + return MAKE_TOKEN(TokenType::Eof, 0); } - return *(current++); + return MAKE_TOKEN(TokenType::Error, "\033[1;36mHow did you get here?\033[0m"); } -TokenType Lexer::checkKeyword(int startIndex, const std::string &restOfKeyword, - TokenType typeIfMatch) { - if (current - (start + startIndex) != restOfKeyword.length()) { - return TokenType::Identifier; +void skipWhitespace(Source &file) { + while (!file.stream.eof()) { + switch (file.stream.peek()) { + case '\r': + case '\n': { + file.line++; + file.advance(); + file.column = 0; + } break; + case ' ': { + file.advance(); + } break; + case '\t': { + file.column += 3; + file.advance(); + } break; + case '#': { + while (file.stream.peek() != '\n' || file.stream.peek() != '\r') { + file.advance(); + } + } break; + default: + return; + } } +} - for (int i = 0; i < restOfKeyword.length(); i++) { - if (*(start + startIndex + i) != restOfKeyword[i]) { - return TokenType::Identifier; +std::shared_ptr makeNumberToken(Source &file) { + size_t size = 0; + char *strlexeme = 0; + bool is_float = false; + + while (isDigit(file.stream.peek())) { + strlexeme = (char *)realloc(strlexeme, ++size); + strlexeme[size - 1] = file.advance(); + } + + if (file.stream.peek() == '.') { + is_float = true; + strlexeme = (char *)realloc(strlexeme, ++size); + strlexeme[size - 1] = file.advance(); + + if (isDigit(file.stream.peek())) { + strlexeme = (char *)realloc(strlexeme, ++size); + strlexeme[size - 1] = file.advance(); + + while (isDigit(file.stream.peek())) { + strlexeme = (char *)realloc(strlexeme, ++size); + strlexeme[size - 1] = file.advance(); + } } } - return typeIfMatch; + strlexeme = (char *)realloc(strlexeme, ++size); + strlexeme[size - 1] = 0; + + return MAKE_TOKEN(TokenType::Number, + (is_float ? std::stof(strlexeme) : std::stoi(strlexeme))); +} + +std::shared_ptr makeIdentifierToken(Source &file) { + size_t size = 0; + char *strlexeme = 0; + + while (isAlpha(file.stream.peek()) || isDigit(file.stream.peek())) { + strlexeme = (char *)realloc(strlexeme, ++size); + strlexeme[size - 1] = file.advance(); + } + + strlexeme = (char *)realloc(strlexeme, ++size); + strlexeme[size - 1] = 0; + + return MAKE_TOKEN(isIdentifierOrKeyword(strlexeme, size), strlexeme); } -TokenType Lexer::isIdentifierOrKeywork() { - switch (*start) { +TokenType isIdentifierOrKeyword(const char *lexeme, size_t len) { + switch (lexeme[0]) { case 'b': - return checkKeyword(1, "reak", TokenType::Break); + return checkKeyword(lexeme, len, 1, "reak", TokenType::Break); case 'c': - if (current - start > 1) { - switch (*(start + 1)) { + if (len > 1) { + switch (lexeme[1]) { case 'l': - return checkKeyword(2, "ass", TokenType::Class); + return checkKeyword(lexeme, len, 2, "ass", TokenType::Break); case 'o': - return checkKeyword(2, "ntinue", TokenType::Continue); + return checkKeyword(lexeme, len, 2, "ntinue", TokenType::Continue); } } break; case 'e': - return checkKeyword(1, "lse", TokenType::Else); + return checkKeyword(lexeme, len, 1, "lse", TokenType::Else); case 'f': - if (current - start > 1) { - switch (*(start + 1)) { + if (len > 1) { + switch (lexeme[1]) { case 'a': - return checkKeyword(2, "lse", TokenType::False); + return checkKeyword(lexeme, len, 2, "lse", TokenType::False); case 'o': - return checkKeyword(2, "r", TokenType::For); + return checkKeyword(lexeme, len, 2, "r", TokenType::For); case 'n': - return checkKeyword(2, "", TokenType::Fn); + return checkKeyword(lexeme, len, 2, "", TokenType::Fn); } } break; case 'i': - if (current - start > 1) { - switch (*(start + 1)) { + if (len > 1) { + switch (lexeme[1]) { case 'f': - return checkKeyword(2, "", TokenType::If); + return checkKeyword(lexeme, len, 2, "", TokenType::If); case 'm': - return checkKeyword(2, "port", TokenType::Import); + return checkKeyword(lexeme, len, 2, "port", TokenType::Import); } } break; case 'l': - if (current - start > 1) { - switch (*(start + 1)) { + if (len > 1) { + switch (lexeme[1]) { case 'e': - return checkKeyword(2, "t", TokenType::Let); + return checkKeyword(lexeme, len, 2, "t", TokenType::Let); case 'o': - return checkKeyword(2, "op", TokenType::Loop); + return checkKeyword(lexeme, len, 2, "op", TokenType::Loop); } } break; case 'n': - return checkKeyword(1, "il", TokenType::Nil); + return checkKeyword(lexeme, len, 1, "il", TokenType::Nil); case 'r': - return checkKeyword(1, "eturn", TokenType::Return); + return checkKeyword(lexeme, len, 1, "eturn", TokenType::Return); case 's': - return checkKeyword(1, "uper", TokenType::Super); + return checkKeyword(lexeme, len, 1, "uper", TokenType::Super); case 't': - if (current - start > 1) { - switch (*(start + 1)) { + if (len > 1) { + switch (lexeme[1]) { case 'r': - return checkKeyword(2, "ue", TokenType::True); + return checkKeyword(lexeme, len, 2, "ue", TokenType::True); case 'h': - return checkKeyword(2, "is", TokenType::This); + return checkKeyword(lexeme, len, 2, "is", TokenType::This); } } break; case 'w': - return checkKeyword(1, "hile", TokenType::While); + return checkKeyword(lexeme, len, 1, "hile", TokenType::While); } return TokenType::Identifier; } -std::shared_ptr Lexer::makeToken(TokenType type, - std::string value) { - if (value != "") { - return std::make_shared(type, value, line, - current-start, filename); - } - - return std::make_shared(type, std::string(start, current), line, - current-start, filename); -} - -std::shared_ptr Lexer::makeNumberToken() { - while (isDigit(peek())) { - advance(); - } - - if (peek() == '.' && isDigit(peekNext())) { - advance(); - while (isDigit(peek())) { - advance(); - } - } - - return makeToken(TokenType::Number); -} - -std::shared_ptr Lexer::makeIdentifierToken() { - while (isAlpha(peek()) || isDigit(peek()) || peek() < 0) { - advance(); - } - - return makeToken(isIdentifierOrKeywork()); -} - -std::shared_ptr Lexer::makeErrorToken(std::string msg) { - hadError = true; - return std::make_shared(TokenType::Error, msg, line, - current-start, filename); -} - -void Lexer::skipWhitespace() { - do { - switch (peek()) { - case '\n': - line++; - case ' ': - case '\r': - case '\t': - advance(); - break; - case '#': - while (peek() != '\n') { - advance(); - } - - break; - default: - return; - } - } while (!isAtEnd()); -} - -std::shared_ptr Lexer::getNextToken() { - skipWhitespace(); - start = current; - - char ch = advance(); - - if (isDigit(ch)) { - return makeNumberToken(); - } else if (isAlpha(ch) || (int)ch < 0) { - return makeIdentifierToken(); - } - - switch (ch) { - case '(': - return makeToken(TokenType::LeftParen); - case ')': - return makeToken(TokenType::RightParen); - case '{': - return makeToken(TokenType::LeftBrace); - case '}': - return makeToken(TokenType::RightBrace); - case '?': - return makeToken(TokenType::Question); - case ',': - return makeToken(TokenType::Comma); - case '.': - return makeToken(TokenType::Dot); - case ':': - return makeToken(TokenType::Colon); - case ';': - return makeToken(TokenType::Semicolon); - case '%': - return makeToken(TokenType::ModOp); - case '&': - if (match('&')) { - return makeToken(TokenType::And); - } - return makeErrorToken("Invalid token '" + std::to_string(*current) + "'."); - case '|': - if (match('|')) { - return makeToken(TokenType::Or); - } - return makeErrorToken("Invalid token '" + std::to_string(*current) + "'."); - case '-': - return makeToken(TokenType::Minus); - case '+': - return makeToken(TokenType::Plus); - case '/': - return makeToken(TokenType::Slash); - case '*': - return makeToken(TokenType::Star); - case '!': - if (match('=')) { - return makeToken(TokenType::BangEqual); - } - return makeToken(TokenType::Bang); - case '=': - if (match('=')) { - return makeToken(TokenType::EqualEqual); - } - return makeToken(TokenType::Equal); - case '>': - if (match('>')) { - return makeToken(TokenType::ShiftRight); - } - if (match('=')) { - return makeToken(TokenType::GreaterEqual); - } - return makeToken(TokenType::Greater); - case '<': - if (match('<')) { - return makeToken(TokenType::ShiftLeft); - } - if (match('=')) { - return makeToken(TokenType::LessEqual); - } - return makeToken(TokenType::Less); - case '\'': { - std::string val(1, advance()); - if (!match('\'')) { - auto res = - makeErrorToken("A char can't be more then one character long."); - return res; - } - return makeToken(TokenType::Char, val); - } - case '"': { - std::string lexeme; - - while (peek() != '"' && !isAtEnd()) { - if (peek() == '\\') { - advance(); - - switch (peek()) { - case 'n': - lexeme += "\n"; - break; - case 't': - lexeme += "\t"; - break; - case 'r': - lexeme += "\r"; - break; - case '\'': - lexeme += "'"; - break; - case '\"': - lexeme += "\""; - break; - case '\\': - lexeme += "\\"; - break; - } - - advance(); - } else { - lexeme += advance(); - } - } - - if (isAtEnd()) { - return makeErrorToken("Unterminated string."); - } - - advance(); - return makeToken(TokenType::String, lexeme); - } +TokenType checkKeyword(const char *token, size_t token_len, int startIndex, + const std::string_view &restOfKeyword, + TokenType typeIfMatch) { + if ((token_len - startIndex) != (restOfKeyword.length() + 1)) { + return TokenType::Identifier; } - if (isAtEnd()) { - return makeToken(TokenType::Eof, "EoF"); + if (strcmp(token + startIndex, restOfKeyword.data())) { + return TokenType::Identifier; } - return makeErrorToken("\033[1;36mHow did you get here?\033[0m"); + return typeIfMatch; } - -int Lexer::getLine() { return line; } -int Lexer::getColumn() { return current - start; } -std::string Lexer::getFilename() { return filename; } +} // namespace Lexer diff --git a/src/lexer.hpp b/src/lexer.hpp index d18b7fb..e1a7b56 100755 --- a/src/lexer.hpp +++ b/src/lexer.hpp @@ -4,49 +4,86 @@ #include "token.hpp" #include "token_type.hpp" -#include +#include +#include #include -class Lexer { -private: - int line; - std::string filename; - const char *start; - const char *current; +namespace Lexer { +struct Source { + Source(std::ifstream &file, const char *filepath) + : stream(std::move(file)), filepath(filepath), line(1), column(0) {} + + inline char advance() { + char res = this->stream.get(); + ++column; + return res; + } public: - bool hadError; + std::ifstream stream; + uint64_t line, column; + const char *filepath; +}; -private: - void skipWhitespace(); - void goToNextLine(); +std::shared_ptr getNextToken(Source &file); - std::shared_ptr makeToken(TokenType, std::string = ""); - std::shared_ptr makeNumberToken(); - std::shared_ptr makeIdentifierToken(); - std::shared_ptr makeErrorToken(std::string); +std::shared_ptr makeIdentifierToken(Source &file); +std::shared_ptr makeNumberToken(Source &file); - TokenType isIdentifierOrKeywork(); - TokenType checkKeyword(int startIndex, const std::string &restOfKeyword, - TokenType typeIfMatch); +void skipWhitespace(Source &file); - char advance(); - char peek() const; - char peekNext() const; +TokenType isIdentifierOrKeyword(const char *lexeme, size_t len); +TokenType checkKeyword(const char *token, size_t token_len, int startIndex, + const std::string_view &restOfKeyword, + TokenType typeIfMatch); - bool isAtEnd() const; - bool isDigit(char ch) const; - bool isAlpha(char ch) const; - bool match(char ch); +inline constexpr bool isDigit(char ch) { return ch >= '0' && ch <= '9'; } +inline constexpr bool isAlpha(char ch) { + return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch == '_' || + ch < -1; +} +} // namespace Lexer -public: - Lexer(const char *, const std::string &); +// class Lexer { +// private: +// int line; +// std::string filename; +// const char *start; +// const char *current; - int getLine(); - int getColumn(); - std::string getFilename(); +// public: +// bool hadError; - std::shared_ptr getNextToken(); -}; +// private: +// void skipWhitespace(); +// void goToNextLine(); + +// std::shared_ptr makeToken(TokenType, std::string = ""); +// std::shared_ptr makeNumberToken(); +// std::shared_ptr makeIdentifierToken(); +// std::shared_ptr makeErrorToken(std::string); + +// TokenType isIdentifierOrKeywork(); +// TokenType checkKeyword(int startIndex, const std::string &restOfKeyword, +// TokenType typeIfMatch); + +// char advance(); +// char peek() const; +// char peekNext() const; + +// bool isAtEnd() const; +// bool isDigit(char ch) const; +// bool isAlpha(char ch) const; +// bool match(char ch); + +// public: +// Lexer(std::ifstream &, const std::string &); + +// int getLine(); +// int getColumn(); +// std::string getFilename(); + +// std::shared_ptr getNextToken(); +// }; #endif diff --git a/src/main.cpp b/src/main.cpp index 77608cf..4beb625 100755 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,8 +1,5 @@ -#include +#include #include -#include -#include -#include #include "parser.hpp" #include "interpreter.hpp" @@ -15,17 +12,13 @@ int main(const int argc, const char **argv) { return 1; } - int fd = open(argv[1], O_RDONLY, S_IRUSR | S_IWUSR); - struct stat sb; - - if (fstat(fd, &sb) == -1) { - perror("Couldn't get file size.\n"); + std::ifstream file(argv[1]); + if (!file.good()) { + std::cerr << "I/O error: couldn't load file `" << argv[1] << "`."; + return 1; } - char *file_in_memory = - (char *)mmap(nullptr, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - - Parser *parser = new Parser(file_in_memory, argv[1]); + Parser *parser = new Parser(file, argv[1]); std::vector> statements = parser->parse(); // AST_Printer test; @@ -47,6 +40,4 @@ int main(const int argc, const char **argv) { } delete parser; - munmap(file_in_memory, sb.st_size); - close(fd); } diff --git a/src/parser.cpp b/src/parser.cpp index 88caeb8..3b9fd47 100755 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1,8 +1,11 @@ #include "parser.hpp" #include "syntax_error.hpp" +#include "token_type.hpp" #include +#include #include +#include #include #include @@ -43,41 +46,31 @@ std::unique_ptr Parser::declaration() { std::vector> Parser::importStmt() { std::shared_ptr path = consume("Expected path to file to import but instead got: '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::String); - if (importedFiles.contains(path->lexeme)) { + const char *filepath = std::get(path->lexeme); + if (importedFiles.contains(filepath)) { throw SyntaxError(previous.get(), - "Recursive file import: '" + path->lexeme + + "Recursive file import: '" + std::string(filepath) + "' has already been imported or is the main file."); } - importedFiles.insert(path->lexeme); - int fd = open(path->lexeme.c_str(), O_RDONLY, S_IRUSR | S_IWUSR); - struct stat sb; - - if (fstat(fd, &sb) == -1) { - perror("Couldn't get file size.\n"); - } - - char *sourceFile = - (char *)mmap(nullptr, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - - CONSUME_SEMICOLON("import"); - - return (new Parser(sourceFile, path->lexeme, importedFiles))->parse(); + importedFiles.insert(filepath); + std::ifstream sourceFile(filepath); CONSUME_SEMICOLON("import"); - return (new Parser(sourceFile, path->lexeme, importedFiles))->parse(); + return (new Parser(sourceFile, filepath, importedFiles))->parse(); } std::unique_ptr Parser::varDecl() { - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; std::shared_ptr name = consume("Expected variable name after 'let' keyword but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::Identifier); std::unique_ptr value; @@ -86,58 +79,59 @@ std::unique_ptr Parser::varDecl() { } consume("Expected ';' at the end of a statement but instead got: '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::Semicolon); - return std::make_unique(line, col, lexer->getFilename(), name, - value); + return std::make_unique(line, col, filename, name, value); } std::unique_ptr Parser::functionDecl(const std::string &kind) { std::vector> args; - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; + std::shared_ptr name = consume("Expected " + kind + " name.", TokenType::Identifier); consume("Expected argument list after trait function name but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::LeftParen); if (!check(TokenType::RightParen)) { do { std::shared_ptr arg = consume( "Expected name of trait function parameter but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::Identifier); args.push_back(arg); } while (match(TokenType::Comma)); } - consume("Expected ')' but instead got '" + current->lexeme + "'.", + + consume("Expected ')' but instead got '" + type2str(current->type) + "'.", TokenType::RightParen); consume("Expected '{' after function signature but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::LeftBrace); - return std::make_unique(line, col, lexer->getFilename(), name, args, + return std::make_unique(line, col, filename, name, args, stmtSequence()); } std::unique_ptr Parser::classDecl() { - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; - std::shared_ptr name = - consume("Expected class name but instead got '" + current->lexeme + "'.", - TokenType::Identifier); + std::shared_ptr name = consume( + "Expected class name but instead got '" + type2str(current->type) + "'.", + TokenType::Identifier); std::unique_ptr superclass; if (match(TokenType::Colon)) { std::shared_ptr supername = consume("Expected superclass name.", TokenType::Identifier); - superclass = std::make_unique(line, col, lexer->getFilename(), - supername); + superclass = std::make_unique(line, col, filename, supername); } - auto clas = std::make_unique(line, col, lexer->getFilename(), name, - superclass, + auto clas = std::make_unique(line, col, filename, name, superclass, std::vector>()); if (match(TokenType::Semicolon)) { return clas; @@ -152,7 +146,7 @@ std::unique_ptr Parser::classDecl() { throw SyntaxError(current.get(), "Expected either a semicolon for an inline class " "definition or a sequence of methods but instead got '" + - current->lexeme + "'."); + type2str(current->type) + "'."); } } @@ -175,13 +169,14 @@ std::unique_ptr Parser::statement() { } std::unique_ptr Parser::scopedStmt() { - int line = lexer->getLine(), col = lexer->getColumn(); - return std::make_unique(line, col, lexer->getFilename(), - stmtSequence()); + return std::make_unique(current->line, current->column, + current->filename, stmtSequence()); } std::unique_ptr Parser::ifStmt() { - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; + std::unique_ptr cond = expression(); std::unique_ptr trueBranch = statement(); std::unique_ptr falseBranch; @@ -190,35 +185,36 @@ std::unique_ptr Parser::ifStmt() { falseBranch = statement(); } - return std::make_unique(line, col, lexer->getFilename(), cond, - trueBranch, falseBranch); + return std::make_unique(line, col, filename, cond, trueBranch, + falseBranch); } std::unique_ptr Parser::whileStmt() { - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; auto condition = expression(); auto body = statement(); - return std::make_unique(line, col, lexer->getFilename(), condition, - body); + return std::make_unique(line, col, filename, condition, body); } std::unique_ptr Parser::loopStmt() { - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; std::unique_ptr cond = std::make_unique( - line, col, lexer->getFilename(), + line, col, filename, std::make_shared(TokenType::True, "true", previous->line, previous->column, previous->filename)); - return std::make_unique(line, col, lexer->getFilename(), cond, - statement()); + return std::make_unique(line, col, filename, cond, statement()); } std::unique_ptr Parser::forStmt() { - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; consume("Expected '(' at the beginning of for statement but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::LeftParen); std::unique_ptr initializer; @@ -233,7 +229,7 @@ std::unique_ptr Parser::forStmt() { cond = expression(); } consume("Expected ';' after loop condition but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::Semicolon); std::unique_ptr increment; @@ -241,39 +237,40 @@ std::unique_ptr Parser::forStmt() { increment = expression(); } consume("Expected ')' after loop clauses but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::RightParen); std::unique_ptr body = statement(); if (!cond) { cond = std::make_unique( - line, col, lexer->getFilename(), + line, col, filename, std::make_shared(TokenType::True, "true", previous->line, previous->column, previous->filename)); } - return std::make_unique(line, col, lexer->getFilename(), initializer, - cond, increment, body); + return std::make_unique(line, col, filename, initializer, cond, + increment, body); } std::unique_ptr Parser::returnStmt() { - int line = lexer->getLine(), col = lexer->getColumn(); std::unique_ptr value; + int line = current->line, col = current->column; + const char *filename = current->filename; if (!check(TokenType::Semicolon)) { value = expression(); } CONSUME_SEMICOLON("return"); - return std::make_unique(line, col, lexer->getFilename(), value); + return std::make_unique(line, col, filename, value); } std::unique_ptr Parser::expressionStmt() { - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; - auto expr = - std::make_unique(line, col, lexer->getFilename(), expression()); + auto expr = std::make_unique(line, col, filename, expression()); CONSUME_SEMICOLON("expression"); return expr; @@ -281,11 +278,11 @@ std::unique_ptr Parser::expressionStmt() { std::unique_ptr Parser::expression() { if (match(TokenType::Break)) { - int line = lexer->getLine(), col = lexer->getColumn(); - return std::make_unique(line, col, lexer->getFilename()); + return std::make_unique(current->line, current->column, + current->filename); } else if (match(TokenType::Break)) { - int line = lexer->getLine(), col = lexer->getColumn(); - return std::make_unique(line, col, lexer->getFilename()); + return std::make_unique(current->line, current->column, + current->filename); } return assignment(); @@ -293,27 +290,28 @@ std::unique_ptr Parser::expression() { std::unique_ptr Parser::assignment() { std::unique_ptr left = orExpr(); - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; if (match(TokenType::Equal)) { std::unique_ptr value = assignment(); if (auto var = dynamic_cast(left.get())) { - return std::make_unique(line, col, lexer->getFilename(), - var->variable, value); + return std::make_unique(line, col, filename, var->variable, + value); } else if (auto get = dynamic_cast(left.get())) { - return std::make_unique(line, col, lexer->getFilename(), - get->instance, get->field, value); + return std::make_unique(line, col, filename, get->instance, + get->field, value); } throw SyntaxError(value.get(), "Invalid assignment value."); } else if (match(TokenType::Question)) { std::unique_ptr trueExpr = assignment(); consume("Expected ':' between ternary expressions but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::Colon); - return std::make_unique(line, col, lexer->getFilename(), left, - trueExpr, assignment()); + return std::make_unique(line, col, filename, left, trueExpr, + assignment()); } return left; @@ -321,13 +319,14 @@ std::unique_ptr Parser::assignment() { std::unique_ptr Parser::orExpr() { std::unique_ptr left = andExpr(); - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; while (match(TokenType::Or)) { std::shared_ptr op = previous; - left = std::make_unique(line, col, lexer->getFilename(), left, - andExpr(), op); + left = + std::make_unique(line, col, filename, left, andExpr(), op); } return left; @@ -335,13 +334,14 @@ std::unique_ptr Parser::orExpr() { std::unique_ptr Parser::andExpr() { std::unique_ptr left = equality(); - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; while (match(TokenType::And)) { std::shared_ptr op = previous; - left = std::make_unique(line, col, lexer->getFilename(), left, - equality(), op); + left = + std::make_unique(line, col, filename, left, equality(), op); } return left; @@ -349,13 +349,14 @@ std::unique_ptr Parser::andExpr() { std::unique_ptr Parser::equality() { std::unique_ptr left = comparison(); - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; while (match(TokenType::EqualEqual, TokenType::BangEqual)) { std::shared_ptr op = previous; - left = std::make_unique(line, col, lexer->getFilename(), left, - comparison(), op); + left = std::make_unique(line, col, filename, left, comparison(), + op); } return left; @@ -363,14 +364,14 @@ std::unique_ptr Parser::equality() { std::unique_ptr Parser::comparison() { std::unique_ptr left = term(); - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; while (match(TokenType::Greater, TokenType::GreaterEqual, TokenType::Less, TokenType::LessEqual)) { std::shared_ptr op = previous; - left = std::make_unique(line, col, lexer->getFilename(), left, - term(), op); + left = std::make_unique(line, col, filename, left, term(), op); } return left; @@ -378,13 +379,14 @@ std::unique_ptr Parser::comparison() { std::unique_ptr Parser::term() { std::unique_ptr left = factor(); - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; while (match(TokenType::Plus, TokenType::Minus, TokenType::ModOp)) { std::shared_ptr op = previous; - left = std::make_unique(line, col, lexer->getFilename(), left, - factor(), op); + left = + std::make_unique(line, col, filename, left, factor(), op); } return left; @@ -392,32 +394,32 @@ std::unique_ptr Parser::term() { std::unique_ptr Parser::factor() { std::unique_ptr left = unary(); - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; while (match(TokenType::Star, TokenType::Slash)) { std::shared_ptr op = previous; - left = std::make_unique(line, col, lexer->getFilename(), left, - unary(), op); + left = std::make_unique(line, col, filename, left, unary(), op); } return left; } std::unique_ptr Parser::unary() { - int line = lexer->getLine(), col = lexer->getColumn(); if (match(TokenType::Bang, TokenType::Minus)) { std::shared_ptr op = previous; - return std::make_unique(line, col, lexer->getFilename(), unary(), - op); + return std::make_unique(current->line, current->column, + current->filename, unary(), op); + } else { + return call(); } - - return call(); } std::unique_ptr Parser::call() { std::unique_ptr expr = primary(); - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; while (1) { if (match(TokenType::LeftParen)) { @@ -429,17 +431,15 @@ std::unique_ptr Parser::call() { } consume("Expected ')' after function call but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::RightParen); - expr = std::make_unique(line, col, lexer->getFilename(), expr, - args); + expr = std::make_unique(line, col, filename, expr, args); } else if (match(TokenType::Dot)) { std::shared_ptr prop = consume("Expected class property or method but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::Identifier); - expr = std::make_unique(line, col, lexer->getFilename(), - expr, prop); + expr = std::make_unique(line, col, filename, expr, prop); } else { break; } @@ -449,33 +449,30 @@ std::unique_ptr Parser::call() { } std::unique_ptr Parser::primary() { - int line = lexer->getLine(), col = lexer->getColumn(); + int line = current->line, col = current->column; + const char *filename = current->filename; if (match(TokenType::Super)) { consume("Expected '.' after super keyword.", TokenType::Dot); std::shared_ptr field = consume("Expected superclass field.", TokenType::Identifier); - return std::make_unique(line, col, lexer->getFilename(), field); + return std::make_unique(line, col, filename, field); } else if (match(TokenType::This)) { - return std::make_unique(line, col, lexer->getFilename(), - previous); + return std::make_unique(line, col, filename, previous); } else if (match(TokenType::Identifier)) { - return std::make_unique(line, col, lexer->getFilename(), - previous); + return std::make_unique(line, col, filename, previous); } else if (match(TokenType::Number, TokenType::Char, TokenType::String, TokenType::True, TokenType::False, TokenType::Nil)) { - return std::make_unique(line, col, lexer->getFilename(), - previous); + return std::make_unique(line, col, filename, previous); } else if (match(TokenType::LeftParen)) { std::unique_ptr expr = expression(); consume("Missing closing parenthesis ')' after expression.", TokenType::RightParen); - return std::make_unique(line, col, lexer->getFilename(), - expr); + return std::make_unique(line, col, filename, expr); } - throw SyntaxError(current.get(), "'" + current->lexeme + + throw SyntaxError(current.get(), "'" + type2str(current->type) + "' not a valid expression statement."); } @@ -505,23 +502,6 @@ void Parser::synchronize() { } } -bool Parser::isAtEnd() { return current->type == TokenType::Eof; } - -template -bool Parser::check(const TokenTypes &...types) { - return ((current->type == types) || ...); -} - -template -bool Parser::match(const TokenTypes &...types) { - if (!(check(types) || ...)) { - return false; - } - - advance(); - return true; -} - std::shared_ptr Parser::advance() { previous = current; @@ -529,23 +509,12 @@ std::shared_ptr Parser::advance() { return current; } - current = lexer->getNextToken(); - if (lexer->hadError) { - throw SyntaxError(current.get(), current->lexeme); - } - return previous; -} - -template -std::shared_ptr Parser::consume(const std::string &msg, - const TokenTypes &...types) { - if (!(check(types) || ...)) { - throw SyntaxError(current.get(), msg); + current = Lexer::getNextToken(this->source); + if (current->type == TokenType::Error) { + throw SyntaxError(current.get(), std::get(current->lexeme)); + } else { + return previous; } - - std::shared_ptr res = current; - advance(); - return res; } std::vector> Parser::stmtSequence() { @@ -562,7 +531,28 @@ std::vector> Parser::stmtSequence() { } consume("Expected '}' at scope block end but instead got '" + - current->lexeme + "'.", + type2str(current->type) + "'.", TokenType::RightBrace); return stmts; } + +template +bool Parser::match(const TokenTypes &...types) { + if (!(check(types) || ...)) { + return false; + } + + advance(); + return true; +} + +std::shared_ptr Parser::consume(const std::string &msg, + const TokenType &expected) { + if (!check(expected)) { + throw SyntaxError(current.get(), msg); + } + + std::shared_ptr res = current; + advance(); + return res; +} diff --git a/src/parser.hpp b/src/parser.hpp index 94f5867..a579adc 100755 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -4,7 +4,10 @@ #include "expressions.hpp" #include "lexer.hpp" #include "statements.hpp" +#include "syntax_error.hpp" +#include "token_type.hpp" +#include #include #include #include @@ -14,24 +17,34 @@ TokenType::Semicolon); class Parser { -private: - Lexer *lexer; - std::shared_ptr current; - std::shared_ptr previous; - std::unordered_set importedFiles; - public: - bool hadError; + Parser(std::ifstream &file, const char *filename) + : source(Lexer::Source(file, filename)), + current(Lexer::getNextToken(this->source)), previous(current) { + importedFiles.insert(filename); + } + + Parser(std::ifstream &file, const char *filename, + std::unordered_set &importedFiles) + : importedFiles(importedFiles), source(Lexer::Source(file, filename)), + current(Lexer::getNextToken(this->source)), previous(current) {} + + std::vector> parse(); private: - bool isAtEnd(); void synchronize(); std::shared_ptr advance(); - template bool check(const TokenTypes &...); + inline bool isAtEnd() { return current->type == TokenType::Eof; } + template - std::shared_ptr consume(const std::string &msg, const TokenTypes &...); - template bool match(const TokenTypes &...); + inline bool check(const TokenTypes &...types) { + return ((current->type == types) || ...); + } + + template bool match(const TokenTypes &...types); + std::shared_ptr consume(const std::string &msg, + const TokenType &expected); std::vector> importStmt(); std::unique_ptr functionDecl(const std::string &); @@ -62,19 +75,14 @@ private: std::unique_ptr call(); std::unique_ptr primary(); -public: - Parser(const char *file, const std::string &filename) - : lexer(new Lexer(file, filename)), current(lexer->getNextToken()), - previous(current) { - importedFiles.insert(filename); - } - - Parser(const char *file, const std::string &filename, - std::unordered_set &importedFiles) - : importedFiles(importedFiles), lexer(new Lexer(file, filename)), - current(lexer->getNextToken()), previous(current) {} +private: + Lexer::Source source; + std::shared_ptr current; + std::shared_ptr previous; + std::unordered_set importedFiles; - std::vector> parse(); +public: + bool hadError; }; #endif diff --git a/src/resolver.cpp b/src/resolver.cpp index 05c9541..c515c9e 100644 --- a/src/resolver.cpp +++ b/src/resolver.cpp @@ -1,5 +1,6 @@ #include "resolver.hpp" #include "syntax_error.hpp" +#include void Resolver::resolve(std::vector> &stmts) { for (auto &&stmt : stmts) { @@ -17,12 +18,13 @@ void Resolver::declare(const Token *name) { return; } + auto namestr = std::get(name->lexeme); std::map &scope = scopes.back(); - if (scope.contains(name->lexeme)) { + if (scope.contains(namestr)) { throw SyntaxError(name, "Variable with this name already exists."); } - scope.insert(std::make_pair(name->lexeme, VarState::Init)); + scope.insert(std::make_pair(namestr, VarState::Init)); } void Resolver::define(const Token *name) { @@ -30,12 +32,13 @@ void Resolver::define(const Token *name) { return; } - scopes.back().insert_or_assign(name->lexeme, VarState::Ready); + scopes.back().insert_or_assign(std::get(name->lexeme), + VarState::Ready); } void Resolver::resolveLocal(Expr *expr, const Token *name) { for (int i = scopes.size() - 1; i >= 0; i--) { - if (scopes[i].contains(name->lexeme)) { + if (scopes[i].contains(std::get(name->lexeme))) { interpreter->resolve(expr, scopes.size() - 1 - i); return; } @@ -105,7 +108,8 @@ void Resolver::visitClassStmt(ClassStmt *clas) { for (auto &&stmt : clas->body) { auto method = dynamic_cast(stmt.get()); - if (method && method->name->lexeme == "init") { + if (method && std::string_view( + std::get(method->name->lexeme)) == "init") { resolveFunction(method, FunctionType::Initializer); } else { resolveFunction(method, FunctionType::Function); @@ -242,8 +246,11 @@ Value Resolver::visitTernaryExpr(TernaryExpr *expr) { } Value Resolver::visitVarExpr(VariableExpr *expr) { - if (!scopes.empty() && scopes.back().contains(expr->variable->lexeme) && - scopes.back().find(expr->variable->lexeme)->second != VarState::Ready) { + if (!scopes.empty() && + scopes.back().contains(std::get(expr->variable->lexeme)) && + scopes.back() + .find(std::get(expr->variable->lexeme)) + ->second != VarState::Ready) { throw SyntaxError(expr, "You are trying to read the value of a variable " "that hasn't been defined yet."); } diff --git a/src/token.cpp b/src/token.cpp new file mode 100644 index 0000000..a20a238 --- /dev/null +++ b/src/token.cpp @@ -0,0 +1,24 @@ +#include "token.hpp" +#include + +std::ostream &operator<<(std::ostream &os, const Token tk) { + os << "Token { line: " << tk.line << ", column: " << tk.column + << ", filename: " << tk.filename << ", type: " << type2str(tk.type); + if (!std::holds_alternative(tk.lexeme)) { + os << ", lexeme: "; + std::visit( + [&](auto &&arg) { + using T = std::decay_t; + if constexpr (std::is_same_v || + std::is_same_v || std::is_same_v) { + os << arg; + } else if constexpr (std::is_same_v) { + os << std::quoted(arg); + } + }, + tk.lexeme); + } + + os << " }"; + return os; +} diff --git a/src/token.hpp b/src/token.hpp index 8c2ff6b..6747108 100755 --- a/src/token.hpp +++ b/src/token.hpp @@ -3,18 +3,28 @@ #include "token_type.hpp" -#include +#include +#include +#include +#include + +using literal_t = + std::variant; + +struct Token { + Token(TokenType type, literal_t lexeme, int32_t line, int32_t column, + const char *filename) + : type(type), lexeme(lexeme), line(line), column(column), + filename(filename) {} -class Token { public: - int line; - int column; + int32_t line; + int32_t column; TokenType type; - std::string lexeme; - std::string filename; - - Token(TokenType type, const std::string &lexeme, int line, int column, const std::string &filename) - : type(type), lexeme(lexeme), line(line), column(column), filename(filename) {} + literal_t lexeme; + const char *filename; }; +std::ostream &operator<<(std::ostream &os, const Token tk); + #endif diff --git a/src/token_type.hpp b/src/token_type.hpp index b30e872..30c58a6 100755 --- a/src/token_type.hpp +++ b/src/token_type.hpp @@ -1,7 +1,13 @@ #ifndef TOKEN_TYPE_H #define TOKEN_TYPE_H -enum TokenType { +#include +#include +#include +#include +#include + +enum class TokenType { // Single-character tokens. LeftParen, RightParen, @@ -61,4 +67,73 @@ enum TokenType { Error, }; +static constexpr std::array, 50> + type2str_map = {{ + {TokenType::LeftParen, "("}, + {TokenType::RightParen, ")"}, + {TokenType::LeftBrace, "{"}, + {TokenType::RightBrace, "}"}, + {TokenType::Question, "?"}, + {TokenType::Comma, ","}, + {TokenType::Dot, "."}, + {TokenType::Colon, ","}, + {TokenType::Semicolon, ";"}, + {TokenType::ModOp, "%"}, + {TokenType::Minus, "-"}, + {TokenType::Plus, "+"}, + {TokenType::Slash, "/"}, + {TokenType::Star, "*"}, + + {TokenType::Bang, "!"}, + {TokenType::BangEqual, "!="}, + {TokenType::Equal, "="}, + {TokenType::EqualEqual, "=="}, + {TokenType::ShiftRight, ">>"}, + {TokenType::Greater, ">"}, + {TokenType::GreaterEqual, ">="}, + {TokenType::ShiftLeft, "<<"}, + {TokenType::Less, "<"}, + {TokenType::LessEqual, "<="}, + + {TokenType::Identifier, "Identifier"}, + {TokenType::String, "String"}, + {TokenType::Char, "Char"}, + {TokenType::Number, "Number"}, + + {TokenType::Import, "import"}, + {TokenType::Let, "let"}, + {TokenType::And, "and"}, + {TokenType::Class, "class"}, + {TokenType::Else, "else"}, + {TokenType::False, "false"}, + {TokenType::Fn, "fn"}, + {TokenType::For, "for"}, + {TokenType::If, "if"}, + {TokenType::Nil, "nil"}, + {TokenType::Or, "or"}, + {TokenType::Return, "return"}, + {TokenType::Break, "break"}, + {TokenType::Continue, "continue"}, + {TokenType::Super, "super"}, + {TokenType::This, "this"}, + {TokenType::True, "true"}, + {TokenType::While, "while"}, + {TokenType::Loop, "loop"}, + {TokenType::Main, "main"}, + {TokenType::Eof, "eof"}, + {TokenType::Error, "error"}, + }}; + +inline std::string type2str(TokenType type) { + const auto it = + std::find_if(type2str_map.begin(), type2str_map.end(), + [&](const auto &value) { return value.first == type; }); + + if (it != type2str_map.end()) { + return it->second; + } else { + __builtin_unreachable(); + } +} + #endif -- 2.52.0