/** * @file Lexer.cpp * @brief ST 子集词法分析 * @author * @date 2026-08-21 */ #include "compiler/Lexer.h" #include #include #include #include namespace compiler { namespace { // 25 个关键字:小写键 → Tok(大小写不敏感,见 Doc/compiler/词法.md) const struct { const char* key; Tok tok; } kKeywords[] = { {"program", Tok::PROGRAM}, {"function", Tok::FUNCTION}, {"function_block", Tok::FUNCTION_BLOCK}, {"end_program", Tok::END_PROGRAM}, {"end_function", Tok::END_FUNCTION}, {"end_function_block", Tok::END_FUNCTION_BLOCK}, {"var", Tok::VAR}, {"var_input", Tok::VAR_INPUT}, {"var_output", Tok::VAR_OUTPUT}, {"var_global", Tok::VAR_GLOBAL}, {"var_external", Tok::VAR_EXTERNAL}, {"end_var", Tok::END_VAR}, {"bool", Tok::BOOL}, {"int", Tok::INT}, {"time", Tok::TIME}, {"if", Tok::IF}, {"elsif", Tok::ELSIF}, {"else", Tok::ELSE}, {"end_if", Tok::END_IF}, {"while", Tok::WHILE}, {"end_while", Tok::END_WHILE}, {"then", Tok::THEN}, {"do", Tok::DO}, {"and", Tok::AND}, {"or", Tok::OR}, {"not", Tok::NOT}, {"true", Tok::TRUE}, {"false", Tok::FALSE}, {"ton", Tok::TON}, {"tof", Tok::TOF}, {"ctu", Tok::CTU}, }; // TIME 字面量单位 → 毫秒倍数(不敏感) int64_t time_unit_ms(const std::string& unit) { if (unit == "ms") return 1; if (unit == "s") return 1000; if (unit == "m") return 60000; if (unit == "h") return 3600000; if (unit == "d") return 86400000; return -1; } class Lexer { public: Lexer(const std::string& source_file, const std::string& content, std::vector* out, std::string* err) : src_(content), sf_(source_file), out_(out), err_(err), pos_(0), line_(1), col_(1) {} bool run() { while (true) { if (!skip_space_and_comments()) { return false; } if (pos_ >= src_.size()) { push(Tok::END, "", 0); return true; } if (!lex_one()) { return false; } } } private: bool fail(const std::string& msg) { if (err_) { char buf[128]; std::snprintf(buf, sizeof buf, " (%s, line %u, col %u)", sf_.c_str(), line_, col_); *err_ = "lex error: " + msg + buf; } return false; } char cur() const { return src_[pos_]; } bool at_end() const { return pos_ >= src_.size(); } void advance() { if (cur() == '\n') { ++line_; col_ = 1; } else { ++col_; } ++pos_; } // 跳过空白与 (* *) 注释(不嵌套);未闭合返回 false bool skip_space_and_comments() { while (!at_end()) { if (std::isspace(static_cast(cur()))) { advance(); } else if (cur() == '(' && pos_ + 1 < src_.size() && src_[pos_ + 1] == '*') { advance(); advance(); // 跳过 "(*" bool closed = false; while (!at_end()) { if (cur() == '*' && pos_ + 1 < src_.size() && src_[pos_ + 1] == ')') { advance(); advance(); closed = true; break; } advance(); } if (!closed) { return fail("unterminated comment"); } } else { break; } } return true; } void push(Tok type, const std::string& text, int64_t value) { Token t; t.type = type; t.text = text; t.int_value = value; t.source_file = sf_; t.line = tok_line_; t.col = tok_col_; out_->push_back(t); } bool lex_one() { // 记录本 token 起始位置(push 时消费已经推进了行列) tok_line_ = line_; tok_col_ = col_; const char c = cur(); // TIME 字面量:T#(大小写不敏感)优先于标识符 if ((c == 't' || c == 'T') && pos_ + 1 < src_.size() && src_[pos_ + 1] == '#') { return lex_time(); } // 标识符 / 关键字(统一折小写) if (std::isalpha(static_cast(c)) || c == '_') { const uint32_t start = pos_; while (!at_end() && (std::isalnum(static_cast(cur())) || cur() == '_')) { advance(); } std::string word = src_.substr(start, pos_ - start); for (char& ch : word) { ch = static_cast(std::tolower(static_cast(ch))); } for (const auto& kw : kKeywords) { if (word == kw.key) { push(kw.tok, kw.key, 0); return true; } } push(Tok::IDENT, word, 0); return true; } // 整数 if (std::isdigit(static_cast(c))) { const uint32_t start = pos_; while (!at_end() && std::isdigit(static_cast(cur()))) { advance(); } const std::string digits = src_.substr(start, pos_ - start); uint64_t v = 0; for (const char ch : digits) { const uint64_t d = static_cast(ch - '0'); if (v > (static_cast(INT64_MAX) - d) / 10) { return fail("integer literal too large"); } v = v * 10 + d; } push(Tok::INT_LIT, digits, static_cast(v)); return true; } // 符号(最长匹配优先) switch (c) { case ':': if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') { advance(); advance(); push(Tok::ASSIGN, ":=", 0); } else { advance(); push(Tok::COLON, ":", 0); } return true; case '=': advance(); push(Tok::EQ, "=", 0); return true; case '<': if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') { advance(); advance(); push(Tok::LE, "<=", 0); } else if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '>') { advance(); advance(); push(Tok::NE, "<>", 0); } else { advance(); push(Tok::LT, "<", 0); } return true; case '>': if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') { advance(); advance(); push(Tok::GE, ">=", 0); } else { advance(); push(Tok::GT, ">", 0); } return true; case '+': advance(); push(Tok::PLUS, "+", 0); return true; case '-': advance(); push(Tok::MINUS, "-", 0); return true; case '*': advance(); push(Tok::STAR, "*", 0); return true; case '/': advance(); push(Tok::SLASH, "/", 0); return true; case '(': advance(); push(Tok::LPAREN, "(", 0); return true; case ')': advance(); push(Tok::RPAREN, ")", 0); return true; case ',': advance(); push(Tok::COMMA, ",", 0); return true; case ';': advance(); push(Tok::SEMI, ";", 0); return true; case '.': advance(); push(Tok::DOT, ".", 0); return true; default: return fail(std::string("illegal character '") + c + "'"); } } // T#... 段式解析,累加毫秒 bool lex_time() { const uint32_t start = pos_; advance(); // T advance(); // # int64_t total = 0; bool any = false; while (!at_end() && std::isdigit(static_cast(cur()))) { uint64_t v = 0; while (!at_end() && std::isdigit(static_cast(cur()))) { const uint64_t d = static_cast(cur() - '0'); if (v > (static_cast(INT64_MAX) - d) / 10) { return fail("bad time literal"); } v = v * 10 + d; advance(); } std::string unit; while (!at_end() && std::isalpha(static_cast(cur()))) { unit += static_cast( std::tolower(static_cast(cur()))); advance(); } const int64_t mult = time_unit_ms(unit); if (mult < 0) { return fail("bad time literal"); } total += static_cast(v) * mult; any = true; } if (!any) { return fail("bad time literal"); } push(Tok::TIME_LIT, src_.substr(start, pos_ - start), total); return true; } const std::string& src_; const std::string& sf_; std::vector* out_; std::string* err_; size_t pos_; uint32_t line_; uint32_t col_; uint32_t tok_line_ = 1; uint32_t tok_col_ = 1; }; } // namespace bool lex(const std::string& source_file, const std::string& content, std::vector* out, std::string* err) { out->clear(); Lexer l(source_file, content, out, err); return l.run(); } bool lex_file(const std::string& path, std::vector* out, std::string* err) { std::ifstream in(path, std::ios::binary); if (!in) { if (err) { *err = "lex error: cannot open file (" + path + ")"; } return false; } std::string content((std::istreambuf_iterator(in)), std::istreambuf_iterator()); return lex(path, content, out, err); } } // namespace compiler