Admin
4ba44459e2
- Parser.h/cpp：POU 外壳（PROGRAM/FUNCTION/FUNCTION_BLOCK）、变量段、语句（赋值/IF/WHILE/FB 调用）、表达式优先级爬升（NOT/AND/OR 短路/比较/算术/函数调用/负号） - Unit 结构：顶层 VAR_GLOBAL 段（GVL 文件形态）+ POU 列表 - 拒绝：VAR_IN_OUT/REF/CLASS/ANY 等保留名、链式比较、FB 字段赋值、顶层非 GVL 段 - THEN/DO 补入关键字表（12.5 修订，同步 12.1 与词法文档） - parser_test：98 断言（line1 三文件 AST、FUNCTION 调用、IF/WHILE/TON、9 类负例），ctest 6/6
356 lines
12 KiB
C++
356 lines
12 KiB
C++
/**
|
||
* @file Lexer.cpp
|
||
* @brief ST 子集词法分析
|
||
* @author
|
||
* @date 2026-08-21
|
||
*/
|
||
|
||
#include "compiler/Lexer.h"
|
||
|
||
#include <cctype>
|
||
#include <cstdio>
|
||
#include <fstream>
|
||
#include <string>
|
||
|
||
namespace compiler {
|
||
namespace {
|
||
|
||
// 25 个关键字：小写键 → Tok（大小写不敏感，见 Doc/compiler/词法.md）
|
||
const struct {
|
||
const char* key;
|
||
Tok tok;
|
||
} kKeywords[] = {
|
||
{"program", Tok::PROGRAM},
|
||
{"function", Tok::FUNCTION},
|
||
{"function_block", Tok::FUNCTION_BLOCK},
|
||
{"end_program", Tok::END_PROGRAM},
|
||
{"end_function", Tok::END_FUNCTION},
|
||
{"end_function_block", Tok::END_FUNCTION_BLOCK},
|
||
{"var", Tok::VAR},
|
||
{"var_input", Tok::VAR_INPUT},
|
||
{"var_output", Tok::VAR_OUTPUT},
|
||
{"var_global", Tok::VAR_GLOBAL},
|
||
{"var_external", Tok::VAR_EXTERNAL},
|
||
{"end_var", Tok::END_VAR},
|
||
{"bool", Tok::BOOL},
|
||
{"int", Tok::INT},
|
||
{"time", Tok::TIME},
|
||
{"if", Tok::IF},
|
||
{"elsif", Tok::ELSIF},
|
||
{"else", Tok::ELSE},
|
||
{"end_if", Tok::END_IF},
|
||
{"while", Tok::WHILE},
|
||
{"end_while", Tok::END_WHILE},
|
||
{"then", Tok::THEN},
|
||
{"do", Tok::DO},
|
||
{"and", Tok::AND},
|
||
{"or", Tok::OR},
|
||
{"not", Tok::NOT},
|
||
{"true", Tok::TRUE},
|
||
{"false", Tok::FALSE},
|
||
{"ton", Tok::TON},
|
||
{"tof", Tok::TOF},
|
||
{"ctu", Tok::CTU},
|
||
};
|
||
|
||
// TIME 字面量单位 → 毫秒倍数（不敏感）
|
||
int64_t time_unit_ms(const std::string& unit) {
|
||
if (unit == "ms") return 1;
|
||
if (unit == "s") return 1000;
|
||
if (unit == "m") return 60000;
|
||
if (unit == "h") return 3600000;
|
||
if (unit == "d") return 86400000;
|
||
return -1;
|
||
}
|
||
|
||
class Lexer {
|
||
public:
|
||
Lexer(const std::string& source_file, const std::string& content,
|
||
std::vector<Token>* out, std::string* err)
|
||
: src_(content), sf_(source_file), out_(out), err_(err),
|
||
pos_(0), line_(1), col_(1) {}
|
||
|
||
bool run() {
|
||
while (true) {
|
||
if (!skip_space_and_comments()) {
|
||
return false;
|
||
}
|
||
if (pos_ >= src_.size()) {
|
||
push(Tok::END, "", 0);
|
||
return true;
|
||
}
|
||
if (!lex_one()) {
|
||
return false;
|
||
}
|
||
}
|
||
}
|
||
|
||
private:
|
||
bool fail(const std::string& msg) {
|
||
if (err_) {
|
||
char buf[128];
|
||
std::snprintf(buf, sizeof buf, " (%s, line %u, col %u)",
|
||
sf_.c_str(), line_, col_);
|
||
*err_ = "lex error: " + msg + buf;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
char cur() const { return src_[pos_]; }
|
||
bool at_end() const { return pos_ >= src_.size(); }
|
||
|
||
void advance() {
|
||
if (cur() == '\n') {
|
||
++line_;
|
||
col_ = 1;
|
||
} else {
|
||
++col_;
|
||
}
|
||
++pos_;
|
||
}
|
||
|
||
// 跳过空白与 (* *) 注释（不嵌套）；未闭合返回 false
|
||
bool skip_space_and_comments() {
|
||
while (!at_end()) {
|
||
if (std::isspace(static_cast<unsigned char>(cur()))) {
|
||
advance();
|
||
} else if (cur() == '(' && pos_ + 1 < src_.size() &&
|
||
src_[pos_ + 1] == '*') {
|
||
advance();
|
||
advance(); // 跳过 "(*"
|
||
bool closed = false;
|
||
while (!at_end()) {
|
||
if (cur() == '*' && pos_ + 1 < src_.size() &&
|
||
src_[pos_ + 1] == ')') {
|
||
advance();
|
||
advance();
|
||
closed = true;
|
||
break;
|
||
}
|
||
advance();
|
||
}
|
||
if (!closed) {
|
||
return fail("unterminated comment");
|
||
}
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
return true;
|
||
}
|
||
|
||
void push(Tok type, const std::string& text, int64_t value) {
|
||
Token t;
|
||
t.type = type;
|
||
t.text = text;
|
||
t.int_value = value;
|
||
t.source_file = sf_;
|
||
t.line = tok_line_;
|
||
t.col = tok_col_;
|
||
out_->push_back(t);
|
||
}
|
||
|
||
bool lex_one() {
|
||
// 记录本 token 起始位置（push 时消费已经推进了行列）
|
||
tok_line_ = line_;
|
||
tok_col_ = col_;
|
||
const char c = cur();
|
||
|
||
// TIME 字面量：T#（大小写不敏感）优先于标识符
|
||
if ((c == 't' || c == 'T') && pos_ + 1 < src_.size() && src_[pos_ + 1] == '#') {
|
||
return lex_time();
|
||
}
|
||
|
||
// 标识符 / 关键字（统一折小写）
|
||
if (std::isalpha(static_cast<unsigned char>(c)) || c == '_') {
|
||
const uint32_t start = pos_;
|
||
while (!at_end() &&
|
||
(std::isalnum(static_cast<unsigned char>(cur())) || cur() == '_')) {
|
||
advance();
|
||
}
|
||
std::string word = src_.substr(start, pos_ - start);
|
||
for (char& ch : word) {
|
||
ch = static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
|
||
}
|
||
for (const auto& kw : kKeywords) {
|
||
if (word == kw.key) {
|
||
push(kw.tok, kw.key, 0);
|
||
return true;
|
||
}
|
||
}
|
||
push(Tok::IDENT, word, 0);
|
||
return true;
|
||
}
|
||
|
||
// 整数
|
||
if (std::isdigit(static_cast<unsigned char>(c))) {
|
||
const uint32_t start = pos_;
|
||
while (!at_end() && std::isdigit(static_cast<unsigned char>(cur()))) {
|
||
advance();
|
||
}
|
||
const std::string digits = src_.substr(start, pos_ - start);
|
||
uint64_t v = 0;
|
||
for (const char ch : digits) {
|
||
const uint64_t d = static_cast<uint64_t>(ch - '0');
|
||
if (v > (static_cast<uint64_t>(INT64_MAX) - d) / 10) {
|
||
return fail("integer literal too large");
|
||
}
|
||
v = v * 10 + d;
|
||
}
|
||
push(Tok::INT_LIT, digits, static_cast<int64_t>(v));
|
||
return true;
|
||
}
|
||
|
||
// 符号（最长匹配优先）
|
||
switch (c) {
|
||
case ':':
|
||
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') {
|
||
advance();
|
||
advance();
|
||
push(Tok::ASSIGN, ":=", 0);
|
||
} else {
|
||
advance();
|
||
push(Tok::COLON, ":", 0);
|
||
}
|
||
return true;
|
||
case '=':
|
||
advance();
|
||
push(Tok::EQ, "=", 0);
|
||
return true;
|
||
case '<':
|
||
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') {
|
||
advance();
|
||
advance();
|
||
push(Tok::LE, "<=", 0);
|
||
} else if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '>') {
|
||
advance();
|
||
advance();
|
||
push(Tok::NE, "<>", 0);
|
||
} else {
|
||
advance();
|
||
push(Tok::LT, "<", 0);
|
||
}
|
||
return true;
|
||
case '>':
|
||
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') {
|
||
advance();
|
||
advance();
|
||
push(Tok::GE, ">=", 0);
|
||
} else {
|
||
advance();
|
||
push(Tok::GT, ">", 0);
|
||
}
|
||
return true;
|
||
case '+':
|
||
advance();
|
||
push(Tok::PLUS, "+", 0);
|
||
return true;
|
||
case '-':
|
||
advance();
|
||
push(Tok::MINUS, "-", 0);
|
||
return true;
|
||
case '*':
|
||
advance();
|
||
push(Tok::STAR, "*", 0);
|
||
return true;
|
||
case '/':
|
||
advance();
|
||
push(Tok::SLASH, "/", 0);
|
||
return true;
|
||
case '(':
|
||
advance();
|
||
push(Tok::LPAREN, "(", 0);
|
||
return true;
|
||
case ')':
|
||
advance();
|
||
push(Tok::RPAREN, ")", 0);
|
||
return true;
|
||
case ',':
|
||
advance();
|
||
push(Tok::COMMA, ",", 0);
|
||
return true;
|
||
case ';':
|
||
advance();
|
||
push(Tok::SEMI, ";", 0);
|
||
return true;
|
||
case '.':
|
||
advance();
|
||
push(Tok::DOT, ".", 0);
|
||
return true;
|
||
default:
|
||
return fail(std::string("illegal character '") + c + "'");
|
||
}
|
||
}
|
||
|
||
// T#<int><unit>... 段式解析，累加毫秒
|
||
bool lex_time() {
|
||
const uint32_t start = pos_;
|
||
advance(); // T
|
||
advance(); // #
|
||
int64_t total = 0;
|
||
bool any = false;
|
||
while (!at_end() && std::isdigit(static_cast<unsigned char>(cur()))) {
|
||
uint64_t v = 0;
|
||
while (!at_end() && std::isdigit(static_cast<unsigned char>(cur()))) {
|
||
const uint64_t d = static_cast<uint64_t>(cur() - '0');
|
||
if (v > (static_cast<uint64_t>(INT64_MAX) - d) / 10) {
|
||
return fail("bad time literal");
|
||
}
|
||
v = v * 10 + d;
|
||
advance();
|
||
}
|
||
std::string unit;
|
||
while (!at_end() && std::isalpha(static_cast<unsigned char>(cur()))) {
|
||
unit += static_cast<char>(
|
||
std::tolower(static_cast<unsigned char>(cur())));
|
||
advance();
|
||
}
|
||
const int64_t mult = time_unit_ms(unit);
|
||
if (mult < 0) {
|
||
return fail("bad time literal");
|
||
}
|
||
total += static_cast<int64_t>(v) * mult;
|
||
any = true;
|
||
}
|
||
if (!any) {
|
||
return fail("bad time literal");
|
||
}
|
||
push(Tok::TIME_LIT, src_.substr(start, pos_ - start), total);
|
||
return true;
|
||
}
|
||
|
||
const std::string& src_;
|
||
const std::string& sf_;
|
||
std::vector<Token>* out_;
|
||
std::string* err_;
|
||
size_t pos_;
|
||
uint32_t line_;
|
||
uint32_t col_;
|
||
uint32_t tok_line_ = 1;
|
||
uint32_t tok_col_ = 1;
|
||
};
|
||
|
||
} // namespace
|
||
|
||
bool lex(const std::string& source_file, const std::string& content,
|
||
std::vector<Token>* out, std::string* err) {
|
||
out->clear();
|
||
Lexer l(source_file, content, out, err);
|
||
return l.run();
|
||
}
|
||
|
||
bool lex_file(const std::string& path, std::vector<Token>* out, std::string* err) {
|
||
std::ifstream in(path, std::ios::binary);
|
||
if (!in) {
|
||
if (err) {
|
||
*err = "lex error: cannot open file (" + path + ")";
|
||
}
|
||
return false;
|
||
}
|
||
std::string content((std::istreambuf_iterator<char>(in)),
|
||
std::istreambuf_iterator<char>());
|
||
return lex(path, content, out, err);
|
||
}
|
||
|
||
} // namespace compiler
|