实现 12.4 词法：25 关键字、小写折叠、TIME 字面量、lex error。
- Lexer.h/cpp：关键字表（大小写不敏感折小写）、(* *) 注释不嵌套、整数/T# 段式字面量、最长匹配符号 - token 带 source_file + 行列（起始位置，修了消费后行列的偏差） - lexer_test：75 断言（line1 三文件 token 流 21/34/27、大小写、符号、注释、行列、5 类负例） - Doc/compiler/词法.md：设计文档 + 执行计划
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* @file Lexer.h
|
||||
* @brief ST 子集词法分析
|
||||
* @author
|
||||
* @date 2026-08-21
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace compiler {
|
||||
|
||||
// token 类型。关键字与 token 集见 Doc/compiler/词法.md。
|
||||
enum class Tok {
|
||||
// 关键字（25 个，与 12.1 冻结表一致）
|
||||
PROGRAM,
|
||||
FUNCTION,
|
||||
FUNCTION_BLOCK,
|
||||
END_PROGRAM,
|
||||
END_FUNCTION,
|
||||
END_FUNCTION_BLOCK,
|
||||
VAR,
|
||||
VAR_INPUT,
|
||||
VAR_OUTPUT,
|
||||
VAR_GLOBAL,
|
||||
VAR_EXTERNAL,
|
||||
END_VAR,
|
||||
BOOL,
|
||||
INT,
|
||||
TIME,
|
||||
IF,
|
||||
ELSIF,
|
||||
ELSE,
|
||||
END_IF,
|
||||
WHILE,
|
||||
END_WHILE,
|
||||
AND,
|
||||
OR,
|
||||
NOT,
|
||||
TRUE,
|
||||
FALSE,
|
||||
TON,
|
||||
TOF,
|
||||
CTU,
|
||||
// 字面量
|
||||
IDENT, // text = 小写名
|
||||
INT_LIT, // int_value
|
||||
TIME_LIT, // int_value = 毫秒
|
||||
// 符号
|
||||
ASSIGN, // :=
|
||||
EQ, // =
|
||||
NE, // <>
|
||||
LT, // <
|
||||
LE, // <=
|
||||
GT, // >
|
||||
GE, // >=
|
||||
PLUS, // +
|
||||
MINUS, // -
|
||||
STAR, // *
|
||||
SLASH, // /
|
||||
LPAREN, // (
|
||||
RPAREN, // )
|
||||
COMMA, // ,
|
||||
SEMI, // ;
|
||||
COLON, // :
|
||||
DOT, // .
|
||||
END, // 文件结束
|
||||
};
|
||||
|
||||
struct Token {
|
||||
Tok type;
|
||||
std::string text; // 标识符/关键字的小写文本；字面量原文
|
||||
int64_t int_value; // INT_LIT / TIME_LIT（毫秒）
|
||||
std::string source_file;
|
||||
uint32_t line; // 从 1 起
|
||||
uint32_t col; // 从 1 起
|
||||
};
|
||||
|
||||
// 对源文本 lex 成 token 流（含末尾 END）。失败返回 false，
|
||||
// err 前缀 "lex error"，带文件与行列。
|
||||
bool lex(const std::string& source_file, const std::string& content,
|
||||
std::vector<Token>* out, std::string* err);
|
||||
|
||||
// 读文件后 lex（文件不存在 / 读取失败也算 lex error）
|
||||
bool lex_file(const std::string& path, std::vector<Token>* out, std::string* err);
|
||||
}
|
||||
+347
-2
@@ -1,8 +1,353 @@
|
||||
/**
|
||||
* @file Lexer.cpp
|
||||
* @brief 词法分析（12.4 实现）
|
||||
* @date 2026-08-19
|
||||
* @brief ST 子集词法分析
|
||||
* @author
|
||||
* @date 2026-08-21
|
||||
*/
|
||||
|
||||
#include "compiler/Lexer.h"
|
||||
|
||||
#include <cctype>
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
|
||||
namespace compiler {
|
||||
namespace {
|
||||
|
||||
// 25 个关键字：小写键 → Tok（大小写不敏感，见 Doc/compiler/词法.md）
|
||||
const struct {
|
||||
const char* key;
|
||||
Tok tok;
|
||||
} kKeywords[] = {
|
||||
{"program", Tok::PROGRAM},
|
||||
{"function", Tok::FUNCTION},
|
||||
{"function_block", Tok::FUNCTION_BLOCK},
|
||||
{"end_program", Tok::END_PROGRAM},
|
||||
{"end_function", Tok::END_FUNCTION},
|
||||
{"end_function_block", Tok::END_FUNCTION_BLOCK},
|
||||
{"var", Tok::VAR},
|
||||
{"var_input", Tok::VAR_INPUT},
|
||||
{"var_output", Tok::VAR_OUTPUT},
|
||||
{"var_global", Tok::VAR_GLOBAL},
|
||||
{"var_external", Tok::VAR_EXTERNAL},
|
||||
{"end_var", Tok::END_VAR},
|
||||
{"bool", Tok::BOOL},
|
||||
{"int", Tok::INT},
|
||||
{"time", Tok::TIME},
|
||||
{"if", Tok::IF},
|
||||
{"elsif", Tok::ELSIF},
|
||||
{"else", Tok::ELSE},
|
||||
{"end_if", Tok::END_IF},
|
||||
{"while", Tok::WHILE},
|
||||
{"end_while", Tok::END_WHILE},
|
||||
{"and", Tok::AND},
|
||||
{"or", Tok::OR},
|
||||
{"not", Tok::NOT},
|
||||
{"true", Tok::TRUE},
|
||||
{"false", Tok::FALSE},
|
||||
{"ton", Tok::TON},
|
||||
{"tof", Tok::TOF},
|
||||
{"ctu", Tok::CTU},
|
||||
};
|
||||
|
||||
// TIME 字面量单位 → 毫秒倍数（不敏感）
|
||||
int64_t time_unit_ms(const std::string& unit) {
|
||||
if (unit == "ms") return 1;
|
||||
if (unit == "s") return 1000;
|
||||
if (unit == "m") return 60000;
|
||||
if (unit == "h") return 3600000;
|
||||
if (unit == "d") return 86400000;
|
||||
return -1;
|
||||
}
|
||||
|
||||
class Lexer {
|
||||
public:
|
||||
Lexer(const std::string& source_file, const std::string& content,
|
||||
std::vector<Token>* out, std::string* err)
|
||||
: src_(content), sf_(source_file), out_(out), err_(err),
|
||||
pos_(0), line_(1), col_(1) {}
|
||||
|
||||
bool run() {
|
||||
while (true) {
|
||||
if (!skip_space_and_comments()) {
|
||||
return false;
|
||||
}
|
||||
if (pos_ >= src_.size()) {
|
||||
push(Tok::END, "", 0);
|
||||
return true;
|
||||
}
|
||||
if (!lex_one()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
bool fail(const std::string& msg) {
|
||||
if (err_) {
|
||||
char buf[128];
|
||||
std::snprintf(buf, sizeof buf, " (%s, line %u, col %u)",
|
||||
sf_.c_str(), line_, col_);
|
||||
*err_ = "lex error: " + msg + buf;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
char cur() const { return src_[pos_]; }
|
||||
bool at_end() const { return pos_ >= src_.size(); }
|
||||
|
||||
void advance() {
|
||||
if (cur() == '\n') {
|
||||
++line_;
|
||||
col_ = 1;
|
||||
} else {
|
||||
++col_;
|
||||
}
|
||||
++pos_;
|
||||
}
|
||||
|
||||
// 跳过空白与 (* *) 注释（不嵌套）；未闭合返回 false
|
||||
bool skip_space_and_comments() {
|
||||
while (!at_end()) {
|
||||
if (std::isspace(static_cast<unsigned char>(cur()))) {
|
||||
advance();
|
||||
} else if (cur() == '(' && pos_ + 1 < src_.size() &&
|
||||
src_[pos_ + 1] == '*') {
|
||||
advance();
|
||||
advance(); // 跳过 "(*"
|
||||
bool closed = false;
|
||||
while (!at_end()) {
|
||||
if (cur() == '*' && pos_ + 1 < src_.size() &&
|
||||
src_[pos_ + 1] == ')') {
|
||||
advance();
|
||||
advance();
|
||||
closed = true;
|
||||
break;
|
||||
}
|
||||
advance();
|
||||
}
|
||||
if (!closed) {
|
||||
return fail("unterminated comment");
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void push(Tok type, const std::string& text, int64_t value) {
|
||||
Token t;
|
||||
t.type = type;
|
||||
t.text = text;
|
||||
t.int_value = value;
|
||||
t.source_file = sf_;
|
||||
t.line = tok_line_;
|
||||
t.col = tok_col_;
|
||||
out_->push_back(t);
|
||||
}
|
||||
|
||||
bool lex_one() {
|
||||
// 记录本 token 起始位置（push 时消费已经推进了行列）
|
||||
tok_line_ = line_;
|
||||
tok_col_ = col_;
|
||||
const char c = cur();
|
||||
|
||||
// TIME 字面量：T#（大小写不敏感）优先于标识符
|
||||
if ((c == 't' || c == 'T') && pos_ + 1 < src_.size() && src_[pos_ + 1] == '#') {
|
||||
return lex_time();
|
||||
}
|
||||
|
||||
// 标识符 / 关键字（统一折小写）
|
||||
if (std::isalpha(static_cast<unsigned char>(c)) || c == '_') {
|
||||
const uint32_t start = pos_;
|
||||
while (!at_end() &&
|
||||
(std::isalnum(static_cast<unsigned char>(cur())) || cur() == '_')) {
|
||||
advance();
|
||||
}
|
||||
std::string word = src_.substr(start, pos_ - start);
|
||||
for (char& ch : word) {
|
||||
ch = static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
|
||||
}
|
||||
for (const auto& kw : kKeywords) {
|
||||
if (word == kw.key) {
|
||||
push(kw.tok, kw.key, 0);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
push(Tok::IDENT, word, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
// 整数
|
||||
if (std::isdigit(static_cast<unsigned char>(c))) {
|
||||
const uint32_t start = pos_;
|
||||
while (!at_end() && std::isdigit(static_cast<unsigned char>(cur()))) {
|
||||
advance();
|
||||
}
|
||||
const std::string digits = src_.substr(start, pos_ - start);
|
||||
uint64_t v = 0;
|
||||
for (const char ch : digits) {
|
||||
const uint64_t d = static_cast<uint64_t>(ch - '0');
|
||||
if (v > (static_cast<uint64_t>(INT64_MAX) - d) / 10) {
|
||||
return fail("integer literal too large");
|
||||
}
|
||||
v = v * 10 + d;
|
||||
}
|
||||
push(Tok::INT_LIT, digits, static_cast<int64_t>(v));
|
||||
return true;
|
||||
}
|
||||
|
||||
// 符号（最长匹配优先）
|
||||
switch (c) {
|
||||
case ':':
|
||||
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') {
|
||||
advance();
|
||||
advance();
|
||||
push(Tok::ASSIGN, ":=", 0);
|
||||
} else {
|
||||
advance();
|
||||
push(Tok::COLON, ":", 0);
|
||||
}
|
||||
return true;
|
||||
case '=':
|
||||
advance();
|
||||
push(Tok::EQ, "=", 0);
|
||||
return true;
|
||||
case '<':
|
||||
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') {
|
||||
advance();
|
||||
advance();
|
||||
push(Tok::LE, "<=", 0);
|
||||
} else if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '>') {
|
||||
advance();
|
||||
advance();
|
||||
push(Tok::NE, "<>", 0);
|
||||
} else {
|
||||
advance();
|
||||
push(Tok::LT, "<", 0);
|
||||
}
|
||||
return true;
|
||||
case '>':
|
||||
if (pos_ + 1 < src_.size() && src_[pos_ + 1] == '=') {
|
||||
advance();
|
||||
advance();
|
||||
push(Tok::GE, ">=", 0);
|
||||
} else {
|
||||
advance();
|
||||
push(Tok::GT, ">", 0);
|
||||
}
|
||||
return true;
|
||||
case '+':
|
||||
advance();
|
||||
push(Tok::PLUS, "+", 0);
|
||||
return true;
|
||||
case '-':
|
||||
advance();
|
||||
push(Tok::MINUS, "-", 0);
|
||||
return true;
|
||||
case '*':
|
||||
advance();
|
||||
push(Tok::STAR, "*", 0);
|
||||
return true;
|
||||
case '/':
|
||||
advance();
|
||||
push(Tok::SLASH, "/", 0);
|
||||
return true;
|
||||
case '(':
|
||||
advance();
|
||||
push(Tok::LPAREN, "(", 0);
|
||||
return true;
|
||||
case ')':
|
||||
advance();
|
||||
push(Tok::RPAREN, ")", 0);
|
||||
return true;
|
||||
case ',':
|
||||
advance();
|
||||
push(Tok::COMMA, ",", 0);
|
||||
return true;
|
||||
case ';':
|
||||
advance();
|
||||
push(Tok::SEMI, ";", 0);
|
||||
return true;
|
||||
case '.':
|
||||
advance();
|
||||
push(Tok::DOT, ".", 0);
|
||||
return true;
|
||||
default:
|
||||
return fail(std::string("illegal character '") + c + "'");
|
||||
}
|
||||
}
|
||||
|
||||
// T#<int><unit>... 段式解析，累加毫秒
|
||||
bool lex_time() {
|
||||
const uint32_t start = pos_;
|
||||
advance(); // T
|
||||
advance(); // #
|
||||
int64_t total = 0;
|
||||
bool any = false;
|
||||
while (!at_end() && std::isdigit(static_cast<unsigned char>(cur()))) {
|
||||
uint64_t v = 0;
|
||||
while (!at_end() && std::isdigit(static_cast<unsigned char>(cur()))) {
|
||||
const uint64_t d = static_cast<uint64_t>(cur() - '0');
|
||||
if (v > (static_cast<uint64_t>(INT64_MAX) - d) / 10) {
|
||||
return fail("bad time literal");
|
||||
}
|
||||
v = v * 10 + d;
|
||||
advance();
|
||||
}
|
||||
std::string unit;
|
||||
while (!at_end() && std::isalpha(static_cast<unsigned char>(cur()))) {
|
||||
unit += static_cast<char>(
|
||||
std::tolower(static_cast<unsigned char>(cur())));
|
||||
advance();
|
||||
}
|
||||
const int64_t mult = time_unit_ms(unit);
|
||||
if (mult < 0) {
|
||||
return fail("bad time literal");
|
||||
}
|
||||
total += static_cast<int64_t>(v) * mult;
|
||||
any = true;
|
||||
}
|
||||
if (!any) {
|
||||
return fail("bad time literal");
|
||||
}
|
||||
push(Tok::TIME_LIT, src_.substr(start, pos_ - start), total);
|
||||
return true;
|
||||
}
|
||||
|
||||
const std::string& src_;
|
||||
const std::string& sf_;
|
||||
std::vector<Token>* out_;
|
||||
std::string* err_;
|
||||
size_t pos_;
|
||||
uint32_t line_;
|
||||
uint32_t col_;
|
||||
uint32_t tok_line_ = 1;
|
||||
uint32_t tok_col_ = 1;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
bool lex(const std::string& source_file, const std::string& content,
|
||||
std::vector<Token>* out, std::string* err) {
|
||||
out->clear();
|
||||
Lexer l(source_file, content, out, err);
|
||||
return l.run();
|
||||
}
|
||||
|
||||
bool lex_file(const std::string& path, std::vector<Token>* out, std::string* err) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
if (!in) {
|
||||
if (err) {
|
||||
*err = "lex error: cannot open file (" + path + ")";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
std::string content((std::istreambuf_iterator<char>(in)),
|
||||
std::istreambuf_iterator<char>());
|
||||
return lex(path, content, out, err);
|
||||
}
|
||||
|
||||
} // namespace compiler
|
||||
|
||||
Reference in New Issue
Block a user