add essential token preprocessing

This commit is contained in:
2026-10-14 12:00:41 +03:00
parent 1d056a736f
commit 35e38127e7
4 changed files with 66 additions and 6 deletions
+3 -2
View File
@@ -6,8 +6,9 @@
namespace Syntax namespace Syntax
{ {
int check_syntax(std::vector<Expression::Token>& tv); void pre_process(std::vector<Expression::Token>& tv);
void apply_corrections(std::vector<Expression::Token>& tv); int check_syntax(const std::vector<Expression::Token>& tv);
void post_process(std::vector<Expression::Token>& tv);
std::vector<Expression::Operation> build_op_vector(const std::vector<Expression::Token>& tv); std::vector<Expression::Operation> build_op_vector(const std::vector<Expression::Token>& tv);
Expression::TreeNode ov_to_tree(const std::vector<Expression::Token> tv, const std::vector<Expression::Operation>& ov); Expression::TreeNode ov_to_tree(const std::vector<Expression::Token> tv, const std::vector<Expression::Operation>& ov);
void print_op_tree(const std::vector<Expression::Token> tv, const std::vector<Expression::Operation>& ov); void print_op_tree(const std::vector<Expression::Token> tv, const std::vector<Expression::Operation>& ov);
+3 -1
View File
@@ -10,12 +10,14 @@ int main(int argc, char **argv)
std::vector<Expression::Token> tv = Parser::tokenize_input(argv[1]); std::vector<Expression::Token> tv = Parser::tokenize_input(argv[1]);
Syntax::pre_process(tv);
if (Syntax::check_syntax(tv)) if (Syntax::check_syntax(tv))
{ {
return 0; return 0;
} }
Syntax::apply_corrections(tv); Syntax::post_process(tv);
std::vector<Expression::Operation> ov = Syntax::build_op_vector(tv); std::vector<Expression::Operation> ov = Syntax::build_op_vector(tv);
+59 -2
View File
@@ -8,6 +8,8 @@
namespace Syntax { namespace Syntax {
static int add_bracket_to(std::vector<Expression::Token>& tv, const int location, const int direction);
static void highlight_issue(const std::vector<Expression::Token>& tv, const int location) static void highlight_issue(const std::vector<Expression::Token>& tv, const int location)
{ {
int start = location - 7 < 0 ? 0 : location - 7; int start = location - 7 < 0 ? 0 : location - 7;
@@ -30,7 +32,62 @@ static void highlight_issue(const std::vector<Expression::Token>& tv, const int
std::cout << "\n"; std::cout << "\n";
} }
int check_syntax(std::vector<Expression::Token>& tv) void pre_process(std::vector<Expression::Token>& tv)
{
pr_debug("Pre-process: remove empty brackets");
for (int i = 0; i < tv.size() - 1; i++)
{
if (tv[i].t == Expression::Token::OPERATOR
&& tv[i].v.o == Expression::Token::OPA
&& tv[i+1].t == Expression::Token::OPERATOR
&& tv[i+1].v.o == Expression::Token::CPA)
{
std::vector<int> highlight;
highlight.emplace_back(i);
highlight.emplace_back(i+1);
Parser::print_math_vector(tv, highlight);
tv.erase(tv.begin() + i, tv.begin() + i + 2);
i--;
}
}
pr_debug("Pre-process: unary minus");
int change = 1;
while (change)
{
change = 0;
for (int i = 0; i < tv.size() - 2; i++)
{
if (tv[i].t == Expression::Token::OPERATOR
&& tv[i].v.o != Expression::Token::CPA
&& tv[i+1].t == Expression::Token::OPERATOR
&& tv[i+1].v.o == Expression::Token::SUB
&& (tv[i+2].t == Expression::Token::CONST
|| tv[i+2].t == Expression::Token::VAR
|| (tv[i+2].t == Expression::Token::OPERATOR
&& tv[i+2].v.o == Expression::Token::OPA)))
{
std::vector<int> highlight;
tv.insert(tv.begin() + i + 1, Expression::Token(0.0));
tv.insert(tv.begin() + i + 1, Expression::Token(Expression::Token::OPA));
highlight.emplace_back(add_bracket_to(tv, i + 3, 1));
highlight.emplace(highlight.begin(), i+2);
highlight.emplace(highlight.begin(), i+1);
Parser::print_math_vector(tv, highlight);
i += 4;
change++;
}
}
}
}
int check_syntax(const std::vector<Expression::Token>& tv)
{ {
int problem_cnt = 0; int problem_cnt = 0;
@@ -639,7 +696,7 @@ static void weight_operators(std::vector<int>& operator_weights, const std::vect
} }
} }
void apply_corrections(std::vector<Expression::Token>& tv) void post_process(std::vector<Expression::Token>& tv)
{ {
pr_debug("Post processing:"); pr_debug("Post processing:");
Parser::print_math_vector(tv); Parser::print_math_vector(tv);
+1 -1
View File
@@ -99,7 +99,7 @@ TEST(tree_proper_test_1)
TEST(tree_proper_test_2) TEST(tree_proper_test_2)
{ {
std::vector<Expression::Token> o = Parser::tokenize_input("(1**5 * (1-2-3-4-5-6-7-8))"); std::vector<Expression::Token> o = Parser::tokenize_input("(1**5 * (1-2-3-4-5-6-7-8))");
Syntax::apply_corrections(o); Syntax::post_process(o);
std::vector<Expression::Operation> ov = Syntax::build_op_vector(o); std::vector<Expression::Operation> ov = Syntax::build_op_vector(o);
return 0; return 0;