fix parser issues

inc/expression.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <iostream>
+#include <sstream>
 #include <memory>
 #include <string>
 
@@ -104,7 +105,25 @@ public:
 		}
 	}
 
-	const char *type_str(void)
+	std::shared_ptr<std::string> to_str(void) const
+	{
+		switch (t)
+		{
+		case VAR:
+			return v.s;
+		case CONST:
+		{
+			char buffer[128] = {0};
+			return std::make_shared<std::string>((std::ostringstream() << v.d).str());
+		}
+		case OPERATOR:
+			return std::make_shared<std::string>("OP " + std::string(op_str()));
+		default:
+			return std::make_shared<std::string>("Not a valid token!");
+		}
+	}
+
+	const char *type_str(void) const
 	{
 		switch (t)
 		{
@@ -119,7 +138,7 @@ public:
 		}
 	}
 
-	const char *op_str(void)
+	const char *op_str(void) const
 	{
 		if (t != OPERATOR)
 		{

inc/parser.hpp

@@ -5,7 +5,8 @@
 
 namespace Parser {
 
-std::shared_ptr<std::vector<Expression::Token>> tokenize_input(char *input_string);
+std::shared_ptr<std::vector<Expression::Token>> tokenize_input(const char * const input_string);
-int assert_same(std::vector<Expression::Token>& a, std::vector<Expression::Token>& b);
+int assert_same(const std::vector<Expression::Token>& a, const std::vector<Expression::Token>& b);
+void print_token_vector(const std::vector<Expression::Token>& tv);
 
 }

src/parser.cpp

@@ -34,22 +34,25 @@ class Parser
 		}
 	}
 
-	void switch_state(enum State e)
+	int switch_state(enum State e)
 	{
 		pr_debug("Leaving " << state_str(s) << ", entering " << state_str(e));
+
+		int cumulative_backtrack = 0;
+
 		switch (s)
 		{
 		case I:
-			leave_i();
+			cumulative_backtrack += leave_i();
 			break;
 		case RC:
-			leave_rc();
+			cumulative_backtrack += leave_rc();
 			break;
 		case RV:
-			leave_rv();
+			cumulative_backtrack += leave_rv();
 			break;
 		case OP:
-			leave_op();
+			cumulative_backtrack += leave_op();
 			break;
 		default:
 			pr_warn("Warning: broken state machine on leave");
@@ -76,6 +79,10 @@ class Parser
 		}
 
 		s = e;
+
+		pr_debug("Cumulative backtrack is " << cumulative_backtrack);
+
+		return cumulative_backtrack;
 	}
 
 public:
@@ -121,26 +128,26 @@ public:
 	{
 		if (isdigit(c) || c == '.')
 		{
-			switch_state(RC);
+			return switch_state(RC) + 1;
-			return 1;
 		}
 		else if (isalpha(c))
 		{
-			switch_state(RV);
+			return switch_state(RV) + 1;
-			return 1;
 		}
 		else if (c == '+' || c == '-' || c == '*' || c == '/' || c == '(' || c == ')')
 		{
-			switch_state(OP);
+			return switch_state(OP) + 1;
-			return 1;
 		}
 		else
 		{
-			pr_warn("Unknown character skipped (" << c << "), still in idle");
+			pr_debug("Unknown character skipped (" << c << "), still in idle");
 			return 0;
 		}
 	}
-	void leave_i(void) {}
+	int leave_i(void)
+	{
+		return 0;
+	}
 
 	void enter_rc(void)
 	{
@@ -150,8 +157,7 @@ public:
 	{
 		if (!(isdigit(c) || c == '.'))
 		{
-			switch_state(I);
+			return switch_state(I) + 1;
-			return 1;
 		}
 		else
 		{
@@ -159,7 +165,7 @@ public:
 			return 0;
 		}
 	}
-	void leave_rc(void)
+	int leave_rc(void)
 	{
 		double value;
 
@@ -171,6 +177,8 @@ public:
 		{
 			pv->push_back(Expression::Token(value));
 		}
+
+		return 0;
 	}
 
 	void enter_rv(void)
@@ -181,8 +189,7 @@ public:
 	{
 		if (!isalpha(c))
 		{
-			switch_state(I);
+			return switch_state(I) + 1;
-			return 1;
 		}
 		else
 		{
@@ -190,9 +197,10 @@ public:
 			return 0;
 		}
 	}
-	void leave_rv(void)
+	int leave_rv(void)
 	{
 		pv->push_back(Expression::Token(buffer));
+		return 0;
 	}
 
 	void enter_op(void)
@@ -203,8 +211,7 @@ public:
 	{
 		if (!(c == '+' || c == '-' || c == '*' || c == '/' || c == '(' || c == ')'))
 		{
-			switch_state(I);
+			return switch_state(I) + 1;
-			return 1;
 		}
 		else
 		{
@@ -212,59 +219,74 @@ public:
 			return 0;
 		}
 	}
-	void leave_op(void)
+	int leave_op(void)
 	{
-		if      (!strcmp(buffer, "("))
+		int i, downsize;
+		for (i = strlen(buffer), downsize = 0; i > 0; buffer[--i] = '\0', downsize++)
 		{
-			pv->push_back(Expression::Token(Expression::Token::OPA));
+			if      (!strcmp(buffer, "("))
+			{
+				pv->push_back(Expression::Token(Expression::Token::OPA));
+				break;
+			}
+			else if (!strcmp(buffer, ")"))
+			{
+				pv->push_back(Expression::Token(Expression::Token::CPA));
+				break;
+			}
+			else if (!strcmp(buffer, "+"))
+			{
+				pv->push_back(Expression::Token(Expression::Token::ADD));
+				break;
+			}
+			else if (!strcmp(buffer, "-"))
+			{
+				pv->push_back(Expression::Token(Expression::Token::SUB));
+				break;
+			}
+			else if (!strcmp(buffer, "*"))
+			{
+				pv->push_back(Expression::Token(Expression::Token::MUL));
+				break;
+			}
+			else if (!strcmp(buffer, "/"))
+			{
+				pv->push_back(Expression::Token(Expression::Token::DIV));
+				break;
+			}
+			else if (!strcmp(buffer, "**"))
+			{
+				pv->push_back(Expression::Token(Expression::Token::POW));
+				break;
+			}
 		}
-		else if (!strcmp(buffer, ")"))
+
+		if (i)
 		{
-			pv->push_back(Expression::Token(Expression::Token::CPA));
+			return downsize;
-		}
-		else if (!strcmp(buffer, "+"))
-		{
-			pv->push_back(Expression::Token(Expression::Token::ADD));
-		}
-		else if (!strcmp(buffer, "-"))
-		{
-			pv->push_back(Expression::Token(Expression::Token::SUB));
-		}
-		else if (!strcmp(buffer, "*"))
-		{
-			pv->push_back(Expression::Token(Expression::Token::MUL));
-		}
-		else if (!strcmp(buffer, "/"))
-		{
-			pv->push_back(Expression::Token(Expression::Token::DIV));
-		}
-		else if (!strcmp(buffer, "**"))
-		{
-			pv->push_back(Expression::Token(Expression::Token::POW));
 		}
 		else
 		{
 			pr_warn("Skip unknown operator: " << buffer);
+			return 0;
 		}
 	}
 };
 
-std::shared_ptr<std::vector<Expression::Token>> tokenize_input(char *input_string)
+std::shared_ptr<std::vector<Expression::Token>> tokenize_input(const char * const input_string)
 {
 	std::shared_ptr<std::vector<Expression::Token>> pv = std::make_shared<std::vector<Expression::Token>>();
 
 	Parser p(pv);
-	for (int i = 0, l = strlen(input_string); i < l; i++)
+	for (int i = 0, l = strlen(input_string) + 1; i < l; i++)
 	{
 		i -= p.dispatch_to_current(input_string[i]);
 	}
 
-	(void) p.dispatch_to_current('\0');
-
 	return pv;
 }
 
-int assert_same(std::vector<Expression::Token>& a, std::vector<Expression::Token>& b)
+int assert_same(const std::vector<Expression::Token>& a, const std::vector<Expression::Token>& b)
 {
 	if (a.size() != b.size())
 	{
@@ -309,4 +331,15 @@ int assert_same(std::vector<Expression::Token>& a, std::vector<Expression::Token
 	return 0;
 }
 
+void print_token_vector(const std::vector<Expression::Token>& tv)
+{
+	int i, l;
+	for (i = 0, l = tv.size() - 1; i < l; i++)
+	{
+		std::cout << "(" << *tv[i].to_str() << ") ";
+	}
+
+	std::cout << "(" << *tv[i].to_str() << ")\n";
+}
+
 }

src/test-cases.cpp

@@ -4,7 +4,9 @@
 
 TEST(parser_proper_test_1)
 {
-	auto o = Parser::tokenize_input((char *) "(1+2-c)");
+	auto o = Parser::tokenize_input("(1+2-c)");
+	Parser::print_token_vector(*o);
+
 	std::vector<Expression::Token> r;
 
 	r.push_back(Expression::Token(Expression::Token::OPA));
@@ -22,7 +24,9 @@ TEST(parser_proper_test_1)
 
 TEST(parser_proper_test_2)
 {
-	auto o = Parser::tokenize_input((char *) "(1**5,-snake+marble*34*(35 / 63.2))");
+	auto o = Parser::tokenize_input("(1**5,-snake+marble*34*(35 / 63.2))");
+	Parser::print_token_vector(*o);
+
 	std::vector<Expression::Token> r;
 
 	r.push_back(Expression::Token(Expression::Token::OPA));
@@ -48,8 +52,45 @@ TEST(parser_proper_test_2)
 	return 0;
 }
 
+TEST(parser_proper_test_3)
+{
+	auto o = Parser::tokenize_input(")))25.4,, 33..4 35 37 39 41 // +-*/***()()()");
+	Parser::print_token_vector(*o);
+
+	std::vector<Expression::Token> r;
+
+	r.push_back(Expression::Token(Expression::Token::CPA));
+	r.push_back(Expression::Token(Expression::Token::CPA));
+	r.push_back(Expression::Token(Expression::Token::CPA));
+	r.push_back(Expression::Token(25.4));
+	r.push_back(Expression::Token(33));
+	r.push_back(Expression::Token(35));
+	r.push_back(Expression::Token(37));
+	r.push_back(Expression::Token(39));
+	r.push_back(Expression::Token(41));
+	r.push_back(Expression::Token(Expression::Token::DIV));
+	r.push_back(Expression::Token(Expression::Token::DIV));
+	r.push_back(Expression::Token(Expression::Token::ADD));
+	r.push_back(Expression::Token(Expression::Token::SUB));
+	r.push_back(Expression::Token(Expression::Token::MUL));
+	r.push_back(Expression::Token(Expression::Token::DIV));
+	r.push_back(Expression::Token(Expression::Token::POW));
+	r.push_back(Expression::Token(Expression::Token::MUL));
+	r.push_back(Expression::Token(Expression::Token::OPA));
+	r.push_back(Expression::Token(Expression::Token::CPA));
+	r.push_back(Expression::Token(Expression::Token::OPA));
+	r.push_back(Expression::Token(Expression::Token::CPA));
+	r.push_back(Expression::Token(Expression::Token::OPA));
+	r.push_back(Expression::Token(Expression::Token::CPA));
+
+	CHECK(Parser::assert_same(*o, r));
+
+	return 0;
+}
+
 void register_tests(void)
 {
 	REGISTER(parser_proper_test_1);
 	REGISTER(parser_proper_test_2);
+	REGISTER(parser_proper_test_3);
 }