unhandled exception using Boost Spirit to parse grammar

Question

I am trying to use Boost Spirit to parse the following grammar: sentence: noun verb sentence conjunction sentence

conjunction: "and"

noun: "birds" "cats"

verb: "fly" "meow"

parsing succeeds when the grammar only includes noun >> verb rule. when grammar is modified to include sentence>>conjunction>>sentence rule and i supply an invalid input such as "birds fly" instead of "birdsfly" i get an unhandled exception when the program runs.

here is the code which is modified from examples found on boost doc

#define BOOST_VARIANT_MINIMIZE_SIZE
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_statement.hpp>
#include <boost/spirit/include/phoenix_container.hpp>
#include <iostream>
#include <string>

using namespace boost::spirit;
using namespace boost::spirit::ascii;

template <typename Lexer>
struct token_list : lex::lexer<Lexer>
{
    token_list()
    {
        noun = "birds|cats";    
        verb =  "fly|meow";
        conjunction = "and";

        this->self.add
            (noun)         
            (verb) 
            (conjunction)
        ;
    }
    lex::token_def<std::string> noun, verb, conjunction;
};

template <typename Iterator>
struct Grammar : qi::grammar<Iterator>
{
    template <typename TokenDef>
    Grammar(TokenDef const& tok)
      : Grammar::base_type(sentence)
    {
        sentence = (tok.noun>>tok.verb)
        |
        (sentence>>tok.conjunction>>sentence)>>eoi
    ;
    }
    qi::rule<Iterator> sentence;
};

int main()
{
typedef lex::lexertl::token<char const*, boost::mpl::vector<std::string>> token_type;
typedef lex::lexertl::lexer<token_type> lexer_type;
typedef token_list<lexer_type>::iterator_type iterator_type;

     token_list<lexer_type> word_count;         
     Grammar<iterator_type> g (word_count); 

     std::string str = "birdsfly"; 
 //std::string str = "birds fly"; this input caused unhandled exception

     char const* first = str.c_str();
     char const* last = &first[str.size()];

     bool r = lex::tokenize_and_parse(first, last, word_count, g);

     if (r) {
         std::cout << "Parsing passed"<< "\n";
     }
     else {
         std::string rest(first, last);
         std::cerr << "Parsing failed\n" << "stopped at: \"" 
                   << rest << "\"\n";
     }
    system("PAUSE");
    return 0;
}

sehe sehe · Accepted Answer · 2013-07-30T10:21:03

You have left-recursion in the second branch of the sentence rule.

sentence = sentence >> ....

will always recurse on sentence, so you're seeing a stackoverflow.

I suggest writing the rule like, e.g:

sentence = 
      (tok.noun >> tok.verb) 
  >> *(tok.conjunction >> sentence) 
  >> qi::eoi
  ;

Now the result reads

g++ -Wall -pedantic -std=c++0x -g -O0 test.cpp -o test
Parsing failed
stopped at: " fly"

(and the inevitable "sh: PAUSE: command not found" of course...)

PS. Don't using namespace please. Instead:

namespace qi  = boost::spirit::qi;
namespace lex = boost::spirit::lex;

Here's a cleaned up version with some other stuff removed/fixed: http://coliru.stacked-crooked.com/view?id=1fb26ca3e8c207979eaaf4592c319316-e223fd4a885a77b520bbfe69dda8fb91

#define BOOST_VARIANT_MINIMIZE_SIZE
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
// #include <boost/spirit/include/phoenix.hpp>
#include <iostream>
#include <string>

namespace qi  = boost::spirit::qi;
namespace lex = boost::spirit::lex;

template <typename Lexer>
struct token_list : lex::lexer<Lexer>
{
    token_list()
    {
        noun        = "birds|cats";    
        verb        = "fly|meow";
        conjunction = "and";

        this->self.add
            (noun)         
            (verb) 
            (conjunction)
        ;
    }

    lex::token_def<std::string> noun, verb, conjunction;
};

template <typename Iterator>
struct Grammar : qi::grammar<Iterator>
{
    template <typename TokenDef>
    Grammar(TokenDef const& tok) : Grammar::base_type(sentence)
    {
        sentence = 
              (tok.noun >> tok.verb) 
          >> *(tok.conjunction >> sentence) 
          >> qi::eoi
          ;
    }
    qi::rule<Iterator> sentence;
};

int main()
{
    typedef std::string::const_iterator It;
    typedef lex::lexertl::token<It, boost::mpl::vector<std::string>> token_type;
    typedef lex::lexertl::lexer<token_type> lexer_type;
    typedef token_list<lexer_type>::iterator_type iterator_type;

    token_list<lexer_type> word_count;         
    Grammar<iterator_type> g(word_count); 

    //std::string str = "birdsfly"; 
    const std::string str = "birds fly";

    It first = str.begin();
    It last  = str.end();

    bool r = lex::tokenize_and_parse(first, last, word_count, g);

    if (r) {
        std::cout << "Parsing passed"<< "\n";
    }
    else {
        std::string rest(first, last);
        std::cerr << "Parsing failed\n" << "stopped at: \"" << rest << "\"\n";
    }
}

unhandled exception using Boost Spirit to parse grammar

1 Answers