Embedding a boost-spirit grammar parsing into a struct into another grammar gives compilation errors

Question

I'm using boost spirit to parse some text. For this I have two grammars. The first one parses a string into a struct, the second one, takes a grammar as template argument and uses it to parse a sequence of data. The second parser should be flexible enough to also handle other grammar return types. Since the original parser is too large to act as a minimal example, I have reduced the code as much as I can, leaving me with something that would not parse anything, but still results in the same compilation errors: (Code on Coliru)

#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/include/qi.hpp>

#include <vector>

namespace ascii = boost::spirit::ascii;
namespace qi = boost::spirit::qi;


struct Struct1
{
  float f;
};
BOOST_FUSION_ADAPT_STRUCT(
    Struct1,
    (float, f))

struct Struct2
{
  float f;
  int i;
};
BOOST_FUSION_ADAPT_STRUCT(
    Struct2,
    (float, f)
    (int, i))


template<typename Iterator,
         typename Result>
class ElementParser : public qi::grammar<Iterator, Result(), ascii::space_type>
{
public:
  using ValueType = Result;
  
  ElementParser() : ElementParser::base_type(element) {}
  
private:
  qi::rule<Iterator, Result(), ascii::space_type> element;
};


template<typename Iterator,
         typename ElementParser,
         typename Element = typename ElementParser::ValueType>
class SP : public qi::grammar<Iterator, std::vector<Element>(), ascii::space_type>
{
public:
  SP()
    : SP::base_type(sequence)
  {
    sequence %= simpleVector % ',';
    // The simpleVector hack is really needed because of some other parsing
    // stuff, that is going on, but has been left out here.
    simpleVector %= qi::repeat(1)[simple];
  }
  
private:
  using Rule = qi::rule<Iterator, std::vector<Element>(), ascii::space_type>;
  Rule sequence;
  Rule simpleVector;
  ElementParser simple;
};


void sequenceTest()
{
  using Iterator = std::string::const_iterator;
  
  SP<Iterator, qi::uint_parser<>, std::size_t> uintParser;                  // OK
  SP<Iterator, ElementParser<Iterator, float>> floatParser;                 // OK
  SP<Iterator, ElementParser<Iterator, std::vector<float>>> vectorParser;   // OK
  
// error: invalid static_cast from type 'const std::vector<Struct1, std::allocator<Struct1> >' to type 'element_type' {aka 'float'}
  SP<Iterator, ElementParser<Iterator, Struct1>> struct1Parser;
  
// error: no matching function for call to 'Struct2::Struct2(const std::vector<Struct2, std::allocator<Struct2> >&)'
  SP<Iterator, ElementParser<Iterator, Struct2>> struct2Parser;
}

As long, as I am using simple types or vectors as return types of the ElementParser, everything is working fine, but as soon as I'm parsing into a struct (which in itself is working fine), the sequence parser SP seems to try some stange assignments. Why do the struct versions result in compilation errors?

Answer 1

I think you're skirting the age-old single-element-sequence compatibility rules. Especially with Struct1 which is indeed adapted as a single-element sequence.

However, in your code I could easily make it work by removing the unnecessary repeat(1)[] contraption:

Live On Coliru

#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/include/qi.hpp>

namespace ascii = boost::spirit::ascii;
namespace qi = boost::spirit::qi;

struct Struct1 { float f; };
struct Struct2 { float f; int i; };
BOOST_FUSION_ADAPT_STRUCT(Struct1, f)
BOOST_FUSION_ADAPT_STRUCT(Struct2, f, i)

template <typename Iterator, typename Result>
class ElementParser
    : public qi::grammar<Iterator, Result(), ascii::space_type> {
  public:
    using ValueType = Result;

    ElementParser() : ElementParser::base_type(element) {
    }

  private:
    qi::rule<Iterator, Result(), ascii::space_type> element;
};

template <typename Iterator, typename ElementParser,
          typename Element = typename ElementParser::ValueType>
class SequenceParser
    : public qi::grammar<Iterator, std::vector<Element>(), ascii::space_type> {
  public:
    SequenceParser() : SequenceParser::base_type(sequence) {
        sequence = simple % ',';
    }

  private:
    qi::rule<Iterator, std::vector<Element>(), ascii::space_type> sequence;
    ElementParser simple;
};

void sequenceTest() {
    using It = std::string::const_iterator;

    SequenceParser<It, qi::uint_parser<>, std::size_t> uintParser;  // OK
    SequenceParser<It, ElementParser<It, float>> floatParser; // OK
    SequenceParser<It, ElementParser<It, std::vector<float>>>
        vectorParser; // OK

    SequenceParser<It, ElementParser<It, Struct1>> struct1Parser;
    SequenceParser<It, ElementParser<It, Struct2>> struct2Parser;
}

int main() {
    sequenceTest();
}

BONUS: JustParseIt magic function

Note that it looks like you are, to an extent, religitigating the library design. Have a look at qi::auto_.

Add to that the ideas from here:

• Using the auto_ expression in boost::spirit with std::vectors
• create_parser trait

For example by specializing the trait to make comma-separated sequence parsers:

template <typename... T>
struct create_parser<std::vector<T...>> : comma_separated_sequence {};

struct comma_separated_sequence {
    using type = decltype(qi::copy(qi::auto_ % ','));
    static type call() { return qi::copy(qi::auto_ % ','); }
};

Now you can implement a JustParseIt function that works with /the world/:

bool JustParseIt(std::string_view input, auto& val) {
    return qi::phrase_parse(input.begin(), input.end(), qi::auto_, qi::space, val);
}

You'll be surprised to see what it parses:

Live On Compiler Explorer

#include <boost/fusion/include/adapted.hpp>
#include <boost/fusion/include/io.hpp>
#include <boost/spirit/include/qi.hpp>
#include <fmt/ranges.h>
#include <fmt/ostream.h>
namespace qi = boost::spirit::qi;

namespace MyLib {
    struct Struct1 { float f; };
    struct Struct2 { float f; int i; };
    using boost::fusion::operator<<;
}
BOOST_FUSION_ADAPT_STRUCT(MyLib::Struct1, f)
BOOST_FUSION_ADAPT_STRUCT(MyLib::Struct2, f, i)

namespace {
    struct comma_separated_sequence {
        using type = decltype(qi::copy(qi::auto_ % ','));
        static type call() { return qi::copy(qi::auto_ % ','); }
    };
}

namespace boost::spirit::traits {
    template <typename... T>
    struct create_parser<std::list<T...>> : comma_separated_sequence {};

    template <typename... T>
    struct create_parser<std::vector<T...>> : comma_separated_sequence {};
}

bool JustParseIt(std::string_view input, auto& val) {
#ifdef BOOST_SPIRIT_DEBUG
    using It      = decltype(input.begin());
    using Skipper = qi::space_type;
    using Attr    = std::decay_t<decltype(val)>;
    static qi::rule<It, Attr(), Skipper> parser = qi::auto_;
    BOOST_SPIRIT_DEBUG_NODE(parser);
    return qi::phrase_parse(input.begin(), input.end(), parser, qi::space, val);
#else
    return qi::phrase_parse(input.begin(), input.end(), qi::auto_, qi::space, val);
#endif
}

int main() {
    using namespace MyLib;
    std::cerr << std::boolalpha; // make debug easier to read

    float f;
    JustParseIt("3.1415", f);

    uint64_t u;
    JustParseIt("00897823", u);

    Struct1 s1;
    JustParseIt("3.1415", s1);

    Struct2 s2;
    JustParseIt("3.1415 00897823", s2);

    std::list<float> floats;;
    JustParseIt("1.2,3.4", floats);

    std::list<Struct1> list1;
    JustParseIt("1.2", list1);
    JustParseIt("1.2, -inf, 9e10, NaN", list1);

    std::vector<boost::variant<Struct2, bool> > variants;
    JustParseIt("true, 9e10 123, NaN 234, false, false", variants);

    std::vector<Struct2> vec2;
    JustParseIt("9e10 123, NaN 234", vec2);

    // this is pushing it - for lack of structurual syntax
    std::vector<std::tuple<bool, Struct1, std::vector<Struct2>>> insane;
    JustParseIt("true 3.14 1e1 1, 2e2 2, 3e3 3, false +inf 4e4 4", insane);

    fmt::print("float f: {}\n"
            "uint64_t u: {}\n"
            "std::list<float> floats: {}\n"
            "std::list<Struct1> list1: {}\n"
            "std::vector<Struct2> vec2: {}\n"
            "Struct1 s1: {}\n"
            "Struct2 s2: {}\n"
            "std::vector<boost::variant<Struct2, bool> > variants: {}\n"
            "std::vector<std::tuple<bool, Struct1, std::vector<Struct2>>> "
            "insane: {}\n",
            f, u, floats, list1, vec2, s1, s2, variants, insane);
}

Prints

float f: 3.1415
uint64_t u: 897823
std::list<float> floats: {1.2, 3.4}
std::list<Struct1> list1: {(1.2), (1.2), (-inf), (9e+10), (nan)}
std::vector<Struct2> vec2: {(9e+10 123), (nan 234)}
Struct1 s1: (3.1415)
Struct2 s2: (3.1415 897823)
std::vector<boost::variant<Struct2, bool> > variants: {1, (9e+10 123), (nan 234), 0, 0}
std::vector<std::tuple<bool, Struct1, std::vector<Struct2>>> insane: {(true, (3.14), {(10 1), (200 2), (3000 3)}), (false, (inf), {(40000 4)})}

Note that you can define BOOST_SPIRIT_DEBUG to get parser debugging to stderr, like e.g.

<parser>
  <try>true, 9e10 123, NaN </try>
  <success></success>
  <attributes>[[true, [9e+10, 123], [nan, 234], false, false]]</attributes>
</parser>

Answer 2

Here is an even shorter example, demonstrating the same problem (compiler explorer):

#include <boost/fusion/adapted/std_tuple.hpp>
#include <boost/spirit/include/qi.hpp>

#include <vector>
#include <tuple>

namespace ascii = boost::spirit::ascii;
namespace qi = boost::spirit::qi;

void test()
{
  using Iterator = std::string::const_iterator;
  
  // OK
  qi::rule<Iterator, std::vector<int>(), ascii::space_type> vecI_src;
  qi::rule<Iterator, std::vector<int>(), ascii::space_type> vecI_dst = *vecI_src;
  
  // error: no matching function for call to 'std::tuple<int, float>::tuple(const std::vector<std::tuple<int, float> >&)'
  qi::rule<Iterator, std::vector<std::tuple<int, float>>(), ascii::space_type> vecT_src;
  qi::rule<Iterator, std::vector<std::tuple<int, float>>(), ascii::space_type> vecT_dst = *vecT_src;
}

I think, the problem is, that vectors and tuples are handles quite similarly in the underlying boost::fusion library, so when it comes to flattening the vector, boost::fusion overshoots the goal and assignment fails. (Possibly by some kind of SFINAE mechanism.) Now, that flattening the vector does not work, the right-hand-side tuple parser's synthesized attribute is of type vector<vector<tuple<int, float>>>, as opposed to the expected vector<tuple<int, float>>.

Knowing this, the (not very pretty) solution I've found (for the original example) is to manually create assignment function overloads for both expected forms:

  static
  void flattenAndAppend(std::vector<Element>& into,
                        std::vector<std::vector<Element>> const& vector)
  {
    for(auto const& subvector: vector)
    {
      into.insert(into.end(), subvector.begin(), subvector.end());
    }
  }
  
  static
  void flattenAndAppend(std::vector<Element>& into,
                        std::vector<Element> const& vector)
  {
    into.insert(into.end(), vector.begin(), vector.end());
  }

and call these in a semantic action via a boost::phoenix function:

    ph::function append = [](auto& into,
                             auto const& a1)
    {
      flattenAndAppend(into, a1);
    };
    
    sequence = (simpleVector % ',')[append(qi::_val, ql::_1)];

Here is the whole working example (compiler explorer):

#include <boost/fusion/adapted/std_tuple.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/include/qi.hpp>

#include <vector>
#include <tuple>

namespace ascii = boost::spirit::ascii;
namespace qi = boost::spirit::qi;
namespace ql = qi::labels;
namespace ph = boost::phoenix;


struct Struct1
{
  float f;
};
BOOST_FUSION_ADAPT_STRUCT(
    Struct1,
    (float, f))

struct Struct2
{
  float f;
  int i;
};
BOOST_FUSION_ADAPT_STRUCT(
    Struct2,
    (float, f)
    (int, i))


template<typename Iterator,
         typename Result>
class ElementParser : public qi::grammar<Iterator, Result(), ascii::space_type>
{
public:
  using ValueType = Result;
  
  ElementParser() : ElementParser::base_type(element) {}
  
private:
  qi::rule<Iterator, Result(), ascii::space_type> element;
};


template<typename Iterator>
class Struct2Tuple : public qi::grammar<Iterator, std::tuple<float, int>(), ascii::space_type>
{
public:
  using ValueType = std::tuple<float, int>;
  
  Struct2Tuple() : Struct2Tuple::base_type(tupleElement)
  {
    ph::function convert = [](auto const& s,
                              auto& t)
    {
      t = std::make_tuple(s.f, s.i);
    };
    
    tupleElement = structElement[convert(ql::_1, qi::_val)];
  }
  
private:
  qi::rule<Iterator, ValueType(), ascii::space_type> tupleElement;
  ElementParser<Iterator, Struct2> structElement;
};


template<typename Iterator,
         typename ElementParser,
         typename Element = typename ElementParser::ValueType>
class SP : public qi::grammar<Iterator, std::vector<Element>(), ascii::space_type>
{
private:
  static
  void flattenAndAppend(std::vector<Element>& into,
                        std::vector<std::vector<Element>> const& vector)
  {
    for(auto const& subvector: vector)
    {
      into.insert(into.end(), subvector.begin(), subvector.end());
    }
  }
  
  static
  void flattenAndAppend(std::vector<Element>& into,
                        std::vector<Element> const& vector)
  {
    into.insert(into.end(), vector.begin(), vector.end());
  }
  
public:
  SP()
    : SP::base_type(sequence)
  {
    ph::function append = [](auto& into,
                             auto const& a1)
    {
      flattenAndAppend(into, a1);
    };
    
    sequence = (simpleVector % ',')[append(qi::_val, ql::_1)];
    simpleVector = qi::repeat(1)[simple];
  }
  
private:
  using Rule = qi::rule<Iterator, std::vector<Element>(), ascii::space_type>;
  Rule sequence;
  Rule simpleVector;
  ElementParser simple;
};


void sequenceTest()
{
  using Iterator = std::string::const_iterator;
  
  SP<Iterator, qi::uint_parser<>, std::size_t> uintParser;                  // OK
  SP<Iterator, ElementParser<Iterator, float>> floatParser;                 // OK
  SP<Iterator, ElementParser<Iterator, std::vector<float>>> vectorParser;   // OK
  
  SP<Iterator, Struct2Tuple<Iterator>> struct2tupleParser;                  // OK.
  
  SP<Iterator, ElementParser<Iterator, std::tuple<float, float>>> tupleParser;   // now OK
  SP<Iterator, ElementParser<Iterator, Struct1>> struct1Parser;                  // now OK
  SP<Iterator, ElementParser<Iterator, Struct2>> struct2Parser;                  // now OK
}