| // Copyright (c) 2001-2011 Hartmut Kaiser |
| // |
| // Distributed under the Boost Software License, Version 1.0. (See accompanying |
| // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) |
| |
| #if !defined(BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM) |
| #define BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM |
| |
| #if defined(_MSC_VER) |
| #pragma once |
| #endif |
| |
| #include <boost/config.hpp> |
| #include <boost/detail/workaround.hpp> |
| #include <boost/spirit/home/qi/detail/assign_to.hpp> |
| #include <boost/spirit/home/support/attributes.hpp> |
| #include <boost/spirit/home/support/argument.hpp> |
| #include <boost/spirit/home/support/detail/lexer/generator.hpp> |
| #include <boost/spirit/home/support/detail/lexer/rules.hpp> |
| #include <boost/spirit/home/support/detail/lexer/consts.hpp> |
| #include <boost/spirit/home/support/utree/utree_traits_fwd.hpp> |
| #include <boost/fusion/include/vector.hpp> |
| #include <boost/fusion/include/at.hpp> |
| #include <boost/fusion/include/value_at.hpp> |
| #include <boost/detail/iterator.hpp> |
| #include <boost/variant.hpp> |
| #include <boost/mpl/vector.hpp> |
| #include <boost/mpl/bool.hpp> |
| #include <boost/mpl/is_sequence.hpp> |
| #include <boost/mpl/begin.hpp> |
| #include <boost/mpl/insert.hpp> |
| #include <boost/mpl/vector.hpp> |
| #include <boost/mpl/if.hpp> |
| #include <boost/mpl/or.hpp> |
| #include <boost/type_traits/is_same.hpp> |
| #include <boost/range/iterator_range.hpp> |
| #if !BOOST_WORKAROUND(BOOST_MSVC, <= 1300) |
| #include <boost/static_assert.hpp> |
| #endif |
| |
| #if defined(BOOST_SPIRIT_DEBUG) |
| #include <iosfwd> |
| #endif |
| |
| namespace boost { namespace spirit { namespace lex { namespace lexertl |
| { |
| /////////////////////////////////////////////////////////////////////////// |
| // |
| // The position_token is the type of the objects returned by the |
| // iterator if it has been specified while instantiating the lexer object. |
| // |
| // template parameters: |
| // Iterator The type of the iterator used to access the |
| // underlying character stream. |
| // AttributeTypes A mpl sequence containing the types of all |
| // required different token values to be supported |
| // by this token type. |
| // HasState A mpl::bool_ indicating, whether this token type |
| // should support lexer states. |
| // Idtype The type to use for the token id (defaults to |
| // std::size_t). |
| // |
| // It is possible to use other token types with the spirit::lex |
| // framework as well. If you plan to use a different type as your token |
| // type, you'll need to expose the following things from your token type |
| // to make it compatible with spirit::lex: |
| // |
| // typedefs |
| // iterator_type The type of the iterator used to access the |
| // underlying character stream. |
| // |
| // id_type The type of the token id used. |
| // |
| // methods |
| // default constructor |
| // This should initialize the token as an end of |
| // input token. |
| // constructors The prototype of the other required |
| // constructors should be: |
| // |
| // token(int) |
| // This constructor should initialize the token as |
| // an invalid token (not carrying any specific |
| // values) |
| // |
| // where: the int is used as a tag only and its value is |
| // ignored |
| // |
| // and: |
| // |
| // token(Idtype id, std::size_t state, |
| // iterator_type first, iterator_type last); |
| // |
| // where: id: token id |
| // state: lexer state this token was matched in |
| // first, last: pair of iterators marking the matched |
| // range in the underlying input stream |
| // |
| // accessors |
| // id() return the token id of the matched input sequence |
| // id(newid) set the token id of the token instance |
| // |
| // state() return the lexer state this token was matched in |
| // |
| // value() return the token value |
| // |
| // Additionally, you will have to implement a couple of helper functions |
| // in the same namespace as the token type: a comparison operator==() to |
| // compare your token instances, a token_is_valid() function and different |
| // specializations of the Spirit customization point |
| // assign_to_attribute_from_value as shown below. |
| // |
| /////////////////////////////////////////////////////////////////////////// |
| template <typename Iterator = char const* |
| , typename AttributeTypes = mpl::vector0<> |
| , typename HasState = mpl::true_ |
| , typename Idtype = std::size_t> |
| struct position_token; |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // This specialization of the token type doesn't contain any item data and |
| // doesn't support working with lexer states. Although, like all other |
| // variants of position_token, it carries a pair of iterators marking the |
| // begin and the end of the matched character sequence. |
| /////////////////////////////////////////////////////////////////////////// |
| template <typename Iterator, typename Idtype> |
| struct position_token<Iterator, lex::omit, mpl::false_, Idtype> |
| { |
| typedef Iterator iterator_type; |
| typedef iterator_range<iterator_type> iterpair_type; |
| typedef mpl::false_ has_state; |
| typedef Idtype id_type; |
| typedef unused_type token_value_type; |
| |
| // default constructed tokens correspond to EOI tokens |
| position_token() |
| : id_(id_type(boost::lexer::npos)) {} |
| |
| // construct an invalid token |
| explicit position_token(int) |
| : id_(id_type(0)) {} |
| |
| position_token(id_type id, std::size_t) |
| : id_(id) {} |
| |
| position_token(id_type id, std::size_t, token_value_type) |
| : id_(id) {} |
| |
| position_token(id_type id, std::size_t, Iterator const& first |
| , Iterator const& last) |
| : id_(id), matched_(first, last) {} |
| |
| // this default conversion operator is needed to allow the direct |
| // usage of tokens in conjunction with the primitive parsers defined |
| // in Qi |
| operator id_type() const { return id_; } |
| |
| // Retrieve or set the token id of this token instance. |
| id_type id() const { return id_; } |
| void id(id_type newid) { id_ = newid; } |
| |
| std::size_t state() const { return 0; } // always '0' (INITIAL state) |
| |
| bool is_valid() const |
| { |
| return 0 != id_ && id_type(boost::lexer::npos) != id_; |
| } |
| |
| // access the stored iterator range of the matched input sequence |
| iterator_type begin() const { return matched_.begin(); } |
| iterator_type end() const { return matched_.end(); } |
| |
| iterpair_type& matched() { return matched_; } |
| iterpair_type const& matched() const { return matched_; } |
| |
| token_value_type& value() { return unused; } |
| token_value_type const& value() const { return unused; } |
| |
| #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) |
| // workaround for MSVC10 which has problems copying a default |
| // constructed iterator_range |
| position_token& operator= (position_token const& rhs) |
| { |
| if (this != &rhs) |
| { |
| id_ = rhs.id_; |
| if (is_valid()) |
| matched_ = rhs.matched_; |
| } |
| return *this; |
| } |
| #endif |
| |
| protected: |
| id_type id_; // token id, 0 if nothing has been matched |
| iterpair_type matched_; // matched input sequence |
| }; |
| |
| #if defined(BOOST_SPIRIT_DEBUG) |
| template <typename Char, typename Traits, typename Iterator |
| , typename AttributeTypes, typename HasState, typename Idtype> |
| inline std::basic_ostream<Char, Traits>& |
| operator<< (std::basic_ostream<Char, Traits>& os |
| , position_token<Iterator, AttributeTypes, HasState, Idtype> const& t) |
| { |
| if (t.is_valid()) { |
| Iterator end = t.end(); |
| for (Iterator it = t.begin(); it != end; ++it) |
| os << *it; |
| } |
| else { |
| os << "<invalid token>"; |
| } |
| return os; |
| } |
| #endif |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // This specialization of the token type doesn't contain any item data but |
| // supports working with lexer states. |
| /////////////////////////////////////////////////////////////////////////// |
| template <typename Iterator, typename Idtype> |
| struct position_token<Iterator, lex::omit, mpl::true_, Idtype> |
| : position_token<Iterator, lex::omit, mpl::false_, Idtype> |
| { |
| private: |
| typedef position_token<Iterator, lex::omit, mpl::false_, Idtype> |
| base_type; |
| |
| public: |
| typedef typename base_type::id_type id_type; |
| typedef Iterator iterator_type; |
| typedef mpl::true_ has_state; |
| typedef unused_type token_value_type; |
| |
| // default constructed tokens correspond to EOI tokens |
| position_token() : state_(boost::lexer::npos) {} |
| |
| // construct an invalid token |
| explicit position_token(int) |
| : base_type(0), state_(boost::lexer::npos) {} |
| |
| position_token(id_type id, std::size_t state) |
| : base_type(id, boost::lexer::npos), state_(state) {} |
| |
| position_token(id_type id, std::size_t state, token_value_type) |
| : base_type(id, boost::lexer::npos, unused) |
| , state_(state) {} |
| |
| position_token(id_type id, std::size_t state |
| , Iterator const& first, Iterator const& last) |
| : base_type(id, boost::lexer::npos, first, last) |
| , state_(state) {} |
| |
| std::size_t state() const { return state_; } |
| |
| #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) |
| // workaround for MSVC10 which has problems copying a default |
| // constructed iterator_range |
| position_token& operator= (position_token const& rhs) |
| { |
| if (this != &rhs) |
| { |
| this->base_type::operator=(static_cast<base_type const&>(rhs)); |
| state_ = rhs.state_; |
| } |
| return *this; |
| } |
| #endif |
| |
| protected: |
| std::size_t state_; // lexer state this token was matched in |
| }; |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // These specializations for an empty attribute list cause all token |
| // instances to expose as it attribute the iterator_range pointing to the |
| // matched input sequence. |
| /////////////////////////////////////////////////////////////////////////// |
| template <typename Iterator, typename HasState, typename Idtype> |
| struct position_token<Iterator, mpl::vector<>, HasState, Idtype> |
| : position_token<Iterator, lex::omit, HasState, Idtype> |
| { |
| private: |
| typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; |
| |
| public: |
| typedef typename base_type::id_type id_type; |
| typedef typename base_type::iterator_type iterator_type; |
| typedef typename base_type::iterpair_type iterpair_type; |
| typedef HasState has_state; |
| typedef iterpair_type token_value_type; |
| |
| // default constructed tokens correspond to EOI tokens |
| position_token() {} |
| |
| // construct an invalid token |
| explicit position_token(int) |
| : base_type(0) {} |
| |
| position_token(id_type id, std::size_t state) |
| : base_type(id, state) {} |
| |
| position_token(id_type id, std::size_t state, token_value_type) |
| : base_type(id, state, unused) {} |
| |
| position_token(id_type id, std::size_t state |
| , Iterator const& first, Iterator const& last) |
| : base_type(id, state, first, last) {} |
| |
| token_value_type& value() { return this->base_type::matched(); } |
| token_value_type const& value() const { return this->base_type::matched(); } |
| }; |
| |
| template <typename Iterator, typename HasState, typename Idtype> |
| struct position_token<Iterator, mpl::vector0<>, HasState, Idtype> |
| : position_token<Iterator, lex::omit, HasState, Idtype> |
| { |
| private: |
| typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; |
| |
| public: |
| typedef typename base_type::id_type id_type; |
| typedef typename base_type::iterator_type iterator_type; |
| typedef typename base_type::iterpair_type iterpair_type; |
| typedef HasState has_state; |
| typedef iterpair_type token_value_type; |
| |
| // default constructed tokens correspond to EOI tokens |
| position_token() {} |
| |
| // construct an invalid token |
| explicit position_token(int) |
| : base_type(0) {} |
| |
| position_token(id_type id, std::size_t state) |
| : base_type(id, state) {} |
| |
| position_token(id_type id, std::size_t state, token_value_type) |
| : base_type(id, state, unused) {} |
| |
| position_token(id_type id, std::size_t state |
| , Iterator const& first, Iterator const& last) |
| : base_type(id, state, first, last) {} |
| |
| token_value_type& value() { return this->base_type::matched(); } |
| token_value_type const& value() const { return this->base_type::matched(); } |
| }; |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // These specializations for an attribute list of length one cause all token |
| // instances to expose the specified type as its attribute. |
| /////////////////////////////////////////////////////////////////////////// |
| template <typename Iterator, typename Attribute, typename HasState |
| , typename Idtype> |
| struct position_token<Iterator, mpl::vector<Attribute>, HasState, Idtype> |
| : position_token<Iterator, lex::omit, HasState, Idtype> |
| { |
| private: |
| typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; |
| |
| public: |
| typedef typename base_type::id_type id_type; |
| typedef typename base_type::iterator_type iterator_type; |
| typedef typename base_type::iterpair_type iterpair_type; |
| typedef HasState has_state; |
| typedef boost::optional<Attribute> token_value_type; |
| |
| // default constructed tokens correspond to EOI tokens |
| position_token() {} |
| |
| // construct an invalid token |
| explicit position_token(int) |
| : base_type(0) {} |
| |
| position_token(id_type id, std::size_t state) |
| : base_type(id, state) {} |
| |
| position_token(id_type id, std::size_t state, token_value_type const& v) |
| : base_type(id, state, unused), value_(v) {} |
| |
| position_token(id_type id, std::size_t state |
| , Iterator const& first, Iterator const& last) |
| : base_type(id, state, first, last) {} |
| |
| token_value_type& value() { return value_; } |
| token_value_type const& value() const { return value_; } |
| |
| bool has_value() const { return value_; } |
| |
| #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) |
| // workaround for MSVC10 which has problems copying a default |
| // constructed iterator_range |
| position_token& operator= (position_token const& rhs) |
| { |
| if (this != &rhs) |
| { |
| this->base_type::operator=(static_cast<base_type const&>(rhs)); |
| if (this->is_valid()) |
| value_ = rhs.value_; |
| } |
| return *this; |
| } |
| #endif |
| |
| protected: |
| token_value_type value_; // token value |
| }; |
| |
| template <typename Iterator, typename Attribute, typename HasState |
| , typename Idtype> |
| struct position_token<Iterator, mpl::vector1<Attribute>, HasState, Idtype> |
| : position_token<Iterator, lex::omit, HasState, Idtype> |
| { |
| private: |
| typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; |
| |
| public: |
| typedef typename base_type::id_type id_type; |
| typedef typename base_type::iterator_type iterator_type; |
| typedef typename base_type::iterpair_type iterpair_type; |
| typedef HasState has_state; |
| typedef boost::optional<Attribute> token_value_type; |
| |
| // default constructed tokens correspond to EOI tokens |
| position_token() {} |
| |
| // construct an invalid token |
| explicit position_token(int) |
| : base_type(0) {} |
| |
| position_token(id_type id, std::size_t state) |
| : base_type(id, state) {} |
| |
| position_token(id_type id, std::size_t state, token_value_type const& v) |
| : base_type(id, state, unused), value_(v) {} |
| |
| position_token(id_type id, std::size_t state |
| , Iterator const& first, Iterator const& last) |
| : base_type(id, state, first, last) {} |
| |
| token_value_type& value() { return value_; } |
| token_value_type const& value() const { return value_; } |
| |
| bool has_value() const { return value_; } |
| |
| #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) |
| // workaround for MSVC10 which has problems copying a default |
| // constructed iterator_range |
| position_token& operator= (position_token const& rhs) |
| { |
| if (this != &rhs) |
| { |
| this->base_type::operator=(static_cast<base_type const&>(rhs)); |
| if (this->is_valid()) |
| value_ = rhs.value_; |
| } |
| return *this; |
| } |
| #endif |
| |
| protected: |
| token_value_type value_; // token value |
| }; |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // The generic version of the position_token type derives from the |
| // specialization above and adds a single data member holding the item |
| // data carried by the token instance. |
| /////////////////////////////////////////////////////////////////////////// |
| namespace detail |
| { |
| /////////////////////////////////////////////////////////////////////// |
| // Meta-function to calculate the type of the variant data item to be |
| // stored with each token instance. |
| // |
| // Note: The iterator pair needs to be the first type in the list of |
| // types supported by the generated variant type (this is being |
| // used to identify whether the stored data item in a particular |
| // token instance needs to be converted from the pair of |
| // iterators (see the first of the assign_to_attribute_from_value |
| // specializations below). |
| /////////////////////////////////////////////////////////////////////// |
| template <typename IteratorPair, typename AttributeTypes> |
| struct position_token_value_typesequence |
| { |
| typedef typename mpl::insert< |
| AttributeTypes |
| , typename mpl::begin<AttributeTypes>::type |
| , IteratorPair |
| >::type sequence_type; |
| typedef typename make_variant_over<sequence_type>::type type; |
| }; |
| |
| /////////////////////////////////////////////////////////////////////// |
| // The type of the data item stored with a token instance is defined |
| // by the template parameter 'AttributeTypes' and may be: |
| // |
| // lex::omit: no data item is stored with the token |
| // instance (this is handled by the |
| // specializations of the token class |
| // below) |
| // mpl::vector0<>: each token instance stores a pair of |
| // iterators pointing to the matched input |
| // sequence |
| // mpl::vector<...>: each token instance stores a variant being |
| // able to store the pair of iterators pointing |
| // to the matched input sequence, or any of the |
| // types a specified in the mpl::vector<> |
| // |
| // All this is done to ensure the token type is as small (in terms |
| // of its byte-size) as possible. |
| /////////////////////////////////////////////////////////////////////// |
| template <typename IteratorPair, typename AttributeTypes> |
| struct position_token_value |
| : mpl::eval_if< |
| mpl::or_< |
| is_same<AttributeTypes, mpl::vector0<> > |
| , is_same<AttributeTypes, mpl::vector<> > > |
| , mpl::identity<IteratorPair> |
| , position_token_value_typesequence<IteratorPair, AttributeTypes> > |
| {}; |
| } |
| |
| template <typename Iterator, typename AttributeTypes, typename HasState |
| , typename Idtype> |
| struct position_token |
| : position_token<Iterator, lex::omit, HasState, Idtype> |
| { |
| private: // precondition assertions |
| #if !BOOST_WORKAROUND(BOOST_MSVC, <= 1300) |
| BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value || |
| is_same<AttributeTypes, lex::omit>::value)); |
| #endif |
| typedef position_token<Iterator, lex::omit, HasState, Idtype> |
| base_type; |
| |
| protected: |
| // If no additional token value types are given, the the token will |
| // hold no token value at all as the base class already has the |
| // iterator pair of the matched range in the underlying input sequence. |
| // Otherwise the token value is stored as a variant and will |
| // initially hold an unused_type but is able to hold any of |
| // the given data types as well. The conversion from the iterator pair |
| // to the required data type is done when it is accessed for the first |
| // time. |
| typedef iterator_range<Iterator> iterpair_type; |
| |
| public: |
| typedef typename base_type::id_type id_type; |
| typedef typename detail::position_token_value< |
| iterpair_type, AttributeTypes>::type token_value_type; |
| |
| typedef Iterator iterator_type; |
| |
| // default constructed tokens correspond to EOI tokens |
| position_token() |
| : value_(iterpair_type(iterator_type(), iterator_type())) |
| {} |
| |
| // construct an invalid token |
| explicit position_token(int) |
| : base_type(0) |
| , value_(iterpair_type(iterator_type(), iterator_type())) |
| {} |
| |
| position_token(id_type id, std::size_t state, token_value_type const& value) |
| : base_type(id, state, value), value_(value) |
| {} |
| |
| position_token(id_type id, std::size_t state, Iterator const& first |
| , Iterator const& last) |
| : base_type(id, state, first, last) |
| , value_(iterpair_type(iterator_type(), iterator_type())) |
| {} |
| |
| token_value_type& value() { return value_; } |
| token_value_type const& value() const { return value_; } |
| |
| #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) |
| // workaround for MSVC10 which has problems copying a default |
| // constructed iterator_range |
| position_token& operator= (position_token const& rhs) |
| { |
| if (this != &rhs) |
| { |
| this->base_type::operator=(static_cast<base_type const&>(rhs)); |
| if (this->is_valid()) |
| value_ = rhs.value_; |
| } |
| return *this; |
| } |
| #endif |
| |
| protected: |
| token_value_type value_; // token value, by default a pair of iterators |
| }; |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // tokens are considered equal, if their id's match (these are unique) |
| template <typename Iterator, typename AttributeTypes, typename HasState |
| , typename Idtype> |
| inline bool |
| operator== (position_token<Iterator, AttributeTypes, HasState, Idtype> const& lhs, |
| position_token<Iterator, AttributeTypes, HasState, Idtype> const& rhs) |
| { |
| return lhs.id() == rhs.id(); |
| } |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // This overload is needed by the multi_pass/functor_input_policy to |
| // validate a token instance. It has to be defined in the same namespace |
| // as the token class itself to allow ADL to find it. |
| /////////////////////////////////////////////////////////////////////////// |
| template <typename Iterator, typename AttributeTypes, typename HasState |
| , typename Idtype> |
| inline bool |
| token_is_valid(position_token<Iterator, AttributeTypes, HasState, Idtype> const& t) |
| { |
| return t.is_valid(); |
| } |
| }}}} |
| |
| namespace boost { namespace spirit { namespace traits |
| { |
| /////////////////////////////////////////////////////////////////////////// |
| // We have to provide specializations for the customization point |
| // assign_to_attribute_from_value allowing to extract the needed value |
| // from the token. |
| /////////////////////////////////////////////////////////////////////////// |
| |
| // This is called from the parse function of token_def if the token_def |
| // has been defined to carry a special attribute type |
| template <typename Attribute, typename Iterator, typename AttributeTypes |
| , typename HasState, typename Idtype> |
| struct assign_to_attribute_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > |
| { |
| static void |
| call(lex::lexertl::position_token< |
| Iterator, AttributeTypes, HasState, Idtype> const& t |
| , Attribute& attr) |
| { |
| // The goal of this function is to avoid the conversion of the pair of |
| // iterators (to the matched character sequence) into the token value |
| // of the required type being done more than once. For this purpose it |
| // checks whether the stored value type is still the default one (pair |
| // of iterators) and if yes, replaces the pair of iterators with the |
| // converted value to be returned from subsequent calls. |
| |
| if (0 == t.value().which()) { |
| // first access to the token value |
| typedef iterator_range<Iterator> iterpair_type; |
| iterpair_type const& ip = t.matched(); |
| |
| // Interestingly enough we use the assign_to() framework defined in |
| // Spirit.Qi allowing to convert the pair of iterators to almost any |
| // required type (assign_to(), if available, uses the standard Spirit |
| // parsers to do the conversion). |
| spirit::traits::assign_to(ip.begin(), ip.end(), attr); |
| |
| // If you get an error during the compilation of the following |
| // assignment expression, you probably forgot to list one or more |
| // types used as token value types (in your token_def<...> |
| // definitions) in your definition of the token class. I.e. any token |
| // value type used for a token_def<...> definition has to be listed |
| // during the declaration of the token type to use. For instance let's |
| // assume we have two token_def's: |
| // |
| // token_def<int> number; number = "..."; |
| // token_def<std::string> identifier; identifier = "..."; |
| // |
| // Then you'll have to use the following token type definition |
| // (assuming you are using the token class): |
| // |
| // typedef mpl::vector<int, std::string> token_values; |
| // typedef token<base_iter_type, token_values> token_type; |
| // |
| // where: base_iter_type is the iterator type used to expose the |
| // underlying input stream. |
| // |
| // This token_type has to be used as the second template parameter |
| // to the lexer class: |
| // |
| // typedef lexer<base_iter_type, token_type> lexer_type; |
| // |
| // again, assuming you're using the lexer<> template for your |
| // tokenization. |
| |
| typedef lex::lexertl::position_token< |
| Iterator, AttributeTypes, HasState, Idtype> token_type; |
| spirit::traits::assign_to( |
| attr, const_cast<token_type&>(t).value()); // re-assign value |
| } |
| else { |
| // reuse the already assigned value |
| spirit::traits::assign_to(get<Attribute>(t.value()), attr); |
| } |
| } |
| }; |
| |
| template <typename Attribute, typename Iterator, typename AttributeTypes |
| , typename HasState, typename Idtype> |
| struct assign_to_container_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > |
| : assign_to_attribute_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > |
| {}; |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // These are called from the parse function of token_def if the token type |
| // has no special attribute type assigned |
| template <typename Attribute, typename Iterator, typename HasState |
| , typename Idtype> |
| struct assign_to_attribute_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > |
| { |
| static void |
| call(lex::lexertl::position_token< |
| Iterator, mpl::vector0<>, HasState, Idtype> const& t |
| , Attribute& attr) |
| { |
| // The default type returned by the token_def parser component (if |
| // it has no token value type assigned) is the pair of iterators |
| // to the matched character sequence. |
| spirit::traits::assign_to(t.begin(), t.end(), attr); |
| } |
| }; |
| |
| // template <typename Attribute, typename Iterator, typename HasState |
| // , typename Idtype> |
| // struct assign_to_container_from_value<Attribute |
| // , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > |
| // : assign_to_attribute_from_value<Attribute |
| // , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > |
| // {}; |
| |
| // same as above but using mpl::vector<> instead of mpl::vector0<> |
| template <typename Attribute, typename Iterator, typename HasState |
| , typename Idtype> |
| struct assign_to_attribute_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > |
| { |
| static void |
| call(lex::lexertl::position_token< |
| Iterator, mpl::vector<>, HasState, Idtype> const& t |
| , Attribute& attr) |
| { |
| // The default type returned by the token_def parser component (if |
| // it has no token value type assigned) is the pair of iterators |
| // to the matched character sequence. |
| spirit::traits::assign_to(t.begin(), t.end(), attr); |
| } |
| }; |
| |
| // template <typename Attribute, typename Iterator, typename HasState |
| // , typename Idtype> |
| // struct assign_to_container_from_value<Attribute |
| // , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > |
| // : assign_to_attribute_from_value<Attribute |
| // , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > |
| // {}; |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // These are called from the parse function of token_def if the token type |
| // has no special attribute type assigned |
| template <typename Attribute, typename Iterator, typename Attr |
| , typename HasState, typename Idtype> |
| struct assign_to_attribute_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > |
| { |
| static void |
| call(lex::lexertl::position_token< |
| Iterator, mpl::vector1<Attr>, HasState, Idtype> const& t |
| , Attribute& attr) |
| { |
| // The goal of this function is to avoid the conversion of the pair of |
| // iterators (to the matched character sequence) into the token value |
| // of the required type being done more than once. |
| |
| if (!t.has_value()) { |
| // first access to the token value |
| typedef iterator_range<Iterator> iterpair_type; |
| iterpair_type const& ip = t.matched(); |
| |
| // Interestingly enough we use the assign_to() framework defined in |
| // Spirit.Qi allowing to convert the pair of iterators to almost any |
| // required type (assign_to(), if available, uses the standard Spirit |
| // parsers to do the conversion). |
| spirit::traits::assign_to(ip.begin(), ip.end(), attr); |
| |
| // Re-assign the attribute to the stored value |
| typedef lex::lexertl::position_token< |
| Iterator, mpl::vector1<Attr>, HasState, Idtype> token_type; |
| spirit::traits::assign_to( |
| attr, const_cast<token_type&>(t).value()); |
| } |
| else { |
| // reuse the already assigned value |
| spirit::traits::assign_to(t.value(), attr); |
| } |
| } |
| }; |
| |
| // template <typename Attribute, typename Iterator, typename Attr |
| // , typename HasState, typename Idtype> |
| // struct assign_to_container_from_value<Attribute |
| // , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > |
| // : assign_to_attribute_from_value<Attribute |
| // , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > |
| // {}; |
| |
| // same as above but using mpl::vector<Attr> instead of mpl::vector1<Attr> |
| template <typename Attribute, typename Iterator, typename Attr |
| , typename HasState, typename Idtype> |
| struct assign_to_attribute_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > |
| { |
| static void |
| call(lex::lexertl::position_token< |
| Iterator, mpl::vector<Attr>, HasState, Idtype> const& t |
| , Attribute& attr) |
| { |
| // The goal of this function is to avoid the conversion of the pair of |
| // iterators (to the matched character sequence) into the token value |
| // of the required type being done more than once. |
| |
| if (!t.has_value()) { |
| // first access to the token value |
| typedef iterator_range<Iterator> iterpair_type; |
| iterpair_type const& ip = t.matched(); |
| |
| // Interestingly enough we use the assign_to() framework defined in |
| // Spirit.Qi allowing to convert the pair of iterators to almost any |
| // required type (assign_to(), if available, uses the standard Spirit |
| // parsers to do the conversion). |
| spirit::traits::assign_to(ip.begin(), ip.end(), attr); |
| |
| // Re-assign the attribute to the stored value |
| typedef lex::lexertl::position_token< |
| Iterator, mpl::vector<Attr>, HasState, Idtype> token_type; |
| spirit::traits::assign_to( |
| attr, const_cast<token_type&>(t).value()); |
| } |
| else { |
| // reuse the already assigned value |
| spirit::traits::assign_to(t.value(), attr); |
| } |
| } |
| }; |
| |
| // template <typename Attribute, typename Iterator, typename Attr |
| // , typename HasState, typename Idtype> |
| // struct assign_to_container_from_value<Attribute |
| // , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > |
| // : assign_to_attribute_from_value<Attribute |
| // , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > |
| // {}; |
| |
| // This is called from the parse function of token_def if the token type |
| // has been explicitly omitted (i.e. no attribute value is used), which |
| // essentially means that every attribute gets initialized using default |
| // constructed values. |
| template <typename Attribute, typename Iterator, typename HasState |
| , typename Idtype> |
| struct assign_to_attribute_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > |
| { |
| static void |
| call(lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> const& t |
| , Attribute& attr) |
| { |
| // do nothing |
| } |
| }; |
| |
| template <typename Attribute, typename Iterator, typename HasState |
| , typename Idtype> |
| struct assign_to_container_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > |
| : assign_to_attribute_from_value<Attribute |
| , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > |
| {}; |
| |
| // This is called from the parse function of lexer_def_ |
| template <typename Iterator, typename AttributeTypes, typename HasState |
| , typename Idtype_, typename Idtype> |
| struct assign_to_attribute_from_value< |
| fusion::vector2<Idtype_, iterator_range<Iterator> > |
| , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > |
| { |
| static void |
| call(lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> const& t |
| , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr) |
| { |
| // The type returned by the lexer_def_ parser components is a |
| // fusion::vector containing the token id of the matched token |
| // and the pair of iterators to the matched character sequence. |
| typedef iterator_range<Iterator> iterpair_type; |
| typedef fusion::vector2<Idtype_, iterator_range<Iterator> > |
| attribute_type; |
| |
| iterpair_type const& ip = t.matched(); |
| attr = attribute_type(t.id(), ip); |
| } |
| }; |
| |
| template <typename Iterator, typename AttributeTypes, typename HasState |
| , typename Idtype_, typename Idtype> |
| struct assign_to_container_from_value< |
| fusion::vector2<Idtype_, iterator_range<Iterator> > |
| , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > |
| : assign_to_attribute_from_value< |
| fusion::vector2<Idtype_, iterator_range<Iterator> > |
| , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > |
| {}; |
| |
| /////////////////////////////////////////////////////////////////////////// |
| // Overload debug output for a single token, this integrates lexer tokens |
| // with Qi's simple_trace debug facilities |
| template <typename Iterator, typename Attribute, typename HasState |
| , typename Idtype> |
| struct token_printer_debug< |
| lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> > |
| { |
| typedef lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> token_type; |
| |
| template <typename Out> |
| static void print(Out& out, token_type const& val) |
| { |
| out << '['; |
| spirit::traits::print_token(out, val.value()); |
| out << ']'; |
| } |
| }; |
| }}} |
| |
| #endif |
