1// Copyright (c) 2001-2011 Hartmut Kaiser
2//
3// Distributed under the Boost Software License, Version 1.0. (See accompanying
4// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
5
6#include <boost/spirit/include/lex_lexertl.hpp>
7#include <boost/spirit/include/lex_lexertl_position_token.hpp>
8
9#include <boost/core/lightweight_test.hpp>
10#include <boost/phoenix/object.hpp>
11#include <boost/phoenix/operator.hpp>
12#include <boost/phoenix/stl/container.hpp>
13#include <boost/spirit/include/qi_numeric.hpp>
14
15namespace spirit = boost::spirit;
16namespace lex = boost::spirit::lex;
17namespace phoenix = boost::phoenix;
18namespace mpl = boost::mpl;
19
20///////////////////////////////////////////////////////////////////////////////
21enum tokenids
22{
23 ID_INT = 1000,
24 ID_DOUBLE
25};
26
27template <typename Lexer>
28struct token_definitions : lex::lexer<Lexer>
29{
30 token_definitions()
31 {
32 this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
33 this->self.add_pattern("OCTALDIGIT", "[0-7]");
34 this->self.add_pattern("DIGIT", "[0-9]");
35
36 this->self.add_pattern("OPTSIGN", "[-+]?");
37 this->self.add_pattern("EXPSTART", "[eE][-+]");
38 this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
39
40 // define tokens and associate them with the lexer
41 int_ = "{OPTSIGN}[1-9]{DIGIT}*";
42 int_.id(id: ID_INT);
43
44 double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
45 double_.id(id: ID_DOUBLE);
46
47 whitespace = "[ \t\n]+";
48
49 this->self =
50 double_
51 | int_
52 | whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
53 ;
54 }
55
56 lex::token_def<double> int_;
57 lex::token_def<double> double_;
58 lex::token_def<lex::omit> whitespace;
59};
60
61template <typename Lexer>
62struct token_definitions_with_state : lex::lexer<Lexer>
63{
64 token_definitions_with_state()
65 {
66 this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
67 this->self.add_pattern("OCTALDIGIT", "[0-7]");
68 this->self.add_pattern("DIGIT", "[0-9]");
69
70 this->self.add_pattern("OPTSIGN", "[-+]?");
71 this->self.add_pattern("EXPSTART", "[eE][-+]");
72 this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
73
74 this->self.add_state();
75 this->self.add_state("INT");
76 this->self.add_state("DOUBLE");
77
78 // define tokens and associate them with the lexer
79 int_ = "{OPTSIGN}[1-9]{DIGIT}*";
80 int_.id(id: ID_INT);
81
82 double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
83 double_.id(id: ID_DOUBLE);
84
85 whitespace = "[ \t\n]+";
86
87 this->self("*") =
88 double_ [ lex::_state = "DOUBLE"]
89 | int_ [ lex::_state = "INT" ]
90 | whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
91 ;
92 }
93
94 lex::token_def<double> int_;
95 lex::token_def<double> double_;
96 lex::token_def<lex::omit> whitespace;
97};
98
99///////////////////////////////////////////////////////////////////////////////
100template <typename Token>
101inline bool
102test_token_ids(int const* ids, std::vector<Token> const& tokens)
103{
104 for (std::size_t i = 0, len = tokens.size(); i < len; ++i)
105 {
106 if (*ids == -1)
107 return false; // reached end of expected data
108
109 if (tokens[i].id() != static_cast<std::size_t>(*ids)) // token id must match
110 return false;
111
112 ++ids;
113 }
114
115 return (*ids == -1) ? true : false;
116}
117
118///////////////////////////////////////////////////////////////////////////////
119template <typename Token>
120inline bool
121test_token_states(std::size_t const* states, std::vector<Token> const& tokens)
122{
123 for (std::size_t i = 0, len = tokens.size(); i < len; ++i)
124 {
125 if (*states == std::size_t(-1))
126 return false; // reached end of expected data
127
128 if (tokens[i].state() != *states) // token state must match
129 return false;
130
131 ++states;
132 }
133
134 return (*states == std::size_t(-1)) ? true : false;
135}
136
137///////////////////////////////////////////////////////////////////////////////
138struct position_type
139{
140 std::size_t begin, end;
141};
142
143template <typename Iterator, typename Token>
144inline bool
145test_token_positions(Iterator begin, position_type const* positions,
146 std::vector<Token> const& tokens)
147{
148 for (std::size_t i = 0, len = tokens.size(); i < len; ++i)
149 {
150 if (positions->begin == std::size_t(-1) &&
151 positions->end == std::size_t(-1))
152 {
153 return false; // reached end of expected data
154 }
155
156 boost::iterator_range<Iterator> matched = tokens[i].matched();
157 std::size_t start = std::distance(begin, matched.begin());
158 std::size_t end = std::distance(begin, matched.end());
159
160 // position must match
161 if (start != positions->begin || end != positions->end)
162 return false;
163
164 ++positions;
165 }
166
167 return (positions->begin == std::size_t(-1) &&
168 positions->end == std::size_t(-1)) ? true : false;
169}
170
171///////////////////////////////////////////////////////////////////////////////
172template <typename Token>
173inline bool
174test_token_values(double const* values, std::vector<Token> const& tokens)
175{
176 for (std::size_t i = 0, len = tokens.size(); i < len; ++i)
177 {
178 if (*values == 0.0)
179 return false; // reached end of expected data
180
181 double val;
182 spirit::traits::assign_to(tokens[i], val);
183 if (val != *values) // token value must match
184 return false;
185
186 ++values;
187 }
188
189 return (*values == 0.0) ? true : false;
190}
191
192///////////////////////////////////////////////////////////////////////////////
193int main()
194{
195 typedef std::string::iterator base_iterator_type;
196 std::string input(" 1 1.2 -2 3 2.3e6 -3.4");
197 int ids[] = { ID_INT, ID_DOUBLE, ID_INT, ID_INT, ID_DOUBLE, ID_DOUBLE, -1 };
198 std::size_t states[] = { 0, 1, 2, 1, 1, 2, std::size_t(-1) };
199 position_type positions[] =
200 {
201 { .begin: 2, .end: 3 }, { .begin: 4, .end: 7 }, { .begin: 8, .end: 10 }, { .begin: 13, .end: 14 }, { .begin: 15, .end: 20 }, { .begin: 21, .end: 25 },
202 { .begin: std::size_t(-1), .end: std::size_t(-1) }
203 };
204 double values[] = { 1.0, 1.2, -2.0, 3.0, 2.3e6, -3.4, 0.0 };
205
206 // token type: token id, iterator_pair as token value, no state
207 {
208 typedef lex::lexertl::token<
209 base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
210 typedef lex::lexertl::actor_lexer<token_type> lexer_type;
211
212 token_definitions<lexer_type> lexer;
213 std::vector<token_type> tokens;
214 base_iterator_type first = input.begin();
215
216 using phoenix::arg_names::_1;
217 BOOST_TEST(lex::tokenize(first, input.end(), lexer
218 , phoenix::push_back(phoenix::ref(tokens), _1)));
219
220 BOOST_TEST(test_token_ids(ids, tokens));
221 BOOST_TEST(test_token_values(values, tokens));
222 }
223
224 {
225 typedef lex::lexertl::position_token<
226 base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
227 typedef lex::lexertl::actor_lexer<token_type> lexer_type;
228
229 token_definitions<lexer_type> lexer;
230 std::vector<token_type> tokens;
231 base_iterator_type first = input.begin();
232
233 using phoenix::arg_names::_1;
234 BOOST_TEST(lex::tokenize(first, input.end(), lexer
235 , phoenix::push_back(phoenix::ref(tokens), _1)));
236
237 BOOST_TEST(test_token_ids(ids, tokens));
238 BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
239 BOOST_TEST(test_token_values(values, tokens));
240 }
241
242 // token type: holds token id, state, iterator_pair as token value
243 {
244 typedef lex::lexertl::token<
245 base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
246 typedef lex::lexertl::actor_lexer<token_type> lexer_type;
247
248 token_definitions_with_state<lexer_type> lexer;
249 std::vector<token_type> tokens;
250 base_iterator_type first = input.begin();
251
252 using phoenix::arg_names::_1;
253 BOOST_TEST(lex::tokenize(first, input.end(), lexer
254 , phoenix::push_back(phoenix::ref(tokens), _1)));
255
256 BOOST_TEST(test_token_ids(ids, tokens));
257 BOOST_TEST(test_token_states(states, tokens));
258 BOOST_TEST(test_token_values(values, tokens));
259 }
260
261 {
262 typedef lex::lexertl::position_token<
263 base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
264 typedef lex::lexertl::actor_lexer<token_type> lexer_type;
265
266 token_definitions_with_state<lexer_type> lexer;
267 std::vector<token_type> tokens;
268 base_iterator_type first = input.begin();
269
270 using phoenix::arg_names::_1;
271 BOOST_TEST(lex::tokenize(first, input.end(), lexer
272 , phoenix::push_back(phoenix::ref(tokens), _1)));
273
274 BOOST_TEST(test_token_ids(ids, tokens));
275 BOOST_TEST(test_token_states(states, tokens));
276 BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
277 BOOST_TEST(test_token_values(values, tokens));
278 }
279
280 return boost::report_errors();
281}
282

source code of boost/libs/spirit/test/lex/token_onetype.cpp