1 | // Boost tokenizer examples -------------------------------------------------// |
2 | |
3 | // (c) Copyright John R. Bandela 2001. |
4 | |
5 | // Distributed under the Boost Software License, Version 1.0. (See |
6 | // accompanying file LICENSE_1_0.txt or copy at |
7 | // http://www.boost.org/LICENSE_1_0.txt) |
8 | |
9 | // See http://www.boost.org for updates, documentation, and revision history. |
10 | |
11 | #include <iostream> |
12 | #include <iterator> |
13 | #include <string> |
14 | #include <algorithm> |
15 | #include <boost/tokenizer.hpp> |
16 | #include <boost/array.hpp> |
17 | |
18 | #include <boost/core/lightweight_test.hpp> |
19 | |
20 | int main() |
21 | { |
22 | using namespace boost; |
23 | |
24 | // Use tokenizer |
25 | { |
26 | const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|" ; |
27 | std::string answer[] = { "Hello" , "world" , "foo" , "bar" , "yow" , "baz" }; |
28 | typedef tokenizer<char_separator<char> > Tok; |
29 | char_separator<char> sep("-;|" ); |
30 | Tok t(test_string, sep); |
31 | BOOST_TEST(std::equal(t.begin(),t.end(),answer)); |
32 | } |
33 | { |
34 | const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|" ; |
35 | std::string answer[] = { "" , "" , "Hello" , "|" , "world" , "|" , "" , "|" , "" , |
36 | "foo" , "" , "bar" , "yow" , "baz" , "|" , "" }; |
37 | typedef tokenizer<char_separator<char> > Tok; |
38 | char_separator<char> sep("-;" , "|" , boost::keep_empty_tokens); |
39 | Tok t(test_string, sep); |
40 | BOOST_TEST(std::equal(t.begin(), t.end(), answer)); |
41 | } |
42 | { |
43 | const std::string test_string = "This,,is, a.test.." ; |
44 | std::string answer[] = {"This" ,"is" ,"a" ,"test" }; |
45 | typedef tokenizer<> Tok; |
46 | Tok t(test_string); |
47 | BOOST_TEST(std::equal(t.begin(),t.end(),answer)); |
48 | } |
49 | |
50 | { |
51 | const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\" ; |
52 | std::string answer[] = {"Field 1" ,"embedded,comma" ,"quote \"" ," escape \\" }; |
53 | typedef tokenizer<escaped_list_separator<char> > Tok; |
54 | Tok t(test_string); |
55 | BOOST_TEST(std::equal(t.begin(),t.end(),answer)); |
56 | } |
57 | |
58 | { |
59 | const std::string test_string = ",1,;2\\\";3\\;,4,5^\\,\'6,7\';" ; |
60 | std::string answer[] = {"" ,"1" ,"" ,"2\"" ,"3;" ,"4" ,"5\\" ,"6,7" ,"" }; |
61 | typedef tokenizer<escaped_list_separator<char> > Tok; |
62 | escaped_list_separator<char> sep("\\^" ,",;" ,"\"\'" ); |
63 | Tok t(test_string,sep); |
64 | BOOST_TEST(std::equal(t.begin(),t.end(),answer)); |
65 | } |
66 | |
67 | { |
68 | const std::string test_string = "12252001" ; |
69 | std::string answer[] = {"12" ,"25" ,"2001" }; |
70 | typedef tokenizer<offset_separator > Tok; |
71 | boost::array<int,3> offsets = {.elems: {2,2,4}}; |
72 | offset_separator func(offsets.begin(),offsets.end()); |
73 | Tok t(test_string,func); |
74 | BOOST_TEST(std::equal(t.begin(),t.end(),answer)); |
75 | } |
76 | |
77 | // Use token_iterator_generator |
78 | { |
79 | const std::string test_string = "This,,is, a.test.." ; |
80 | std::string answer[] = {"This" ,"is" ,"a" ,"test" }; |
81 | typedef token_iterator_generator<char_delimiters_separator<char> >::type Iter; |
82 | Iter begin = make_token_iterator<std::string>(begin: test_string.begin(), |
83 | end: test_string.end(),fun: char_delimiters_separator<char>()); |
84 | Iter end; |
85 | BOOST_TEST(std::equal(begin,end,answer)); |
86 | } |
87 | |
88 | { |
89 | const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\" ; |
90 | std::string answer[] = {"Field 1" ,"embedded,comma" ,"quote \"" ," escape \\" }; |
91 | typedef token_iterator_generator<escaped_list_separator<char> >::type Iter; |
92 | Iter begin = make_token_iterator<std::string>(begin: test_string.begin(), |
93 | end: test_string.end(),fun: escaped_list_separator<char>()); |
94 | Iter begin_c(begin); |
95 | Iter end; |
96 | BOOST_TEST(std::equal(begin,end,answer)); |
97 | |
98 | while(begin_c != end) |
99 | { |
100 | BOOST_TEST(begin_c.at_end() == 0); |
101 | ++begin_c; |
102 | } |
103 | BOOST_TEST(begin_c.at_end()); |
104 | } |
105 | |
106 | { |
107 | const std::string test_string = "12252001" ; |
108 | std::string answer[] = {"12" ,"25" ,"2001" }; |
109 | typedef token_iterator_generator<offset_separator>::type Iter; |
110 | boost::array<int,3> offsets = {.elems: {2,2,4}}; |
111 | offset_separator func(offsets.begin(),offsets.end()); |
112 | Iter begin = make_token_iterator<std::string>(begin: test_string.begin(), |
113 | end: test_string.end(),fun: func); |
114 | Iter end= make_token_iterator<std::string>(begin: test_string.end(), |
115 | end: test_string.end(),fun: func); |
116 | BOOST_TEST(std::equal(begin,end,answer)); |
117 | } |
118 | |
119 | // Test copying |
120 | { |
121 | const std::string test_string = "abcdef" ; |
122 | token_iterator_generator<offset_separator>::type beg, end, other; |
123 | boost::array<int,3> ar = {.elems: {1,2,3}}; |
124 | offset_separator f(ar.begin(),ar.end()); |
125 | beg = make_token_iterator<std::string>(begin: test_string.begin(),end: test_string.end(),fun: f); |
126 | |
127 | ++beg; |
128 | other = beg; |
129 | ++other; |
130 | |
131 | BOOST_TEST(*beg=="bc" ); |
132 | BOOST_TEST(*other=="def" ); |
133 | |
134 | other = make_token_iterator<std::string>(begin: test_string.begin(), |
135 | end: test_string.end(),fun: f); |
136 | |
137 | BOOST_TEST(*other=="a" ); |
138 | } |
139 | |
140 | // Test non-default constructed char_delimiters_separator |
141 | { |
142 | const std::string test_string = "how,are you, doing" ; |
143 | std::string answer[] = {"how" ,"," ,"are you" ,"," ," doing" }; |
144 | tokenizer<> t(test_string,char_delimiters_separator<char>(true,"," ,"" )); |
145 | BOOST_TEST(std::equal(t.begin(),t.end(),answer)); |
146 | } |
147 | |
148 | return boost::report_errors(); |
149 | } |
150 | |