Scanner.cc
Go to the documentation of this file.
1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2 
3 Modifications copyright 2017 Universite catholique de Louvain (UCL), Belgium
4 
5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License.
7 You may obtain a copy of the License at
8 
9  http://www.apache.org/licenses/LICENSE-2.0
10 
11 Unless required by applicable law or agreed to in writing, software
12 distributed under the License is distributed on an "AS IS" BASIS,
13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 See the License for the specific language governing permissions and
15 limitations under the License.
16 ==============================================================================*/
17 
24 #include <catch.hpp>
25 
26 #include <strings/Scanner.h>
27 #include <strings/StringPiece.h>
28 
29 using namespace momemta;
30 using namespace momemta::strings;
31 
32 /*class ScannerTest : public ::testing::Test {
33  protected:
34  // Returns a string with all chars that are in <clz>, in byte value order.
35  string ClassStr(Scanner::CharClass clz) {
36  string s;
37  for (int i = 0; i < 256; ++i) {
38  char ch = i;
39  if (Scanner::Matches(clz, ch)) {
40  s += ch;
41  }
42  }
43  return s;
44  }
45 };*/
46 
47 TEST_CASE("Scanner", "[core/strings]") {
48  SECTION("Any") {
49  StringPiece remaining, match;
50  REQUIRE(Scanner(" horse0123")
51  .Any(Scanner::SPACE)
52  .Any(Scanner::DIGIT)
53  .Any(Scanner::LETTER)
54  .GetResult(&remaining, &match));
55  REQUIRE(" horse" == match);
56  REQUIRE("0123" == remaining);
57 
58  REQUIRE(Scanner("")
59  .Any(Scanner::SPACE)
60  .Any(Scanner::DIGIT)
61  .Any(Scanner::LETTER)
62  .GetResult(&remaining, &match));
63  REQUIRE("" == remaining);
64  REQUIRE("" == match);
65 
66  REQUIRE(Scanner("----")
67  .Any(Scanner::SPACE)
68  .Any(Scanner::DIGIT)
69  .Any(Scanner::LETTER)
70  .GetResult(&remaining, &match));
71  REQUIRE("----" == remaining);
72  REQUIRE("" == match);
73  }
74 
75  SECTION("AnySpace") {
76  StringPiece remaining, match;
77  REQUIRE(Scanner(" a b ")
78  .AnySpace()
79  .One(Scanner::LETTER)
80  .AnySpace()
81  .GetResult(&remaining, &match));
82  REQUIRE(" a " == match);
83  REQUIRE("b " == remaining);
84  }
85 
86  SECTION("AnyEscapedNewline") {
87  StringPiece remaining, match;
88  REQUIRE(Scanner("\\\n")
89  .Any(Scanner::LETTER_DIGIT_UNDERSCORE)
90  .GetResult(&remaining, &match));
91  REQUIRE("\\\n" == remaining);
92  REQUIRE("" == match);
93  }
94 
95  SECTION("AnyEmptyString") {
96  StringPiece remaining, match;
97  REQUIRE(Scanner("")
98  .Any(Scanner::LETTER_DIGIT_UNDERSCORE)
99  .GetResult(&remaining, &match));
100  REQUIRE("" == remaining);
101  REQUIRE("" == match);
102  }
103 
104  SECTION("Eos") {
105  REQUIRE_FALSE(Scanner("a").Eos().GetResult());
106  REQUIRE(Scanner("").Eos().GetResult());
107  REQUIRE_FALSE(Scanner("abc").OneLiteral("ab").Eos().GetResult());
108  REQUIRE(Scanner("abc").OneLiteral("abc").Eos().GetResult());
109  }
110 
111  SECTION("Many") {
112  StringPiece remaining, match;
113  REQUIRE(Scanner("abc").Many(Scanner::LETTER).GetResult());
114  REQUIRE_FALSE(Scanner("0").Many(Scanner::LETTER).GetResult());
115  REQUIRE_FALSE(Scanner("").Many(Scanner::LETTER).GetResult());
116 
117  REQUIRE(
118  Scanner("abc ").Many(Scanner::LETTER).GetResult(&remaining, &match));
119  REQUIRE(" " == remaining);
120  REQUIRE("abc" == match);
121  REQUIRE(
122  Scanner("abc").Many(Scanner::LETTER).GetResult(&remaining, &match));
123  REQUIRE("" == remaining);
124  REQUIRE("abc" == match);
125  }
126 
127  SECTION("One") {
128  StringPiece remaining, match;
129  REQUIRE(Scanner("abc").One(Scanner::LETTER).GetResult());
130  REQUIRE_FALSE(Scanner("0").One(Scanner::LETTER).GetResult());
131  REQUIRE_FALSE(Scanner("").One(Scanner::LETTER).GetResult());
132 
133  REQUIRE(Scanner("abc")
134  .One(Scanner::LETTER)
135  .One(Scanner::LETTER)
136  .GetResult(&remaining, &match));
137  REQUIRE("c" == remaining);
138  REQUIRE("ab" == match);
139  REQUIRE(Scanner("a").One(Scanner::LETTER).GetResult(&remaining, &match));
140  REQUIRE("" == remaining);
141  REQUIRE("a" == match);
142  }
143 
144  SECTION("OneLiteral") {
145  REQUIRE_FALSE(Scanner("abc").OneLiteral("abC").GetResult());
146  REQUIRE(Scanner("abc").OneLiteral("ab").OneLiteral("c").GetResult());
147  }
148 
149  SECTION("ScanUntil") {
150  StringPiece remaining, match;
151  REQUIRE(Scanner(R"(' \1 \2 \3 \' \\'rest)")
152  .OneLiteral("'")
153  .ScanUntil('\'')
154  .OneLiteral("'")
155  .GetResult(&remaining, &match));
156  REQUIRE(R"( \\'rest)" == remaining);
157  REQUIRE(R"(' \1 \2 \3 \')" == match);
158 
159  // The "scan until" character is not present.
160  remaining = match = "unset";
161  REQUIRE_FALSE(Scanner(R"(' \1 \2 \3 \\rest)")
162  .OneLiteral("'")
163  .ScanUntil('\'')
164  .GetResult(&remaining, &match));
165  REQUIRE("unset" == remaining);
166  REQUIRE("unset" == match);
167 
168  // Scan until an escape character.
169  remaining = match = "";
170  REQUIRE(
171  Scanner(R"(123\456)").ScanUntil('\\').GetResult(&remaining, &match));
172  REQUIRE(R"(\456)" == remaining);
173  REQUIRE("123" == match);
174  }
175 
176  SECTION("ScanEscapedUntil") {
177  StringPiece remaining, match;
178  REQUIRE(Scanner(R"(' \1 \2 \3 \' \\'rest)")
179  .OneLiteral("'")
180  .ScanEscapedUntil('\'')
181  .OneLiteral("'")
182  .GetResult(&remaining, &match));
183  REQUIRE("rest" == remaining);
184  REQUIRE(R"(' \1 \2 \3 \' \\')" == match);
185 
186  // The "scan until" character is not present.
187  remaining = match = "unset";
188  REQUIRE_FALSE(Scanner(R"(' \1 \2 \3 \' \\rest)")
189  .OneLiteral("'")
190  .ScanEscapedUntil('\'')
191  .GetResult(&remaining, &match));
192  REQUIRE("unset" == remaining);
193  REQUIRE("unset" == match);
194  }
195 
196  SECTION("ZeroOrOneLiteral") {
197  StringPiece remaining, match;
198  REQUIRE(
199  Scanner("abc").ZeroOrOneLiteral("abC").GetResult(&remaining, &match));
200  REQUIRE("abc" == remaining);
201  REQUIRE("" == match);
202 
203  REQUIRE(
204  Scanner("abcd").ZeroOrOneLiteral("ab").ZeroOrOneLiteral("c").GetResult(
205  &remaining, &match));
206  REQUIRE("d" == remaining);
207  REQUIRE("abc" == match);
208 
209  REQUIRE(
210  Scanner("").ZeroOrOneLiteral("abc").GetResult(&remaining, &match));
211  REQUIRE("" == remaining);
212  REQUIRE("" == match);
213  }
214 
215 // Test output of GetResult (including the forms with optional params),
216 // and that it can be called multiple times.
217  SECTION("CaptureAndGetResult") {
218  StringPiece remaining, match;
219 
220  Scanner scan(" first second");
221  REQUIRE(scan.Any(Scanner::SPACE)
222  .RestartCapture()
223  .One(Scanner::LETTER)
224  .Any(Scanner::LETTER_DIGIT)
225  .StopCapture()
226  .Any(Scanner::SPACE)
227  .GetResult(&remaining, &match));
228  REQUIRE("second" == remaining);
229  REQUIRE("first" == match);
230  REQUIRE(scan.GetResult());
231  remaining = "";
232  REQUIRE(scan.GetResult(&remaining));
233  REQUIRE("second" == remaining);
234  remaining = "";
235  match = "";
236  REQUIRE(scan.GetResult(&remaining, &match));
237  REQUIRE("second" == remaining);
238  REQUIRE("first" == match);
239 
240  scan.RestartCapture().One(Scanner::LETTER).One(Scanner::LETTER);
241  remaining = "";
242  match = "";
243  REQUIRE(scan.GetResult(&remaining, &match));
244  REQUIRE("cond" == remaining);
245  REQUIRE("se" == match);
246  }
247 
248 // Tests that if StopCapture is not called, then calling GetResult, then
249 // scanning more, then GetResult again will update the capture.
250  SECTION("MultipleGetResultExtendsCapture") {
251  StringPiece remaining, match;
252 
253  Scanner scan("one2three");
254  REQUIRE(scan.Many(Scanner::LETTER).GetResult(&remaining, &match));
255  REQUIRE("2three" == remaining);
256  REQUIRE("one" == match);
257  REQUIRE(scan.Many(Scanner::DIGIT).GetResult(&remaining, &match));
258  REQUIRE("three" == remaining);
259  REQUIRE("one2" == match);
260  REQUIRE(scan.Many(Scanner::LETTER).GetResult(&remaining, &match));
261  REQUIRE("" == remaining);
262  REQUIRE("one2three" == match);
263  }
264 
265  SECTION("FailedMatchDoesntChangeResult") {
266  // A failed match doesn't change pointers passed to GetResult.
267  Scanner scan("name");
268  StringPiece remaining = "rem";
269  StringPiece match = "match";
270  REQUIRE_FALSE(scan.One(Scanner::SPACE).GetResult(&remaining, &match));
271  REQUIRE("rem" == remaining);
272  REQUIRE("match" == match);
273  }
274 
275  SECTION("DefaultCapturesAll") {
276  // If RestartCapture() is not called, the whole string is used.
277  Scanner scan("a b");
278  StringPiece remaining = "rem";
279  StringPiece match = "match";
280  REQUIRE(scan.Any(Scanner::LETTER)
281  .AnySpace()
282  .Any(Scanner::LETTER)
283  .GetResult(&remaining, &match));
284  REQUIRE("" == remaining);
285  REQUIRE("a b" == match);
286  }
287 
288  SECTION("AllCharClasses") {
289 
290  auto ClassStr = [](Scanner::CharClass clz) -> std::string {
291  std::string s;
292  for (int i = 0; i < 256; ++i) {
293  char ch = i;
294  if (Scanner::Matches(clz, ch)) {
295  s += ch;
296  }
297  }
298  return s;
299  };
300 
301  REQUIRE(256 == ClassStr(Scanner::ALL).size());
302  REQUIRE("0123456789" == ClassStr(Scanner::DIGIT));
303  REQUIRE("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" ==
304  ClassStr(Scanner::LETTER));
305  REQUIRE("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" ==
306  ClassStr(Scanner::LETTER_DIGIT));
307  REQUIRE(
308  "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
309  "abcdefghijklmnopqrstuvwxyz" ==
310  ClassStr(Scanner::LETTER_DIGIT_DASH_UNDERSCORE));
311  REQUIRE(
312  "-./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
313  "abcdefghijklmnopqrstuvwxyz" ==
314  ClassStr(Scanner::LETTER_DIGIT_DASH_DOT_SLASH));
315  REQUIRE(
316  "-./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
317  "abcdefghijklmnopqrstuvwxyz" ==
318  ClassStr(Scanner::LETTER_DIGIT_DASH_DOT_SLASH_UNDERSCORE));
319  REQUIRE(".0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" ==
320  ClassStr(Scanner::LETTER_DIGIT_DOT));
321  REQUIRE("+-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" ==
322  ClassStr(Scanner::LETTER_DIGIT_DOT_PLUS_MINUS));
323  REQUIRE(".0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ==
324  ClassStr(Scanner::LETTER_DIGIT_DOT_UNDERSCORE));
325  REQUIRE("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ==
326  ClassStr(Scanner::LETTER_DIGIT_UNDERSCORE));
327  REQUIRE("abcdefghijklmnopqrstuvwxyz" == ClassStr(Scanner::LOWERLETTER));
328  REQUIRE("0123456789abcdefghijklmnopqrstuvwxyz" ==
329  ClassStr(Scanner::LOWERLETTER_DIGIT));
330  REQUIRE("0123456789_abcdefghijklmnopqrstuvwxyz" ==
331  ClassStr(Scanner::LOWERLETTER_DIGIT_UNDERSCORE));
332  REQUIRE("123456789" == ClassStr(Scanner::NON_ZERO_DIGIT));
333  REQUIRE("\t\n\v\f\r " == ClassStr(Scanner::SPACE));
334  REQUIRE("ABCDEFGHIJKLMNOPQRSTUVWXYZ" == ClassStr(Scanner::UPPERLETTER));
335  }
336 
337  SECTION("Peek") {
338  REQUIRE('a' == Scanner("abc").Peek());
339  REQUIRE('a' == Scanner("abc").Peek('b'));
340  REQUIRE('\0' == Scanner("").Peek());
341  REQUIRE('z' == Scanner("").Peek('z'));
342  REQUIRE('A' == Scanner("0123A").Any(Scanner::DIGIT).Peek());
343  REQUIRE('\0' == Scanner("0123A").Any(Scanner::LETTER_DIGIT).Peek());
344  }
345 }
momemta
Definition: Graph.h:21
Generated by   doxygen 1.8.13