diff options
Diffstat (limited to 'src/lib/dns/tests/master_lexer_state_unittest.cc')
| -rw-r--r-- | src/lib/dns/tests/master_lexer_state_unittest.cc | 606 |
1 files changed, 0 insertions, 606 deletions
diff --git a/src/lib/dns/tests/master_lexer_state_unittest.cc b/src/lib/dns/tests/master_lexer_state_unittest.cc index e810136068..8b13789179 100644 --- a/src/lib/dns/tests/master_lexer_state_unittest.cc +++ b/src/lib/dns/tests/master_lexer_state_unittest.cc @@ -1,607 +1 @@ -// Copyright (C) 2012-2015 Internet Systems Consortium, Inc. ("ISC") -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#include <config.h> - -#include <dns/master_lexer.h> -#include <dns/master_lexer_inputsource.h> -#include <dns/master_lexer_state.h> - -#include <gtest/gtest.h> - -#include <sstream> - -using namespace isc::dns; -using namespace master_lexer_internal; - -namespace { -typedef MasterToken Token; // shortcut - -class MasterLexerStateTest : public ::testing::Test { -protected: - MasterLexerStateTest() : common_options(MasterLexer::INITIAL_WS), - s_null(NULL), - s_crlf(State::getInstance(State::CRLF)), - s_string(State::getInstance(State::String)), - s_qstring(State::getInstance(State::QString)), - s_number(State::getInstance(State::Number)), - options(MasterLexer::NONE), - orig_options(options) - {} - - // Specify INITIAL_WS as common initial options. - const MasterLexer::Options common_options; - MasterLexer lexer; - const State* const s_null; - const State& s_crlf; - const State& s_string; - const State& s_qstring; - const State& s_number; - std::stringstream ss; - MasterLexer::Options options, orig_options; -}; - -// Common check for the end-of-file condition. -// Token is set to END_OF_FILE, and the lexer was NOT last eol state. -// Passed state can be any valid one; they are stateless, just providing the -// interface for inspection. -void -eofCheck(const State& state, MasterLexer& lexer) { - EXPECT_EQ(Token::END_OF_FILE, state.getToken(lexer).getType()); - EXPECT_FALSE(state.wasLastEOL(lexer)); -} - -TEST_F(MasterLexerStateTest, startAndEnd) { - // A simple case: the input is empty, so we begin with start and - // are immediately done. - lexer.pushSource(ss); - EXPECT_EQ(s_null, State::start(lexer, common_options)); - eofCheck(s_crlf, lexer); -} - -TEST_F(MasterLexerStateTest, startToEOL) { - ss << "\n"; - lexer.pushSource(ss); - - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_TRUE(s_crlf.wasLastEOL(lexer)); - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - - // The next lexer session will reach EOF. Same eof check should pass. - EXPECT_EQ(s_null, State::start(lexer, common_options)); - eofCheck(s_crlf, lexer); -} - -TEST_F(MasterLexerStateTest, space) { - // repeat '\t\n' twice (see below), then space after EOL - ss << " \t\n\t\n "; - lexer.pushSource(ss); - - // by default space characters and tabs will be ignored. We check this - // twice; at the second iteration, it's a white space at the beginning - // of line, but since we don't specify INITIAL_WS option, it's treated as - // normal space and ignored. - for (size_t i = 0; i < 2; ++i) { - EXPECT_EQ(s_null, State::start(lexer, MasterLexer::NONE)); - EXPECT_TRUE(s_crlf.wasLastEOL(lexer)); - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - } - - // Now we specify the INITIAL_WS option. It will be recognized and the - // corresponding token will be returned. - EXPECT_EQ(s_null, State::start(lexer, MasterLexer::INITIAL_WS)); - EXPECT_FALSE(s_crlf.wasLastEOL(lexer)); - EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType()); -} - -TEST_F(MasterLexerStateTest, parentheses) { - ss << "\n(\na\n )\n "; // 1st \n is to check if 'was EOL' is set to false - lexer.pushSource(ss); - - EXPECT_EQ(s_null, State::start(lexer, common_options)); // handle \n - - // Now handle '('. It skips \n and recognize 'a' as string - EXPECT_EQ(0, s_crlf.getParenCount(lexer)); // check pre condition - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - EXPECT_EQ(1, s_crlf.getParenCount(lexer)); // check post condition - EXPECT_FALSE(s_crlf.wasLastEOL(lexer)); - - // skip 'a' - s_string.handle(lexer); - - // Then handle ')'. '\n' before ')' isn't recognized because - // it's canceled due to the '('. Likewise, the space after the '\n' - // shouldn't be recognized but should be just ignored. - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(0, s_crlf.getParenCount(lexer)); - - // Now, temporarily disabled options are restored: Both EOL and the - // initial WS are recognized - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType()); -} - -TEST_F(MasterLexerStateTest, nestedParentheses) { - // This is an unusual, but allowed (in this implementation) case. - ss << "(a(b)\n c)\n "; - lexer.pushSource(ss); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '(' - s_string.handle(lexer); // consume 'a' - EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '(' - s_string.handle(lexer); // consume 'b' - EXPECT_EQ(2, s_crlf.getParenCount(lexer)); // now the count is 2 - - // Close the inner most parentheses. count will be decreased, but option - // shouldn't be restored yet, so the intermediate EOL or initial WS won't - // be recognized. - EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume ')' - s_string.handle(lexer); // consume 'c' - EXPECT_EQ(1, s_crlf.getParenCount(lexer)); - - // Close the outermost parentheses. count will be reset to 0, and original - // options are restored. - EXPECT_EQ(s_null, State::start(lexer, common_options)); - - // Now, temporarily disabled options are restored: Both EOL and the - // initial WS are recognized - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType()); -} - -TEST_F(MasterLexerStateTest, unbalancedParentheses) { - // Only closing paren is provided. We prepend a \n to check if it's - // correctly canceled after detecting the error. - ss << "\n)"; - ss << "(a"; - lexer.pushSource(ss); - - EXPECT_EQ(s_null, State::start(lexer, common_options)); // consume '\n' - EXPECT_TRUE(s_crlf.wasLastEOL(lexer)); // this \n was remembered - - // Now checking ')'. The result should be error, count shouldn't be - // changed. "last EOL" should be canceled. - EXPECT_EQ(0, s_crlf.getParenCount(lexer)); - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(0, s_crlf.getParenCount(lexer)); - ASSERT_EQ(Token::ERROR, s_crlf.getToken(lexer).getType()); - EXPECT_EQ(Token::UNBALANCED_PAREN, s_crlf.getToken(lexer).getErrorCode()); - EXPECT_FALSE(s_crlf.wasLastEOL(lexer)); - - // Reach EOF with a dangling open parenthesis. - EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '(' - s_string.handle(lexer); // consume 'a' - EXPECT_EQ(1, s_crlf.getParenCount(lexer)); - EXPECT_EQ(s_null, State::start(lexer, common_options)); // reach EOF - ASSERT_EQ(Token::ERROR, s_crlf.getToken(lexer).getType()); - EXPECT_EQ(Token::UNBALANCED_PAREN, s_crlf.getToken(lexer).getErrorCode()); - EXPECT_EQ(0, s_crlf.getParenCount(lexer)); // should be reset to 0 -} - -TEST_F(MasterLexerStateTest, startToComment) { - // Begin with 'start', detect space, then encounter a comment. Skip - // the rest of the line, and recognize the new line. Note that the - // second ';' is simply ignored. - ss << " ;a;\n"; - ss << ";a;"; // Likewise, but the comment ends with EOF. - lexer.pushSource(ss); - - // Initial whitespace (asked for in common_options) - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType()); - // Comment ending with EOL - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - - // Comment ending with EOF - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType()); -} - -TEST_F(MasterLexerStateTest, commentAfterParen) { - // comment after an opening parenthesis. The code that is tested by - // other tests should also ensure that it works correctly, but we - // check it explicitly. - ss << "( ;this is a comment\na)\n"; - lexer.pushSource(ss); - - // consume '(', skip comments, consume 'a', then consume ')' - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); -} - -TEST_F(MasterLexerStateTest, crlf) { - ss << "\r\n"; // case 1 - ss << "\r "; // case 2 - ss << "\r;comment\na"; // case 3 - ss << "\r"; // case 4 - lexer.pushSource(ss); - - // 1. A sequence of \r, \n is recognized as a single 'end-of-line' - EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r' - s_crlf.handle(lexer); // recognize '\n' - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - EXPECT_TRUE(s_crlf.wasLastEOL(lexer)); - - // 2. Single '\r' (not followed by \n) is recognized as a single - // 'end-of-line'. then there will be "initial WS" - EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r' - // see ' ', "unget" it - s_crlf.handle(lexer); - EXPECT_EQ(s_null, State::start(lexer, common_options)); // recognize ' ' - EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType()); - - // 3. comment between \r and \n - EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r' - // skip comments, recognize '\n' - s_crlf.handle(lexer); - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // skip 'a' - - // 4. \r then EOF - EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r' - // see EOF, then "unget" it - s_crlf.handle(lexer); - EXPECT_EQ(s_null, State::start(lexer, common_options)); // recognize EOF - EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType()); -} - -// Commonly used check for string related test cases, checking if the given -// token has expected values. -void -stringTokenCheck(const std::string& expected, const MasterToken& token, - bool quoted = false) -{ - EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType()); - EXPECT_EQ(expected, token.getString()); - const std::string actual(token.getStringRegion().beg, - token.getStringRegion().beg + - token.getStringRegion().len); - EXPECT_EQ(expected, actual); - - // There should be "hidden" nul-terminator after the string data. - ASSERT_NE(static_cast<const char*>(NULL), token.getStringRegion().beg); - EXPECT_EQ(0, *(token.getStringRegion().beg + token.getStringRegion().len)); -} - -TEST_F(MasterLexerStateTest, string) { - // Check with simple strings followed by separate characters - ss << "followed-by-EOL\n"; - ss << "followed-by-CR\r"; - ss << "followed-by-space "; - ss << "followed-by-tab\t"; - ss << "followed-by-comment;this is comment and ignored\n"; - ss << "followed-by-paren(closing)"; - ss << "followed-by-EOF"; - lexer.pushSource(ss); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see \n - EXPECT_FALSE(s_string.wasLastEOL(lexer)); - stringTokenCheck("followed-by-EOL", s_string.getToken(lexer)); - EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see \r - stringTokenCheck("followed-by-CR", s_string.getToken(lexer)); - EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // handle \r... - s_crlf.handle(lexer); // ...and skip it - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see ' ' - stringTokenCheck("followed-by-space", s_string.getToken(lexer)); - - // skip ' ', then recognize the next string - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see \t - stringTokenCheck("followed-by-tab", s_string.getToken(lexer)); - - // skip \t, then recognize the next string - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see comment - stringTokenCheck("followed-by-comment", s_string.getToken(lexer)); - EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n after it - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see '(' - stringTokenCheck("followed-by-paren", s_string.getToken(lexer)); - EXPECT_EQ(&s_string, State::start(lexer, common_options)); // str in () - s_string.handle(lexer); // recognize the str, see ')' - stringTokenCheck("closing", s_string.getToken(lexer)); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see EOF - stringTokenCheck("followed-by-EOF", s_string.getToken(lexer)); -} - -TEST_F(MasterLexerStateTest, stringEscape) { - // some of the separate characters should be considered part of the - // string if escaped. - ss << "escaped\\ space "; - ss << "escaped\\\ttab "; - ss << "escaped\\(paren "; - ss << "escaped\\)close "; - ss << "escaped\\;comment "; - ss << "escaped\\\\ backslash "; // second '\' shouldn't escape ' ' - lexer.pushSource(ss); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("escaped\\ space", s_string.getToken(lexer)); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("escaped\\\ttab", s_string.getToken(lexer)); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("escaped\\(paren", s_string.getToken(lexer)); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("escaped\\)close", s_string.getToken(lexer)); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("escaped\\;comment", s_string.getToken(lexer)); - - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see ' ' in mid - stringTokenCheck("escaped\\\\", s_string.getToken(lexer)); - - // Confirm the word that follows the escaped '\' is correctly recognized. - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("backslash", s_string.getToken(lexer)); -} - -TEST_F(MasterLexerStateTest, quotedString) { - ss << "\"ignore-quotes\"\n"; - ss << "\"quoted string\" "; // space is part of the qstring - ss << "\"\" "; // empty quoted string - // also check other separator characters. note that \r doesn't cause - // UNBALANCED_QUOTES. Not sure if it's intentional, but that's how the - // BIND 9 version works, so we follow it (it should be too minor to matter - // in practice anyway) - ss << "\"quoted()\t\rstring\" "; - ss << "\"escape\\ in quote\" "; - ss << "\"escaped\\\"\" "; - ss << "\"escaped backslash\\\\\" "; - ss << "\"no;comment\""; - lexer.pushSource(ss); - - // by default, '"' is unexpected (when QSTRING is not specified), - // and it returns MasterToken::UNEXPECTED_QUOTES. - EXPECT_EQ(s_null, State::start(lexer, common_options)); - EXPECT_EQ(Token::UNEXPECTED_QUOTES, s_string.getToken(lexer).getErrorCode()); - // Read it as a QSTRING. - s_qstring.handle(lexer); // recognize quoted str, see \n - stringTokenCheck("ignore-quotes", s_qstring.getToken(lexer), true); - EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n after it - EXPECT_TRUE(s_string.wasLastEOL(lexer)); - - // If QSTRING is specified in option, '"' is regarded as a beginning of - // a quoted string. - const MasterLexer::Options options = common_options | MasterLexer::QSTRING; - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - EXPECT_FALSE(s_string.wasLastEOL(lexer)); // EOL is canceled due to '"' - s_qstring.handle(lexer); - stringTokenCheck("quoted string", s_string.getToken(lexer), true); - - // Empty string is okay as qstring - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - stringTokenCheck("", s_string.getToken(lexer), true); - - // Also checks other separator characters within a qstring - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - stringTokenCheck("quoted()\t\rstring", s_string.getToken(lexer), true); - - // escape character mostly doesn't have any effect in the qstring - // processing - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - stringTokenCheck("escape\\ in quote", s_string.getToken(lexer), true); - - // The only exception is the quotation mark itself. Note that the escape - // only works on the quotation mark immediately after it. - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - stringTokenCheck("escaped\"", s_string.getToken(lexer), true); - - // quoted '\' then '"'. Unlike the previous case '"' shouldn't be - // escaped. - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - stringTokenCheck("escaped backslash\\\\", s_string.getToken(lexer), true); - - // ';' has no meaning in a quoted string (not indicating a comment) - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - stringTokenCheck("no;comment", s_string.getToken(lexer), true); -} - -TEST_F(MasterLexerStateTest, brokenQuotedString) { - ss << "\"unbalanced-quote\n"; - ss << "\"quoted\\\n\" "; - ss << "\"unclosed quote and EOF"; - lexer.pushSource(ss); - - // EOL is encountered without closing the quote - const MasterLexer::Options options = common_options | MasterLexer::QSTRING; - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - ASSERT_EQ(Token::ERROR, s_qstring.getToken(lexer).getType()); - EXPECT_EQ(Token::UNBALANCED_QUOTES, - s_qstring.getToken(lexer).getErrorCode()); - // We can resume after the error from the '\n' - EXPECT_EQ(s_null, State::start(lexer, options)); - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - - // \n is okay in a quoted string if escaped - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - stringTokenCheck("quoted\\\n", s_string.getToken(lexer), true); - - // EOF is encountered without closing the quote - EXPECT_EQ(&s_qstring, State::start(lexer, options)); - s_qstring.handle(lexer); - ASSERT_EQ(Token::ERROR, s_qstring.getToken(lexer).getType()); - EXPECT_EQ(Token::UNEXPECTED_END, s_qstring.getToken(lexer).getErrorCode()); - // If we continue we'll simply see the EOF - EXPECT_EQ(s_null, State::start(lexer, options)); - EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType()); -} - -TEST_F(MasterLexerStateTest, basicNumbers) { - ss << "0 "; - ss << "1 "; - ss << "12345 "; - ss << "4294967295 "; // 2^32-1 - ss << "4294967296 "; // Out of range - ss << "340282366920938463463374607431768211456 "; - // Very much out of range (2^128) - ss << "005 "; // Leading zeroes are ignored - ss << "42;asdf\n"; // Number with comment - ss << "37"; // Simple number again, here to make - // sure none of the above messed up - // the tokenizer - lexer.pushSource(ss); - - // Ask the lexer to recognize numbers as well - const MasterLexer::Options options = common_options | MasterLexer::NUMBER; - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(0, s_number.getToken(lexer).getNumber()); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(1, s_number.getToken(lexer).getNumber()); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(12345, s_number.getToken(lexer).getNumber()); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(4294967295u, s_number.getToken(lexer).getNumber()); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(Token::NUMBER_OUT_OF_RANGE, - s_number.getToken(lexer).getErrorCode()); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(Token::NUMBER_OUT_OF_RANGE, - s_number.getToken(lexer).getErrorCode()); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(5, s_number.getToken(lexer).getNumber()); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(42, s_number.getToken(lexer).getNumber()); - - EXPECT_EQ(s_null, State::start(lexer, options)); - EXPECT_TRUE(s_crlf.wasLastEOL(lexer)); - EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType()); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - EXPECT_EQ(37, s_number.getToken(lexer).getNumber()); - - // If we continue we'll simply see the EOF - EXPECT_EQ(s_null, State::start(lexer, options)); - EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType()); -} - -// Test tokens that look like (or start out as) numbers, -// but turn out to be strings. Tests include escaped characters. -TEST_F(MasterLexerStateTest, stringNumbers) { - ss << "123 "; // Should be read as a string if the - // NUMBER option is not given - ss << "-1 "; // Negative numbers are interpreted - // as strings (unsigned integers only) - ss << "123abc456 "; // 'Numbers' containing non-digits should - // be interpreted as strings - ss << "123\\456 "; // Numbers containing escaped digits are - // interpreted as strings - ss << "3scaped\\ space "; - ss << "3scaped\\\ttab "; - ss << "3scaped\\(paren "; - ss << "3scaped\\)close "; - ss << "3scaped\\;comment "; - ss << "3scaped\\\\ 8ackslash "; // second '\' shouldn't escape ' ' - - lexer.pushSource(ss); - - // Note that common_options does not include MasterLexer::NUMBER, - // so the token should be recognized as a string - EXPECT_EQ(&s_string, State::start(lexer, common_options)); - s_string.handle(lexer); - stringTokenCheck("123", s_string.getToken(lexer), false); - - // Ask the lexer to recognize numbers as well - const MasterLexer::Options options = common_options | MasterLexer::NUMBER; - - EXPECT_EQ(&s_string, State::start(lexer, options)); - s_string.handle(lexer); - stringTokenCheck("-1", s_string.getToken(lexer), false); - - // Starts out as a number, but ends up being a string - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - stringTokenCheck("123abc456", s_number.getToken(lexer), false); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); - stringTokenCheck("123\\456", s_number.getToken(lexer), false); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("3scaped\\ space", s_number.getToken(lexer)); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("3scaped\\\ttab", s_number.getToken(lexer)); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("3scaped\\(paren", s_number.getToken(lexer)); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("3scaped\\)close", s_number.getToken(lexer)); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("3scaped\\;comment", s_number.getToken(lexer)); - - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); // recognize str, see ' ' in mid - stringTokenCheck("3scaped\\\\", s_number.getToken(lexer)); - - // Confirm the word that follows the escaped '\' is correctly recognized. - EXPECT_EQ(&s_number, State::start(lexer, options)); - s_number.handle(lexer); // recognize str, see ' ' at end - stringTokenCheck("8ackslash", s_number.getToken(lexer)); - - // If we continue we'll simply see the EOF - EXPECT_EQ(s_null, State::start(lexer, options)); - EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType()); -} - -} // end anonymous namespace |