BIND 10 master, updated. e66472386a716a31089a2306b9e5d51f7618feeb [master] resolve additional conflicts due to change of token class name

BIND 10 source code commits bind10-changes at lists.isc.org
Tue Dec 4 18:19:18 UTC 2012


The branch, master has been updated
       via  e66472386a716a31089a2306b9e5d51f7618feeb (commit)
       via  90641997b368ea093686c97d839b2794b927180c (commit)
       via  2f554072bc7b82a18d898073802f545305a90110 (commit)
       via  191d96a58702a5a3059993a1287136833ec94a4e (commit)
       via  d4b9c33282199c912e6f795f6bbfddde12510795 (commit)
       via  d27729277c9e4c9bb4cf121700d67181d6c0aadc (commit)
       via  09cd4b9b32415144a2542171558b56226322bd7e (commit)
       via  597ed99f869f9d170c93467e8abcc2b03959f75f (commit)
       via  ae3a1e5d17d5c8ab8c9dcf20cf60a68744f5cca0 (commit)
       via  d52186367e1f8b49dcb5a3b4825841a95251412f (commit)
       via  f70d131fd20085a472de32f86299a77647f18fd5 (commit)
       via  b991213a389213fe42ef0c87bde56e462c025229 (commit)
       via  c6c7cdee4088d6be3728ffaf07a25b920b0e8b5c (commit)
       via  ebea1d79bcd55536f89e250ce1be13a8369171f7 (commit)
       via  610720a5bef22ba7193c9ad6e3cbe318eebdbf14 (commit)
      from  1545ba5efdff605b12188442e34b5845c925256f (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
commit e66472386a716a31089a2306b9e5d51f7618feeb
Author: JINMEI Tatuya <jinmei at isc.org>
Date:   Tue Dec 4 10:18:48 2012 -0800

    [master] resolve additional conflicts due to change of token class name

commit 90641997b368ea093686c97d839b2794b927180c
Merge: 1545ba5 2f55407
Author: JINMEI Tatuya <jinmei at isc.org>
Date:   Tue Dec 4 09:58:54 2012 -0800

    [master] Merge branch 'trac2506'

-----------------------------------------------------------------------

Summary of changes:
 src/lib/dns/master_lexer.cc                      |  109 +++--
 src/lib/dns/master_lexer.h                       |  534 +++++++++++++---------
 src/lib/dns/master_lexer_state.h                 |   10 +-
 src/lib/dns/rdata.cc                             |    6 +-
 src/lib/dns/rrparamregistry-placeholder.cc       |    6 +-
 src/lib/dns/tests/master_lexer_state_unittest.cc |    4 +-
 src/lib/dns/tests/master_lexer_token_unittest.cc |   90 ++--
 src/lib/dns/tests/master_lexer_unittest.cc       |  201 ++++++--
 8 files changed, 619 insertions(+), 341 deletions(-)

-----------------------------------------------------------------------
diff --git a/src/lib/dns/master_lexer.cc b/src/lib/dns/master_lexer.cc
index 2bf0254..f1f5749 100644
--- a/src/lib/dns/master_lexer.cc
+++ b/src/lib/dns/master_lexer.cc
@@ -36,7 +36,7 @@ using namespace master_lexer_internal;
 
 
 struct MasterLexer::MasterLexerImpl {
-    MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
+    MasterLexerImpl() : source_(NULL), token_(MasterToken::NOT_STARTED),
                         paren_count_(0), last_was_eol_(false),
                         has_previous_(false),
                         previous_paren_count_(0),
@@ -82,7 +82,7 @@ struct MasterLexer::MasterLexerImpl {
 
     std::vector<InputSourcePtr> sources_;
     InputSource* source_;       // current source (NULL if sources_ is empty)
-    Token token_;               // currently recognized token (set by a state)
+    MasterToken token_;         // currently recognized token (set by a state)
     std::vector<char> data_;    // placeholder for string data
 
     // These are used in states, and defined here only as a placeholder.
@@ -165,9 +165,8 @@ MasterLexer::getSourceLine() const {
     return (impl_->sources_.back()->getCurrentLine());
 }
 
-const MasterLexer::Token&
+const MasterToken&
 MasterLexer::getNextToken(Options options) {
-    // If the source is not available
     if (impl_->source_ == NULL) {
         isc_throw(isc::InvalidOperation, "No source to read tokens from");
     }
@@ -178,7 +177,7 @@ MasterLexer::getNextToken(Options options) {
     impl_->has_previous_ = true;
     // Reset the token now. This is to check a token was actually produced.
     // This is debugging aid.
-    impl_->token_ = Token(Token::NO_TOKEN_PRODUCED);
+    impl_->token_ = MasterToken(MasterToken::NO_TOKEN_PRODUCED);
     // And get the token
 
     // This actually handles EOF internally too.
@@ -188,8 +187,62 @@ MasterLexer::getNextToken(Options options) {
     }
     // Make sure a token was produced. Since this Can Not Happen, we assert
     // here instead of throwing.
-    assert(impl_->token_.getType() != Token::ERROR ||
-           impl_->token_.getErrorCode() != Token::NO_TOKEN_PRODUCED);
+    assert(impl_->token_.getType() != MasterToken::ERROR ||
+           impl_->token_.getErrorCode() != MasterToken::NO_TOKEN_PRODUCED);
+    return (impl_->token_);
+}
+
+namespace {
+inline MasterLexer::Options
+optionsForTokenType(MasterToken::Type expect) {
+    switch (expect) {
+    case MasterToken::STRING:
+        return (MasterLexer::NONE);
+    case MasterToken::QSTRING:
+        return (MasterLexer::QSTRING);
+    case MasterToken::NUMBER:
+        return (MasterLexer::NUMBER);
+    default:
+        isc_throw(InvalidParameter,
+                  "expected type for getNextToken not supported: " << expect);
+    }
+}
+}
+
+const MasterToken&
+MasterLexer::getNextToken(MasterToken::Type expect, bool eol_ok) {
+    // Get the next token, specifying an appropriate option corresponding to
+    // the expected type.  The result should be set in impl_->token_.
+    getNextToken(optionsForTokenType(expect));
+
+    if (impl_->token_.getType() == MasterToken::ERROR) {
+        if (impl_->token_.getErrorCode() == MasterToken::NUMBER_OUT_OF_RANGE) {
+            ungetToken();
+        }
+        throw LexerError(__FILE__, __LINE__, impl_->token_);
+    }
+
+    const bool is_eol_like =
+        (impl_->token_.getType() == MasterToken::END_OF_LINE ||
+         impl_->token_.getType() == MasterToken::END_OF_FILE);
+    if (eol_ok && is_eol_like) {
+        return (impl_->token_);
+    }
+    if (impl_->token_.getType() == MasterToken::STRING &&
+        expect == MasterToken::QSTRING) {
+        return (impl_->token_);
+    }
+    if (impl_->token_.getType() != expect) {
+        ungetToken();
+        if (is_eol_like) {
+            throw LexerError(__FILE__, __LINE__,
+                             MasterToken(MasterToken::UNEXPECTED_END));
+        }
+        assert(expect == MasterToken::NUMBER);
+        throw LexerError(__FILE__, __LINE__,
+                         MasterToken(MasterToken::BAD_NUMBER));
+    }
+
     return (impl_->token_);
 }
 
@@ -212,16 +265,17 @@ const char* const error_text[] = {
     "unexpected end of input",  // UNEXPECTED_END
     "unbalanced quotes",        // UNBALANCED_QUOTES
     "no token produced",        // NO_TOKEN_PRODUCED
-    "number out of range"       // NUMBER_OUT_OF_RANGE
+    "number out of range",      // NUMBER_OUT_OF_RANGE
+    "not a valid number"        // BAD_NUMBER
 };
 const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
 } // end unnamed namespace
 
 std::string
-MasterLexer::Token::getErrorText() const {
+MasterToken::getErrorText() const {
     if (type_ != ERROR) {
         isc_throw(InvalidOperation,
-                  "Token::getErrorText() for non error type");
+                  "MasterToken::getErrorText() for non error type");
     }
 
     // The class integrity ensures the following:
@@ -234,14 +288,12 @@ namespace master_lexer_internal {
 // Note that these need to be defined here so that they can refer to
 // the details of MasterLexerImpl.
 
-typedef MasterLexer::Token Token; // convenience shortcut
-
 bool
 State::wasLastEOL(const MasterLexer& lexer) const {
     return (lexer.impl_->last_was_eol_);
 }
 
-const MasterLexer::Token&
+const MasterToken&
 State::getToken(const MasterLexer& lexer) const {
     return (lexer.impl_->token_);
 }
@@ -271,7 +323,7 @@ public:
         if (c != '\n') {
             getLexerImpl(lexer)->source_->ungetChar();
         }
-        getLexerImpl(lexer)->token_ = Token(Token::END_OF_LINE);
+        getLexerImpl(lexer)->token_ = MasterToken(MasterToken::END_OF_LINE);
         getLexerImpl(lexer)->last_was_eol_ = true;
     }
 };
@@ -342,24 +394,24 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
         if (c == InputSource::END_OF_STREAM) {
             lexerimpl.last_was_eol_ = false;
             if (paren_count != 0) {
-                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
                 paren_count = 0; // reset to 0; this helps in lenient mode.
                 return (NULL);
             }
-            lexerimpl.token_ = Token(Token::END_OF_FILE);
+            lexerimpl.token_ = MasterToken(MasterToken::END_OF_FILE);
             return (NULL);
         } else if (c == ' ' || c == '\t') {
             // If requested and we are not in (), recognize the initial space.
             if (lexerimpl.last_was_eol_ && paren_count == 0 &&
                 (options & MasterLexer::INITIAL_WS) != 0) {
                 lexerimpl.last_was_eol_ = false;
-                lexerimpl.token_ = Token(Token::INITIAL_WS);
+                lexerimpl.token_ = MasterToken(MasterToken::INITIAL_WS);
                 return (NULL);
             }
         } else if (c == '\n') {
             lexerimpl.last_was_eol_ = true;
             if (paren_count == 0) { // we don't recognize EOL if we are in ()
-                lexerimpl.token_ = Token(Token::END_OF_LINE);
+                lexerimpl.token_ = MasterToken(MasterToken::END_OF_LINE);
                 return (NULL);
             }
         } else if (c == '\r') {
@@ -375,7 +427,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
         } else if (c == ')') {
             lexerimpl.last_was_eol_ = false;
             if (paren_count == 0) {
-                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
                 return (NULL);
             }
             --paren_count;
@@ -407,7 +459,7 @@ String::handle(MasterLexer& lexer) const {
         if (getLexerImpl(lexer)->isTokenEnd(c, escaped)) {
             getLexerImpl(lexer)->source_->ungetChar();
             getLexerImpl(lexer)->token_ =
-                MasterLexer::Token(&data.at(0), data.size());
+                MasterToken(&data.at(0), data.size());
             return;
         }
         escaped = (c == '\\' && !escaped);
@@ -417,7 +469,7 @@ String::handle(MasterLexer& lexer) const {
 
 void
 QString::handle(MasterLexer& lexer) const {
-    MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+    MasterToken& token = getLexerImpl(lexer)->token_;
     std::vector<char>& data = getLexerImpl(lexer)->data_;
     data.clear();
 
@@ -425,7 +477,7 @@ QString::handle(MasterLexer& lexer) const {
     while (true) {
         const int c = getLexerImpl(lexer)->source_->getChar();
         if (c == InputSource::END_OF_STREAM) {
-            token = Token(Token::UNEXPECTED_END);
+            token = MasterToken(MasterToken::UNEXPECTED_END);
             return;
         } else if (c == '"') {
             if (escaped) {
@@ -434,12 +486,12 @@ QString::handle(MasterLexer& lexer) const {
                 escaped = false;
                 data.back() = '"';
             } else {
-                token = MasterLexer::Token(&data.at(0), data.size(), true);
+                token = MasterToken(&data.at(0), data.size(), true);
                 return;
             }
         } else if (c == '\n' && !escaped) {
             getLexerImpl(lexer)->source_->ungetChar();
-            token = Token(Token::UNBALANCED_QUOTES);
+            token = MasterToken(MasterToken::UNBALANCED_QUOTES);
             return;
         } else {
             escaped = (c == '\\' && !escaped);
@@ -450,7 +502,7 @@ QString::handle(MasterLexer& lexer) const {
 
 void
 Number::handle(MasterLexer& lexer) const {
-    MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+    MasterToken& token = getLexerImpl(lexer)->token_;
 
     // It may yet turn out to be a string, so we first
     // collect all the data
@@ -470,15 +522,14 @@ Number::handle(MasterLexer& lexer) const {
                 try {
                     const uint32_t number32 =
                         boost::lexical_cast<uint32_t, const char*>(&data[0]);
-                    token = MasterLexer::Token(number32);
+                    token = MasterToken(number32);
                 } catch (const boost::bad_lexical_cast&) {
                     // Since we already know we have only digits,
                     // range should be the only possible problem.
-                    token = Token(Token::NUMBER_OUT_OF_RANGE);
+                    token = MasterToken(MasterToken::NUMBER_OUT_OF_RANGE);
                 }
             } else {
-                token = MasterLexer::Token(&data.at(0),
-                                           data.size());
+                token = MasterToken(&data.at(0), data.size());
             }
             return;
         }
diff --git a/src/lib/dns/master_lexer.h b/src/lib/dns/master_lexer.h
index 4a861fc..fdc69fc 100644
--- a/src/lib/dns/master_lexer.h
+++ b/src/lib/dns/master_lexer.h
@@ -28,225 +28,6 @@ namespace master_lexer_internal {
 class State;
 }
 
-/// \brief Tokenizer for parsing DNS master files.
-///
-/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
-/// master files.  It understands some special rules of master files as
-/// defined in RFC 1035, such as comments, character escaping, and multi-line
-/// data, and provides the user application with the actual data in a
-/// more convenient form such as a std::string object.
-///
-/// In order to support the $INCLUDE notation, this class is designed to be
-/// able to operate on multiple files or input streams in the nested way.
-/// The \c pushSource() and \c popSource() methods correspond to the push
-/// and pop operations.
-///
-/// While this class is public, it is less likely to be used by normal
-/// applications; it's mainly expected to be used within this library,
-/// specifically by the \c MasterLoader class and \c Rdata implementation
-/// classes.
-///
-/// \note The error handling policy of this class is slightly different from
-/// that of other classes of this library.  We generally throw an exception
-/// for an invalid input, whether it's more likely to be a program error or
-/// a "user error", which means an invalid input that comes from outside of
-/// the library.  But, this class returns an error code for some certain
-/// types of user errors instead of throwing an exception.  Such cases include
-/// a syntax error identified by the lexer or a misspelled file name that
-/// causes a system error at the time of open.  This is based on the assumption
-/// that the main user of this class is a parser of master files, where
-/// we want to give an option to ignore some non fatal errors and continue
-/// the parsing.  This will be useful if it just performs overall error
-/// checks on a master file.  When the (immediate) caller needs to do explicit
-/// error handling, exceptions are not that a useful tool for error reporting
-/// because we cannot separate the normal and error cases anyway, which would
-/// be one major advantage when we use exceptions.  And, exceptions are
-/// generally more expensive, either when it happens or just by being able
-/// to handle with \c try and \c catch (depending on the underlying
-/// implementation of the exception handling).  For these reasons, some of
-/// this class does not throw for an error that would be reported as an
-/// exception in other classes.
-class MasterLexer {
-    friend class master_lexer_internal::State;
-public:
-    /// \brief Exception thrown when we fail to read from the input
-    /// stream or file.
-    struct ReadError : public Unexpected {
-        ReadError(const char* file, size_t line, const char* what) :
-            Unexpected(file, line, what)
-        {}
-    };
-
-    class Token;       // we define it separately for better readability
-
-    /// \brief Options for getNextToken.
-    ///
-    /// A compound option, indicating multiple options are set, can be
-    /// specified using the logical OR operator (operator|()).
-    enum Options {
-        NONE = 0,               ///< No option
-        INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
-                        ///< end-of-line
-        QSTRING = 2,    ///< recognize quoted string
-        NUMBER = 4   ///< recognize numeric text as integer
-    };
-
-    /// \brief The constructor.
-    ///
-    /// \throw std::bad_alloc Internal resource allocation fails (rare case).
-    MasterLexer();
-
-    /// \brief The destructor.
-    ///
-    /// It internally closes any remaining input sources.
-    ~MasterLexer();
-
-    /// \brief Open a file and make it the current input source of MasterLexer.
-    ///
-    /// The opened file can be explicitly closed by the \c popSource() method;
-    /// if \c popSource() is not called within the lifetime of the
-    /// \c MasterLexer, it will be closed in the destructor.
-    ///
-    /// In the case possible system errors in opening the file (most likely
-    /// because of specifying a non-existent or unreadable file), it returns
-    /// false, and if the optional \c error parameter is non NULL, it will be
-    /// set to a description of the error (any existing content of the string
-    /// will be discarded).  If opening the file succeeds, the given
-    /// \c error parameter will be intact.
-    ///
-    /// Note that this method has two styles of error reporting: one by
-    /// returning \c false (and setting \c error optionally) and the other
-    /// by throwing an exception.  See the note for the class description
-    /// about the distinction.
-    ///
-    /// \throw InvalidParameter filename is NULL
-    /// \param filename A non NULL string specifying a master file
-    /// \param error If non null, a placeholder to set error description in
-    /// case of failure.
-    ///
-    /// \return true if pushing the file succeeds; false otherwise.
-    bool pushSource(const char* filename, std::string* error = NULL);
-
-    /// \brief Make the given stream the current input source of MasterLexer.
-    ///
-    /// The caller still holds the ownership of the passed stream; it's the
-    /// caller's responsibility to keep it valid as long as it's used in
-    /// \c MasterLexer or to release any resource for the stream after that.
-    /// The caller can explicitly tell \c MasterLexer to stop using the
-    /// stream by calling the \c popSource() method.
-    ///
-    /// \param input An input stream object that produces textual
-    /// representation of DNS RRs.
-    void pushSource(std::istream& input);
-
-    /// \brief Stop using the most recently opened input source (file or
-    /// stream).
-    ///
-    /// If it's a file, the previously opened file will be closed internally.
-    /// If it's a stream, \c MasterLexer will simply stop using
-    /// the stream; the caller can assume it will be never used in
-    /// \c MasterLexer thereafter.
-    ///
-    /// This method must not be called when there is no source pushed for
-    /// \c MasterLexer.  This method is otherwise exception free.
-    ///
-    /// \throw isc::InvalidOperation Called with no pushed source.
-    void popSource();
-
-    /// \brief Return the name of the current input source name.
-    ///
-    /// If it's a file, it will be the C string given at the corresponding
-    /// \c pushSource() call, that is, its filename.  If it's a stream, it will
-    /// be formatted as \c "stream-%p" where \c %p is hex representation
-    /// of the address of the stream object.
-    ///
-    /// If there is no opened source at the time of the call, this method
-    /// returns an empty string.
-    ///
-    /// \throw std::bad_alloc Resource allocation failed for string
-    /// construction (rare case)
-    ///
-    /// \return A string representation of the current source (see the
-    /// description)
-    std::string getSourceName() const;
-
-    /// \brief Return the input source line number.
-    ///
-    /// If there is an opened source, the return value will be a non-0
-    /// integer indicating the line number of the current source where
-    /// the \c MasterLexer is currently working.  The expected usage of
-    /// this value is to print a helpful error message when parsing fails
-    /// by specifically identifying the position of the error.
-    ///
-    /// If there is no opened source at the time of the call, this method
-    /// returns 0.
-    ///
-    /// \throw None
-    ///
-    /// \return The current line number of the source (see the description)
-    size_t getSourceLine() const;
-
-    /// \brief Parse and return another token from the input.
-    ///
-    /// It reads a bit of the last opened source and produces another token
-    /// found in it.
-    ///
-    /// This method does not provide the strong exception guarantee. Generally,
-    /// if it throws, the object should not be used any more and should be
-    /// discarded. It was decided all the exceptions thrown from here are
-    /// serious enough that aborting the loading process is the only reasonable
-    /// recovery anyway, so the strong exception guarantee is not needed.
-    ///
-    /// \param options The options can be used to modify the tokenization.
-    ///     The method can be made reporting things which are usually ignored
-    ///     by this parameter. Multiple options can be passed at once by
-    ///     bitwise or (eg. option1 | option 2). See description of available
-    ///     options.
-    /// \return Next token found in the input. Note that the token refers to
-    ///     some internal data in the lexer. It is valid only until
-    ///     getNextToken or ungetToken is called. Also, the token becomes
-    ///     invalid when the lexer is destroyed.
-    /// \throw isc::InvalidOperation in case the source is not available. This
-    ///     may mean the pushSource() has not been called yet, or that the
-    ///     current source has been read past the end.
-    /// \throw ReadError in case there's problem reading from the underlying
-    ///     source (eg. I/O error in the file on the disk).
-    /// \throw std::bad_alloc in case allocation of some internal resources
-    ///     or the token fail.
-    const Token& getNextToken(Options options = NONE);
-
-    /// \brief Return the last token back to the lexer.
-    ///
-    /// The method undoes the lasts call to getNextToken(). If you call the
-    /// getNextToken() again with the same options, it'll return the same
-    /// token. If the options are different, it may return a different token,
-    /// but it acts as if the previous getNextToken() was never called.
-    ///
-    /// It is possible to return only one token back in time (you can't call
-    /// ungetToken() twice in a row without calling getNextToken() in between
-    /// successfully).
-    ///
-    /// It does not work after change of source (by pushSource or popSource).
-    ///
-    /// \throw isc::InvalidOperation If called second time in a row or if
-    ///     getNextToken() was not called since the last change of the source.
-    void ungetToken();
-
-private:
-    struct MasterLexerImpl;
-    MasterLexerImpl* impl_;
-};
-
-/// \brief Operator to combine \c MasterLexer options
-///
-/// This is a trivial shortcut so that compound options can be specified
-/// in an intuitive way.
-inline MasterLexer::Options
-operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
-    return (static_cast<MasterLexer::Options>(
-                static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
-}
-
 /// \brief Tokens for \c MasterLexer
 ///
 /// This is a simple value-class encapsulating a type of a lexer token and
@@ -261,7 +42,7 @@ operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
 /// (using the default version of copy constructor and assignment operator),
 /// but it's mainly for internal implementation convenience.  Applications will
 /// simply refer to Token object as a reference via the \c MasterLexer class.
-class MasterLexer::Token {
+class MasterToken {
 public:
     /// \brief Enumeration for token types
     ///
@@ -293,6 +74,7 @@ public:
         NO_TOKEN_PRODUCED, ///< No token was produced. This means programmer
                            /// error and should never get out of the lexer.
         NUMBER_OUT_OF_RANGE, ///< Number was out of range
+        BAD_NUMBER,    ///< Number is expected but not recognized
         MAX_ERROR_CODE ///< Max integer corresponding to valid error codes.
                        /// (excluding this one). Mainly for internal use.
     };
@@ -318,7 +100,7 @@ public:
     /// \throw InvalidParameter A value type token is specified.
     /// \param type The type of the token.  It must indicate a non-value
     /// type (not larger than \c NOVALUE_TYPE_MAX).
-    explicit Token(Type type) : type_(type) {
+    explicit MasterToken(Type type) : type_(type) {
         if (type > NOVALUE_TYPE_MAX) {
             isc_throw(InvalidParameter, "Token per-type constructor "
                       "called with invalid type: " << type);
@@ -340,7 +122,7 @@ public:
     /// \param str_beg The start address of the string
     /// \param str_len The size of the string in bytes
     /// \param quoted true if it's a quoted string; false otherwise.
-    Token(const char* str_beg, size_t str_len, bool quoted = false) :
+    MasterToken(const char* str_beg, size_t str_len, bool quoted = false) :
         type_(quoted ? QSTRING : STRING)
     {
         val_.str_region_.beg = str_beg;
@@ -351,7 +133,7 @@ public:
     ///
     /// \brief number An unsigned 32-bit integer corresponding to the token
     /// value.
-    explicit Token(uint32_t number) : type_(NUMBER) {
+    explicit MasterToken(uint32_t number) : type_(NUMBER) {
         val_.number_ = number;
     }
 
@@ -359,7 +141,7 @@ public:
     ///
     /// \throw InvalidParameter Invalid error code value is specified.
     /// \brief error_code A pre-defined constant of \c ErrorCode.
-    explicit Token(ErrorCode error_code) : type_(ERROR) {
+    explicit MasterToken(ErrorCode error_code) : type_(ERROR) {
         if (!(error_code < MAX_ERROR_CODE)) {
             isc_throw(InvalidParameter, "Invalid master lexer error code: "
                       << error_code);
@@ -476,6 +258,310 @@ private:
     } val_;
 };
 
+/// \brief Tokenizer for parsing DNS master files.
+///
+/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
+/// master files.  It understands some special rules of master files as
+/// defined in RFC 1035, such as comments, character escaping, and multi-line
+/// data, and provides the user application with the actual data in a
+/// more convenient form such as a std::string object.
+///
+/// In order to support the $INCLUDE notation, this class is designed to be
+/// able to operate on multiple files or input streams in the nested way.
+/// The \c pushSource() and \c popSource() methods correspond to the push
+/// and pop operations.
+///
+/// While this class is public, it is less likely to be used by normal
+/// applications; it's mainly expected to be used within this library,
+/// specifically by the \c MasterLoader class and \c Rdata implementation
+/// classes.
+///
+/// \note The error handling policy of this class is slightly different from
+/// that of other classes of this library.  We generally throw an exception
+/// for an invalid input, whether it's more likely to be a program error or
+/// a "user error", which means an invalid input that comes from outside of
+/// the library.  But, this class returns an error code for some certain
+/// types of user errors instead of throwing an exception.  Such cases include
+/// a syntax error identified by the lexer or a misspelled file name that
+/// causes a system error at the time of open.  This is based on the assumption
+/// that the main user of this class is a parser of master files, where
+/// we want to give an option to ignore some non fatal errors and continue
+/// the parsing.  This will be useful if it just performs overall error
+/// checks on a master file.  When the (immediate) caller needs to do explicit
+/// error handling, exceptions are not that a useful tool for error reporting
+/// because we cannot separate the normal and error cases anyway, which would
+/// be one major advantage when we use exceptions.  And, exceptions are
+/// generally more expensive, either when it happens or just by being able
+/// to handle with \c try and \c catch (depending on the underlying
+/// implementation of the exception handling).  For these reasons, some of
+/// this class does not throw for an error that would be reported as an
+/// exception in other classes.
+class MasterLexer {
+    friend class master_lexer_internal::State;
+public:
+    /// \brief Exception thrown when we fail to read from the input
+    /// stream or file.
+    class ReadError : public Unexpected {
+    public:
+        ReadError(const char* file, size_t line, const char* what) :
+            Unexpected(file, line, what)
+        {}
+    };
+
+    /// \brief Exception thrown from a wrapper version of
+    /// \c MasterLexer::getNextToken() for non fatal errors.
+    ///
+    /// See the method description for more details.
+    ///
+    /// The \c token_ member variable (read-only) is set to a \c MasterToken
+    /// object of type ERROR indicating the reason for the error.
+    class LexerError : public Exception {
+    public:
+        LexerError(const char* file, size_t line, MasterToken error_token) :
+            Exception(file, line, error_token.getErrorText().c_str()),
+            token_(error_token)
+        {}
+        const MasterToken token_;
+    };
+
+    /// \brief Options for getNextToken.
+    ///
+    /// A compound option, indicating multiple options are set, can be
+    /// specified using the logical OR operator (operator|()).
+    enum Options {
+        NONE = 0,               ///< No option
+        INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
+                        ///< end-of-line
+        QSTRING = 2,    ///< recognize quoted string
+        NUMBER = 4   ///< recognize numeric text as integer
+    };
+
+    /// \brief The constructor.
+    ///
+    /// \throw std::bad_alloc Internal resource allocation fails (rare case).
+    MasterLexer();
+
+    /// \brief The destructor.
+    ///
+    /// It internally closes any remaining input sources.
+    ~MasterLexer();
+
+    /// \brief Open a file and make it the current input source of MasterLexer.
+    ///
+    /// The opened file can be explicitly closed by the \c popSource() method;
+    /// if \c popSource() is not called within the lifetime of the
+    /// \c MasterLexer, it will be closed in the destructor.
+    ///
+    /// In the case possible system errors in opening the file (most likely
+    /// because of specifying a non-existent or unreadable file), it returns
+    /// false, and if the optional \c error parameter is non NULL, it will be
+    /// set to a description of the error (any existing content of the string
+    /// will be discarded).  If opening the file succeeds, the given
+    /// \c error parameter will be intact.
+    ///
+    /// Note that this method has two styles of error reporting: one by
+    /// returning \c false (and setting \c error optionally) and the other
+    /// by throwing an exception.  See the note for the class description
+    /// about the distinction.
+    ///
+    /// \throw InvalidParameter filename is NULL
+    /// \param filename A non NULL string specifying a master file
+    /// \param error If non null, a placeholder to set error description in
+    /// case of failure.
+    ///
+    /// \return true if pushing the file succeeds; false otherwise.
+    bool pushSource(const char* filename, std::string* error = NULL);
+
+    /// \brief Make the given stream the current input source of MasterLexer.
+    ///
+    /// The caller still holds the ownership of the passed stream; it's the
+    /// caller's responsibility to keep it valid as long as it's used in
+    /// \c MasterLexer or to release any resource for the stream after that.
+    /// The caller can explicitly tell \c MasterLexer to stop using the
+    /// stream by calling the \c popSource() method.
+    ///
+    /// \param input An input stream object that produces textual
+    /// representation of DNS RRs.
+    void pushSource(std::istream& input);
+
+    /// \brief Stop using the most recently opened input source (file or
+    /// stream).
+    ///
+    /// If it's a file, the previously opened file will be closed internally.
+    /// If it's a stream, \c MasterLexer will simply stop using
+    /// the stream; the caller can assume it will be never used in
+    /// \c MasterLexer thereafter.
+    ///
+    /// This method must not be called when there is no source pushed for
+    /// \c MasterLexer.  This method is otherwise exception free.
+    ///
+    /// \throw isc::InvalidOperation Called with no pushed source.
+    void popSource();
+
+    /// \brief Return the name of the current input source name.
+    ///
+    /// If it's a file, it will be the C string given at the corresponding
+    /// \c pushSource() call, that is, its filename.  If it's a stream, it will
+    /// be formatted as \c "stream-%p" where \c %p is hex representation
+    /// of the address of the stream object.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns an empty string.
+    ///
+    /// \throw std::bad_alloc Resource allocation failed for string
+    /// construction (rare case)
+    ///
+    /// \return A string representation of the current source (see the
+    /// description)
+    std::string getSourceName() const;
+
+    /// \brief Return the input source line number.
+    ///
+    /// If there is an opened source, the return value will be a non-0
+    /// integer indicating the line number of the current source where
+    /// the \c MasterLexer is currently working.  The expected usage of
+    /// this value is to print a helpful error message when parsing fails
+    /// by specifically identifying the position of the error.
+    ///
+    /// If there is no opened source at the time of the call, this method
+    /// returns 0.
+    ///
+    /// \throw None
+    ///
+    /// \return The current line number of the source (see the description)
+    size_t getSourceLine() const;
+
+    /// \brief Parse and return another token from the input.
+    ///
+    /// It reads a bit of the last opened source and produces another token
+    /// found in it.
+    ///
+    /// This method does not provide the strong exception guarantee. Generally,
+    /// if it throws, the object should not be used any more and should be
+    /// discarded. It was decided all the exceptions thrown from here are
+    /// serious enough that aborting the loading process is the only reasonable
+    /// recovery anyway, so the strong exception guarantee is not needed.
+    ///
+    /// \param options The options can be used to modify the tokenization.
+    ///     The method can be made reporting things which are usually ignored
+    ///     by this parameter. Multiple options can be passed at once by
+    ///     bitwise or (eg. option1 | option 2). See description of available
+    ///     options.
+    /// \return Next token found in the input. Note that the token refers to
+    ///     some internal data in the lexer. It is valid only until
+    ///     getNextToken or ungetToken is called. Also, the token becomes
+    ///     invalid when the lexer is destroyed.
+    /// \throw isc::InvalidOperation in case the source is not available. This
+    ///     may mean the pushSource() has not been called yet, or that the
+    ///     current source has been read past the end.
+    /// \throw ReadError in case there's problem reading from the underlying
+    ///     source (eg. I/O error in the file on the disk).
+    /// \throw std::bad_alloc in case allocation of some internal resources
+    ///     or the token fail.
+    const MasterToken& getNextToken(Options options = NONE);
+
+    /// \brief Parse the input for the expected type of token.
+    ///
+    /// This method is a wrapper of the other version, customized for the case
+    /// where a particular type of token is expected as the next one.
+    /// More specifically, it's intended to be used to get tokens for RDATA
+    /// fields.  Since most RDATA types of fixed format, the token type is
+    /// often predictable and the method interface can be simplified.
+    ///
+    /// This method basically works as follows: it gets the type of the
+    /// expected token, calls the other version of \c getNextToken(Options),
+    /// and returns the token if it's of the expected type (due to the usage
+    /// assumption this should be normally the case).  There are some non
+    /// trivial details though:
+    ///
+    /// - If the expected type is MasterToken::QSTRING, both quoted and
+    ///   unquoted strings are recognized and returned.
+    /// - If the optional \c eol_ok parameter is \c true (very rare case),
+    ///   MasterToken::END_OF_LINE and MasterToken::END_OF_FILE are recognized
+    ///   and returned if they are found instead of the expected type of
+    ///   token.
+    /// - If the next token is not of the expected type (including the case
+    ///   a number is expected but it's out of range), ungetToken() is
+    ///   internally called so the caller can re-read that token.
+    /// - If other types or errors (such as unbalanced parentheses) are
+    ///   detected, the erroneous part isn't "ungotten"; the caller can
+    ///   continue parsing after that part.
+    ///
+    /// In some very rare cases where the RDATA has an optional trailing field,
+    /// the \c eol_ok parameter would be set to \c true.  This way the caller
+    /// can handle both cases (the field does or does not exist) by a single
+    /// call to this method.  In all other cases \c eol_ok should be set to
+    /// \c false, and that is the default and can be omitted.
+    ///
+    /// Unlike the other version of \c getNextToken(Options), this method
+    /// throws an exception of type \c LexerError for non fatal errors such as
+    /// broken syntax or encountering an unexpected type of token.  This way
+    /// the caller can write RDATA parser code without bothering to handle
+    /// errors for each field.  For example, pseudo parser code for MX RDATA
+    /// would look like this:
+    /// \code
+    ///    const uint32_t pref =
+    ///        lexer.getNextToken(MasterToken::NUMBER).getNumber();
+    ///    // check if pref is the uint16_t range; no other check is needed.
+    ///    const Name mx(lexer.getNextToken(MasterToken::STRING).getString());
+    /// \endcode
+    ///
+    /// In the case where \c LexerError exception is thrown, it's expected
+    /// to be handled comprehensively for the parser of the RDATA or at a
+    /// higher layer.  The \c token_ member variable of the corresponding
+    /// \c LexerError exception object stores a token of type
+    /// \c MasterToken::ERROR that indicates the reason for the error.
+    ///
+    /// Due to the specific intended usage of this method, only a subset
+    /// of \c MasterToken::Type values are acceptable for the \c expect
+    /// parameter: \c MasterToken::STRING, \c MasterToken::QSTRING, and
+    /// \c MasterToken::NUMBER.  Specifying other values will result in
+    /// an \c InvalidParameter exception.
+    ///
+    /// \throw InvalidParameter The expected token type is not allowed for
+    /// this method.
+    /// \throw LexerError The lexer finds non fatal error or it finds an
+    /// \throw other Anything the other version of getNextToken() can throw.
+    ///
+    /// \param expect Expected type of token.  Must be either STRING, QSTRING,
+    /// or NUMBER.
+    /// \param eol_ok \c true iff END_OF_LINE or END_OF_FILE is acceptable.
+    /// \return The expected type of token.
+    const MasterToken& getNextToken(MasterToken::Type expect,
+                                    bool eol_ok = false);
+
+    /// \brief Return the last token back to the lexer.
+    ///
+    /// The method undoes the lasts call to getNextToken(). If you call the
+    /// getNextToken() again with the same options, it'll return the same
+    /// token. If the options are different, it may return a different token,
+    /// but it acts as if the previous getNextToken() was never called.
+    ///
+    /// It is possible to return only one token back in time (you can't call
+    /// ungetToken() twice in a row without calling getNextToken() in between
+    /// successfully).
+    ///
+    /// It does not work after change of source (by pushSource or popSource).
+    ///
+    /// \throw isc::InvalidOperation If called second time in a row or if
+    ///     getNextToken() was not called since the last change of the source.
+    void ungetToken();
+
+private:
+    struct MasterLexerImpl;
+    MasterLexerImpl* impl_;
+};
+
+/// \brief Operator to combine \c MasterLexer options
+///
+/// This is a trivial shortcut so that compound options can be specified
+/// in an intuitive way.
+inline MasterLexer::Options
+operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
+    return (static_cast<MasterLexer::Options>(
+                static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
+}
+
 } // namespace dns
 } // namespace isc
 #endif  // MASTER_LEXER_H
diff --git a/src/lib/dns/master_lexer_state.h b/src/lib/dns/master_lexer_state.h
index 2a64c9d..f296c1c 100644
--- a/src/lib/dns/master_lexer_state.h
+++ b/src/lib/dns/master_lexer_state.h
@@ -43,10 +43,10 @@ namespace master_lexer_internal {
 /// state, so it makes more sense to separate the interface for the transition
 /// from the initial state.
 ///
-/// When an object of a specific state class completes the session, it
-/// normally sets the identified token in the lexer, and returns NULL;
-/// if more transition is necessary, it returns a pointer to the next state
-/// object.
+/// If the whole lexer transition is completed within start(), it sets the
+/// identified token and returns NULL; otherwise it returns a pointer to
+/// an object of a specific state class that completes the session
+/// on the call of handle().
 ///
 /// As is usual in the state design pattern, the \c State class is made
 /// a friend class of \c MasterLexer and can refer to its internal details.
@@ -119,7 +119,7 @@ public:
     /// purposes.
     ///@{
     bool wasLastEOL(const MasterLexer& lexer) const;
-    const MasterLexer::Token& getToken(const MasterLexer& lexer) const;
+    const MasterToken& getToken(const MasterLexer& lexer) const;
     size_t getParenCount(const MasterLexer& lexer) const;
     ///@}
 
diff --git a/src/lib/dns/rdata.cc b/src/lib/dns/rdata.cc
index f8deec6..46224c6 100644
--- a/src/lib/dns/rdata.cc
+++ b/src/lib/dns/rdata.cc
@@ -211,9 +211,9 @@ Generic::Generic(MasterLexer& lexer, const Name*,
     std::string s;
 
     while (true) {
-        const MasterLexer::Token& token = lexer.getNextToken();
-        if ((token.getType() == MasterLexer::Token::END_OF_FILE) ||
-            (token.getType() == MasterLexer::Token::END_OF_LINE)) {
+        const MasterToken& token = lexer.getNextToken();
+        if ((token.getType() == MasterToken::END_OF_FILE) ||
+            (token.getType() == MasterToken::END_OF_LINE)) {
             break;
         }
 
diff --git a/src/lib/dns/rrparamregistry-placeholder.cc b/src/lib/dns/rrparamregistry-placeholder.cc
index ed59f5d..8160851 100644
--- a/src/lib/dns/rrparamregistry-placeholder.cc
+++ b/src/lib/dns/rrparamregistry-placeholder.cc
@@ -51,9 +51,9 @@ AbstractRdataFactory::create(MasterLexer& lexer, const Name*,
     std::string s;
 
     while (true) {
-        const MasterLexer::Token& token = lexer.getNextToken();
-        if ((token.getType() == MasterLexer::Token::END_OF_FILE) ||
-            (token.getType() == MasterLexer::Token::END_OF_LINE)) {
+        const MasterToken& token = lexer.getNextToken();
+        if ((token.getType() == MasterToken::END_OF_FILE) ||
+            (token.getType() == MasterToken::END_OF_LINE)) {
             break;
         }
 
diff --git a/src/lib/dns/tests/master_lexer_state_unittest.cc b/src/lib/dns/tests/master_lexer_state_unittest.cc
index d8a6b66..7645ede 100644
--- a/src/lib/dns/tests/master_lexer_state_unittest.cc
+++ b/src/lib/dns/tests/master_lexer_state_unittest.cc
@@ -24,7 +24,7 @@ using namespace isc::dns;
 using namespace master_lexer_internal;
 
 namespace {
-typedef MasterLexer::Token Token; // shortcut
+typedef MasterToken Token; // shortcut
 
 class MasterLexerStateTest : public ::testing::Test {
 protected:
@@ -260,7 +260,7 @@ TEST_F(MasterLexerStateTest, crlf) {
 // Commonly used check for string related test cases, checking if the given
 // token has expected values.
 void
-stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
+stringTokenCheck(const std::string& expected, const MasterToken& token,
                  bool quoted = false)
 {
     EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType());
diff --git a/src/lib/dns/tests/master_lexer_token_unittest.cc b/src/lib/dns/tests/master_lexer_token_unittest.cc
index 1f022df..89a4f9c 100644
--- a/src/lib/dns/tests/master_lexer_token_unittest.cc
+++ b/src/lib/dns/tests/master_lexer_token_unittest.cc
@@ -31,27 +31,27 @@ const size_t TEST_STRING_LEN = sizeof(TEST_STRING) - 1;
 class MasterLexerTokenTest : public ::testing::Test {
 protected:
     MasterLexerTokenTest() :
-        token_eof(MasterLexer::Token::END_OF_FILE),
+        token_eof(MasterToken::END_OF_FILE),
         token_str(TEST_STRING, TEST_STRING_LEN),
         token_num(42),
-        token_err(MasterLexer::Token::UNEXPECTED_END)
+        token_err(MasterToken::UNEXPECTED_END)
     {}
 
-    const MasterLexer::Token token_eof; // an example of non-value type token
-    const MasterLexer::Token token_str;
-    const MasterLexer::Token token_num;
-    const MasterLexer::Token token_err;
+    const MasterToken token_eof; // an example of non-value type token
+    const MasterToken token_str;
+    const MasterToken token_num;
+    const MasterToken token_err;
 };
 
 
 TEST_F(MasterLexerTokenTest, strings) {
     // basic construction and getter checks
-    EXPECT_EQ(MasterLexer::Token::STRING, token_str.getType());
+    EXPECT_EQ(MasterToken::STRING, token_str.getType());
     EXPECT_EQ(std::string("string token"), token_str.getString());
     std::string strval = "dummy"; // this should be replaced
     token_str.getString(strval);
     EXPECT_EQ(std::string("string token"), strval);
-    const MasterLexer::Token::StringRegion str_region =
+    const MasterToken::StringRegion str_region =
         token_str.getStringRegion();
     EXPECT_EQ(TEST_STRING, str_region.beg);
     EXPECT_EQ(TEST_STRING_LEN, str_region.len);
@@ -62,17 +62,17 @@ TEST_F(MasterLexerTokenTest, strings) {
     std::string expected_str("string token");
     expected_str.push_back('\0');
     EXPECT_EQ(expected_str,
-              MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString());
-    MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
+              MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString());
+    MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
     EXPECT_EQ(expected_str, strval);
 
     // Construct type of qstring
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
-              MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), true).
+    EXPECT_EQ(MasterToken::QSTRING,
+              MasterToken(TEST_STRING, sizeof(TEST_STRING), true).
               getType());
     // if we explicitly set 'quoted' to false, it should be normal string
-    EXPECT_EQ(MasterLexer::Token::STRING,
-              MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), false).
+    EXPECT_EQ(MasterToken::STRING,
+              MasterToken(TEST_STRING, sizeof(TEST_STRING), false).
               getType());
 
     // getString/StringRegion() aren't allowed for non string(-variant) types
@@ -86,23 +86,23 @@ TEST_F(MasterLexerTokenTest, strings) {
 
 TEST_F(MasterLexerTokenTest, numbers) {
     EXPECT_EQ(42, token_num.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token_num.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token_num.getType());
 
     // It's copyable and assignable.
-    MasterLexer::Token token(token_num);
+    MasterToken token(token_num);
     EXPECT_EQ(42, token.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token.getType());
 
     token = token_num;
     EXPECT_EQ(42, token.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token.getType());
 
     // it's okay to replace it with a different type of token
     token = token_eof;
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token.getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, token.getType());
 
     // Possible max value
-    token = MasterLexer::Token(0xffffffff);
+    token = MasterToken(0xffffffff);
     EXPECT_EQ(4294967295u, token.getNumber());
 
     // getNumber() isn't allowed for non number types
@@ -112,58 +112,52 @@ TEST_F(MasterLexerTokenTest, numbers) {
 
 TEST_F(MasterLexerTokenTest, novalues) {
     // Just checking we can construct them and getType() returns correct value.
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token_eof.getType());
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE,
-              MasterLexer::Token(MasterLexer::Token::END_OF_LINE).getType());
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
-              MasterLexer::Token(MasterLexer::Token::INITIAL_WS).getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, token_eof.getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE,
+              MasterToken(MasterToken::END_OF_LINE).getType());
+    EXPECT_EQ(MasterToken::INITIAL_WS,
+              MasterToken(MasterToken::INITIAL_WS).getType());
 
     // Special types of tokens cannot have value-based types
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::STRING),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::QSTRING),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::NUMBER),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::ERROR),
-                 isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::STRING), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::QSTRING), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::NUMBER), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::ERROR), isc::InvalidParameter);
 }
 
 TEST_F(MasterLexerTokenTest, errors) {
-    EXPECT_EQ(MasterLexer::Token::ERROR, token_err.getType());
-    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END, token_err.getErrorCode());
+    EXPECT_EQ(MasterToken::ERROR, token_err.getType());
+    EXPECT_EQ(MasterToken::UNEXPECTED_END, token_err.getErrorCode());
     EXPECT_EQ("unexpected end of input", token_err.getErrorText());
-    EXPECT_EQ("lexer not started",
-              MasterLexer::Token(MasterLexer::Token::NOT_STARTED).
+    EXPECT_EQ("lexer not started", MasterToken(MasterToken::NOT_STARTED).
               getErrorText());
     EXPECT_EQ("unbalanced parentheses",
-              MasterLexer::Token(MasterLexer::Token::UNBALANCED_PAREN).
+              MasterToken(MasterToken::UNBALANCED_PAREN).
               getErrorText());
-    EXPECT_EQ("unbalanced quotes",
-              MasterLexer::Token(MasterLexer::Token::UNBALANCED_QUOTES).
+    EXPECT_EQ("unbalanced quotes", MasterToken(MasterToken::UNBALANCED_QUOTES).
               getErrorText());
-    EXPECT_EQ("no token produced",
-              MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
+    EXPECT_EQ("no token produced", MasterToken(MasterToken::NO_TOKEN_PRODUCED).
               getErrorText());
     EXPECT_EQ("number out of range",
-              MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE).
+              MasterToken(MasterToken::NUMBER_OUT_OF_RANGE).
               getErrorText());
+    EXPECT_EQ("not a valid number",
+              MasterToken(MasterToken::BAD_NUMBER).getErrorText());
 
     // getErrorCode/Text() isn't allowed for non number types
     EXPECT_THROW(token_num.getErrorCode(), isc::InvalidOperation);
     EXPECT_THROW(token_num.getErrorText(), isc::InvalidOperation);
 
-    // Only the pre-defined error code is accepted.  Hardcoding '6' (max code
+    // Only the pre-defined error code is accepted.  Hardcoding '7' (max code
     // + 1) is intentional; it'd be actually better if we notice it when we
     // update the enum list (which shouldn't happen too often).
-    EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(6)),
+    EXPECT_THROW(MasterToken(MasterToken::ErrorCode(7)),
                  isc::InvalidParameter);
 
     // Check the coexistence of "from number" and "from error-code"
     // constructors won't cause confusion.
-    EXPECT_EQ(MasterLexer::Token::NUMBER,
-              MasterLexer::Token(static_cast<uint32_t>(
-                                     MasterLexer::Token::NOT_STARTED)).
+    EXPECT_EQ(MasterToken::NUMBER,
+              MasterToken(static_cast<uint32_t>(MasterToken::NOT_STARTED)).
               getType());
 }
 }
diff --git a/src/lib/dns/tests/master_lexer_unittest.cc b/src/lib/dns/tests/master_lexer_unittest.cc
index eca6a73..b2415da 100644
--- a/src/lib/dns/tests/master_lexer_unittest.cc
+++ b/src/lib/dns/tests/master_lexer_unittest.cc
@@ -141,19 +141,19 @@ TEST_F(MasterLexerTest, getNextToken) {
     lexer.pushSource(ss);
 
     // First, the newline should get out.
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Then the whitespace, if we specify the option.
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
     // The newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // The (quoted) string
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
 
     // And the end of line and file
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Test we correctly find end of file.
@@ -162,12 +162,12 @@ TEST_F(MasterLexerTest, eof) {
     lexer.pushSource(ss);
 
     // The first one is found to be EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
     // And it stays on EOF for any following attempts
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
     // And we can step back one token, but that is the EOF too.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened parentheses and no
@@ -177,12 +177,12 @@ TEST_F(MasterLexerTest, getUnbalancedParen) {
     lexer.pushSource(ss);
 
     // The string gets out first
-    EXPECT_EQ(MasterLexer::Token::STRING, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
     // Then an unbalanced parenthesis
-    EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN,
+    EXPECT_EQ(MasterToken::UNBALANCED_PAREN,
               lexer.getNextToken().getErrorCode());
     // And then EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened quoted string and no
@@ -192,10 +192,10 @@ TEST_F(MasterLexerTest, getUnbalancedString) {
     lexer.pushSource(ss);
 
     // Then an unbalanced qstring (reported as an unexpected end)
-    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END,
+    EXPECT_EQ(MasterToken::UNEXPECTED_END,
               lexer.getNextToken(MasterLexer::QSTRING).getErrorCode());
     // And then EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Test ungetting tokens works
@@ -204,28 +204,28 @@ TEST_F(MasterLexerTest, ungetToken) {
     lexer.pushSource(ss);
 
     // Try getting the newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Return it and get again
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Get the string and return it back
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
     lexer.ungetToken();
     // But if we change the options, it honors them
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::QSTRING |
                                  MasterLexer::INITIAL_WS).getType());
     // Get to the "more" string
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
-    EXPECT_EQ(MasterLexer::Token::STRING,
+    EXPECT_EQ(MasterToken::STRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
     // Return it back. It should get inside the parentheses.
     // Upon next attempt to get it again, the newline inside the parentheses
     // should be still ignored.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::STRING,
+    EXPECT_EQ(MasterToken::STRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
 }
 
@@ -235,16 +235,16 @@ TEST_F(MasterLexerTest, ungetRealOptions) {
     ss << "\n    \n";
     lexer.pushSource(ss);
     // Skip the first newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
 
     // If we call it the usual way, it skips up to the newline and returns
     // it
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
 
     // Now we return it. If we call it again, but with different options,
     // we get the initial whitespace.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
 }
 
@@ -253,7 +253,7 @@ TEST_F(MasterLexerTest, ungetTwice) {
     ss << "\n";
     lexer.pushSource(ss);
 
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Unget the token. It can be done once
     lexer.ungetToken();
     // But not twice
@@ -271,17 +271,164 @@ TEST_F(MasterLexerTest, ungetBeforeGet) {
 TEST_F(MasterLexerTest, ungetAfterSwitch) {
     ss << "\n\n";
     lexer.pushSource(ss);
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Switch the source
     std::stringstream ss2;
     ss2 << "\n\n";
     lexer.pushSource(ss2);
     EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
     // We can get from the new source
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // And when we drop the current source, we can't unget again
     lexer.popSource();
     EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
 }
 
+// Common checks for the case when getNextToken() should result in LexerError
+void
+lexerErrorCheck(MasterLexer& lexer, MasterToken::Type expect,
+                MasterToken::ErrorCode expected_error)
+{
+    bool thrown = false;
+    try {
+        lexer.getNextToken(expect);
+    } catch (const MasterLexer::LexerError& error) {
+        EXPECT_EQ(expected_error, error.token_.getErrorCode());
+        thrown = true;
+    }
+    EXPECT_TRUE(thrown);
+}
+
+// Common checks regarding expected/unexpected end-of-line
+//
+// The 'lexer' should be at a position before two consecutive '\n's.
+// The first one will be recognized, and the second one will be considered an
+// unexpected token.  Then this helper consumes the second '\n', so the caller
+// can continue the test after these '\n's.
+void
+eolCheck(MasterLexer& lexer, MasterToken::Type expect) {
+    // If EOL is found and eol_ok is true, we get it.
+    EXPECT_EQ(MasterToken::END_OF_LINE,
+              lexer.getNextToken(expect, true).getType());
+    // We'll see the second '\n'; by default it will fail.
+    EXPECT_THROW(lexer.getNextToken(expect), MasterLexer::LexerError);
+    // Same if eol_ok is explicitly set to false.  This also checks the
+    // offending '\n' was "ungotten".
+    EXPECT_THROW(lexer.getNextToken(expect, false), MasterLexer::LexerError);
+
+    // And also check the error token set in the exception object.
+    lexerErrorCheck(lexer, expect, MasterToken::UNEXPECTED_END);
+
+    // Then skip the 2nd '\n'
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
+}
+
+// Common checks regarding expected/unexpected end-of-file
+//
+// The 'lexer' should be at a position just before an end-of-file.
+void
+eofCheck(MasterLexer& lexer, MasterToken::Type expect) {
+    EXPECT_EQ(MasterToken::END_OF_FILE,
+              lexer.getNextToken(expect, true).getType());
+    EXPECT_THROW(lexer.getNextToken(expect), MasterLexer::LexerError);
+    EXPECT_THROW(lexer.getNextToken(expect, false), MasterLexer::LexerError);
+}
+
+TEST_F(MasterLexerTest, getNextTokenString) {
+    ss << "normal-string\n";
+    ss << "\n";
+    ss << "another-string";
+    lexer.pushSource(ss);
+
+    // Normal successful case: Expecting a string and get one.
+    EXPECT_EQ("normal-string",
+              lexer.getNextToken(MasterToken::STRING).getString());
+    eolCheck(lexer, MasterToken::STRING);
+
+    // Same set of tests but for end-of-file
+    EXPECT_EQ("another-string",
+              lexer.getNextToken(MasterToken::STRING, true).getString());
+    eofCheck(lexer, MasterToken::STRING);
+}
+
+TEST_F(MasterLexerTest, getNextTokenQString) {
+    ss << "\"quoted-string\"\n";
+    ss << "\n";
+    ss << "normal-string";
+    lexer.pushSource(ss);
+
+    // Expecting a quoted string and get one.
+    EXPECT_EQ("quoted-string",
+              lexer.getNextToken(MasterToken::QSTRING).getString());
+    eolCheck(lexer, MasterToken::QSTRING);
+
+    // Expecting a quoted string but see a normal string.  It's okay.
+    EXPECT_EQ("normal-string",
+              lexer.getNextToken(MasterToken::QSTRING).getString());
+    eofCheck(lexer, MasterToken::QSTRING);
+}
+
+TEST_F(MasterLexerTest, getNextTokenNumber) {
+    ss << "3600\n";
+    ss << "\n";
+    ss << "4294967296 ";        // =2^32, out of range
+    ss << "not-a-number ";
+    ss << "123abc "; // starting with digits, but resulting in a string
+    ss << "86400";
+    lexer.pushSource(ss);
+
+    // Expecting a number string and get one.
+    EXPECT_EQ(3600,
+              lexer.getNextToken(MasterToken::NUMBER).getNumber());
+    eolCheck(lexer, MasterToken::NUMBER);
+
+    // Expecting a number, but it's too big for uint32.
+    lexerErrorCheck(lexer, MasterToken::NUMBER,
+                    MasterToken::NUMBER_OUT_OF_RANGE);
+    // The token should have been "ungotten".  Re-read and skip it.
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
+
+    // Expecting a number, but see a string.
+    lexerErrorCheck(lexer, MasterToken::NUMBER, MasterToken::BAD_NUMBER);
+    // The unexpected string should have been "ungotten".  Re-read and skip it.
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
+
+    // Expecting a number, but see a string.
+    lexerErrorCheck(lexer, MasterToken::NUMBER, MasterToken::BAD_NUMBER);
+    // The unexpected string should have been "ungotten".  Re-read and skip it.
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
+
+    // Unless we specify NUMBER, decimal number string should be recognized
+    // as a string.
+    EXPECT_EQ("86400",
+              lexer.getNextToken(MasterToken::STRING).getString());
+    eofCheck(lexer, MasterToken::NUMBER);
+}
+
+TEST_F(MasterLexerTest, getNextTokenErrors) {
+    // Check miscellaneous error cases
+
+    ss << ") ";                 // unbalanced parenthesis
+    ss << "string-after-error ";
+    lexer.pushSource(ss);
+
+    // Only string/qstring/number can be "expected".
+    EXPECT_THROW(lexer.getNextToken(MasterToken::END_OF_LINE),
+                 isc::InvalidParameter);
+    EXPECT_THROW(lexer.getNextToken(MasterToken::END_OF_FILE),
+                 isc::InvalidParameter);
+    EXPECT_THROW(lexer.getNextToken(MasterToken::INITIAL_WS),
+                 isc::InvalidParameter);
+    EXPECT_THROW(lexer.getNextToken(MasterToken::ERROR),
+                 isc::InvalidParameter);
+
+    // If it encounters a syntax error, it results in LexerError exception.
+    lexerErrorCheck(lexer, MasterToken::STRING, MasterToken::UNBALANCED_PAREN);
+
+    // Unlike the NUMBER_OUT_OF_RANGE case, the error part has been skipped
+    // within getNextToken().  We should be able to get the next token.
+    EXPECT_EQ("string-after-error",
+              lexer.getNextToken(MasterToken::STRING).getString());
+}
+
 }



More information about the bind10-changes mailing list