add antlr source code and ReadMe

[toc] / antlr4-cpp-runtime-4.9.2-source / runtime / src / Token.h

diff --git a/antlr4-cpp-runtime-4.9.2-source/runtime/src/Token.h b/antlr4-cpp-runtime-4.9.2-source/runtime/src/Token.h
new file mode 100644 (file)
index 0000000..1878b28
--- /dev/null
+++ b/antlr4-cpp-runtime-4.9.2-source/runtime/src/Token.h
@@ -0,0 +1,124 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "IntStream.h"
+
+namespace antlr4 {
+
+  /// A token has properties: text, type, line, character position in the line
+  /// (so we can ignore tabs), token channel, index, and source from which
+  /// we obtained this token.
+  class ANTLR4CPP_PUBLIC Token {
+  public:
+#if __cplusplus >= 201703L
+    static constexpr size_t INVALID_TYPE = 0;
+#else
+    enum : size_t {
+      INVALID_TYPE = 0,
+    };
+#endif
+
+    /// During lookahead operations, this "token" signifies we hit rule end ATN state
+    /// and did not follow it despite needing to.
+#if __cplusplus >= 201703L
+    static constexpr size_t EPSILON = std::numeric_limits<size_t>::max() - 1;
+    static constexpr size_t MIN_USER_TOKEN_TYPE = 1;
+    static constexpr size_t EOF = IntStream::EOF;
+#else
+    enum : size_t {
+      EPSILON = static_cast<size_t>(-2), // std::numeric_limits<size_t>::max() - 1; doesn't work in VS 2013
+      MIN_USER_TOKEN_TYPE = 1,
+      EOF = IntStream::EOF,
+    };
+#endif
+
+    virtual ~Token();
+
+    /// All tokens go to the parser (unless skip() is called in that rule)
+    /// on a particular "channel".  The parser tunes to a particular channel
+    /// so that whitespace etc... can go to the parser on a "hidden" channel.
+#if __cplusplus >= 201703L
+    static constexpr size_t DEFAULT_CHANNEL = 0;
+#else
+    enum : size_t {
+      DEFAULT_CHANNEL = 0,
+    };
+#endif
+
+    /// Anything on different channel than DEFAULT_CHANNEL is not parsed
+    /// by parser.
+#if __cplusplus >= 201703L
+    static constexpr size_t HIDDEN_CHANNEL = 1;
+#else
+    enum : size_t {
+      HIDDEN_CHANNEL = 1,
+    };
+#endif
+
+    /**
+     * This is the minimum constant value which can be assigned to a
+     * user-defined token channel.
+     *
+     * <p>
+     * The non-negative numbers less than {@link #MIN_USER_CHANNEL_VALUE} are
+     * assigned to the predefined channels {@link #DEFAULT_CHANNEL} and
+     * {@link #HIDDEN_CHANNEL}.</p>
+     *
+     * @see Token#getChannel()
+     */
+#if __cplusplus >= 201703L
+    static constexpr size_t MIN_USER_CHANNEL_VALUE = 2;
+#else
+    enum : size_t {
+      MIN_USER_CHANNEL_VALUE = 2,
+    };
+#endif
+
+    /// Get the text of the token.
+    virtual std::string getText() const = 0;
+
+    /// Get the token type of the token
+    virtual size_t getType() const = 0;
+
+    /// The line number on which the 1st character of this token was matched,  line=1..n
+    virtual size_t getLine() const = 0;
+
+    /// The index of the first character of this token relative to the
+    /// beginning of the line at which it occurs, 0..n-1
+    virtual size_t getCharPositionInLine() const = 0;
+
+    /// Return the channel this token. Each token can arrive at the parser
+    /// on a different channel, but the parser only "tunes" to a single channel.
+    /// The parser ignores everything not on DEFAULT_CHANNEL.
+    virtual size_t getChannel() const = 0;
+
+    /// An index from 0..n-1 of the token object in the input stream.
+    /// This must be valid in order to print token streams and
+    /// use TokenRewriteStream.
+    ///
+    /// Return INVALID_INDEX to indicate that this token was conjured up since
+    /// it doesn't have a valid index.
+    virtual size_t getTokenIndex() const = 0;
+
+    /// The starting character index of the token
+    /// This method is optional; return INVALID_INDEX if not implemented.
+    virtual size_t getStartIndex() const = 0;
+
+    /// The last character index of the token.
+    /// This method is optional; return INVALID_INDEX if not implemented.
+    virtual size_t getStopIndex() const = 0;
+
+    /// Gets the <seealso cref="TokenSource"/> which created this token.
+    virtual TokenSource *getTokenSource() const = 0;
+
+    /// Gets the <seealso cref="CharStream"/> from which this token was derived.
+    virtual CharStream *getInputStream() const = 0;
+
+    virtual std::string toString() const = 0;
+  };
+
+} // namespace antlr4