source: trunk/yao/share/antlr-2.7.7/lib/cpp/antlr/TokenBuffer.hpp @ 1

Last change on this file since 1 was 1, checked in by lnalod, 15 years ago

Initial import of YAO sources

File size: 2.8 KB
Line 
1#ifndef INC_TokenBuffer_hpp__
2#define INC_TokenBuffer_hpp__
3
4/* ANTLR Translator Generator
5 * Project led by Terence Parr at http://www.jGuru.com
6 * Software rights: http://www.antlr.org/license.html
7 *
8 * $Id: //depot/code/org.antlr/release/antlr-2.7.7/lib/cpp/antlr/TokenBuffer.hpp#2 $
9 */
10
11#include <antlr/config.hpp>
12#include <antlr/TokenStream.hpp>
13#include <antlr/CircularQueue.hpp>
14
15#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
16namespace antlr {
17#endif
18
19/**A Stream of Token objects fed to the parser from a TokenStream that can
20 * be rewound via mark()/rewind() methods.
21 * <p>
22 * A dynamic array is used to buffer up all the input tokens.  Normally,
23 * "k" tokens are stored in the buffer.  More tokens may be stored during
24 * guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
25 * Consumption of tokens is deferred.  In other words, reading the next
26 * token is not done by conume(), but deferred until needed by LA or LT.
27 * <p>
28 *
29 * @todo: see if we can integrate this one with InputBuffer into one template
30 * or so.
31 *
32 * @see antlr.Token
33 * @see antlr.TokenStream
34 * @see antlr.TokenQueue
35 */
36class ANTLR_API TokenBuffer {
37public:
38        /** Create a token buffer */
39        TokenBuffer(TokenStream& input_);
40        virtual ~TokenBuffer();
41
42        /// Reset the input buffer to empty state
43        inline void reset( void )
44        {
45                nMarkers = 0;
46                markerOffset = 0;
47                numToConsume = 0;
48                queue.clear();
49        }
50
51        /** Get a lookahead token value */
52        int LA( unsigned int i );
53
54        /** Get a lookahead token */
55        RefToken LT( unsigned int i );
56
57        /** Return an integer marker that can be used to rewind the buffer to
58         * its current state.
59         */
60        unsigned int mark();
61
62        /**Rewind the token buffer to a marker.
63         * @param mark Marker returned previously from mark()
64         */
65        void rewind(unsigned int mark);
66
67        /** Mark another token for deferred consumption */
68        inline void consume()
69        {
70                numToConsume++;
71        }
72
73        /// Return the number of entries in the TokenBuffer
74        virtual unsigned int entries() const;
75
76private:
77        /** Ensure that the token buffer is sufficiently full */
78        void fill(unsigned int amount);
79        /** Sync up deferred consumption */
80        void syncConsume();
81
82protected:
83        /// Token source
84        TokenStream& input;
85
86        /// Number of active markers
87        unsigned int nMarkers;
88
89        /// Additional offset used when markers are active
90        unsigned int markerOffset;
91
92        /// Number of calls to consume() since last LA() or LT() call
93        unsigned int numToConsume;
94
95        /// Circular queue with Tokens
96        CircularQueue<RefToken> queue;
97
98private:
99        TokenBuffer(const TokenBuffer& other);
100        const TokenBuffer& operator=(const TokenBuffer& other);
101};
102
103/** Sync up deferred consumption */
104inline void TokenBuffer::syncConsume()
105{
106        if (numToConsume > 0)
107        {
108                if (nMarkers > 0)
109                        markerOffset += numToConsume;
110                else
111                        queue.removeItems( numToConsume );
112
113                numToConsume = 0;
114        }
115}
116
117#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
118}
119#endif
120
121#endif //INC_TokenBuffer_hpp__
Note: See TracBrowser for help on using the repository browser.