Context Navigation

← Previous Revision
Latest Revision
Next Revision →
Blame
Revision Log

antlr.py @ 1

Last change on this file since 1 was 1, checked in by lnalod, 15 years ago
Initial import of YAO sources
File size: 80.7 KB

Line
1	## This file is part of PyANTLR. See LICENSE.txt for license
2	## details..........Copyright (C) Wolfgang Haefelinger, 2004.
3
4	## get sys module
5	import sys
6
7	version = sys.version.split()[0]
8	if version < '2.2.1':
9	False = 0
10	if version < '2.3':
11	True = not False
12
13	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
14	### global symbols ###
15	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
16
17	### ANTLR Standard Tokens
18	SKIP = -1
19	INVALID_TYPE = 0
20	EOF_TYPE = 1
21	EOF = 1
22	NULL_TREE_LOOKAHEAD = 3
23	MIN_USER_TYPE = 4
24
25	### ANTLR's EOF Symbol
26	EOF_CHAR = ''
27
28	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
29	### general functions ###
30	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
31
32	## Version should be automatically derived from configure.in. For now,
33	## we need to bump it ourselfs. Don't remove the <version> tags.
34	## <version>
35	def version():
36	r = {
37	'major' : '2',
38	'minor' : '7',
39	'micro' : '5',
40	'patch' : '' ,
41	'version': '2.7.5'
42	}
43	return r
44	## </version>
45
46	def error(fmt,*args):
47	if fmt:
48	print "error: ", fmt % tuple(args)
49
50	def ifelse(cond,_then,_else):
51	if cond :
52	r = _then
53	else:
54	r = _else
55	return r
56
57	def is_string_type(x):
58	return (isinstance(x,str) or isinstance(x,unicode))
59
60	def assert_string_type(x):
61	assert is_string_type(x)
62	pass
63
64	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
65	### ANTLR Exceptions ###
66	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
67
68	class ANTLRException(Exception):
69
70	def __init__(self, *args):
71	Exception.__init__(self, *args)
72
73
74	class RecognitionException(ANTLRException):
75
76	def __init__(self, *args):
77	ANTLRException.__init__(self, *args)
78	self.fileName = None
79	self.line = -1
80	self.column = -1
81	if len(args) >= 2:
82	self.fileName = args[1]
83	if len(args) >= 3:
84	self.line = args[2]
85	if len(args) >= 4:
86	self.column = args[3]
87
88	def __str__(self):
89	buf = ['']
90	if self.fileName:
91	buf.append(self.fileName + ":")
92	if self.line != -1:
93	if not self.fileName:
94	buf.append("line ")
95	buf.append(str(self.line))
96	if self.column != -1:
97	buf.append(":" + str(self.column))
98	buf.append(":")
99	buf.append(" ")
100	return str('').join(buf)
101
102	__repr__ = __str__
103
104
105	class NoViableAltException(RecognitionException):
106
107	def __init__(self, *args):
108	RecognitionException.__init__(self, *args)
109	self.token = None
110	self.node = None
111	if isinstance(args[0],AST):
112	self.node = args[0]
113	elif isinstance(args[0],Token):
114	self.token = args[0]
115	else:
116	raise TypeError("NoViableAltException requires Token or AST argument")
117
118	def __str__(self):
119	if self.token:
120	line = self.token.getLine()
121	col = self.token.getColumn()
122	text = self.token.getText()
123	return "unexpected symbol at line %s (column %s): \"%s\"" % (line,col,text)
124	if self.node == ASTNULL:
125	return "unexpected end of subtree"
126	assert self.node
127	### hackish, we assume that an AST contains method getText
128	return "unexpected node: %s" % (self.node.getText())
129
130	__repr__ = __str__
131
132
133	class NoViableAltForCharException(RecognitionException):
134
135	def __init__(self, *args):
136	self.foundChar = None
137	if len(args) == 2:
138	self.foundChar = args[0]
139	scanner = args[1]
140	RecognitionException.__init__(self, "NoViableAlt",
141	scanner.getFilename(),
142	scanner.getLine(),
143	scanner.getColumn())
144	elif len(args) == 4:
145	self.foundChar = args[0]
146	fileName = args[1]
147	line = args[2]
148	column = args[3]
149	RecognitionException.__init__(self, "NoViableAlt",
150	fileName, line, column)
151	else:
152	RecognitionException.__init__(self, "NoViableAlt",
153	'', -1, -1)
154
155	def __str__(self):
156	mesg = "unexpected char: "
157	if self.foundChar >= ' ' and self.foundChar <= '~':
158	mesg += "'" + self.foundChar + "'"
159	elif self.foundChar:
160	mesg += "0x" + hex(ord(self.foundChar)).upper()[2:]
161	else:
162	mesg += "<None>"
163	return mesg
164
165	__repr__ = __str__
166
167
168	class SemanticException(RecognitionException):
169
170	def __init__(self, *args):
171	RecognitionException.__init__(self, *args)
172
173
174	class MismatchedCharException(RecognitionException):
175
176	NONE = 0
177	CHAR = 1
178	NOT_CHAR = 2
179	RANGE = 3
180	NOT_RANGE = 4
181	SET = 5
182	NOT_SET = 6
183
184	def __init__(self, *args):
185	self.args = args
186	if len(args) == 5:
187	# Expected range / not range
188	if args[3]:
189	self.mismatchType = MismatchedCharException.NOT_RANGE
190	else:
191	self.mismatchType = MismatchedCharException.RANGE
192	self.foundChar = args[0]
193	self.expecting = args[1]
194	self.upper = args[2]
195	self.scanner = args[4]
196	RecognitionException.__init__(self, "Mismatched char range",
197	self.scanner.getFilename(),
198	self.scanner.getLine(),
199	self.scanner.getColumn())
200	elif len(args) == 4 and is_string_type(args[1]):
201	# Expected char / not char
202	if args[2]:
203	self.mismatchType = MismatchedCharException.NOT_CHAR
204	else:
205	self.mismatchType = MismatchedCharException.CHAR
206	self.foundChar = args[0]
207	self.expecting = args[1]
208	self.scanner = args[3]
209	RecognitionException.__init__(self, "Mismatched char",
210	self.scanner.getFilename(),
211	self.scanner.getLine(),
212	self.scanner.getColumn())
213	elif len(args) == 4 and isinstance(args[1], BitSet):
214	# Expected BitSet / not BitSet
215	if args[2]:
216	self.mismatchType = MismatchedCharException.NOT_SET
217	else:
218	self.mismatchType = MismatchedCharException.SET
219	self.foundChar = args[0]
220	self.set = args[1]
221	self.scanner = args[3]
222	RecognitionException.__init__(self, "Mismatched char set",
223	self.scanner.getFilename(),
224	self.scanner.getLine(),
225	self.scanner.getColumn())
226	else:
227	self.mismatchType = MismatchedCharException.NONE
228	RecognitionException.__init__(self, "Mismatched char")
229
230	## Append a char to the msg buffer. If special,
231	# then show escaped version
232	#
233	def appendCharName(self, sb, c):
234	if not c or c == 65535:
235	# 65535 = (char) -1 = EOF
236	sb.append("'<EOF>'")
237	elif c == '\n':
238	sb.append("'\\n'")
239	elif c == '\r':
240	sb.append("'\\r'");
241	elif c == '\t':
242	sb.append("'\\t'")
243	else:
244	sb.append('\'' + c + '\'')
245
246	##
247	# Returns an error message with line number/column information
248	#
249	def __str__(self):
250	sb = ['']
251	sb.append(RecognitionException.__str__(self))
252
253	if self.mismatchType == MismatchedCharException.CHAR:
254	sb.append("expecting ")
255	self.appendCharName(sb, self.expecting)
256	sb.append(", found ")
257	self.appendCharName(sb, self.foundChar)
258	elif self.mismatchType == MismatchedCharException.NOT_CHAR:
259	sb.append("expecting anything but '")
260	self.appendCharName(sb, self.expecting)
261	sb.append("'; got it anyway")
262	elif self.mismatchType in [MismatchedCharException.RANGE, MismatchedCharException.NOT_RANGE]:
263	sb.append("expecting char ")
264	if self.mismatchType == MismatchedCharException.NOT_RANGE:
265	sb.append("NOT ")
266	sb.append("in range: ")
267	appendCharName(sb, self.expecting)
268	sb.append("..")
269	appendCharName(sb, self.upper)
270	sb.append(", found ")
271	appendCharName(sb, self.foundChar)
272	elif self.mismatchType in [MismatchedCharException.SET, MismatchedCharException.NOT_SET]:
273	sb.append("expecting ")
274	if self.mismatchType == MismatchedCharException.NOT_SET:
275	sb.append("NOT ")
276	sb.append("one of (")
277	for i in range(len(self.set)):
278	self.appendCharName(sb, self.set[i])
279	sb.append("), found ")
280	self.appendCharName(sb, self.foundChar)
281
282	return str().join(sb).strip()
283
284	__repr__ = __str__
285
286
287	class MismatchedTokenException(RecognitionException):
288
289	NONE = 0
290	TOKEN = 1
291	NOT_TOKEN = 2
292	RANGE = 3
293	NOT_RANGE = 4
294	SET = 5
295	NOT_SET = 6
296
297	def __init__(self, *args):
298	self.args = args
299	self.tokenNames = []
300	self.token = None
301	self.tokenText = ''
302	self.node = None
303	if len(args) == 6:
304	# Expected range / not range
305	if args[3]:
306	self.mismatchType = MismatchedTokenException.NOT_RANGE
307	else:
308	self.mismatchType = MismatchedTokenException.RANGE
309	self.tokenNames = args[0]
310	self.expecting = args[2]
311	self.upper = args[3]
312	self.fileName = args[5]
313
314	elif len(args) == 4 and isinstance(args[2], int):
315	# Expected token / not token
316	if args[3]:
317	self.mismatchType = MismatchedTokenException.NOT_TOKEN
318	else:
319	self.mismatchType = MismatchedTokenException.TOKEN
320	self.tokenNames = args[0]
321	self.expecting = args[2]
322
323	elif len(args) == 4 and isinstance(args[2], BitSet):
324	# Expected BitSet / not BitSet
325	if args[3]:
326	self.mismatchType = MismatchedTokenException.NOT_SET
327	else:
328	self.mismatchType = MismatchedTokenException.SET
329	self.tokenNames = args[0]
330	self.set = args[2]
331
332	else:
333	self.mismatchType = MismatchedTokenException.NONE
334	RecognitionException.__init__(self, "Mismatched Token: expecting any AST node", "<AST>", -1, -1)
335
336	if len(args) >= 2:
337	if isinstance(args[1],Token):
338	self.token = args[1]
339	self.tokenText = self.token.getText()
340	RecognitionException.__init__(self, "Mismatched Token",
341	self.fileName,
342	self.token.getLine(),
343	self.token.getColumn())
344	elif isinstance(args[1],AST):
345	self.node = args[1]
346	self.tokenText = str(self.node)
347	RecognitionException.__init__(self, "Mismatched Token",
348	"<AST>",
349	self.node.getLine(),
350	self.node.getColumn())
351	else:
352	self.tokenText = "<empty tree>"
353	RecognitionException.__init__(self, "Mismatched Token",
354	"<AST>", -1, -1)
355
356	def appendTokenName(self, sb, tokenType):
357	if tokenType == INVALID_TYPE:
358	sb.append("<Set of tokens>")
359	elif tokenType < 0 or tokenType >= len(self.tokenNames):
360	sb.append("<" + str(tokenType) + ">")
361	else:
362	sb.append(self.tokenNames[tokenType])
363
364	##
365	# Returns an error message with line number/column information
366	#
367	def __str__(self):
368	sb = ['']
369	sb.append(RecognitionException.__str__(self))
370
371	if self.mismatchType == MismatchedTokenException.TOKEN:
372	sb.append("expecting ")
373	self.appendTokenName(sb, self.expecting)
374	sb.append(", found " + self.tokenText)
375	elif self.mismatchType == MismatchedTokenException.NOT_TOKEN:
376	sb.append("expecting anything but '")
377	self.appendTokenName(sb, self.expecting)
378	sb.append("'; got it anyway")
379	elif self.mismatchType in [MismatchedTokenException.RANGE, MismatchedTokenException.NOT_RANGE]:
380	sb.append("expecting token ")
381	if self.mismatchType == MismatchedTokenException.NOT_RANGE:
382	sb.append("NOT ")
383	sb.append("in range: ")
384	appendTokenName(sb, self.expecting)
385	sb.append("..")
386	appendTokenName(sb, self.upper)
387	sb.append(", found " + self.tokenText)
388	elif self.mismatchType in [MismatchedTokenException.SET, MismatchedTokenException.NOT_SET]:
389	sb.append("expecting ")
390	if self.mismatchType == MismatchedTokenException.NOT_SET:
391	sb.append("NOT ")
392	sb.append("one of (")
393	for i in range(len(self.set)):
394	self.appendTokenName(sb, self.set[i])
395	sb.append("), found " + self.tokenText)
396
397	return str().join(sb).strip()
398
399	__repr__ = __str__
400
401
402	class TokenStreamException(ANTLRException):
403
404	def __init__(self, *args):
405	ANTLRException.__init__(self, *args)
406
407
408	# Wraps an Exception in a TokenStreamException
409	class TokenStreamIOException(TokenStreamException):
410
411	def __init__(self, *args):
412	if args and isinstance(args[0], Exception):
413	io = args[0]
414	TokenStreamException.__init__(self, str(io))
415	self.io = io
416	else:
417	TokenStreamException.__init__(self, *args)
418	self.io = self
419
420
421	# Wraps a RecognitionException in a TokenStreamException
422	class TokenStreamRecognitionException(TokenStreamException):
423
424	def __init__(self, *args):
425	if args and isinstance(args[0], RecognitionException):
426	recog = args[0]
427	TokenStreamException.__init__(self, str(recog))
428	self.recog = recog
429	else:
430	raise TypeError("TokenStreamRecognitionException requires RecognitionException argument")
431
432	def __str__(self):
433	return str(self.recog)
434
435	__repr__ = __str__
436
437
438	class TokenStreamRetryException(TokenStreamException):
439
440	def __init__(self, *args):
441	TokenStreamException.__init__(self, *args)
442
443
444	class CharStreamException(ANTLRException):
445
446	def __init__(self, *args):
447	ANTLRException.__init__(self, *args)
448
449
450	# Wraps an Exception in a CharStreamException
451	class CharStreamIOException(CharStreamException):
452
453	def __init__(self, *args):
454	if args and isinstance(args[0], Exception):
455	io = args[0]
456	CharStreamException.__init__(self, str(io))
457	self.io = io
458	else:
459	CharStreamException.__init__(self, *args)
460	self.io = self
461
462
463	class TryAgain(Exception):
464	pass
465
466
467	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
468	### Token ###
469	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
470
471	class Token(object):
472	SKIP = -1
473	INVALID_TYPE = 0
474	EOF_TYPE = 1
475	EOF = 1
476	NULL_TREE_LOOKAHEAD = 3
477	MIN_USER_TYPE = 4
478
479	def __init__(self,**argv):
480	try:
481	self.type = argv['type']
482	except:
483	self.type = INVALID_TYPE
484	try:
485	self.text = argv['text']
486	except:
487	self.text = "<no text>"
488
489	def isEOF(self):
490	return (self.type == EOF_TYPE)
491
492	def getColumn(self):
493	return 0
494
495	def getLine(self):
496	return 0
497
498	def getFilename(self):
499	return None
500
501	def setFilename(self,name):
502	return self
503
504	def getText(self):
505	return "<no text>"
506
507	def setText(self,text):
508	if is_string_type(text):
509	pass
510	else:
511	raise TypeError("Token.setText requires string argument")
512	return self
513
514	def setColumn(self,column):
515	return self
516
517	def setLine(self,line):
518	return self
519
520	def getType(self):
521	return self.type
522
523	def setType(self,type):
524	if isinstance(type,int):
525	self.type = type
526	else:
527	raise TypeError("Token.setType requires integer argument")
528	return self
529
530	def toString(self):
531	## not optimal
532	type_ = self.type
533	if type_ == 3:
534	tval = 'NULL_TREE_LOOKAHEAD'
535	elif type_ == 1:
536	tval = 'EOF_TYPE'
537	elif type_ == 0:
538	tval = 'INVALID_TYPE'
539	elif type_ == -1:
540	tval = 'SKIP'
541	else:
542	tval = type_
543	return '["%s",<%s>]' % (self.getText(),tval)
544
545	__str__ = toString
546	__repr__ = toString
547
548	### static attribute ..
549	Token.badToken = Token( type=INVALID_TYPE, text="<no text>")
550
551	if __name__ == "__main__":
552	print "testing .."
553	T = Token.badToken
554	print T
555
556	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
557	### CommonToken ###
558	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
559
560	class CommonToken(Token):
561
562	def __init__(self,**argv):
563	Token.__init__(self,**argv)
564	self.line = 0
565	self.col = 0
566	try:
567	self.line = argv['line']
568	except:
569	pass
570	try:
571	self.col = argv['col']
572	except:
573	pass
574
575	def getLine(self):
576	return self.line
577
578	def getText(self):
579	return self.text
580
581	def getColumn(self):
582	return self.col
583
584	def setLine(self,line):
585	self.line = line
586	return self
587
588	def setText(self,text):
589	self.text = text
590	return self
591
592	def setColumn(self,col):
593	self.col = col
594	return self
595
596	def toString(self):
597	## not optimal
598	type_ = self.type
599	if type_ == 3:
600	tval = 'NULL_TREE_LOOKAHEAD'
601	elif type_ == 1:
602	tval = 'EOF_TYPE'
603	elif type_ == 0:
604	tval = 'INVALID_TYPE'
605	elif type_ == -1:
606	tval = 'SKIP'
607	else:
608	tval = type_
609	d = {
610	'text' : self.text,
611	'type' : tval,
612	'line' : self.line,
613	'colm' : self.col
614	}
615
616	fmt = '["%(text)s",<%(type)s>,line=%(line)s,col=%(colm)s]'
617	return fmt % d
618
619	__str__ = toString
620	__repr__ = toString
621
622
623	if __name__ == '__main__' :
624	T = CommonToken()
625	print T
626	T = CommonToken(col=15,line=1,text="some text", type=5)
627	print T
628	T = CommonToken()
629	T.setLine(1).setColumn(15).setText("some text").setType(5)
630	print T
631	print T.getLine()
632	print T.getColumn()
633	print T.getText()
634	print T.getType()
635
636	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
637	### CommonHiddenStreamToken ###
638	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
639
640	class CommonHiddenStreamToken(CommonToken):
641	def __init__(self,*args):
642	CommonToken.__init__(self,*args)
643	self.hiddenBefore = None
644	self.hiddenAfter = None
645
646	def getHiddenAfter(self):
647	return self.hiddenAfter
648
649	def getHiddenBefore(self):
650	return self.hiddenBefore
651
652	def setHiddenAfter(self,t):
653	self.hiddenAfter = t
654
655	def setHiddenBefore(self, t):
656	self.hiddenBefore = t
657
658	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
659	### Queue ###
660	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
661
662	## Shall be a circular buffer on tokens ..
663	class Queue(object):
664
665	def __init__(self):
666	self.buffer = [] # empty list
667
668	def append(self,item):
669	self.buffer.append(item)
670
671	def elementAt(self,index):
672	return self.buffer[index]
673
674	def reset(self):
675	self.buffer = []
676
677	def removeFirst(self):
678	self.buffer.pop(0)
679
680	def length(self):
681	return len(self.buffer)
682
683	def __str__(self):
684	return str(self.buffer)
685
686	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
687	### InputBuffer ###
688	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
689
690	class InputBuffer(object):
691	def __init__(self):
692	self.nMarkers = 0
693	self.markerOffset = 0
694	self.numToConsume = 0
695	self.queue = Queue()
696
697	def __str__(self):
698	return "(%s,%s,%s,%s)" % (
699	self.nMarkers,
700	self.markerOffset,
701	self.numToConsume,
702	self.queue)
703
704	def __repr__(self):
705	return str(self)
706
707	def commit(self):
708	self.nMarkers -= 1
709
710	def consume(self) :
711	self.numToConsume += 1
712
713	## probably better to return a list of items
714	## because of unicode. Or return a unicode
715	## string ..
716	def getLAChars(self) :
717	i = self.markerOffset
718	n = self.queue.length()
719	s = ''
720	while i<n:
721	s += self.queue.elementAt(i)
722	return s
723
724	## probably better to return a list of items
725	## because of unicode chars
726	def getMarkedChars(self) :
727	s = ''
728	i = 0
729	n = self.markerOffset
730	while i<n:
731	s += self.queue.elementAt(i)
732	return s
733
734	def isMarked(self) :
735	return self.nMarkers != 0
736
737	def fill(self,k):
738	### abstract method
739	raise NotImplementedError()
740
741	def LA(self,k) :
742	self.fill(k)
743	return self.queue.elementAt(self.markerOffset + k - 1)
744
745	def mark(self) :
746	self.syncConsume()
747	self.nMarkers += 1
748	return self.markerOffset
749
750	def rewind(self,mark) :
751	self.syncConsume()
752	self.markerOffset = mark
753	self.nMarkers -= 1
754
755	def reset(self) :
756	self.nMarkers = 0
757	self.markerOffset = 0
758	self.numToConsume = 0
759	self.queue.reset()
760
761	def syncConsume(self) :
762	while self.numToConsume > 0:
763	if self.nMarkers > 0:
764	# guess mode -- leave leading characters and bump offset.
765	self.markerOffset += 1
766	else:
767	# normal mode -- remove first character
768	self.queue.removeFirst()
769	self.numToConsume -= 1
770
771	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
772	### CharBuffer ###
773	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
774
775	class CharBuffer(InputBuffer):
776	def __init__(self,reader):
777	##assert isinstance(reader,file)
778	super(CharBuffer,self).__init__()
779	## a reader is supposed to be anything that has
780	## a method 'read(int)'.
781	self.input = reader
782
783	def __str__(self):
784	base = super(CharBuffer,self).__str__()
785	return "CharBuffer{%s,%s" % (base,str(input))
786
787	def fill(self,amount):
788	try:
789	self.syncConsume()
790	while self.queue.length() < (amount + self.markerOffset) :
791	## retrieve just one char - what happend at end
792	## of input?
793	c = self.input.read(1)
794	### python's behaviour is to return the empty string on
795	### EOF, ie. no exception whatsoever is thrown. An empty
796	### python string has the nice feature that it is of
797	### type 'str' and "not ''" would return true. Contrary,
798	### one can't do this: '' in 'abc'. This should return
799	### false, but all we get is then a TypeError as an
800	### empty string is not a character.
801
802	### Let's assure then that we have either seen a
803	### character or an empty string (EOF).
804	assert len(c) == 0 or len(c) == 1
805
806	### And it shall be of type string (ASCII or UNICODE).
807	assert is_string_type(c)
808
809	### Just append EOF char to buffer. Note that buffer may
810	### contain then just more than one EOF char ..
811
812	### use unicode chars instead of ASCII ..
813	self.queue.append(c)
814	except Exception,e:
815	raise CharStreamIOException(e)
816	##except: # (mk) Cannot happen ...
817	##error ("unexpected exception caught ..")
818	##assert 0
819
820	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
821	### LexerSharedInputState ###
822	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
823
824	class LexerSharedInputState(object):
825	def __init__(self,ibuf):
826	assert isinstance(ibuf,InputBuffer)
827	self.input = ibuf
828	self.column = 1
829	self.line = 1
830	self.tokenStartColumn = 1
831	self.tokenStartLine = 1
832	self.guessing = 0
833	self.filename = None
834
835	def reset(self):
836	self.column = 1
837	self.line = 1
838	self.tokenStartColumn = 1
839	self.tokenStartLine = 1
840	self.guessing = 0
841	self.filename = None
842	self.input.reset()
843
844	def LA(self,k):
845	return self.input.LA(k)
846
847	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
848	### TokenStream ###
849	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
850
851	class TokenStream(object):
852	def nextToken(self):
853	pass
854
855	def __iter__(self):
856	return TokenStreamIterator(self)
857
858	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
859	### TokenStreamIterator ###
860	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
861
862	class TokenStreamIterator(object):
863	def __init__(self,inst):
864	if isinstance(inst,TokenStream):
865	self.inst = inst
866	return
867	raise TypeError("TokenStreamIterator requires TokenStream object")
868
869	def next(self):
870	assert self.inst
871	item = self.inst.nextToken()
872	if not item or item.isEOF():
873	raise StopIteration()
874	return item
875
876	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
877	### TokenStreamSelector ###
878	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
879
880	class TokenStreamSelector(TokenStream):
881
882	def __init__(self):
883	self._input = None
884	self._stmap = {}
885	self._stack = []
886
887	def addInputStream(self,stream,key):
888	self._stmap[key] = stream
889
890	def getCurrentStream(self):
891	return self._input
892
893	def getStream(self,sname):
894	try:
895	stream = self._stmap[sname]
896	except:
897	raise ValueError("TokenStream " + sname + " not found");
898	return stream;
899
900	def nextToken(self):
901	while 1:
902	try:
903	return self._input.nextToken()
904	except TokenStreamRetryException,r:
905	### just retry "forever"
906	pass
907
908	def pop(self):
909	stream = self._stack.pop();
910	self.select(stream);
911	return stream;
912
913	def push(self,arg):
914	self._stack.append(self._input);
915	self.select(arg)
916
917	def retry(self):
918	raise TokenStreamRetryException()
919
920	def select(self,arg):
921	if isinstance(arg,TokenStream):
922	self._input = arg
923	return
924	if is_string_type(arg):
925	self._input = self.getStream(arg)
926	return
927	raise TypeError("TokenStreamSelector.select requires " +
928	"TokenStream or string argument")
929
930	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
931	### TokenStreamBasicFilter ###
932	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
933
934	class TokenStreamBasicFilter(TokenStream):
935
936	def __init__(self,input):
937
938	self.input = input;
939	self.discardMask = BitSet()
940
941	def discard(self,arg):
942	if isinstance(arg,int):
943	self.discardMask.add(arg)
944	return
945	if isinstance(arg,BitSet):
946	self.discardMark = arg
947	return
948	raise TypeError("TokenStreamBasicFilter.discard requires" +
949	"integer or BitSet argument")
950
951	def nextToken(self):
952	tok = self.input.nextToken()
953	while tok and self.discardMask.member(tok.getType()):
954	tok = self.input.nextToken()
955	return tok
956
957	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
958	### TokenStreamHiddenTokenFilter ###
959	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
960
961	class TokenStreamHiddenTokenFilter(TokenStreamBasicFilter):
962
963	def __init__(self,input):
964	TokenStreamBasicFilter.__init__(self,input)
965	self.hideMask = BitSet()
966	self.nextMonitoredToken = None
967	self.lastHiddenToken = None
968	self.firstHidden = None
969
970	def consume(self):
971	self.nextMonitoredToken = self.input.nextToken()
972
973	def consumeFirst(self):
974	self.consume()
975
976	p = None;
977	while self.hideMask.member(self.LA(1).getType()) or \
978	self.discardMask.member(self.LA(1).getType()):
979	if self.hideMask.member(self.LA(1).getType()):
980	if not p:
981	p = self.LA(1)
982	else:
983	p.setHiddenAfter(self.LA(1))
984	self.LA(1).setHiddenBefore(p)
985	p = self.LA(1)
986	self.lastHiddenToken = p
987	if not self.firstHidden:
988	self.firstHidden = p
989	self.consume()
990
991	def getDiscardMask(self):
992	return self.discardMask
993
994	def getHiddenAfter(self,t):
995	return t.getHiddenAfter()
996
997	def getHiddenBefore(self,t):
998	return t.getHiddenBefore()
999
1000	def getHideMask(self):
1001	return self.hideMask
1002
1003	def getInitialHiddenToken(self):
1004	return self.firstHidden
1005
1006	def hide(self,m):
1007	if isinstance(m,int):
1008	self.hideMask.add(m)
1009	return
1010	if isinstance(m.BitMask):
1011	self.hideMask = m
1012	return
1013
1014	def LA(self,i):
1015	return self.nextMonitoredToken
1016
1017	def nextToken(self):
1018	if not self.LA(1):
1019	self.consumeFirst()
1020
1021	monitored = self.LA(1)
1022
1023	monitored.setHiddenBefore(self.lastHiddenToken)
1024	self.lastHiddenToken = None
1025
1026	self.consume()
1027	p = monitored
1028
1029	while self.hideMask.member(self.LA(1).getType()) or \
1030	self.discardMask.member(self.LA(1).getType()):
1031	if self.hideMask.member(self.LA(1).getType()):
1032	p.setHiddenAfter(self.LA(1))
1033	if p != monitored:
1034	self.LA(1).setHiddenBefore(p)
1035	p = self.lastHiddenToken = self.LA(1)
1036	self.consume()
1037	return monitored
1038
1039	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1040	### StringBuffer ###
1041	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1042
1043	class StringBuffer:
1044	def __init__(self,string=None):
1045	if string:
1046	self.text = list(string)
1047	else:
1048	self.text = []
1049
1050	def setLength(self,sz):
1051	if not sz :
1052	self.text = []
1053	return
1054	assert sz>0
1055	if sz >= self.length():
1056	return
1057	### just reset to empty buffer
1058	self.text = self.text[0:sz]
1059
1060	def length(self):
1061	return len(self.text)
1062
1063	def append(self,c):
1064	self.text.append(c)
1065
1066	### return buffer as string. Arg 'a' is used as index
1067	## into the buffer and 2nd argument shall be the length.
1068	## If 2nd args is absent, we return chars till end of
1069	## buffer starting with 'a'.
1070	def getString(self,a=None,length=None):
1071	if not a :
1072	a = 0
1073	assert a>=0
1074	if a>= len(self.text) :
1075	return ""
1076
1077	if not length:
1078	## no second argument
1079	L = self.text[a:]
1080	else:
1081	assert (a+length) <= len(self.text)
1082	b = a + length
1083	L = self.text[a:b]
1084	s = ""
1085	for x in L : s += x
1086	return s
1087
1088	toString = getString ## alias
1089
1090	def __str__(self):
1091	return str(self.text)
1092
1093	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1094	### Reader ###
1095	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1096
1097	## When reading Japanese chars, it happens that a stream returns a
1098	## 'char' of length 2. This looks like a bug in the appropriate
1099	## codecs - but I'm rather unsure about this. Anyway, if this is
1100	## the case, I'm going to split this string into a list of chars
1101	## and put them on hold, ie. on a buffer. Next time when called
1102	## we read from buffer until buffer is empty.
1103	## wh: nov, 25th -> problem does not appear in Python 2.4.0.c1.
1104
1105	class Reader(object):
1106	def __init__(self,stream):
1107	self.cin = stream
1108	self.buf = []
1109
1110	def read(self,num):
1111	assert num==1
1112
1113	if len(self.buf):
1114	return self.buf.pop()
1115
1116	## Read a char - this may return a string.
1117	## Is this a bug in codecs/Python?
1118	c = self.cin.read(1)
1119
1120	if not c or len(c)==1:
1121	return c
1122
1123	L = list(c)
1124	L.reverse()
1125	for x in L:
1126	self.buf.append(x)
1127
1128	## read one char ..
1129	return self.read(1)
1130
1131	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1132	### CharScanner ###
1133	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1134
1135	class CharScanner(TokenStream):
1136	## class members
1137	NO_CHAR = 0
1138	EOF_CHAR = '' ### EOF shall be the empty string.
1139
1140	def __init__(self, argv, *kwargs):
1141	super(CharScanner, self).__init__()
1142	self.saveConsumedInput = True
1143	self.tokenClass = None
1144	self.caseSensitive = True
1145	self.caseSensitiveLiterals = True
1146	self.literals = None
1147	self.tabsize = 8
1148	self._returnToken = None
1149	self.commitToPath = False
1150	self.traceDepth = 0
1151	self.text = StringBuffer()
1152	self.hashString = hash(self)
1153	self.setTokenObjectClass(CommonToken)
1154	self.setInput(*argv)
1155
1156	def __iter__(self):
1157	return CharScannerIterator(self)
1158
1159	def setInput(self,*argv):
1160	## case 1:
1161	## if there's no arg we default to read from
1162	## standard input
1163	if not argv:
1164	import sys
1165	self.setInput(sys.stdin)
1166	return
1167
1168	## get 1st argument
1169	arg1 = argv[0]
1170
1171	## case 2:
1172	## if arg1 is a string, we assume it's a file name
1173	## and open a stream using 2nd argument as open
1174	## mode. If there's no 2nd argument we fall back to
1175	## mode '+rb'.
1176	if is_string_type(arg1):
1177	f = open(arg1,"rb")
1178	self.setInput(f)
1179	self.setFilename(arg1)
1180	return
1181
1182	## case 3:
1183	## if arg1 is a file we wrap it by a char buffer (
1184	## some additional checks?? No, can't do this in
1185	## general).
1186	if isinstance(arg1,file):
1187	self.setInput(CharBuffer(arg1))
1188	return
1189
1190	## case 4:
1191	## if arg1 is of type SharedLexerInputState we use
1192	## argument as is.
1193	if isinstance(arg1,LexerSharedInputState):
1194	self.inputState = arg1
1195	return
1196
1197	## case 5:
1198	## check whether argument type is of type input
1199	## buffer. If so create a SharedLexerInputState and
1200	## go ahead.
1201	if isinstance(arg1,InputBuffer):
1202	self.setInput(LexerSharedInputState(arg1))
1203	return
1204
1205	## case 6:
1206	## check whether argument type has a method read(int)
1207	## If so create CharBuffer ...
1208	try:
1209	if arg1.read:
1210	rd = Reader(arg1)
1211	cb = CharBuffer(rd)
1212	ss = LexerSharedInputState(cb)
1213	self.inputState = ss
1214	return
1215	except:
1216	pass
1217
1218	## case 7:
1219	## raise wrong argument exception
1220	raise TypeError(argv)
1221
1222	def setTabSize(self,size) :
1223	self.tabsize = size
1224
1225	def getTabSize(self) :
1226	return self.tabsize
1227
1228	def setCaseSensitive(self,t) :
1229	self.caseSensitive = t
1230
1231	def setCommitToPath(self,commit) :
1232	self.commitToPath = commit
1233
1234	def setFilename(self,f) :
1235	self.inputState.filename = f
1236
1237	def setLine(self,line) :
1238	self.inputState.line = line
1239
1240	def setText(self,s) :
1241	self.resetText()
1242	self.text.append(s)
1243
1244	def getCaseSensitive(self) :
1245	return self.caseSensitive
1246
1247	def getCaseSensitiveLiterals(self) :
1248	return self.caseSensitiveLiterals
1249
1250	def getColumn(self) :
1251	return self.inputState.column
1252
1253	def setColumn(self,c) :
1254	self.inputState.column = c
1255
1256	def getCommitToPath(self) :
1257	return self.commitToPath
1258
1259	def getFilename(self) :
1260	return self.inputState.filename
1261
1262	def getInputBuffer(self) :
1263	return self.inputState.input
1264
1265	def getInputState(self) :
1266	return self.inputState
1267
1268	def setInputState(self,state) :
1269	assert isinstance(state,LexerSharedInputState)
1270	self.inputState = state
1271
1272	def getLine(self) :
1273	return self.inputState.line
1274
1275	def getText(self) :
1276	return str(self.text)
1277
1278	def getTokenObject(self) :
1279	return self._returnToken
1280
1281	def LA(self,i) :
1282	c = self.inputState.input.LA(i)
1283	if not self.caseSensitive:
1284	### E0006
1285	c = c.__class__.lower(c)
1286	return c
1287
1288	def makeToken(self,type) :
1289	try:
1290	## dynamically load a class
1291	assert self.tokenClass
1292	tok = self.tokenClass()
1293	tok.setType(type)
1294	tok.setColumn(self.inputState.tokenStartColumn)
1295	tok.setLine(self.inputState.tokenStartLine)
1296	return tok
1297	except:
1298	self.panic("unable to create new token")
1299	return Token.badToken
1300
1301	def mark(self) :
1302	return self.inputState.input.mark()
1303
1304	def _match_bitset(self,b) :
1305	if b.member(self.LA(1)):
1306	self.consume()
1307	else:
1308	raise MismatchedCharException(self.LA(1), b, False, self)
1309
1310	def _match_string(self,s) :
1311	for c in s:
1312	if self.LA(1) == c:
1313	self.consume()
1314	else:
1315	raise MismatchedCharException(self.LA(1), c, False, self)
1316
1317	def match(self,item):
1318	if is_string_type(item):
1319	return self._match_string(item)
1320	else:
1321	return self._match_bitset(item)
1322
1323	def matchNot(self,c) :
1324	if self.LA(1) != c:
1325	self.consume()
1326	else:
1327	raise MismatchedCharException(self.LA(1), c, True, self)
1328
1329	def matchRange(self,c1,c2) :
1330	if self.LA(1) < c1 or self.LA(1) > c2 :
1331	raise MismatchedCharException(self.LA(1), c1, c2, False, self)
1332	else:
1333	self.consume()
1334
1335	def newline(self) :
1336	self.inputState.line += 1
1337	self.inputState.column = 1
1338
1339	def tab(self) :
1340	c = self.getColumn()
1341	nc = ( ((c-1)/self.tabsize) + 1) * self.tabsize + 1
1342	self.setColumn(nc)
1343
1344	def panic(self,s='') :
1345	print "CharScanner: panic: " + s
1346	sys.exit(1)
1347
1348	def reportError(self,ex) :
1349	print ex
1350
1351	def reportError(self,s) :
1352	if not self.getFilename():
1353	print "error: " + str(s)
1354	else:
1355	print self.getFilename() + ": error: " + str(s)
1356
1357	def reportWarning(self,s) :
1358	if not self.getFilename():
1359	print "warning: " + str(s)
1360	else:
1361	print self.getFilename() + ": warning: " + str(s)
1362
1363	def resetText(self) :
1364	self.text.setLength(0)
1365	self.inputState.tokenStartColumn = self.inputState.column
1366	self.inputState.tokenStartLine = self.inputState.line
1367
1368	def rewind(self,pos) :
1369	self.inputState.input.rewind(pos)
1370
1371	def setTokenObjectClass(self,cl):
1372	self.tokenClass = cl
1373
1374	def testForLiteral(self,token):
1375	if not token:
1376	return
1377	assert isinstance(token,Token)
1378
1379	_type = token.getType()
1380
1381	## special tokens can't be literals
1382	if _type in [SKIP,INVALID_TYPE,EOF_TYPE,NULL_TREE_LOOKAHEAD] :
1383	return
1384
1385	_text = token.getText()
1386	if not _text:
1387	return
1388
1389	assert is_string_type(_text)
1390	_type = self.testLiteralsTable(_text,_type)
1391	token.setType(_type)
1392	return _type
1393
1394	def testLiteralsTable(self,*args):
1395	if is_string_type(args[0]):
1396	s = args[0]
1397	i = args[1]
1398	else:
1399	s = self.text.getString()
1400	i = args[0]
1401
1402	## check whether integer has been given
1403	if not isinstance(i,int):
1404	assert isinstance(i,int)
1405
1406	## check whether we have a dict
1407	assert isinstance(self.literals,dict)
1408	try:
1409	## E0010
1410	if not self.caseSensitiveLiterals:
1411	s = s.__class__.lower(s)
1412	i = self.literals[s]
1413	except:
1414	pass
1415	return i
1416
1417	def toLower(self,c):
1418	return c.__class__.lower()
1419
1420	def traceIndent(self):
1421	print ' ' * self.traceDepth
1422
1423	def traceIn(self,rname):
1424	self.traceDepth += 1
1425	self.traceIndent()
1426	print "> lexer %s c== %s" % (rname,self.LA(1))
1427
1428	def traceOut(self,rname):
1429	self.traceIndent()
1430	print "< lexer %s c== %s" % (rname,self.LA(1))
1431	self.traceDepth -= 1
1432
1433	def uponEOF(self):
1434	pass
1435
1436	def append(self,c):
1437	if self.saveConsumedInput :
1438	self.text.append(c)
1439
1440	def commit(self):
1441	self.inputState.input.commit()
1442
1443	def consume(self):
1444	if not self.inputState.guessing:
1445	c = self.LA(1)
1446	if self.caseSensitive:
1447	self.append(c)
1448	else:
1449	# use input.LA(), not LA(), to get original case
1450	# CharScanner.LA() would toLower it.
1451	c = self.inputState.input.LA(1)
1452	self.append(c)
1453
1454	if c and c in "\t":
1455	self.tab()
1456	else:
1457	self.inputState.column += 1
1458	self.inputState.input.consume()
1459
1460	## Consume chars until one matches the given char
1461	def consumeUntil_char(self,c):
1462	while self.LA(1) != EOF_CHAR and self.LA(1) != c:
1463	self.consume()
1464
1465	## Consume chars until one matches the given set
1466	def consumeUntil_bitset(self,bitset):
1467	while self.LA(1) != EOF_CHAR and not self.set.member(self.LA(1)):
1468	self.consume()
1469
1470	### If symbol seen is EOF then generate and set token, otherwise
1471	### throw exception.
1472	def default(self,la1):
1473	if not la1 :
1474	self.uponEOF()
1475	self._returnToken = self.makeToken(EOF_TYPE)
1476	else:
1477	self.raise_NoViableAlt(la1)
1478
1479	def filterdefault(self,la1,*args):
1480	if not la1:
1481	self.uponEOF()
1482	self._returnToken = self.makeToken(EOF_TYPE)
1483	return
1484
1485	if not args:
1486	self.consume()
1487	raise TryAgain()
1488	else:
1489	### apply filter object
1490	self.commit();
1491	try:
1492	func=args[0]
1493	args=args[1:]
1494	apply(func,args)
1495	except RecognitionException, e:
1496	## catastrophic failure
1497	self.reportError(e);
1498	self.consume();
1499	raise TryAgain()
1500
1501	def raise_NoViableAlt(self,la1=None):
1502	if not la1: la1 = self.LA(1)
1503	fname = self.getFilename()
1504	line = self.getLine()
1505	col = self.getColumn()
1506	raise NoViableAltForCharException(la1,fname,line,col)
1507
1508	def set_return_token(self,_create,_token,_ttype,_offset):
1509	if _create and not _token and (not _ttype == SKIP):
1510	string = self.text.getString(_offset)
1511	_token = self.makeToken(_ttype)
1512	_token.setText(string)
1513	self._returnToken = _token
1514	return _token
1515
1516	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1517	### CharScannerIterator ###
1518	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1519
1520	class CharScannerIterator:
1521
1522	def __init__(self,inst):
1523	if isinstance(inst,CharScanner):
1524	self.inst = inst
1525	return
1526	raise TypeError("CharScannerIterator requires CharScanner object")
1527
1528	def next(self):
1529	assert self.inst
1530	item = self.inst.nextToken()
1531	if not item or item.isEOF():
1532	raise StopIteration()
1533	return item
1534
1535	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1536	### BitSet ###
1537	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1538
1539	### I'm assuming here that a long is 64bits. It appears however, that
1540	### a long is of any size. That means we can use a single long as the
1541	### bitset (!), ie. Python would do almost all the work (TBD).
1542
1543	class BitSet(object):
1544	BITS = 64
1545	NIBBLE = 4
1546	LOG_BITS = 6
1547	MOD_MASK = BITS -1
1548
1549	def __init__(self,data=None):
1550	if not data:
1551	BitSet.__init__(self,[long(0)])
1552	return
1553	if isinstance(data,int):
1554	BitSet.__init__(self,[long(data)])
1555	return
1556	if isinstance(data,long):
1557	BitSet.__init__(self,[data])
1558	return
1559	if not isinstance(data,list):
1560	raise TypeError("BitSet requires integer, long, or " +
1561	"list argument")
1562	for x in data:
1563	if not isinstance(x,long):
1564	raise TypeError(self,"List argument item is " +
1565	"not a long: %s" % (x))
1566	self.data = data
1567
1568	def __str__(self):
1569	bits = len(self.data) * BitSet.BITS
1570	s = ""
1571	for i in xrange(0,bits):
1572	if self.at(i):
1573	s += "1"
1574	else:
1575	s += "o"
1576	if not ((i+1) % 10):
1577	s += '\|%s\|' % (i+1)
1578	return s
1579
1580	def __repr__(self):
1581	return str(self)
1582
1583	def member(self,item):
1584	if not item:
1585	return False
1586
1587	if isinstance(item,int):
1588	return self.at(item)
1589
1590	if not is_string_type(item):
1591	raise TypeError(self,"char or unichar expected: %s" % (item))
1592
1593	## char is a (unicode) string with at most lenght 1, ie.
1594	## a char.
1595
1596	if len(item) != 1:
1597	raise TypeError(self,"char expected: %s" % (item))
1598
1599	### handle ASCII/UNICODE char
1600	num = ord(item)
1601
1602	### check whether position num is in bitset
1603	return self.at(num)
1604
1605	def wordNumber(self,bit):
1606	return bit >> BitSet.LOG_BITS
1607
1608	def bitMask(self,bit):
1609	pos = bit & BitSet.MOD_MASK ## bit mod BITS
1610	return (1L << pos)
1611
1612	def set(self,bit,on=True):
1613	# grow bitset as required (use with care!)
1614	i = self.wordNumber(bit)
1615	mask = self.bitMask(bit)
1616	if i>=len(self.data):
1617	d = i - len(self.data) + 1
1618	for x in xrange(0,d):
1619	self.data.append(0L)
1620	assert len(self.data) == i+1
1621	if on:
1622	self.data[i] \|= mask
1623	else:
1624	self.data[i] &= (~mask)
1625
1626	### make add an alias for set
1627	add = set
1628
1629	def off(self,bit,off=True):
1630	self.set(bit,not off)
1631
1632	def at(self,bit):
1633	i = self.wordNumber(bit)
1634	v = self.data[i]
1635	m = self.bitMask(bit)
1636	return v & m
1637
1638
1639	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1640	### some further funcs ###
1641	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1642
1643	def illegalarg_ex(func):
1644	raise ValueError(
1645	"%s is only valid if parser is built for debugging" %
1646	(func.func_name))
1647
1648	def runtime_ex(func):
1649	raise RuntimeException(
1650	"%s is only valid if parser is built for debugging" %
1651	(func.func_name))
1652
1653	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1654	### TokenBuffer ###
1655	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1656
1657	class TokenBuffer(object):
1658	def __init__(self,stream):
1659	self.input = stream
1660	self.nMarkers = 0
1661	self.markerOffset = 0
1662	self.numToConsume = 0
1663	self.queue = Queue()
1664
1665	def reset(self) :
1666	self.nMarkers = 0
1667	self.markerOffset = 0
1668	self.numToConsume = 0
1669	self.queue.reset()
1670
1671	def consume(self) :
1672	self.numToConsume += 1
1673
1674	def fill(self, amount):
1675	self.syncConsume()
1676	while self.queue.length() < (amount + self.markerOffset):
1677	self.queue.append(self.input.nextToken())
1678
1679	def getInput(self):
1680	return self.input
1681
1682	def LA(self,k) :
1683	self.fill(k)
1684	return self.queue.elementAt(self.markerOffset + k - 1).type
1685
1686	def LT(self,k) :
1687	self.fill(k)
1688	return self.queue.elementAt(self.markerOffset + k - 1)
1689
1690	def mark(self) :
1691	self.syncConsume()
1692	self.nMarkers += 1
1693	return self.markerOffset
1694
1695	def rewind(self,mark) :
1696	self.syncConsume()
1697	self.markerOffset = mark
1698	self.nMarkers -= 1
1699
1700	def syncConsume(self) :
1701	while self.numToConsume > 0:
1702	if self.nMarkers > 0:
1703	# guess mode -- leave leading characters and bump offset.
1704	self.markerOffset += 1
1705	else:
1706	# normal mode -- remove first character
1707	self.queue.removeFirst()
1708	self.numToConsume -= 1
1709
1710	def __str__(self):
1711	return "(%s,%s,%s,%s,%s)" % (
1712	self.input,
1713	self.nMarkers,
1714	self.markerOffset,
1715	self.numToConsume,
1716	self.queue)
1717
1718	def __repr__(self):
1719	return str(self)
1720
1721	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1722	### ParserSharedInputState ###
1723	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1724
1725	class ParserSharedInputState(object):
1726
1727	def __init__(self):
1728	self.input = None
1729	self.reset()
1730
1731	def reset(self):
1732	self.guessing = 0
1733	self.filename = None
1734	if self.input:
1735	self.input.reset()
1736
1737	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1738	### Parser ###
1739	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1740
1741	class Parser(object):
1742
1743	def __init__(self, args, *kwargs):
1744	self.tokenNames = None
1745	self.returnAST = None
1746	self.astFactory = None
1747	self.tokenTypeToASTClassMap = {}
1748	self.ignoreInvalidDebugCalls = False
1749	self.traceDepth = 0
1750	if not args:
1751	self.inputState = ParserSharedInputState()
1752	return
1753	arg0 = args[0]
1754	assert isinstance(arg0,ParserSharedInputState)
1755	self.inputState = arg0
1756	return
1757
1758	def getTokenTypeToASTClassMap(self):
1759	return self.tokenTypeToASTClassMap
1760
1761
1762	def addMessageListener(self, l):
1763	if not self.ignoreInvalidDebugCalls:
1764	illegalarg_ex(addMessageListener)
1765
1766	def addParserListener(self,l) :
1767	if (not self.ignoreInvalidDebugCalls) :
1768	illegalarg_ex(addParserListener)
1769
1770	def addParserMatchListener(self, l) :
1771	if (not self.ignoreInvalidDebugCalls) :
1772	illegalarg_ex(addParserMatchListener)
1773
1774	def addParserTokenListener(self, l) :
1775	if (not self.ignoreInvalidDebugCalls):
1776	illegalarg_ex(addParserTokenListener)
1777
1778	def addSemanticPredicateListener(self, l) :
1779	if (not self.ignoreInvalidDebugCalls):
1780	illegalarg_ex(addSemanticPredicateListener)
1781
1782	def addSyntacticPredicateListener(self, l) :
1783	if (not self.ignoreInvalidDebugCalls):
1784	illegalarg_ex(addSyntacticPredicateListener)
1785
1786	def addTraceListener(self, l) :
1787	if (not self.ignoreInvalidDebugCalls):
1788	illegalarg_ex(addTraceListener)
1789
1790	def consume(self):
1791	raise NotImplementedError()
1792
1793	def _consumeUntil_type(self,tokenType):
1794	while self.LA(1) != EOF_TYPE and self.LA(1) != tokenType:
1795	self.consume()
1796
1797	def _consumeUntil_bitset(self, set):
1798	while self.LA(1) != EOF_TYPE and not set.member(self.LA(1)):
1799	self.consume()
1800
1801	def consumeUntil(self,arg):
1802	if isinstance(arg,int):
1803	self._consumeUntil_type(arg)
1804	else:
1805	self._consumeUntil_bitset(arg)
1806
1807	def defaultDebuggingSetup(self):
1808	pass
1809
1810	def getAST(self) :
1811	return self.returnAST
1812
1813	def getASTFactory(self) :
1814	return self.astFactory
1815
1816	def getFilename(self) :
1817	return self.inputState.filename
1818
1819	def getInputState(self) :
1820	return self.inputState
1821
1822	def setInputState(self, state) :
1823	self.inputState = state
1824
1825	def getTokenName(self,num) :
1826	return self.tokenNames[num]
1827
1828	def getTokenNames(self) :
1829	return self.tokenNames
1830
1831	def isDebugMode(self) :
1832	return self.false
1833
1834	def LA(self, i):
1835	raise NotImplementedError()
1836
1837	def LT(self, i):
1838	raise NotImplementedError()
1839
1840	def mark(self):
1841	return self.inputState.input.mark()
1842
1843	def _match_int(self,t):
1844	if (self.LA(1) != t):
1845	raise MismatchedTokenException(
1846	self.tokenNames, self.LT(1), t, False, self.getFilename())
1847	else:
1848	self.consume()
1849
1850	def _match_set(self, b):
1851	if (not b.member(self.LA(1))):
1852	raise MismatchedTokenException(
1853	self.tokenNames,self.LT(1), b, False, self.getFilename())
1854	else:
1855	self.consume()
1856
1857	def match(self,set) :
1858	if isinstance(set,int):
1859	self._match_int(set)
1860	return
1861	if isinstance(set,BitSet):
1862	self._match_set(set)
1863	return
1864	raise TypeError("Parser.match requires integer ot BitSet argument")
1865
1866	def matchNot(self,t):
1867	if self.LA(1) == t:
1868	raise MismatchedTokenException(
1869	tokenNames, self.LT(1), t, True, self.getFilename())
1870	else:
1871	self.consume()
1872
1873	def removeMessageListener(self, l) :
1874	if (not self.ignoreInvalidDebugCalls):
1875	runtime_ex(removeMessageListener)
1876
1877	def removeParserListener(self, l) :
1878	if (not self.ignoreInvalidDebugCalls):
1879	runtime_ex(removeParserListener)
1880
1881	def removeParserMatchListener(self, l) :
1882	if (not self.ignoreInvalidDebugCalls):
1883	runtime_ex(removeParserMatchListener)
1884
1885	def removeParserTokenListener(self, l) :
1886	if (not self.ignoreInvalidDebugCalls):
1887	runtime_ex(removeParserTokenListener)
1888
1889	def removeSemanticPredicateListener(self, l) :
1890	if (not self.ignoreInvalidDebugCalls):
1891	runtime_ex(removeSemanticPredicateListener)
1892
1893	def removeSyntacticPredicateListener(self, l) :
1894	if (not self.ignoreInvalidDebugCalls):
1895	runtime_ex(removeSyntacticPredicateListener)
1896
1897	def removeTraceListener(self, l) :
1898	if (not self.ignoreInvalidDebugCalls):
1899	runtime_ex(removeTraceListener)
1900
1901	def reportError(self,x) :
1902	fmt = "syntax error:"
1903	f = self.getFilename()
1904	if f:
1905	fmt = ("%s:" % f) + fmt
1906	if isinstance(x,Token):
1907	line = x.getColumn()
1908	col = x.getLine()
1909	text = x.getText()
1910	fmt = fmt + 'unexpected symbol at line %s (column %s) : "%s"'
1911	print >>sys.stderr, fmt % (line,col,text)
1912	else:
1913	print >>sys.stderr, fmt,str(x)
1914
1915	def reportWarning(self,s):
1916	f = self.getFilename()
1917	if f:
1918	print "%s:warning: %s" % (f,str(x))
1919	else:
1920	print "warning: %s" % (str(x))
1921
1922	def rewind(self, pos) :
1923	self.inputState.input.rewind(pos)
1924
1925	def setASTFactory(self, f) :
1926	self.astFactory = f
1927
1928	def setASTNodeClass(self, cl) :
1929	self.astFactory.setASTNodeType(cl)
1930
1931	def setASTNodeType(self, nodeType) :
1932	self.setASTNodeClass(nodeType)
1933
1934	def setDebugMode(self, debugMode) :
1935	if (not self.ignoreInvalidDebugCalls):
1936	runtime_ex(setDebugMode)
1937
1938	def setFilename(self, f) :
1939	self.inputState.filename = f
1940
1941	def setIgnoreInvalidDebugCalls(self, value) :
1942	self.ignoreInvalidDebugCalls = value
1943
1944	def setTokenBuffer(self, t) :
1945	self.inputState.input = t
1946
1947	def traceIndent(self):
1948	print " " * self.traceDepth
1949
1950	def traceIn(self,rname):
1951	self.traceDepth += 1
1952	self.trace("> ", rname)
1953
1954	def traceOut(self,rname):
1955	self.trace("< ", rname)
1956	self.traceDepth -= 1
1957
1958	### wh: moved from ASTFactory to Parser
1959	def addASTChild(self,currentAST, child):
1960	if not child:
1961	return
1962	if not currentAST.root:
1963	currentAST.root = child
1964	elif not currentAST.child:
1965	currentAST.root.setFirstChild(child)
1966	else:
1967	currentAST.child.setNextSibling(child)
1968	currentAST.child = child
1969	currentAST.advanceChildToEnd()
1970
1971	### wh: moved from ASTFactory to Parser
1972	def makeASTRoot(self,currentAST,root) :
1973	if root:
1974	### Add the current root as a child of new root
1975	root.addChild(currentAST.root)
1976	### The new current child is the last sibling of the old root
1977	currentAST.child = currentAST.root
1978	currentAST.advanceChildToEnd()
1979	### Set the new root
1980	currentAST.root = root
1981
1982	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1983	### LLkParser ###
1984	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
1985
1986	class LLkParser(Parser):
1987
1988	def __init__(self, args, *kwargs):
1989	try:
1990	arg1 = args[0]
1991	except:
1992	arg1 = 1
1993
1994	if isinstance(arg1,int):
1995	super(LLkParser,self).__init__()
1996	self.k = arg1
1997	return
1998
1999	if isinstance(arg1,ParserSharedInputState):
2000	super(LLkParser,self).__init__(arg1)
2001	self.set_k(1,*args)
2002	return
2003
2004	if isinstance(arg1,TokenBuffer):
2005	super(LLkParser,self).__init__()
2006	self.setTokenBuffer(arg1)
2007	self.set_k(1,*args)
2008	return
2009
2010	if isinstance(arg1,TokenStream):
2011	super(LLkParser,self).__init__()
2012	tokenBuf = TokenBuffer(arg1)
2013	self.setTokenBuffer(tokenBuf)
2014	self.set_k(1,*args)
2015	return
2016
2017	### unknown argument
2018	raise TypeError("LLkParser requires integer, " +
2019	"ParserSharedInputStream or TokenStream argument")
2020
2021	def consume(self):
2022	self.inputState.input.consume()
2023
2024	def LA(self,i):
2025	return self.inputState.input.LA(i)
2026
2027	def LT(self,i):
2028	return self.inputState.input.LT(i)
2029
2030	def set_k(self,index,*args):
2031	try:
2032	self.k = args[index]
2033	except:
2034	self.k = 1
2035
2036	def trace(self,ee,rname):
2037	print type(self)
2038	self.traceIndent()
2039	guess = ""
2040	if self.inputState.guessing > 0:
2041	guess = " [guessing]"
2042	print(ee + rname + guess)
2043	for i in xrange(1,self.k+1):
2044	if i != 1:
2045	print(", ")
2046	if self.LT(i) :
2047	v = self.LT(i).getText()
2048	else:
2049	v = "null"
2050	print "LA(%s) == %s" % (i,v)
2051	print("\n")
2052
2053	def traceIn(self,rname):
2054	self.traceDepth += 1;
2055	self.trace("> ", rname);
2056
2057	def traceOut(self,rname):
2058	self.trace("< ", rname);
2059	self.traceDepth -= 1;
2060
2061	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2062	### TreeParserSharedInputState ###
2063	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2064
2065	class TreeParserSharedInputState(object):
2066	def __init__(self):
2067	self.guessing = 0
2068
2069	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2070	### TreeParser ###
2071	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2072
2073	class TreeParser(object):
2074
2075	def __init__(self, args, *kwargs):
2076	self.inputState = TreeParserSharedInputState()
2077	self._retTree = None
2078	self.tokenNames = []
2079	self.returnAST = None
2080	self.astFactory = ASTFactory()
2081	self.traceDepth = 0
2082
2083	def getAST(self):
2084	return self.returnAST
2085
2086	def getASTFactory(self):
2087	return self.astFactory
2088
2089	def getTokenName(self,num) :
2090	return self.tokenNames[num]
2091
2092	def getTokenNames(self):
2093	return self.tokenNames
2094
2095	def match(self,t,set) :
2096	assert isinstance(set,int) or isinstance(set,BitSet)
2097	if not t or t == ASTNULL:
2098	raise MismatchedTokenException(self.getTokenNames(), t,set, False)
2099
2100	if isinstance(set,int) and t.getType() != set:
2101	raise MismatchedTokenException(self.getTokenNames(), t,set, False)
2102
2103	if isinstance(set,BitSet) and not set.member(t.getType):
2104	raise MismatchedTokenException(self.getTokenNames(), t,set, False)
2105
2106	def matchNot(self,t, ttype) :
2107	if not t or (t == ASTNULL) or (t.getType() == ttype):
2108	raise MismatchedTokenException(getTokenNames(), t, ttype, True)
2109
2110	def reportError(self,ex):
2111	print >>sys.stderr,"error:",ex
2112
2113	def reportWarning(self, s):
2114	print "warning:",s
2115
2116	def setASTFactory(self,f):
2117	self.astFactory = f
2118
2119	def setASTNodeType(self,nodeType):
2120	self.setASTNodeClass(nodeType)
2121
2122	def setASTNodeClass(self,nodeType):
2123	self.astFactory.setASTNodeType(nodeType)
2124
2125	def traceIndent(self):
2126	print " " * self.traceDepth
2127
2128	def traceIn(self,rname,t):
2129	self.traceDepth += 1
2130	self.traceIndent()
2131	print("> " + rname + "(" +
2132	ifelse(t,str(t),"null") + ")" +
2133	ifelse(self.inputState.guessing>0,"[guessing]",""))
2134
2135	def traceOut(self,rname,t):
2136	self.traceIndent()
2137	print("< " + rname + "(" +
2138	ifelse(t,str(t),"null") + ")" +
2139	ifelse(self.inputState.guessing>0,"[guessing]",""))
2140	self.traceDepth -= 1
2141
2142	### wh: moved from ASTFactory to TreeParser
2143	def addASTChild(self,currentAST, child):
2144	if not child:
2145	return
2146	if not currentAST.root:
2147	currentAST.root = child
2148	elif not currentAST.child:
2149	currentAST.root.setFirstChild(child)
2150	else:
2151	currentAST.child.setNextSibling(child)
2152	currentAST.child = child
2153	currentAST.advanceChildToEnd()
2154
2155	### wh: moved from ASTFactory to TreeParser
2156	def makeASTRoot(self,currentAST,root):
2157	if root:
2158	### Add the current root as a child of new root
2159	root.addChild(currentAST.root)
2160	### The new current child is the last sibling of the old root
2161	currentAST.child = currentAST.root
2162	currentAST.advanceChildToEnd()
2163	### Set the new root
2164	currentAST.root = root
2165
2166	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2167	### funcs to work on trees ###
2168	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2169
2170	def rightmost(ast):
2171	if ast:
2172	while(ast.right):
2173	ast = ast.right
2174	return ast
2175
2176	def cmptree(s,t,partial):
2177	while(s and t):
2178	### as a quick optimization, check roots first.
2179	if not s.equals(t):
2180	return False
2181
2182	### if roots match, do full list match test on children.
2183	if not cmptree(s.getFirstChild(),t.getFirstChild(),partial):
2184	return False
2185
2186	s = s.getNextSibling()
2187	t = t.getNextSibling()
2188
2189	r = ifelse(partial,not t,not s and not t)
2190	return r
2191
2192	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2193	### AST ###
2194	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2195
2196	class AST(object):
2197	def __init__(self):
2198	pass
2199
2200	def addChild(self, c):
2201	pass
2202
2203	def equals(self, t):
2204	return False
2205
2206	def equalsList(self, t):
2207	return False
2208
2209	def equalsListPartial(self, t):
2210	return False
2211
2212	def equalsTree(self, t):
2213	return False
2214
2215	def equalsTreePartial(self, t):
2216	return False
2217
2218	def findAll(self, tree):
2219	return None
2220
2221	def findAllPartial(self, subtree):
2222	return None
2223
2224	def getFirstChild(self):
2225	return self
2226
2227	def getNextSibling(self):
2228	return self
2229
2230	def getText(self):
2231	return ""
2232
2233	def getType(self):
2234	return INVALID_TYPE
2235
2236	def getLine(self):
2237	return 0
2238
2239	def getColumn(self):
2240	return 0
2241
2242	def getNumberOfChildren(self):
2243	return 0
2244
2245	def initialize(self, t, txt):
2246	pass
2247
2248	def initialize(self, t):
2249	pass
2250
2251	def setFirstChild(self, c):
2252	pass
2253
2254	def setNextSibling(self, n):
2255	pass
2256
2257	def setText(self, text):
2258	pass
2259
2260	def setType(self, ttype):
2261	pass
2262
2263	def toString(self):
2264	self.getText()
2265
2266	__str__ = toString
2267
2268	def toStringList(self):
2269	return self.getText()
2270
2271	def toStringTree(self):
2272	return self.getText()
2273
2274	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2275	### ASTNULLType ###
2276	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2277
2278	### There is only one instance of this class **/
2279	class ASTNULLType(AST):
2280	def __init__(self):
2281	AST.__init__(self)
2282	pass
2283
2284	def getText(self):
2285	return "<ASTNULL>"
2286
2287	def getType(self):
2288	return NULL_TREE_LOOKAHEAD
2289
2290
2291	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2292	### BaseAST ###
2293	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2294
2295	class BaseAST(AST):
2296
2297	verboseStringConversion = False
2298	tokenNames = None
2299
2300	def __init__(self):
2301	self.down = None ## kid
2302	self.right = None ## sibling
2303
2304	def addChild(self,node):
2305	if node:
2306	t = rightmost(self.down)
2307	if t:
2308	t.right = node
2309	else:
2310	assert not self.down
2311	self.down = node
2312
2313	def getNumberOfChildren(self):
2314	t = self.down
2315	n = 0
2316	while t:
2317	n += 1
2318	t = t.right
2319	return n
2320
2321	def doWorkForFindAll(self,v,target,partialMatch):
2322	sibling = self
2323
2324	while sibling:
2325	c1 = partialMatch and sibling.equalsTreePartial(target)
2326	if c1:
2327	v.append(sibling)
2328	else:
2329	c2 = not partialMatch and sibling.equalsTree(target)
2330	if c2:
2331	v.append(sibling)
2332
2333	### regardless of match or not, check any children for matches
2334	if sibling.getFirstChild():
2335	sibling.getFirstChild().doWorkForFindAll(v,target,partialMatch)
2336
2337	sibling = sibling.getNextSibling()
2338
2339	### Is node t equal to 'self' in terms of token type and text?
2340	def equals(self,t):
2341	if not t:
2342	return False
2343	return self.getText() == t.getText() and self.getType() == t.getType()
2344
2345	### Is t an exact structural and equals() match of this tree. The
2346	### 'self' reference is considered the start of a sibling list.
2347	###
2348	def equalsList(self, t):
2349	return cmptree(self, t, partial=False)
2350
2351	### Is 't' a subtree of this list?
2352	### The siblings of the root are NOT ignored.
2353	###
2354	def equalsListPartial(self,t):
2355	return cmptree(self,t,partial=True)
2356
2357	### Is tree rooted at 'self' equal to 't'? The siblings
2358	### of 'self' are ignored.
2359	###
2360	def equalsTree(self, t):
2361	return self.equals(t) and \
2362	cmptree(self.getFirstChild(), t.getFirstChild(), partial=False)
2363
2364	### Is 't' a subtree of the tree rooted at 'self'? The siblings
2365	### of 'self' are ignored.
2366	###
2367	def equalsTreePartial(self, t):
2368	if not t:
2369	return True
2370	return self.equals(t) and cmptree(
2371	self.getFirstChild(), t.getFirstChild(), partial=True)
2372
2373	### Walk the tree looking for all exact subtree matches. Return
2374	### an ASTEnumerator that lets the caller walk the list
2375	### of subtree roots found herein.
2376	def findAll(self,target):
2377	roots = []
2378
2379	### the empty tree cannot result in an enumeration
2380	if not target:
2381	return None
2382	# find all matches recursively
2383	self.doWorkForFindAll(roots, target, False)
2384	return roots
2385
2386	### Walk the tree looking for all subtrees. Return
2387	### an ASTEnumerator that lets the caller walk the list
2388	### of subtree roots found herein.
2389	def findAllPartial(self,sub):
2390	roots = []
2391
2392	### the empty tree cannot result in an enumeration
2393	if not sub:
2394	return None
2395
2396	self.doWorkForFindAll(roots, sub, True) ### find all matches recursively
2397	return roots
2398
2399	### Get the first child of this node None if not children
2400	def getFirstChild(self):
2401	return self.down
2402
2403	### Get the next sibling in line after this one
2404	def getNextSibling(self):
2405	return self.right
2406
2407	### Get the token text for this node
2408	def getText(self):
2409	return ""
2410
2411	### Get the token type for this node
2412	def getType(self):
2413	return 0
2414
2415	def getLine(self):
2416	return 0
2417
2418	def getColumn(self):
2419	return 0
2420
2421	### Remove all children */
2422	def removeChildren(self):
2423	self.down = None
2424
2425	def setFirstChild(self,c):
2426	self.down = c
2427
2428	def setNextSibling(self, n):
2429	self.right = n
2430
2431	### Set the token text for this node
2432	def setText(self, text):
2433	pass
2434
2435	### Set the token type for this node
2436	def setType(self, ttype):
2437	pass
2438
2439	### static
2440	def setVerboseStringConversion(verbose,names):
2441	verboseStringConversion = verbose
2442	tokenNames = names
2443	setVerboseStringConversion = staticmethod(setVerboseStringConversion)
2444
2445	### Return an array of strings that maps token ID to it's text.
2446	## @since 2.7.3
2447	def getTokenNames():
2448	return tokenNames
2449
2450	def toString(self):
2451	return self.getText()
2452
2453	### return tree as lisp string - sibling included
2454	def toStringList(self):
2455	ts = self.toStringTree()
2456	sib = self.getNextSibling()
2457	if sib:
2458	ts += sib.toStringList()
2459	return ts
2460
2461	__str__ = toStringList
2462
2463	### return tree as string - siblings ignored
2464	def toStringTree(self):
2465	ts = ""
2466	kid = self.getFirstChild()
2467	if kid:
2468	ts += " ("
2469	ts += " " + self.toString()
2470	if kid:
2471	ts += kid.toStringList()
2472	ts += " )"
2473	return ts
2474
2475	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2476	### CommonAST ###
2477	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2478
2479	### Common AST node implementation
2480	class CommonAST(BaseAST):
2481	def __init__(self,token=None):
2482	super(CommonAST,self).__init__()
2483	self.ttype = INVALID_TYPE
2484	self.text = "<no text>"
2485	self.line = 0
2486	self.column= 0
2487	self.initialize(token)
2488	#assert self.text
2489
2490	### Get the token text for this node
2491	def getText(self):
2492	return self.text
2493
2494	### Get the token type for this node
2495	def getType(self):
2496	return self.ttype
2497
2498	### Get the line for this node
2499	def getLine(self):
2500	return self.line
2501
2502	### Get the column for this node
2503	def getColumn(self):
2504	return self.column
2505
2506	def initialize(self,*args):
2507	if not args:
2508	return
2509
2510	arg0 = args[0]
2511
2512	if isinstance(arg0,int):
2513	arg1 = args[1]
2514	self.setType(arg0)
2515	self.setText(arg1)
2516	return
2517
2518	if isinstance(arg0,AST) or isinstance(arg0,Token):
2519	self.setText(arg0.getText())
2520	self.setType(arg0.getType())
2521	self.line = arg0.getLine()
2522	self.column = arg0.getColumn()
2523	return
2524
2525	### Set the token text for this node
2526	def setText(self,text_):
2527	assert is_string_type(text_)
2528	self.text = text_
2529
2530	### Set the token type for this node
2531	def setType(self,ttype_):
2532	assert isinstance(ttype_,int)
2533	self.ttype = ttype_
2534
2535	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2536	### CommonASTWithHiddenTokens ###
2537	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2538
2539	class CommonASTWithHiddenTokens(CommonAST):
2540
2541	def __init__(self,*args):
2542	CommonAST.__init__(self,*args)
2543	self.hiddenBefore = None
2544	self.hiddenAfter = None
2545
2546	def getHiddenAfter(self):
2547	return self.hiddenAfter
2548
2549	def getHiddenBefore(self):
2550	return self.hiddenBefore
2551
2552	def initialize(self,*args):
2553	CommonAST.initialize(self,*args)
2554	if args and isinstance(args[0],Token):
2555	assert isinstance(args[0],CommonHiddenStreamToken)
2556	self.hiddenBefore = args[0].getHiddenBefore()
2557	self.hiddenAfter = args[0].getHiddenAfter()
2558
2559	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2560	### ASTPair ###
2561	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2562
2563	class ASTPair(object):
2564	def __init__(self):
2565	self.root = None ### current root of tree
2566	self.child = None ### current child to which siblings are added
2567
2568	### Make sure that child is the last sibling */
2569	def advanceChildToEnd(self):
2570	if self.child:
2571	while self.child.getNextSibling():
2572	self.child = self.child.getNextSibling()
2573
2574	### Copy an ASTPair. Don't call it clone() because we want type-safety */
2575	def copy(self):
2576	tmp = ASTPair()
2577	tmp.root = self.root
2578	tmp.child = self.child
2579	return tmp
2580
2581	def toString(self):
2582	r = ifelse(not root,"null",self.root.getText())
2583	c = ifelse(not child,"null",self.child.getText())
2584	return "[%s,%s]" % (r,c)
2585
2586	__str__ = toString
2587	__repr__ = toString
2588
2589
2590	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2591	### ASTFactory ###
2592	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2593
2594	class ASTFactory(object):
2595	def __init__(self,table=None):
2596	self._class = None
2597	self._classmap = ifelse(table,table,None)
2598
2599	def create(self,*args):
2600	if not args:
2601	return self.create(INVALID_TYPE)
2602
2603	arg0 = args[0]
2604	arg1 = None
2605	arg2 = None
2606
2607	try:
2608	arg1 = args[1]
2609	arg2 = args[2]
2610	except:
2611	pass
2612
2613	# ctor(int)
2614	if isinstance(arg0,int) and not arg2:
2615	### get class for 'self' type
2616	c = self.getASTNodeType(arg0)
2617	t = self.create(c)
2618	if t:
2619	t.initialize(arg0, ifelse(arg1,arg1,""))
2620	return t
2621
2622	# ctor(int,something)
2623	if isinstance(arg0,int) and arg2:
2624	t = self.create(arg2)
2625	if t:
2626	t.initialize(arg0,arg1)
2627	return t
2628
2629	# ctor(AST)
2630	if isinstance(arg0,AST):
2631	t = self.create(arg0.getType())
2632	if t:
2633	t.initialize(arg0)
2634	return t
2635
2636	# ctor(token)
2637	if isinstance(arg0,Token) and not arg1:
2638	ttype = arg0.getType()
2639	assert isinstance(ttype,int)
2640	t = self.create(ttype)
2641	if t:
2642	t.initialize(arg0)
2643	return t
2644
2645	# ctor(token,class)
2646	if isinstance(arg0,Token) and arg1:
2647	assert isinstance(arg1,type)
2648	assert issubclass(arg1,AST)
2649	# this creates instance of 'arg1' using 'arg0' as
2650	# argument. Wow, that's magic!
2651	t = arg1(arg0)
2652	assert t and isinstance(t,AST)
2653	return t
2654
2655	# ctor(class)
2656	if isinstance(arg0,type):
2657	### next statement creates instance of type (!)
2658	t = arg0()
2659	assert isinstance(t,AST)
2660	return t
2661
2662
2663	def setASTNodeClass(self,className=None):
2664	if not className:
2665	return
2666	assert isinstance(className,type)
2667	assert issubclass(className,AST)
2668	self._class = className
2669
2670	### kind of misnomer - use setASTNodeClass instead.
2671	setASTNodeType = setASTNodeClass
2672
2673	def getASTNodeClass(self):
2674	return self._class
2675
2676
2677
2678	def getTokenTypeToASTClassMap(self):
2679	return self._classmap
2680
2681	def setTokenTypeToASTClassMap(self,amap):
2682	self._classmap = amap
2683
2684	def error(self, e):
2685	import sys
2686	print >> sys.stderr, e
2687
2688	def setTokenTypeASTNodeType(self, tokenType, className):
2689	"""
2690	Specify a mapping between a token type and a (AST) class.
2691	"""
2692	if not self._classmap:
2693	self._classmap = {}
2694
2695	if not className:
2696	try:
2697	del self._classmap[tokenType]
2698	except:
2699	pass
2700	else:
2701	### here we should also perform actions to ensure that
2702	### a. class can be loaded
2703	### b. class is a subclass of AST
2704	###
2705	assert isinstance(className,type)
2706	assert issubclass(className,AST) ## a & b
2707	### enter the class
2708	self._classmap[tokenType] = className
2709
2710	def getASTNodeType(self,tokenType):
2711	"""
2712	For a given token type return the AST node type. First we
2713	lookup a mapping table, second we try _class
2714	and finally we resolve to "antlr.CommonAST".
2715	"""
2716
2717	# first
2718	if self._classmap:
2719	try:
2720	c = self._classmap[tokenType]
2721	if c:
2722	return c
2723	except:
2724	pass
2725	# second
2726	if self._class:
2727	return self._class
2728
2729	# default
2730	return CommonAST
2731
2732	### methods that have been moved to file scope - just listed
2733	### here to be somewhat consistent with original API
2734	def dup(self,t):
2735	return antlr.dup(t,self)
2736
2737	def dupList(self,t):
2738	return antlr.dupList(t,self)
2739
2740	def dupTree(self,t):
2741	return antlr.dupTree(t,self)
2742
2743	### methods moved to other classes
2744	### 1. makeASTRoot -> Parser
2745	### 2. addASTChild -> Parser
2746
2747	### non-standard: create alias for longish method name
2748	maptype = setTokenTypeASTNodeType
2749
2750	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2751	### ASTVisitor ###
2752	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2753
2754	class ASTVisitor(object):
2755	def __init__(self,*args):
2756	pass
2757
2758	def visit(self,ast):
2759	pass
2760
2761	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2762	### static methods and variables ###
2763	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###
2764
2765	ASTNULL = ASTNULLType()
2766
2767	### wh: moved from ASTFactory as there's nothing ASTFactory-specific
2768	### in this method.
2769	def make(*nodes):
2770	if not nodes:
2771	return None
2772
2773	for i in xrange(0,len(nodes)):
2774	node = nodes[i]
2775	if node:
2776	assert isinstance(node,AST)
2777
2778	root = nodes[0]
2779	tail = None
2780	if root:
2781	root.setFirstChild(None)
2782
2783	for i in xrange(1,len(nodes)):
2784	if not nodes[i]:
2785	continue
2786	if not root:
2787	root = tail = nodes[i]
2788	elif not tail:
2789	root.setFirstChild(nodes[i])
2790	tail = root.getFirstChild()
2791	else:
2792	tail.setNextSibling(nodes[i])
2793	tail = tail.getNextSibling()
2794
2795	### Chase tail to last sibling
2796	while tail.getNextSibling():
2797	tail = tail.getNextSibling()
2798	return root
2799
2800	def dup(t,factory):
2801	if not t:
2802	return None
2803
2804	if factory:
2805	dup_t = factory.create(t.__class__)
2806	else:
2807	raise TypeError("dup function requires ASTFactory argument")
2808	dup_t.initialize(t)
2809	return dup_t
2810
2811	def dupList(t,factory):
2812	result = dupTree(t,factory)
2813	nt = result
2814	while t:
2815	## for each sibling of the root
2816	t = t.getNextSibling()
2817	nt.setNextSibling(dupTree(t,factory))
2818	nt = nt.getNextSibling()
2819	return result
2820
2821	def dupTree(t,factory):
2822	result = dup(t,factory)
2823	if t:
2824	result.setFirstChild(dupList(t.getFirstChild(),factory))
2825	return result
2826
2827	###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
2828	### $Id: antlr.py,v 1.1.1.1 2005/02/02 10:24:36 geronimo Exp $
2829
2830	# Local Variables: ***
2831	# mode: python ***
2832	# py-indent-offset: 4 ***
2833	# End: ***

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: trunk/yao/share/antlr-2.7.7/lib/python/antlr/antlr.py @ 1

Download in other formats: