[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Help-smalltalk] [PATCH 3/3] stinst: rewrite parsing of literal arrays a
|
From: |
Paolo Bonzini |
|
Subject: |
[Help-smalltalk] [PATCH 3/3] stinst: rewrite parsing of literal arrays and fix compilation of ##(...) within arrays |
|
Date: |
Tue, 7 Jan 2014 12:35:57 +0100 |
packages/stinst/parser:
2014-01-07 Paolo Bonzini <address@hidden>
* RBParser.st: Move parsing of literal arrays to RBParser. The
scanner now returns just a RBLiteralArrayStartToken.
* RBToken.st: Add RBLiteralArrayStartToken.
* RewriteTests.st: Do not inspect the RBLiteralNode that is
returned in TestFormat>>#testArrayRewrite, this will be covered
by the parser tests.
* RBScannerTests.st: Add tests for ##(...).
* RBParserTests.st: Add more literal array tests.
---
packages/stinst/parser/ChangeLog | 11 +++
packages/stinst/parser/RBParser.st | 132 ++++++++++++++++++++-----------
packages/stinst/parser/RBParserTests.st | 42 ++++++++--
packages/stinst/parser/RBScannerTests.st | 53 +++++++++++++
packages/stinst/parser/RBToken.st | 23 ++++++
packages/stinst/parser/RewriteTests.st | 2 -
6 files changed, 208 insertions(+), 55 deletions(-)
diff --git a/packages/stinst/parser/ChangeLog b/packages/stinst/parser/ChangeLog
index a1c35cb..9f44c4d 100644
--- a/packages/stinst/parser/ChangeLog
+++ b/packages/stinst/parser/ChangeLog
@@ -1,5 +1,16 @@
2014-01-07 Paolo Bonzini <address@hidden>
+ * RBParser.st: Move parsing of literal arrays to RBParser. The
+ scanner now returns just a RBLiteralArrayStartToken.
+ * RBToken.st: Add RBLiteralArrayStartToken.
+ * RewriteTests.st: Do not inspect the RBLiteralNode that is
+ returned in TestFormat>>#testArrayRewrite, this will be covered
+ by the parser tests.
+ * RBScannerTests.st: Add tests for ##(...).
+ * RBParserTests.st: Add more literal array tests.
+
+2014-01-07 Paolo Bonzini <address@hidden>
+
* RBFormatter.st: Add #acceptLiteralArrayNode:.
* RBParseNodes.st: Implement RBLiteralArrayNode.
diff --git a/packages/stinst/parser/RBParser.st
b/packages/stinst/parser/RBParser.st
index b953762..99b8d2b 100644
--- a/packages/stinst/parser/RBParser.st
+++ b/packages/stinst/parser/RBParser.st
@@ -538,9 +538,8 @@ Object subclass: RBParser [
parseNegatedNumber [
<category: 'private-parsing'>
| token |
- self step.
token := currentToken.
- (token value respondsTo: #negated) ifFalse: [
+ token value isNumber ifFalse: [
^self parserError: 'Number expected' ].
token value negative ifTrue: [
^self parserError: 'Positive number expected' ].
@@ -562,17 +561,96 @@ Object subclass: RBParser [
currentToken isIdentifier ifTrue: [^self parsePrimitiveIdentifier].
currentToken isLiteral ifTrue: [^self parsePrimitiveLiteral].
(currentToken isBinary and: [ currentToken value == #- ])
- ifTrue: [^self parseNegatedNumber].
+ ifTrue: [^self step; parseNegatedNumber].
currentToken isSpecial
ifTrue:
[currentToken value == $[ ifTrue: [^self parseBlock].
currentToken value == ${ ifTrue: [^self parseArrayConstructor].
currentToken value == $( ifTrue: [^self
parseParenthesizedExpression]].
+ currentToken isLiteralArrayStart ifTrue: [^self parseLiteralArray].
currentToken isPatternBlock ifTrue: [^self parsePatternBlock].
currentToken isOptimized ifTrue: [^self parseOptimizedExpression].
self parserError: 'Variable expected'
]
+ parseLiteralArray [
+ <category: 'private-parsing'>
+ | arrayStream start stop |
+ arrayStream := WriteStream on: (Array new: 10).
+ start := currentToken start.
+ self step.
+
+ [currentToken isSpecial and: [currentToken value == $)]]
+ whileFalse:
+ [arrayStream nextPut: self parseLiteralArrayParts].
+ stop := currentToken stop.
+ self step.
+ ^RBLiteralArrayNode
+ left: start
+ nodes: arrayStream contents
+ right: stop
+ ]
+
+ parseByteArray [
+ "FIXME: it's ugly that this is both here for #( [1 2 3] )
+ and in RBScanner for #[1 2 3]."
+ <category: 'private-parsing'>
+ | byteStream number start stop |
+ byteStream := WriteStream on: (ByteArray new: 100).
+ start := currentToken start.
+
+ [self step.
+ currentToken isLiteral] whileTrue:
+ [number := currentToken value.
+ (number isInteger and: [number between: 0 and: 255])
+ ifFalse: [self scannerError: 'Expecting 8-bit integer'].
+ byteStream nextPut: number].
+ (currentToken isSpecial and: [ currentToken value == $] ])
+ ifFalse: [self scannerError: ''']'' expected'].
+ stop := currentToken stop.
+ self step. "]"
+ ^RBLiteralNode literalToken: (RBLiteralToken
+ value: byteStream contents
+ start: start
+ stop: stop)
+ ]
+
+ parseLiteralArrayParts [
+ <category: 'private-parsing'>
+ currentToken isLiteral ifTrue: [^self parsePrimitiveLiteral].
+ currentToken isIdentifier
+ ifTrue:
+ [| token value |
+ token := currentToken.
+ value := self parsePrimitiveIdentifier.
+ value isVariable ifTrue: [
+ value := RBLiteralNode literalToken: (RBLiteralToken
+ value: token value asSymbol
+ start: token start
+ stop: token stop)].
+ ^value].
+ currentToken isBinary
+ ifTrue: [
+ | token |
+ token := currentToken.
+ self step.
+ (token value == #- and: [token stop + 1 = currentToken start
and: [
+ currentToken isLiteral and: [
+ currentToken value isNumber and: [
+ currentToken value positive ]]]]) ifTrue: [^self
parseNegatedNumber].
+ ^RBLiteralNode literalToken: (RBLiteralToken
+ value: token value
+ start: token start
+ stop: token stop)].
+ currentToken isSpecial
+ ifTrue:
+ [currentToken value == $[ ifTrue: [^self parseByteArray].
+ currentToken value == $( ifTrue: [^self parseLiteralArray]].
+ currentToken isLiteralArrayStart ifTrue: [^self parseLiteralArray].
+ currentToken isOptimized ifTrue: [^self parseOptimizedExpression].
+ ^self parserError: 'literal array element expected'
+ ]
+
parseResourceTag [
<category: 'private-parsing'>
| start |
@@ -1206,57 +1284,15 @@ Stream subclass: RBScanner [
characterType == #binary
ifTrue: [^(self scanBinary: RBLiteralToken) stop: self
previousStepPosition].
currentCharacter == $' ifTrue: [^self scanStringSymbol].
- currentCharacter == $( ifTrue: [^self scanLiteralArray].
+ currentCharacter == $( ifTrue: [
+ self step.
+ ^RBLiteralArrayStartToken start: tokenStart].
currentCharacter == $[ ifTrue: [^self scanByteArray].
currentCharacter == ${ ifTrue: [^self scanQualifier].
currentCharacter == $# ifTrue: [^self scanExtendedLiterals].
self scannerError: 'Expecting a literal type'
]
- scanLiteralArray [
- <category: 'private-scanning'>
- | arrayStream start |
- arrayStream := WriteStream on: (Array new: 10).
- self step.
- start := tokenStart.
-
- [self stripSeparators.
- tokenStart := stream position.
- currentCharacter == $)]
- whileFalse:
- [arrayStream nextPut: self scanLiteralArrayParts.
- buffer reset].
- self step.
- ^RBLiteralToken
- value: arrayStream contents
- start: start
- stop: self previousStepPosition
- ]
-
- scanLiteralArrayParts [
- <category: 'private-scanning'>
- currentCharacter == $# ifTrue: [^self scanLiteral].
- characterType == #alphabetic
- ifTrue:
- [| token value |
- token := self scanSymbol.
- value := token value.
- value == #nil ifTrue: [token value: nil].
- value == #true ifTrue: [token value: true].
- value == #false ifTrue: [token value: false].
- ^token].
- (characterType == #digit
- or: [currentCharacter == $- and: [(self classify: stream peek) ==
#digit]])
- ifTrue: [^self scanNumber].
- characterType == #binary
- ifTrue: [^(self scanBinary: RBLiteralToken) stop: self
previousStepPosition].
- currentCharacter == $' ifTrue: [^self scanLiteralString].
- currentCharacter == $$ ifTrue: [^self scanLiteralCharacter].
- currentCharacter == $( ifTrue: [^self scanLiteralArray].
- currentCharacter == $[ ifTrue: [^self scanByteArray].
- ^self scannerError: 'Unknown character in literal array'
- ]
-
scanLiteralCharacter [
<category: 'private-scanning'>
| token value char tokenStop |
diff --git a/packages/stinst/parser/RBParserTests.st
b/packages/stinst/parser/RBParserTests.st
index f569821..f0df0d1 100644
--- a/packages/stinst/parser/RBParserTests.st
+++ b/packages/stinst/parser/RBParserTests.st
@@ -58,14 +58,46 @@ TestCase subclass: TestParser [
]
testLiteralArrayParsing [
- | node |
+ | node nested |
node := RBParser parseExpression: '#(-3 -2 -16r1)'.
- self assert: node value first = -3.
- self assert: node value second = -2.
- self assert: node value third = -1.
+ self assert: node nodes first value = -3.
+ self assert: node nodes second value = -2.
+ self assert: node nodes third value = -1.
node := RBParser parseExpression: '#(16r-1)'.
- self assert: node value first = -1.
+ self assert: node nodes first value = -1.
+
+ node := RBParser parseExpression: '#(- 2)'.
+ self assert: node nodes first value = #-.
+ self assert: node nodes second value = 2.
+
+ node := RBParser parseExpression: '#(true false nil foo)'.
+ self assert: node nodes first value = true.
+ self assert: node nodes second value = false.
+ self assert: node nodes third value = nil.
+ self assert: node nodes fourth value = #foo.
+
+ node := RBParser parseExpression: '#(1 #[2 3] 4)'.
+ self assert: node nodes first value = 1.
+ self assert: node nodes second value = #[2 3].
+ self assert: node nodes third value = 4.
+
+ node := RBParser parseExpression: '#(1 [2 3] 4)'.
+ self assert: node nodes first value = 1.
+ self assert: node nodes second value = #[2 3].
+ self assert: node nodes third value = 4.
+
+ node := RBParser parseExpression: '#(1 (2 3) 4)'.
+ self assert: node nodes first value = 1.
+ nested := node nodes second.
+ self assert: nested nodes first value = 2.
+ self assert: nested nodes second value = 3.
+ self assert: node nodes third value = 4.
+
+ node := RBParser parseExpression: '#(1 ##(2/3) 4)'.
+ self assert: node nodes first value = 1.
+ self assert: node nodes second class == RBOptimizedNode.
+ self assert: node nodes third value = 4.
]
]
diff --git a/packages/stinst/parser/RBScannerTests.st
b/packages/stinst/parser/RBScannerTests.st
index 41ecfd9..451e876 100644
--- a/packages/stinst/parser/RBScannerTests.st
+++ b/packages/stinst/parser/RBScannerTests.st
@@ -46,5 +46,58 @@ TestCase subclass: TestScanner [
num := scanner next.
self assert: num value = 3.
]
+
+ testEmbeddedCompiletimeConstant [
+ | scanner token value |
+ scanner := RBScanner on: '#(##(1))' readStream.
+
+ "Token is the literal"
+ token := scanner next.
+ self assert: token isLiteralArrayStart.
+
+ "Token is for optimized code"
+ token := scanner next.
+ self assert: token isOptimized.
+
+ "Token is the '1'"
+ token := scanner next.
+ self assert: token isLiteral.
+ self assert: token value equals: 1.
+
+ "Token is the ')'"
+ token := scanner next.
+ self assert: token isSpecial.
+ self assert: token value equals: $).
+
+ "Token is the ')'"
+ token := scanner next.
+ self assert: token isSpecial.
+ self assert: token value equals: $).
+
+ "And we are at the end"
+ self assert: scanner atEnd.
+ ]
+
+ testDirectCompiletimeConstant [
+ | scanner token |
+ scanner := RBScanner on: '##(1)' readStream.
+
+ "Token is for optimized code"
+ token := scanner next.
+ self assert: token isOptimized.
+
+ "Token is the '1'"
+ token := scanner next.
+ self assert: token isLiteral.
+ self assert: token value equals: 1.
+
+ "Token is the ')'"
+ token := scanner next.
+ self assert: token isSpecial.
+ self assert: token value equals: $).
+
+ "And we are at the end"
+ self assert: scanner atEnd.
+ ]
]
diff --git a/packages/stinst/parser/RBToken.st
b/packages/stinst/parser/RBToken.st
index 399a6db..7c22c9c 100644
--- a/packages/stinst/parser/RBToken.st
+++ b/packages/stinst/parser/RBToken.st
@@ -95,6 +95,11 @@ Object subclass: RBToken [
^false
]
+ isLiteralArrayStart [
+ <category: 'testing'>
+ ^false
+ ]
+
isOptimized [
<category: 'testing'>
^false
@@ -391,6 +396,24 @@ RBValueToken subclass: RBKeywordToken [
+RBToken subclass: RBLiteralArrayStartToken [
+
+ <category: 'Refactory-Scanner'>
+ <comment: nil>
+
+ isLiteralArrayStart [
+ <category: 'testing'>
+ ^true
+ ]
+
+ length [
+ <category: 'testing'>
+ ^2
+ ]
+]
+
+
+
RBToken subclass: RBOptimizedToken [
<category: 'Refactory-Scanner'>
diff --git a/packages/stinst/parser/RewriteTests.st
b/packages/stinst/parser/RewriteTests.st
index 322ac38..05130a4 100644
--- a/packages/stinst/parser/RewriteTests.st
+++ b/packages/stinst/parser/RewriteTests.st
@@ -273,8 +273,6 @@ TestCase subclass: TestFormat [
testArrayRewrite [
| inp res |
inp := RBParser parseExpression: '#(16r01 2r01 16rFF )'.
- self assert: inp value = (Array with: 1 with: 1 with: 255).
-
res := RBFormatter new formatAll: (Array with: inp).
self assert: res = '#(16r01 2r01 16rFF)'.
]
--
1.8.4.2