root
/
spring-framework
mirror of https://github.com/spring-projects/spring-framework.git
1
0
Fork 0
Code Issues Actions 6 Packages Projects Releases Wiki Activity

Tokenizer uses single process step and length-based exhaustion check 

Issue: SPR-16032

(cherry picked from commit 03b6828)
This commit is contained in:
Juergen Hoeller 2017-10-04 12:23:16 +02:00
parent 1a1d767d59
commit d0d4421aea
3 changed files with 98 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spring-expression/src/main/java/org/springframework/expression/spel/standard/InternalSpelExpressionParser.java
View File

@ -120,8 +120,7 @@ class InternalSpelExpressionParser extends TemplateAwareExpressionParser {
try {
this.expressionString = expressionString;
Tokenizer tokenizer = new Tokenizer(expressionString);
tokenizer.process();
this.tokenStream = tokenizer.getTokens();
this.tokenStream = tokenizer.process();
this.tokenStreamLength = this.tokenStream.size();
this.tokenStreamPointer = 0;
this.constructedNodes.clear();

119
spring-expression/src/main/java/org/springframework/expression/spel/standard/Tokenizer.java
View File

@ -28,14 +28,15 @@ import org.springframework.expression.spel.SpelParseException;
* Lex some input data into a stream of tokens that can then be parsed.
*
* @author Andy Clement
* @author Juergen Hoeller
* @author Phillip Webb
* @since 3.0
*/
class Tokenizer {
// if this is changed, it must remain sorted
private static final String[] ALTERNATIVE_OPERATOR_NAMES = { "DIV", "EQ", "GE", "GT",
"LE", "LT", "MOD", "NE", "NOT" };
// If this gets changed, it must remain sorted...
private static final String[] ALTERNATIVE_OPERATOR_NAMES =
{"DIV", "EQ", "GE", "GT", "LE", "LT", "MOD", "NE", "NOT"};
private static final byte FLAGS[] = new byte[256];
@ -64,29 +65,28 @@ class Tokenizer {
}
String expressionString;
private String expressionString;
char[] toProcess;
private char[] charsToProcess;
int pos;
private int pos;
int max;
private int max;
List<Token> tokens = new ArrayList<Token>();
private List<Token> tokens = new ArrayList<Token>();
public Tokenizer(String inputData) {
this.expressionString = inputData;
this.toProcess = (inputData + "\0").toCharArray();
this.max = this.toProcess.length;
this.charsToProcess = (inputData + "\0").toCharArray();
this.max = this.charsToProcess.length;
this.pos = 0;
process();
}
public void process() {
public List<Token> process() {
while (this.pos < this.max) {
char ch = this.toProcess[this.pos];
char ch = this.charsToProcess[this.pos];
if (isAlphabetic(ch)) {
lexIdentifier();
}
@ -190,9 +190,7 @@ class Tokenizer {
break;
case '|':
if (!isTwoCharToken(TokenKind.SYMBOLIC_OR)) {
throw new InternalParseException(new SpelParseException(
this.expressionString, this.pos, SpelMessage.MISSING_CHARACTER,
"|"));
raiseParseException(this.pos, SpelMessage.MISSING_CHARACTER, "|");
}
pushPairToken(TokenKind.SYMBOLIC_OR);
break;
@ -261,41 +259,38 @@ class Tokenizer {
break;
case 0:
// hit sentinel at end of value
this.pos++; // will take us to the end
this.pos++; // will take us to the end
break;
case '\\':
throw new InternalParseException(
new SpelParseException(this.expressionString, this.pos, SpelMessage.UNEXPECTED_ESCAPE_CHAR));
raiseParseException(this.pos, SpelMessage.UNEXPECTED_ESCAPE_CHAR);
break;
default:
throw new IllegalStateException("Cannot handle (" + Integer.valueOf(ch) + ") '" + ch + "'");
}
}
}
}
public List<Token> getTokens() {
return this.tokens;
}
// STRING_LITERAL: '\''! (APOS|~'\'')* '\''!;
private void lexQuotedStringLiteral() {
int start = this.pos;
boolean terminated = false;
while (!terminated) {
this.pos++;
char ch = this.toProcess[this.pos];
char ch = this.charsToProcess[this.pos];
if (ch == '\'') {
// may not be the end if the char after is also a '
if (this.toProcess[this.pos + 1] == '\'') {
this.pos++; // skip over that too, and continue
if (this.charsToProcess[this.pos + 1] == '\'') {
this.pos++; // skip over that too, and continue
}
else {
terminated = true;
}
}
if (ch == 0) {
throw new InternalParseException(new SpelParseException(this.expressionString, start,
SpelMessage.NON_TERMINATING_QUOTED_STRING));
if (isExhausted()) {
raiseParseException(start, SpelMessage.NON_TERMINATING_QUOTED_STRING);
}
}
this.pos++;
@ -308,19 +303,18 @@ class Tokenizer {
boolean terminated = false;
while (!terminated) {
this.pos++;
char ch = this.toProcess[this.pos];
char ch = this.charsToProcess[this.pos];
if (ch == '"') {
// may not be the end if the char after is also a "
if (this.toProcess[this.pos + 1] == '"') {
this.pos++; // skip over that too, and continue
if (this.charsToProcess[this.pos + 1] == '"') {
this.pos++; // skip over that too, and continue
}
else {
terminated = true;
}
}
if (ch == 0) {
throw new InternalParseException(new SpelParseException(this.expressionString,
start, SpelMessage.NON_TERMINATING_DOUBLE_QUOTED_STRING));
if (isExhausted()) {
raiseParseException(start, SpelMessage.NON_TERMINATING_DOUBLE_QUOTED_STRING);
}
}
this.pos++;
@ -346,7 +340,7 @@ class Tokenizer {
private void lexNumericLiteral(boolean firstCharIsZero) {
boolean isReal = false;
int start = this.pos;
char ch = this.toProcess[this.pos + 1];
char ch = this.charsToProcess[this.pos + 1];
boolean isHex = ch == 'x' || ch == 'X';
// deal with hexadecimal
@ -355,7 +349,7 @@ class Tokenizer {
do {
this.pos++;
}
while (isHexadecimalDigit(this.toProcess[this.pos]));
while (isHexadecimalDigit(this.charsToProcess[this.pos]));
if (isChar('L', 'l')) {
pushHexIntToken(subarray(start + 2, this.pos), true, start, this.pos);
this.pos++;
@ -372,10 +366,10 @@ class Tokenizer {
do {
this.pos++;
}
while (isDigit(this.toProcess[this.pos]));
while (isDigit(this.charsToProcess[this.pos]));
// a '.' indicates this number is a real
ch = this.toProcess[this.pos];
ch = this.charsToProcess[this.pos];
if (ch == '.') {
isReal = true;
int dotpos = this.pos;
@ -383,7 +377,7 @@ class Tokenizer {
do {
this.pos++;
}
while (isDigit(this.toProcess[this.pos]));
while (isDigit(this.charsToProcess[this.pos]));
if (this.pos == dotpos + 1) {
// the number is something like '3.'. It is really an int but may be
// part of something like '3.toString()'. In this case process it as
@ -398,19 +392,18 @@ class Tokenizer {
// Now there may or may not be an exponent
// is it a long ?
// Is it a long ?
if (isChar('L', 'l')) {
if (isReal) { // 3.4L - not allowed
throw new InternalParseException(new SpelParseException(this.expressionString,
start, SpelMessage.REAL_CANNOT_BE_LONG));
if (isReal) { // 3.4L - not allowed
raiseParseException(start, SpelMessage.REAL_CANNOT_BE_LONG);
}
pushIntToken(subarray(start, endOfNumber), true, start, endOfNumber);
this.pos++;
}
else if (isExponentChar(this.toProcess[this.pos])) {
isReal = true; // if it wasn't before, it is now
else if (isExponentChar(this.charsToProcess[this.pos])) {
isReal = true; // if it wasn't before, it is now
this.pos++;
char possibleSign = this.toProcess[this.pos];
char possibleSign = this.charsToProcess[this.pos];
if (isSign(possibleSign)) {
this.pos++;
}
@ -419,19 +412,19 @@ class Tokenizer {
do {
this.pos++;
}
while (isDigit(this.toProcess[this.pos]));
while (isDigit(this.charsToProcess[this.pos]));
boolean isFloat = false;
if (isFloatSuffix(this.toProcess[this.pos])) {
if (isFloatSuffix(this.charsToProcess[this.pos])) {
isFloat = true;
endOfNumber = ++this.pos;
}
else if (isDoubleSuffix(this.toProcess[this.pos])) {
else if (isDoubleSuffix(this.charsToProcess[this.pos])) {
endOfNumber = ++this.pos;
}
pushRealToken(subarray(start, this.pos), isFloat, start, this.pos);
}
else {
ch = this.toProcess[this.pos];
ch = this.charsToProcess[this.pos];
boolean isFloat = false;
if (isFloatSuffix(ch)) {
isReal = true;
@ -456,7 +449,7 @@ class Tokenizer {
do {
this.pos++;
}
while (isIdentifier(this.toProcess[this.pos]));
while (isIdentifier(this.charsToProcess[this.pos]));
char[] subarray = subarray(start, this.pos);
// Check if this is the alternative (textual) representation of an operator (see
@ -484,14 +477,10 @@ class Tokenizer {
private void pushHexIntToken(char[] data, boolean isLong, int start, int end) {
if (data.length == 0) {
if (isLong) {
throw new InternalParseException(new SpelParseException(this.expressionString,
start, SpelMessage.NOT_A_LONG, this.expressionString.substring(start,
end + 1)));
raiseParseException(start, SpelMessage.NOT_A_LONG, this.expressionString.substring(start, end + 1));
}
else {
throw new InternalParseException(new SpelParseException(this.expressionString,
start, SpelMessage.NOT_AN_INTEGER, this.expressionString.substring(
start, end)));
raiseParseException(start, SpelMessage.NOT_AN_INTEGER, this.expressionString.substring(start, end));
}
}
if (isLong) {
@ -513,7 +502,7 @@ class Tokenizer {
private char[] subarray(int start, int end) {
char[] result = new char[end - start];
System.arraycopy(this.toProcess, start, result, 0, end - start);
System.arraycopy(this.charsToProcess, start, result, 0, end - start);
return result;
}
@ -522,8 +511,8 @@ class Tokenizer {
*/
private boolean isTwoCharToken(TokenKind kind) {
return (kind.tokenChars.length == 2 &&
this.toProcess[this.pos] == kind.tokenChars[0] &&
this.toProcess[this.pos + 1] == kind.tokenChars[1]);
this.charsToProcess[this.pos] == kind.tokenChars[0] &&
this.charsToProcess[this.pos + 1] == kind.tokenChars[1]);
}
/**
@ -552,7 +541,7 @@ class Tokenizer {
}
private boolean isChar(char a, char b) {
char ch = this.toProcess[this.pos];
char ch = this.charsToProcess[this.pos];
return ch == a || ch == b;
}
@ -593,4 +582,12 @@ class Tokenizer {
return (FLAGS[ch] & IS_HEXDIGIT) != 0;
}
private boolean isExhausted() {
return (this.pos == this.max - 1);
}
private void raiseParseException(int start, SpelMessage msg, Object... inserts) {
throw new InternalParseException(new SpelParseException(this.expressionString, start, msg, inserts));
}
}

66
spring-expression/src/test/java/org/springframework/expression/spel/SpelReproTests.java
View File

@ -59,6 +59,7 @@ import org.springframework.expression.spel.support.ReflectivePropertyAccessor;
import org.springframework.expression.spel.support.StandardEvaluationContext;
import org.springframework.expression.spel.support.StandardTypeLocator;
import org.springframework.expression.spel.testresources.le.div.mod.reserved.Reserver;
import org.springframework.util.ObjectUtils;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
@ -228,7 +229,7 @@ public class SpelReproTests extends AbstractExpressionTests {
@Test
public void SPR5804() throws Exception {
Map<String, String> m = new HashMap<String, String>();
Map<String, String> m = new HashMap<>();
m.put("foo", "bar");
StandardEvaluationContext eContext = new StandardEvaluationContext(m); // root is a map instance
eContext.addPropertyAccessor(new MapAccessor());
@ -550,7 +551,7 @@ public class SpelReproTests extends AbstractExpressionTests {
public String floo = "bar";
public XX() {
m = new HashMap<String, String>();
m = new HashMap<>();
m.put("$foo", "wibble");
m.put("bar", "siddle");
}
@ -579,7 +580,7 @@ public class SpelReproTests extends AbstractExpressionTests {
static class Holder {
public Map<String, String> map = new HashMap<String, String>();
public Map<String, String> map = new HashMap<>();
}
@ -783,9 +784,9 @@ public class SpelReproTests extends AbstractExpressionTests {
@Test
public void mapOfMap_SPR7244() throws Exception {
Map<String, Object> map = new LinkedHashMap<String, Object>();
Map<String, Object> map = new LinkedHashMap<>();
map.put("uri", "http:");
Map<String, String> nameMap = new LinkedHashMap<String, String>();
Map<String, String> nameMap = new LinkedHashMap<>();
nameMap.put("givenName", "Arthur");
map.put("value", nameMap);
@ -849,11 +850,11 @@ public class SpelReproTests extends AbstractExpressionTests {
public Map<String, String> ms;
C() {
ls = new ArrayList<String>();
ls = new ArrayList<>();
ls.add("abc");
ls.add("def");
as = new String[] { "abc", "def" };
ms = new HashMap<String, String>();
ms = new HashMap<>();
ms.put("abc", "xyz");
ms.put("def", "pqr");
}
@ -877,7 +878,7 @@ public class SpelReproTests extends AbstractExpressionTests {
@Test
public void greaterThanWithNulls_SPR7840() throws Exception {
List<D> list = new ArrayList<D>();
List<D> list = new ArrayList<>();
list.add(new D("aaa"));
list.add(new D("bbb"));
list.add(new D(null));
@ -937,7 +938,7 @@ public class SpelReproTests extends AbstractExpressionTests {
EvaluationContext emptyEvalContext = new StandardEvaluationContext();
List<TypeDescriptor> args = new ArrayList<TypeDescriptor>();
List<TypeDescriptor> args = new ArrayList<>();
args.add(TypeDescriptor.forObject(new Integer(42)));
ConversionPriority1 target = new ConversionPriority1();
@ -977,7 +978,7 @@ public class SpelReproTests extends AbstractExpressionTests {
WideningPrimitiveConversion target = new WideningPrimitiveConversion();
EvaluationContext emptyEvalContext = new StandardEvaluationContext();
List<TypeDescriptor> args = new ArrayList<TypeDescriptor>();
List<TypeDescriptor> args = new ArrayList<>();
args.add(TypeDescriptor.forObject(INTEGER_VALUE));
MethodExecutor me = new ReflectiveMethodResolver(true).resolve(emptyEvalContext, target, "getX", args);
@ -990,10 +991,10 @@ public class SpelReproTests extends AbstractExpressionTests {
@Test
public void varargsAndPrimitives_SPR8174() throws Exception {
EvaluationContext emptyEvalContext = new StandardEvaluationContext();
List<TypeDescriptor> args = new ArrayList<TypeDescriptor>();
List<TypeDescriptor> args = new ArrayList<>();
args.add(TypeDescriptor.forObject(34L));
ReflectionUtil<Integer> ru = new ReflectionUtil<Integer>();
ReflectionUtil<Integer> ru = new ReflectionUtil<>();
MethodExecutor me = new ReflectiveMethodResolver().resolve(emptyEvalContext, ru, "methodToCall", args);
args.set(0, TypeDescriptor.forObject(23));
@ -1113,7 +1114,7 @@ public class SpelReproTests extends AbstractExpressionTests {
public int DIV = 1;
public int div = 3;
public Map<String, String> m = new HashMap<String, String>();
public Map<String, String> m = new HashMap<>();
Reserver() {
m.put("NE", "xyz");
@ -1229,10 +1230,10 @@ public class SpelReproTests extends AbstractExpressionTests {
class ContextObject {
public Map<String, String> firstContext = new HashMap<String, String>();
public Map<String, String> secondContext = new HashMap<String, String>();
public Map<String, String> thirdContext = new HashMap<String, String>();
public Map<String, String> fourthContext = new HashMap<String, String>();
public Map<String, String> firstContext = new HashMap<>();
public Map<String, String> secondContext = new HashMap<>();
public Map<String, String> thirdContext = new HashMap<>();
public Map<String, String> fourthContext = new HashMap<>();
public ContextObject() {
firstContext.put("shouldBeFirst", "first");
@ -1276,7 +1277,7 @@ public class SpelReproTests extends AbstractExpressionTests {
public void customStaticFunctions_SPR9038() {
ExpressionParser parser = new SpelExpressionParser();
StandardEvaluationContext context = new StandardEvaluationContext();
List<MethodResolver> methodResolvers = new ArrayList<MethodResolver>();
List<MethodResolver> methodResolvers = new ArrayList<>();
methodResolvers.add(new ReflectiveMethodResolver() {
@Override
protected Method[] getMethods(Class<?> type) {
@ -1768,6 +1769,7 @@ public class SpelReproTests extends AbstractExpressionTests {
};
}
});
result = spel.getValue(context);
assertNotNull(result);
assertTrue(result.getClass().isArray());
@ -1802,7 +1804,7 @@ public class SpelReproTests extends AbstractExpressionTests {
public void SPR9194() {
TestClass2 one = new TestClass2("abc");
TestClass2 two = new TestClass2("abc");
Map<String, TestClass2> map = new HashMap<String, TestClass2>();
Map<String, TestClass2> map = new HashMap<>();
map.put("one", one);
map.put("two", two);
@ -1813,7 +1815,7 @@ public class SpelReproTests extends AbstractExpressionTests {
@Test
public void SPR11348() {
Collection<String> coll = new LinkedHashSet<String>();
Collection<String> coll = new LinkedHashSet<>();
coll.add("one");
coll.add("two");
coll = Collections.unmodifiableCollection(coll);
@ -1927,10 +1929,10 @@ public class SpelReproTests extends AbstractExpressionTests {
@Test
@SuppressWarnings("rawtypes")
public void SPR13055() throws Exception {
List<Map<String, Object>> myPayload = new ArrayList<Map<String, Object>>();
List<Map<String, Object>> myPayload = new ArrayList<>();
Map<String, Object> v1 = new HashMap<String, Object>();
Map<String, Object> v2 = new HashMap<String, Object>();
Map<String, Object> v1 = new HashMap<>();
Map<String, Object> v2 = new HashMap<>();
v1.put("test11", "test11");
v1.put("test12", "test12");
@ -1964,7 +1966,7 @@ public class SpelReproTests extends AbstractExpressionTests {
assertEquals("custard", expr.getValue(context));
expr = new SpelExpressionParser().parseRaw("&foo");
assertEquals("foo factory",expr.getValue(context));
try {
expr = new SpelExpressionParser().parseRaw("&@foo");
fail("Illegal syntax, error expected");
@ -1973,7 +1975,7 @@ public class SpelReproTests extends AbstractExpressionTests {
assertEquals(SpelMessage.INVALID_BEAN_REFERENCE,spe.getMessageCode());
assertEquals(0,spe.getPosition());
}
try {
expr = new SpelExpressionParser().parseRaw("@&foo");
fail("Illegal syntax, error expected");
@ -1981,7 +1983,7 @@ public class SpelReproTests extends AbstractExpressionTests {
catch (SpelParseException spe) {
assertEquals(SpelMessage.INVALID_BEAN_REFERENCE,spe.getMessageCode());
assertEquals(0,spe.getPosition());
}
}
}
@Test
@ -2094,6 +2096,16 @@ public class SpelReproTests extends AbstractExpressionTests {
assertEquals(Charset.forName("UTF-8"), result);
}
@Test
public void SPR16032() {
EvaluationContext context = new StandardEvaluationContext();
context.setVariable("str", "a\0b");
Expression ex = parser.parseExpression("#str?.split('\0')");
Object result = ex.getValue(context);
assertTrue(ObjectUtils.nullSafeEquals(result, new String[] {"a", "b"}));
}
public static class ListOf {
@ -2269,7 +2281,7 @@ public class SpelReproTests extends AbstractExpressionTests {
private String name;
private List<Item> children = new ArrayList<Item>();
private List<Item> children = new ArrayList<>();
public void setName(String name) {
this.name = name;