diff --git a/src/main/java/com/hubspot/jinjava/LegacyOverrides.java b/src/main/java/com/hubspot/jinjava/LegacyOverrides.java index 80684d81a..df532a2d2 100644 --- a/src/main/java/com/hubspot/jinjava/LegacyOverrides.java +++ b/src/main/java/com/hubspot/jinjava/LegacyOverrides.java @@ -9,11 +9,13 @@ public class LegacyOverrides { private final boolean evaluateMapKeys; private final boolean iterateOverMapKeys; private final boolean usePyishObjectMapper; + private final boolean whitespaceRequiredWithinTokens; private LegacyOverrides(Builder builder) { evaluateMapKeys = builder.evaluateMapKeys; iterateOverMapKeys = builder.iterateOverMapKeys; usePyishObjectMapper = builder.usePyishObjectMapper; + whitespaceRequiredWithinTokens = builder.whitespaceRequiredWithinTokens; } public static Builder newBuilder() { @@ -32,10 +34,15 @@ public boolean isUsePyishObjectMapper() { return usePyishObjectMapper; } + public boolean isWhitespaceRequiredWithinTokens() { + return whitespaceRequiredWithinTokens; + } + public static class Builder { private boolean evaluateMapKeys = false; private boolean iterateOverMapKeys = false; private boolean usePyishObjectMapper = false; + private boolean whitespaceRequiredWithinTokens = false; private Builder() {} @@ -47,7 +54,8 @@ public static Builder from(LegacyOverrides legacyOverrides) { return new Builder() .withEvaluateMapKeys(legacyOverrides.evaluateMapKeys) .withIterateOverMapKeys(legacyOverrides.iterateOverMapKeys) - .withUsePyishObjectMapper(legacyOverrides.usePyishObjectMapper); + .withUsePyishObjectMapper(legacyOverrides.usePyishObjectMapper) + .withWhitespaceRequiredWithinTokens(legacyOverrides.whitespaceRequiredWithinTokens); } public Builder withEvaluateMapKeys(boolean evaluateMapKeys) { @@ -64,5 +72,10 @@ public Builder withUsePyishObjectMapper(boolean usePyishObjectMapper) { this.usePyishObjectMapper = usePyishObjectMapper; return this; } + + public Builder withWhitespaceRequiredWithinTokens(boolean useWhitespace) { + this.whitespaceRequiredWithinTokens = useWhitespace; + return this; + } } } diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java index fc6ed02a2..78d3ee2cc 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java @@ -88,48 +88,55 @@ private Token getNextToken() { if (c == symbols.getPrefix()) { if (currPost < length) { c = is[currPost]; - - if (c == symbols.getNote()) { - if (inComment == 1 || inRaw == 1) { - continue; - } - inComment = 1; - - tokenLength = currPost - tokenStart - 1; - if (tokenLength > 0) { - // start a new token - lastStart = tokenStart; - tokenStart = --currPost; - tokenKind = c; - inComment = 0; - return newToken(symbols.getFixed()); - } else { - tokenKind = c; - } - } else if (c == symbols.getTag() || c == symbols.getExprStart()) { - if (inComment > 0) { - continue; - } - if (inRaw > 0 && (c == symbols.getExprStart() || !isEndRaw())) { - continue; - } - // match token two ends - if (!matchToken(c) && tokenKind > 0) { - continue; - } - if (inBlock++ > 0) { - continue; - } - - tokenLength = currPost - tokenStart - 1; - if (tokenLength > 0) { - // start a new token - lastStart = tokenStart; - tokenStart = --currPost; - tokenKind = c; - return newToken(symbols.getFixed()); - } else { - tokenKind = c; + boolean startTokenFound = true; + if (config.getLegacyOverrides().isWhitespaceRequiredWithinTokens()) { + boolean hasNextChar = (currPost + 1) < length; + boolean nextCharIsWhitespace = hasNextChar && (' ' == is[currPost + 1]); + startTokenFound = nextCharIsWhitespace; + } + if (startTokenFound) { + if (c == symbols.getNote()) { + if (inComment == 1 || inRaw == 1) { + continue; + } + inComment = 1; + + tokenLength = currPost - tokenStart - 1; + if (tokenLength > 0) { + // start a new token + lastStart = tokenStart; + tokenStart = --currPost; + tokenKind = c; + inComment = 0; + return newToken(symbols.getFixed()); + } else { + tokenKind = c; + } + } else if (c == symbols.getTag() || c == symbols.getExprStart()) { + if (inComment > 0) { + continue; + } + if (inRaw > 0 && (c == symbols.getExprStart() || !isEndRaw())) { + continue; + } + // match token two ends + if (!matchToken(c) && tokenKind > 0) { + continue; + } + if (inBlock++ > 0) { + continue; + } + + tokenLength = currPost - tokenStart - 1; + if (tokenLength > 0) { + // start a new token + lastStart = tokenStart; + tokenStart = --currPost; + tokenKind = c; + return newToken(symbols.getFixed()); + } else { + tokenKind = c; + } } } } else { // reach the stream end diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java index ba117ff7e..030e87fc9 100644 --- a/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java +++ b/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java @@ -7,6 +7,7 @@ import com.google.common.collect.Lists; import com.google.common.io.Resources; import com.hubspot.jinjava.JinjavaConfig; +import com.hubspot.jinjava.LegacyOverrides; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.stream.Collectors; @@ -259,6 +260,24 @@ public void testEscapedQuoteWithinAttrValue() { ); } + @Test + public void testCommentWithWhitespaceChar() { + List tokens = tokens("comment-without-whitespace"); + assertThat(tokens.get(0).content.trim()).isEqualTo("$"); + + LegacyOverrides legacyOverrides = LegacyOverrides + .newBuilder() + .withWhitespaceRequiredWithinTokens(true) + .build(); + JinjavaConfig config = JinjavaConfig + .newBuilder() + .withLegacyOverrides(legacyOverrides) + .build(); + TokenScanner scanner = fixture("comment-without-whitespace", config); + tokens = Lists.newArrayList(scanner); + assertThat(tokens.get(0).content.trim()).isEqualTo("${#array[@]}"); + } + @Test public void testEscapedBackslashWithinAttrValue() { List tokens = tokens("escape-char-tokens"); @@ -292,6 +311,10 @@ private List tokens(String fixture) { } private TokenScanner fixture(String fixture) { + return fixture(fixture, config); + } + + private TokenScanner fixture(String fixture, JinjavaConfig config) { try { return new TokenScanner( Resources.toString( diff --git a/src/test/resources/parse/tokenizer/comment-without-whitespace.jinja b/src/test/resources/parse/tokenizer/comment-without-whitespace.jinja new file mode 100644 index 000000000..cede8a774 --- /dev/null +++ b/src/test/resources/parse/tokenizer/comment-without-whitespace.jinja @@ -0,0 +1 @@ +${#array[@]} \ No newline at end of file