diff --git a/src/main/java/net/sf/jsqlparser/expression/StringValue.java b/src/main/java/net/sf/jsqlparser/expression/StringValue.java index 45afea1dd..54a4f7b46 100644 --- a/src/main/java/net/sf/jsqlparser/expression/StringValue.java +++ b/src/main/java/net/sf/jsqlparser/expression/StringValue.java @@ -234,7 +234,8 @@ public StringValue withUnicodeEscapeCharacter(Character unicodeEscapeCharacter) * returned unchanged. * * @throws IllegalArgumentException if a {@link Escaping#POSTGRESQL_UNICODE_ESCAPE} value has an - * escape that PostgreSQL rejects; the parser rejects such literals + * escape that PostgreSQL rejects, or a {@link Escaping#POSTGRESQL_ESCAPE_STRING} value + * has a Unicode escape that PostgreSQL rejects; the parser rejects such literals */ public String getNotExcapedValue() { if (quoteStr != null && quoteStr.startsWith("$")) { @@ -317,14 +318,14 @@ private static int appendPostgresEscape(Unescaped result, String text, int index result.append('\t'); return index + 1; case 'x': - return appendNumber(result, text, index + 1, 16, 1, 2, c, true); + return appendByte(result, text, index + 1, 16, 1, 2, c); case 'u': - return appendNumber(result, text, index + 1, 16, 4, 4, c, false); + return appendCodePoint(result, text, index, 4); case 'U': - return appendNumber(result, text, index + 1, 16, 8, 8, c, false); + return appendCodePoint(result, text, index, 8); default: if (c >= '0' && c <= '7') { - return appendNumber(result, text, index, 8, 1, 3, c, true); + return appendByte(result, text, index, 8, 1, 3, c); } result.append(c); return index + 1; @@ -332,11 +333,11 @@ private static int appendPostgresEscape(Unescaped result, String text, int index } /** - * Appends the byte or code point spelled by {@code min} to {@code max} digits at {@code start}, - * or {@code fallback} when fewer digits follow, and returns the next index. + * Appends the byte spelled by {@code min} to {@code max} digits at {@code start}, or + * {@code fallback} when fewer digits follow, and returns the next index. */ - private static int appendNumber(Unescaped result, String text, int start, int radix, - int min, int max, char fallback, boolean isByte) { + private static int appendByte(Unescaped result, String text, int start, int radix, int min, + int max, char fallback) { int end = start; while (end < text.length() && end - start < max && Character.digit(text.charAt(end), radix) >= 0) { @@ -346,12 +347,28 @@ private static int appendNumber(Unescaped result, String text, int start, int ra result.append(fallback); return start; } - int number = Integer.parseInt(text.substring(start, end), radix); - if (isByte) { - result.appendByte(number); - } else { - result.appendCodePoint(number); + result.appendByte(Integer.parseInt(text.substring(start, end), radix)); + return end; + } + + /** + * Appends the code point spelled by exactly {@code digits} hexadecimal digits after the escape + * letter at {@code index} and returns the next index. As in a Unicode escape string, the code + * point must be valid and a UTF-16 surrogate pair must be written as two consecutive escapes. + */ + private static int appendCodePoint(Unescaped result, String text, int index, int digits) { + int start = index + 1; + int end = start + digits; + long codePoint = parseHex(text, start, end); + if (codePoint < 0) { + throw new IllegalArgumentException("invalid Unicode escape at position " + (index - 1) + + ": Unicode escapes must be \\u followed by 4 or \\U by 8 hexadecimal digits"); + } + if (codePoint == 0 || codePoint > Character.MAX_CODE_POINT) { + throw new IllegalArgumentException( + "invalid Unicode escape value at position " + (index - 1)); } + result.appendCodePoint((int) codePoint, index - 1); return end; } @@ -393,7 +410,7 @@ private static int appendUnicodeEscape(UnicodeUnescaped result, String text, int boolean plus = position + 1 < text.length() && text.charAt(position + 1) == '+'; int start = plus ? position + 2 : position + 1; int end = start + (plus ? 6 : 4); - int codePoint = parseHex(text, start, end); + long codePoint = parseHex(text, start, end); if (codePoint < 0) { throw new IllegalArgumentException("invalid Unicode escape at position " + position + ": Unicode escapes must be " + escape + "XXXX or " + escape + "+XXXXXX"); @@ -402,7 +419,7 @@ private static int appendUnicodeEscape(UnicodeUnescaped result, String text, int throw new IllegalArgumentException( "invalid Unicode escape value at position " + position); } - result.appendCodePoint(codePoint, position); + result.appendCodePoint((int) codePoint, position); return end; } @@ -416,11 +433,11 @@ private static boolean isHexDigit(char c) { } /** Returns the ASCII hexadecimal number spelled by text[start, end), or -1. */ - private static int parseHex(String text, int start, int end) { + private static long parseHex(String text, int start, int end) { if (end > text.length()) { return -1; } - int number = 0; + long number = 0; for (int i = start; i < end; i++) { char c = text.charAt(i); if (!isHexDigit(c)) { @@ -431,26 +448,50 @@ private static int parseHex(String text, int start, int end) { return number; } - /** Collects unescaped characters; consecutive escaped bytes are decoded as UTF-8. */ + /** + * Collects unescaped characters; consecutive escaped bytes are decoded as UTF-8. A UTF-16 + * surrogate pair must be written as two consecutive Unicode escapes, as PostgreSQL requires. + */ private static final class Unescaped { private final StringBuilder chars; private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + private char highSurrogate; + private int highSurrogatePosition; Unescaped(int capacity) { chars = new StringBuilder(capacity); } void append(char c) { + requireNoPendingSurrogate(); flushBytes(); chars.append(c); } - void appendCodePoint(int codePoint) { + /** Appends the code point of the Unicode escape at {@code position}. */ + void appendCodePoint(int codePoint, int position) { flushBytes(); - chars.appendCodePoint(codePoint); + if (codePoint >= Character.MIN_LOW_SURROGATE + && codePoint <= Character.MAX_LOW_SURROGATE) { + if (highSurrogate == 0) { + throw invalidSurrogatePair(position); + } + chars.appendCodePoint(Character.toCodePoint(highSurrogate, (char) codePoint)); + highSurrogate = 0; + return; + } + requireNoPendingSurrogate(); + if (codePoint >= Character.MIN_HIGH_SURROGATE + && codePoint <= Character.MAX_HIGH_SURROGATE) { + highSurrogate = (char) codePoint; + highSurrogatePosition = position; + } else { + chars.appendCodePoint(codePoint); + } } void appendByte(int value) { + requireNoPendingSurrogate(); bytes.write(value); } @@ -461,8 +502,15 @@ private void flushBytes() { } } + private void requireNoPendingSurrogate() { + if (highSurrogate != 0) { + throw invalidSurrogatePair(highSurrogatePosition); + } + } + @Override public String toString() { + requireNoPendingSurrogate(); flushBytes(); return chars.toString(); } diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 842062229..a9ad7983d 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -3427,8 +3427,11 @@ public class CCJSqlParser extends AbstractJSqlParser { } } - /** Creates a string literal that remembers how this configuration lexed its backslashes. */ - private StringValue stringValue(Token token) { + /** + * Creates a string literal that remembers how this configuration lexed its backslashes. A + * PostgreSQL escape string with a Unicode escape that the server rejects is rejected as well. + */ + private StringValue stringValue(Token token) throws ParseException { StringValue value = new StringValue(token.image); // a U&'...' string brings its own escaping if (value.getQuoteStr() == null || value.getQuoteStr().startsWith("$") @@ -3440,6 +3443,12 @@ public class CCJSqlParser extends AbstractJSqlParser { if (Dialect.POSTGRESQL.name().equals(dialect)) { if (backslashes || "E".equalsIgnoreCase(value.getPrefix())) { value.setEscaping(StringValue.Escaping.POSTGRESQL_ESCAPE_STRING); + try { + value.getNotExcapedValue(); + } catch (IllegalArgumentException e) { + throw new ParseException("Invalid escape string " + token.image + ": " + + e.getMessage()); + } } } else if (backslashes && (dialect == null || isMySqlDialect())) { value.setEscaping(StringValue.Escaping.MYSQL_BACKSLASH); diff --git a/src/test/java/net/sf/jsqlparser/expression/StringValueTest.java b/src/test/java/net/sf/jsqlparser/expression/StringValueTest.java index 9f5a0c573..5089720ba 100644 --- a/src/test/java/net/sf/jsqlparser/expression/StringValueTest.java +++ b/src/test/java/net/sf/jsqlparser/expression/StringValueTest.java @@ -147,6 +147,34 @@ void testPostgresEscapeStrings() throws JSQLParserException { .getNotExcapedValue()); } + @Test + void testPostgresEscapeStringUnicodeEscapes() throws JSQLParserException { + String[][] cases = { + {"E'\\U0010FFFF'", new String(Character.toChars(0x10FFFF))}, + // a surrogate pair as two escapes, in either form + {"E'\\uD83D\\uDE00'", "\uD83D\uDE00"}, + {"E'\\ud83d\\U0000de00'", "\uD83D\uDE00"}, + {"E'\\x41\\uD83D\\uDE00\\101'", "A\uD83D\uDE00A"}}; + for (String[] c : cases) { + assertEquals(c[1], literal(c[0], Dialect.POSTGRESQL).getNotExcapedValue(), c[0]); + } + + // the escapes PostgreSQL rejects: too few digits, invalid code points and a surrogate + // not written as a pair + for (String sql : new String[] {"E'\\u00e'", "E'\\U0001F60'", "E'\\u0000'", + "E'\\U00000000'", "E'\\U00110000'", "E'\\U7FFFFFFF'", "E'\\UFFFFFFFF'", + "E'\\uD800'", "E'\\uD800x'", "E'\\uD800\\u0041'", "E'\\uD800\\x41'", + "E'\\uD800'''", "E'\\uDE00'", "E'\\uD800\\uD800\\uDE00'"}) { + assertThrows(IllegalArgumentException.class, () -> new StringValue(sql) + .withEscaping(StringValue.Escaping.POSTGRESQL_ESCAPE_STRING) + .getNotExcapedValue(), sql); + assertThrows(JSQLParserException.class, () -> literal(sql, Dialect.POSTGRESQL), sql); + } + // only the escapes of PostgreSQL strings are checked + assertEquals("\\uD800", literal("'\\uD800'", Dialect.POSTGRESQL).getNotExcapedValue()); + assertEquals("\\uD800", literal("E'\\uD800'", null).getNotExcapedValue()); + } + @Test void testPostgresUnicodeEscapeStrings() throws JSQLParserException { StringValue value = literal("U&'\\0041BC \\00e9'", Dialect.POSTGRESQL);