diff --git a/xyz-hub-test/src/test/java/com/here/xyz/hub/rest/ApiParamTest.java b/xyz-hub-test/src/test/java/com/here/xyz/hub/rest/ApiParamTest.java index b1dfb1bfdf..ea08ce1e01 100644 --- a/xyz-hub-test/src/test/java/com/here/xyz/hub/rest/ApiParamTest.java +++ b/xyz-hub-test/src/test/java/com/here/xyz/hub/rest/ApiParamTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017-2019 HERE Europe B.V. + * Copyright (C) 2017-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package com.here.xyz.hub.rest; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.here.xyz.events.PropertiesQuery; import com.here.xyz.events.PropertyQuery; @@ -69,7 +70,6 @@ public void parsePropertiesQuery() { assertEquals(2, query.getValues().size()); assertEquals("string", query.getValues().get(0)); assertEquals("5", query.getValues().get(1)); - } @Test @@ -148,4 +148,93 @@ public void parsePropertiesQuerySpace() { assertEquals(1, query.getValues().size()); assertEquals(3L, query.getValues().get(0)); } + + @Test + public void parsePropertiesQueryJsonPath() { + String URIquery = "p.jsonPath=$.properties.address.city"; + PropertiesQuery pq = PropertiesQuery.fromString(URIquery, "", false); + + assertEquals("1 OR block is expected", 1, pq.size()); + PropertyQueryList pql = pq.get(0); + assertEquals("1 AND block is expected.", 1, pql.size()); + + PropertyQuery query = pql.stream() + .filter(q -> q.getKey().equals("properties.jsonPath")) + .findFirst() + .get(); + + assertEquals(QueryOperation.EQUALS, query.getOperation()); + assertEquals(1, query.getValues().size()); + Object value = query.getValues().get(0); + assertTrue("JSONPath value should be a String", value instanceof String); + assertEquals("$.properties.address.city", value); + } + + @Test + public void parsePropertiesQueryJsonPathWithOtherFilters() { + String URIquery = + "p.jsonPath=$.properties.address.city" + + "&p.a=3" + + "&p.boolean=true" + + "&f.createdAt>0"; + + PropertiesQuery pq = PropertiesQuery.fromString(URIquery, "", false); + assertEquals("1 OR block is expected", 1, pq.size()); + + PropertyQueryList pql = pq.get(0); + // 4 AND blocks: jsonPath, a, boolean, createdAt + assertEquals("4 AND blocks are expected.", 4, pql.size()); + + // jsonPath + PropertyQuery query = pql.stream() + .filter(q -> q.getKey().equals("properties.jsonPath")) + .findFirst() + .get(); + assertEquals(QueryOperation.EQUALS, query.getOperation()); + assertEquals(1, query.getValues().size()); + assertTrue(query.getValues().get(0) instanceof String); + assertEquals("$.properties.address.city", query.getValues().get(0)); + + // properties.a (still parsed as number) + query = pql.stream().filter(q -> q.getKey().equals("properties.a")).findFirst().get(); + assertEquals(QueryOperation.EQUALS, query.getOperation()); + assertEquals(1, query.getValues().size()); + assertEquals(3L, query.getValues().get(0)); + + // properties.boolean (still parsed as boolean) + query = pql.stream().filter(q -> q.getKey().equals("properties.boolean")).findFirst().get(); + assertEquals(QueryOperation.EQUALS, query.getOperation()); + assertEquals(1, query.getValues().size()); + assertEquals(true, query.getValues().get(0)); + + // createdAt (mapped via SEARCH_KEY_REPLACEMENTS) + query = pql.stream() + .filter(q -> q.getKey().equals("properties.@ns:com:here:xyz.createdAt")) + .findFirst() + .get(); + assertEquals(QueryOperation.GREATER_THAN, query.getOperation()); + assertEquals(1, query.getValues().size()); + assertEquals(0L, query.getValues().get(0)); + } + + @Test + public void parsePropertiesQuerySpaceJsonPath() { + String URISpaceQuery = "a=1&b=2&jsonPath=$.space.properties.version"; + PropertiesQuery pq = PropertiesQuery.fromString(URISpaceQuery, "jsonPath", true); + + assertEquals("1 OR block is expected", 1, pq.size()); + PropertyQueryList pql = pq.get(0); + assertEquals("1 AND block is expected.", 1, pql.size()); + + PropertyQuery query = pql.stream() + .filter(q -> q.getKey().equals("jsonPath")) + .findFirst() + .get(); + + assertEquals(QueryOperation.EQUALS, query.getOperation()); + assertEquals(1, query.getValues().size()); + Object value = query.getValues().get(0); + assertTrue("JSONPath value should be a String", value instanceof String); + assertEquals("$.space.properties.version", value); + } } diff --git a/xyz-models/src/main/java/com/here/xyz/events/PropertiesQuery.java b/xyz-models/src/main/java/com/here/xyz/events/PropertiesQuery.java index 20d0ea3011..ae20b7ea0f 100644 --- a/xyz-models/src/main/java/com/here/xyz/events/PropertiesQuery.java +++ b/xyz-models/src/main/java/com/here/xyz/events/PropertiesQuery.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017-2019 HERE Europe B.V. + * Copyright (C) 2017-2025 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -155,6 +155,11 @@ public static String getConvertedKey(String rawKey) { } public static Object getConvertedValue(String rawValue) { + // JSONPath + if (rawValue != null && rawValue.startsWith("$")) { + return rawValue; + } + // Boolean if (rawValue.equals("true")) { return true; @@ -196,4 +201,23 @@ private static String transformLegacyTags(String legacyTagsQuery) { return F_PREFIX + "tags" + "=cs=" + tags; } + + public List getJsonPathValues() { + List jsonPaths = new ArrayList<>(); + + for (PropertyQueryList queries : this) { + for (PropertyQuery query : queries) { + for (Object value : query.getValues()) { + if (value instanceof String) { + String s = (String) value; + if (s.startsWith("$")) { + jsonPaths.add(s); + } + } + } + } + } + + return jsonPaths; + } } diff --git a/xyz-models/src/main/java/com/here/xyz/models/hub/Space.java b/xyz-models/src/main/java/com/here/xyz/models/hub/Space.java index 9b926ea573..a2883cdf46 100644 --- a/xyz-models/src/main/java/com/here/xyz/models/hub/Space.java +++ b/xyz-models/src/main/java/com/here/xyz/models/hub/Space.java @@ -32,10 +32,9 @@ import com.here.xyz.XyzSerializable; import com.here.xyz.XyzSerializable.Public; import com.here.xyz.XyzSerializable.Static; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import com.here.xyz.util.JsonPathValidator; + +import java.util.*; /** * The space configuration. @@ -531,10 +530,13 @@ public Space withReadOnlyHeadVersion(long readOnlyHeadVersion) { return this; } + // -------- Internal canonical accessors (used by server-side code) -------- + @JsonIgnore public Map getSearchableProperties() { return searchableProperties; } + @JsonIgnore public void setSearchableProperties(final Map searchableProperties) { this.searchableProperties = searchableProperties; } @@ -544,6 +546,44 @@ public Space withSearchableProperties(final Map searchablePrope return this; } + // -------- JSON-facing alias accessors -------- + + @JsonInclude(Include.NON_EMPTY) + @JsonView({Public.class, Static.class}) + @JsonProperty("searchableProperties") + public Map getSearchablePropertiesByAlias() { + if (searchableProperties == null || searchableProperties.isEmpty()) + return null; + + Map byAlias = new LinkedHashMap<>(); + + for (Map.Entry e : searchableProperties.entrySet()) { + String key = e.getKey(); + Boolean value = e.getValue(); + + try { + NormalizedProperty np = parseAndNormalizeKey(key); + byAlias.put(np.alias, value); + } + catch (IllegalArgumentException ex) { + // In case of unexpected format, fallback + byAlias.put(key, value); + } + } + + return byAlias; + } + + @JsonProperty("searchableProperties") + public void setSearchablePropertiesByAlias(final Map searchablePropertiesByAlias) { + if (searchablePropertiesByAlias == null || searchablePropertiesByAlias.isEmpty()) { + this.searchableProperties = searchablePropertiesByAlias; + return; + } + + this.searchableProperties = normalizeSearchableProperties(searchablePropertiesByAlias); + } + public List> getSortableProperties() { return sortableProperties; } @@ -611,6 +651,167 @@ public Space withMimeType(String mimeType) { return this; } + /** + * Normalizes all searchable property definitions into the canonical form: + * $:[]::scalar|array + */ + private Map normalizeSearchableProperties(Map rawProps) { + Map normalized = new LinkedHashMap<>(); + Set aliases = new HashSet<>(); + + for (Map.Entry entry : rawProps.entrySet()) { + String rawKey = entry.getKey(); + Boolean value = entry.getValue(); + + if (rawKey == null || rawKey.trim().isEmpty()) { + throw new IllegalArgumentException("Searchable property key must not be null or empty"); + } + + NormalizedProperty np = parseAndNormalizeKey(rawKey.trim()); + + // Enforce alias uniqueness + if (!aliases.add(np.alias)) { + throw new IllegalArgumentException("Duplicate alias detected: " + np.alias); + } + + String canonicalKey = buildCanonicalKey(np); + normalized.put(canonicalKey, value); + } + + return normalized; + } + + private String buildCanonicalKey(NormalizedProperty np) { + return "$" + np.alias + ":[" + np.jsonPathExpression + "]::" + np.resultType; + } + + private static class NormalizedProperty { + String alias; + String jsonPathExpression; + String resultType; + } + + /** + * Parse a raw key and normalize it to (alias, jsonPathExpression, resultType). + * - New-style keys: "$alias:[$.path]::scalar" (or without []) + * - Legacy keys: "path", "path::scalar", "path::array" + */ + private NormalizedProperty parseAndNormalizeKey(String key) { + if (key.startsWith("$") && key.contains("::")) { + NormalizedProperty np = tryParseNewStyleKey(key); + if (np != null) { + return np; + } + } + + // Fallback + return parseLegacyKey(key); + } + + /** + * Parses a new-style key of form + * $alias:[$.jsonPath]::scalar|array + */ + private NormalizedProperty tryParseNewStyleKey(String key) { + String[] typeSplit = key.split("::", 2); + if (typeSplit.length != 2) { + return null; + } + + String leftPart = typeSplit[0].trim(); + String typePart = typeSplit[1].trim(); + + if (!"scalar".equals(typePart) && !"array".equals(typePart)) { + return null; + } + + int colonIdx = leftPart.indexOf(':'); + if (colonIdx <= 1) { + return null; + } + + String aliasPart = leftPart.substring(1, colonIdx).trim(); + String exprPart = leftPart.substring(colonIdx + 1).trim(); + + if (aliasPart.isEmpty() || exprPart.isEmpty()) { + return null; + } + + String expression = exprPart; + if (expression.startsWith("[") && expression.endsWith("]") && expression.length() >= 2) { + expression = expression.substring(1, expression.length() - 1).trim(); + } + + // Validate JSONPath on the stripped expression + validateJsonPath(expression); + + NormalizedProperty np = new NormalizedProperty(); + np.alias = aliasPart; + np.jsonPathExpression = expression; + np.resultType = typePart; + return np; + } + + /** + * Parses legacy keys like: + * - "foo" + * - "foo::array" + */ + private NormalizedProperty parseLegacyKey(String key) { + String base = key; + String resultType = "scalar"; //default + + int sepIdx = key.lastIndexOf("::"); + if (sepIdx > -1 && sepIdx + 2 < key.length()) { + String maybeType = key.substring(sepIdx + 2).trim(); + if ("scalar".equals(maybeType) || "array".equals(maybeType)) { + base = key.substring(0, sepIdx).trim(); + resultType = maybeType; + } + } + + if (base.isEmpty()) { + throw new IllegalArgumentException("Malformed searchable property key: '" + key + "'"); + } + + String expression; + if (base.startsWith("$")) { + expression = base; + } + else { + expression = "$." + base; + } + + // Derive alias from the path (without the leading '$' or '$.') + String alias; + if (expression.startsWith("$.") && expression.length() > 2) { + alias = expression.substring(2); + } + else if (expression.startsWith("$") && expression.length() > 1) { + alias = expression.substring(1); + } + else { + alias = base; + } + + validateJsonPath(expression); + + NormalizedProperty np = new NormalizedProperty(); + np.alias = alias; + np.jsonPathExpression = expression; + np.resultType = resultType; + return np; + } + + private void validateJsonPath(String expression) { + JsonPathValidator.ValidationResult result = JsonPathValidator.validate(expression); + if (!result.isValid()) { + int pos = result.errorPosition().orElse(-1); + throw new IllegalArgumentException( + String.format("Invalid JSONPath expression '%s'. Error at position %d.", expression, pos)); + } + } + /** * Used as a JsonView on a {@link Space} to indicate that a property should be part of a response which was requested to contain * connector information. diff --git a/xyz-models/src/main/java/com/here/xyz/util/JsonPathValidator.java b/xyz-models/src/main/java/com/here/xyz/util/JsonPathValidator.java new file mode 100644 index 0000000000..d81592c20b --- /dev/null +++ b/xyz-models/src/main/java/com/here/xyz/util/JsonPathValidator.java @@ -0,0 +1,701 @@ +/* + * Copyright (C) 2017-2025 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.xyz.util; + +import java.util.*; + +public final class JsonPathValidator { + private static final boolean REQUIRE_STRING_FOR_REGEX = true; + + public static ValidationResult validate(String input) { + if (input == null) + return ValidationResult.error(0, "Input is null"); + + try { + Lexer lexer = new Lexer(input); + List tokens = lexer.lex(); + Parser parser = new Parser(tokens); + parser.parsePath(); + parser.expect(TokenType.EOF, "Unexpected trailing"); + return ValidationResult.ok(); + } catch (ParseException pe) { + return ValidationResult.error(pe.position, pe.getMessage()); + } + } + + public static final class ValidationResult { + private final boolean valid; + private final int position; + private final String message; + + private ValidationResult(boolean valid, int position, String message) { + this.valid = valid; + this.position = position; + this.message = message; + } + + public static ValidationResult ok() { + return new ValidationResult(true, -1, null); + } + + public static ValidationResult error(int pos, String msg) { + return new ValidationResult(false, pos, msg); + } + + public boolean isValid() { + return valid; + } + + public OptionalInt errorPosition() { + return valid ? OptionalInt.empty() : OptionalInt.of(position); + } + + public Optional errorMessage() { + return Optional.ofNullable(message); + } + + } + + private enum TokenType { + // structural characters + DOLLAR, DOT, STAR, LBRACKET, RBRACKET, LPAREN, RPAREN, COMMA, COLON, QUESTION, AT, + // operators + EQ, NE, LT, LE, GT, GE, AND, OR, NOT, REGEXMATCH, + // literals + NUMBER, STRING, TRUE, FALSE, NULL, + // identifiers(unquoted names after dot) + IDENT, + EOF + } + + private static final class Token { + final TokenType type; + final String text; + final int position; + + Token(TokenType t, String text, int pos) { + this.type = t; + this.text = text; + this.position = pos; + } + + public String toString() { + return type + (text != null ? ("(" + text + ")") : "") + "@" + position; + } + } + + private static final class Lexer { + private final String str; + private final int length; + private int idx; + + Lexer(String s) { + this.str = s; + this.length = s.length(); + } + + List lex() { + List out = new ArrayList<>(); + while (true) { + skipWhitespace(); + + if (idx >= length) { + out.add(new Token(TokenType.EOF, "", idx)); + break; + } + + char c = str.charAt(idx); + int pos = idx; + switch (c) { + case '$': + idx++; + out.add(new Token(TokenType.DOLLAR, "$", pos)); + break; + case '.': + idx++; + out.add(new Token(TokenType.DOT, ".", pos)); + break; + case '*': + idx++; + out.add(new Token(TokenType.STAR, "*", pos)); + break; + case '[': + idx++; + out.add(new Token(TokenType.LBRACKET, "[", pos)); + break; + case ']': + idx++; + out.add(new Token(TokenType.RBRACKET, "]", pos)); + break; + case '(': + idx++; + out.add(new Token(TokenType.LPAREN, "(", pos)); + break; + case ')': + idx++; + out.add(new Token(TokenType.RPAREN, ")", pos)); + break; + case ',': + idx++; + out.add(new Token(TokenType.COMMA, ",", pos)); + break; + case ':': + idx++; + out.add(new Token(TokenType.COLON, ":", pos)); + break; + case '?': + idx++; + out.add(new Token(TokenType.QUESTION, "?", pos)); + break; + case '@': + idx++; + out.add(new Token(TokenType.AT, "@", pos)); + break; + case '!': + idx++; + if (match('=')) + out.add(new Token(TokenType.NE, "!=", pos)); + else + out.add(new Token(TokenType.NOT, "!", pos)); + break; + case '=': + idx++; + if (match('=')) + out.add(new Token(TokenType.EQ, "==", pos)); + else if (match('~')) + out.add(new Token(TokenType.REGEXMATCH, "=~", pos)); + else + throw err(pos, "expected '=' or '~' after '='"); + break; + case '<': + idx++; + if (match('=')) + out.add(new Token(TokenType.LE, "<=", pos)); + else + out.add(new Token(TokenType.LT, "<", pos)); + break; + case '>': + idx++; + if (match('=')) + out.add(new Token(TokenType.GE, ">=", pos)); + else + out.add(new Token(TokenType.GT, ">", pos)); + break; + case '\'': + out.add(readString('\'', pos)); + break; + case '"': + out.add(readString('"', pos)); + break; + case '&': + idx++; + if (match('&')) + out.add(new Token(TokenType.AND, "&&", pos)); + else + throw err(pos, "single '&' not allowed"); + break; + case '|': + idx++; + if (match('|')) + out.add(new Token(TokenType.OR, "||", pos)); + else + throw err(pos, "single '|' not allowed"); + break; + default: + if (isDigit(c) || (c == '-' && (idx + 1 < length) && isDigit(str.charAt(idx + 1)))) { + out.add(readNumber(pos)); + } else if (isIdentStart(c)) { + String ident = readIdent(); + TokenType kw = keyword(ident); + out.add(new Token(kw == null ? TokenType.IDENT : kw, ident, pos)); + } else { + throw err(pos, "unexpected character '" + c + "'"); + } + } + } + return out; + } + + private void skipWhitespace() { + while (idx < length) { + char c = str.charAt(idx); + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') + idx++; + else + break; + } + } + + private boolean match(char ch) { + if (idx < length && str.charAt(idx) == ch) { + idx++; + return true; + } + return false; + } + + private Token readString(char quote, int startPos) { + StringBuilder sb = new StringBuilder(); + idx++; // consume opening quote + boolean escaped = false; + while (idx < length) { + char c = str.charAt(idx++); + if (escaped) { + switch (c) { + case '"': + case '\'': + case '\\': + case '/': + sb.append(c); + break; + case 'b': + sb.append('\b'); + break; + case 'f': + sb.append('\f'); + break; + case 'n': + sb.append('\n'); + break; + case 'r': + sb.append('\r'); + break; + case 't': + sb.append('\t'); + break; + case 'u': + if (idx + 4 > length) + throw err(startPos, "unterminated unicode escape"); + String hex = str.substring(idx, idx + 4); + if (!hex.matches("[0-9A-Fa-f]{4}")) + throw err(startPos, "invalid unicode escape"); + sb.append((char) Integer.parseInt(hex, 16)); + idx += 4; + break; + default: + throw err(startPos, "invalid escape \\" + c); + } + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == quote) { + return new Token(TokenType.STRING, sb.toString(), startPos); + } else { + sb.append(c); + } + } + throw err(startPos, "unterminated string literal"); + } + + private Token readNumber(int startPos) { + int j = idx; + if (str.charAt(j) == '-') + j++; + + if (j >= length || !isDigit(str.charAt(j))) + throw err(startPos, "invalid number"); + + if (str.charAt(j) == '0') { + j++; + } else { + while (j < length && isDigit(str.charAt(j))) + j++; + } + + if (j < length && str.charAt(j) == '.') { + j++; + if (j >= length || !isDigit(str.charAt(j))) + throw err(startPos, "invalid fraction"); + while (j < length && isDigit(str.charAt(j))) + j++; + } + + if (j < length && (str.charAt(j) == 'e' || str.charAt(j) == 'E')) { + j++; + if (j < length && (str.charAt(j) == '+' || str.charAt(j) == '-')) + j++; + if (j >= length || !isDigit(str.charAt(j))) + throw err(startPos, "invalid exponent"); + while (j < length && isDigit(str.charAt(j))) + j++; + } + + String num = str.substring(idx, j); + idx = j; + return new Token(TokenType.NUMBER, num, startPos); + } + + private String readIdent() { + int j = idx; + j++; + while (j < length && isIdentPart(str.charAt(j))) + j++; + + String id = str.substring(idx, j); + idx = j; + return id; + } + + private static boolean isDigit(char c) { + return c >= '0' && c <= '9'; + } + + private static boolean isIdentStart(char c) { + return (c == '_' || Character.isLetter(c)); + } + + private static boolean isIdentPart(char c) { + return (c == '_' || Character.isLetterOrDigit(c)); + } + + private static TokenType keyword(String ident) { + if ("true".equals(ident)) + return TokenType.TRUE; + if ("false".equals(ident)) + return TokenType.FALSE; + if ("null".equals(ident)) + return TokenType.NULL; + + return null; + } + + private static ParseException err(int pos, String msg) { + return new ParseException(pos, msg); + } + } + + private static final class Parser { + private final List tokens; + private int idx = 0; + + Parser(List tokens) { + this.tokens = tokens; + } + + void parsePath() { + expect(TokenType.DOLLAR, "path must start with '$'"); + while (!peek(TokenType.EOF)) { + if (accept(TokenType.DOT)) { + if (accept(TokenType.STAR)) { + // $.* (wildcard member) + } else if (peek(TokenType.STRING)) { + // $."quoted" + next(); + } else { + // $.name(unquoted identifier) + expect(TokenType.IDENT, "expected member name after '.'"); + } + } else if (accept(TokenType.LBRACKET)) { + if (accept(TokenType.QUESTION)) { // filter + expect(TokenType.LPAREN, "expected '(' after '?'"); + parseBooleanExpr(); + expect(TokenType.RPAREN, "expected ')' to close filter expression"); + expect(TokenType.RBRACKET, "expected ']' to close filter"); + } else if (accept(TokenType.STAR)) { // [*] + expect(TokenType.RBRACKET, "expected ']' after '*'"); + } else if (peek(TokenType.STRING)) { // ['name'] or ["name"] or unions + parseUnionOrMember(); + expect(TokenType.RBRACKET, "expected ']' after bracket member/union"); + } else if (peek(TokenType.NUMBER) || peek(TokenType.COLON)) { // index/slice/union starting with number (negatives included) + parseIndexSliceOrUnion(); + expect(TokenType.RBRACKET, "expected ']' after array selector"); + } else if (accept(TokenType.RBRACKET)) { + throw err(prev().position, "empty bracket selector '[]' is not allowed"); + } else { + throw err(curr().position, "unexpected token in bracket selector: " + curr().type); + } + } else { + break; + } + } + } + + // Parses ['name'] or ["name"] or union of strings/numbers/* separated by commas + private void parseUnionOrMember() { + next(); + while (accept(TokenType.COMMA)) { + if (accept(TokenType.STAR)) + continue; + if (peek(TokenType.STRING) || peek(TokenType.NUMBER)) { + next(); + } else + throw err(curr().position, "expected string/number/* in union"); + } + } + + // Parses [index], [start:end[:step]], or unions like [0,1,2] + private void parseIndexSliceOrUnion() { + if (accept(TokenType.COLON)) { + if (peek(TokenType.NUMBER)) + next(); + if (accept(TokenType.COLON)) { + Token step = expect(TokenType.NUMBER, "expected slice step after second ':'"); + if ("0".equals(step.text)) + throw err(step.position, "slice step cannot be 0"); + } + + return; + } + + parseNumericOrWildcard(); + if (accept(TokenType.COLON)) { // slice + if (peek(TokenType.NUMBER)) + next(); + if (accept(TokenType.COLON)) { + Token step = expect(TokenType.NUMBER, "expected slice step after second ':'"); + if ("0".equals(step.text)) + throw err(step.position, "slice step cannot be 0"); + } + + return; + } + + // possible union continuation + while (accept(TokenType.COMMA)) { + parseNumericOrWildcardOrString(); + } + } + + private void parseNumericOrWildcard() { + if (accept(TokenType.STAR)) + return; + expect(TokenType.NUMBER, "expected number"); + } + + private void parseNumericOrWildcardOrString() { + if (accept(TokenType.STAR)) + return; + if (accept(TokenType.STRING)) + return; + expect(TokenType.NUMBER, "expected number"); + } + + // Boolean expression for filters (precedence: ! > && > ||) + private void parseBooleanExpr() { + parseOr(); + } + + private void parseOr() { + parseAnd(); + while (accept(TokenType.OR)) { + parseAnd(); + } + } + + private void parseAnd() { + parseNot(); + while (accept(TokenType.AND)) { + parseNot(); + } + } + + private void parseNot() { + while (accept(TokenType.NOT)) { + } + parsePrimaryBool(); + } + + private void parsePrimaryBool() { + if (accept(TokenType.LPAREN)) { + parseBooleanExpr(); + expect(TokenType.RPAREN, "expected ')' in expression"); + return; + } + + // Try to consume a left value + boolean leftConsumed = false; + + if (peek(TokenType.IDENT) && lookaheadIs(TokenType.LPAREN)) { + parseFunctionCall(); + leftConsumed = true; + } else if (peek(TokenType.AT)) { // @-path + parseRelativePath(); + leftConsumed = true; + } else if (peek(TokenType.STRING) || peek(TokenType.NUMBER) || + peek(TokenType.TRUE) || peek(TokenType.FALSE) || peek(TokenType.NULL)) { + next(); // literal + leftConsumed = true; + } + + if (leftConsumed) { + // Optional operator (enables comparisons) + boolean isRegex = false; + if (accept(TokenType.EQ) || accept(TokenType.NE) || accept(TokenType.LT) || accept(TokenType.LE) || + accept(TokenType.GT) || accept(TokenType.GE) || (isRegex = accept(TokenType.REGEXMATCH))) { + // Right operand: function() | @-path | literal + if (peek(TokenType.IDENT) && lookaheadIs(TokenType.LPAREN)) { + if (isRegex && REQUIRE_STRING_FOR_REGEX) + throw err(curr().position, "right operand of '=~' must be a string"); + parseFunctionCall(); + } else if (peek(TokenType.AT)) { + if (isRegex && REQUIRE_STRING_FOR_REGEX) + throw err(curr().position, "right operand of '=~' must be a string"); + parseRelativePath(); + } else if (peek(TokenType.STRING) || peek(TokenType.NUMBER) || + peek(TokenType.TRUE) || peek(TokenType.FALSE) || peek(TokenType.NULL)) { + if (isRegex && REQUIRE_STRING_FOR_REGEX && !peek(TokenType.STRING)) + throw err(curr().position, "right operand of '=~' must be a string"); + next(); // consume rhs literal + } else { + throw err(curr().position, "expected value, @-path, or function call after operator"); + } + } + + return; + } + + throw err(curr().position, "expected boolean expression"); + } + + private void parseRelativePath() { + expect(TokenType.AT, "expected '@' for relative path"); + + while (true) { + if (accept(TokenType.DOT)) { + if (accept(TokenType.STAR)) { + // @.* (wildcard) + } else if (accept(TokenType.STRING)) { + // @."quoted" + } else { + expect(TokenType.IDENT, "expected member name after '.'"); + } + } else if (accept(TokenType.LBRACKET)) { + if (accept(TokenType.STAR)) { + expect(TokenType.RBRACKET, "expected ']' after '*'"); + } else if (peek(TokenType.STRING)) { + parseUnionOrMember(); + expect(TokenType.RBRACKET, "expected ']' after bracket member"); + } else if (peek(TokenType.NUMBER) || peek(TokenType.COLON)) { + parseIndexSliceOrUnion(); + expect(TokenType.RBRACKET, "expected ']' after array selector"); + } else { + throw err(curr().position, "unexpected token in relative bracket selector"); + } + } else { + break; + } + } + } + + private void parseFunctionCall() { + Token name = expect(TokenType.IDENT, "expected function name"); + String fn = name.text; + expect(TokenType.LPAREN, "expected '(' after function name"); + + switch (fn) { + case "length": + case "count": + case "value": + parseValue(); + expect(TokenType.RPAREN, "expected ')' to close " + fn + "()"); + return; + + case "match": + case "search": + parseValue(); + expect(TokenType.COMMA, "expected ',' after first argument of " + fn + "()"); + if (!peek(TokenType.STRING)) + throw err(curr().position, "expected string regex as second argument of " + fn + "()"); + next(); // consume regex string + if (accept(TokenType.COMMA)) { + if (!peek(TokenType.STRING)) + throw err(curr().position, "expected string flags as third argument of " + fn + "()"); + next(); // consume flags string + } + expect(TokenType.RPAREN, "expected ')' to close " + fn + "()"); + return; + + default: + throw err(name.position, "unknown function '" + fn + "'"); + } + } + + // Value-like expression: literals, @-paths, function calls, or parenthesized value + private void parseValue() { + if (accept(TokenType.LPAREN)) { + parseValue(); + expect(TokenType.RPAREN, "expected ')' in value"); + return; + } + if (peek(TokenType.AT)) { + parseRelativePath(); + return; + } + if (peek(TokenType.STRING) || peek(TokenType.NUMBER) || + peek(TokenType.TRUE) || peek(TokenType.FALSE) || peek(TokenType.NULL)) { + next(); + return; + } + if (peek(TokenType.IDENT) && lookaheadIs(TokenType.LPAREN)) { + parseFunctionCall(); + return; + } + + throw err(curr().position, "expected value"); + } + + private boolean accept(TokenType t) { + if (peek(t)) { + idx++; + return true; + } + return false; + } + + private Token expect(TokenType t, String msg) { + if (!peek(t)) + throw err(curr().position, msg); + return next(); + } + + private boolean lookaheadIs(TokenType t) { + return (idx + 1 < tokens.size()) && (tokens.get(idx + 1).type == t); + } + + private boolean peek(TokenType t) { + return tokens.get(idx).type == t; + } + + private Token next() { + return tokens.get(idx++); + } + + private Token curr() { + return tokens.get(idx); + } + + private Token prev() { + return tokens.get(idx - 1); + } + + private static ParseException err(int pos, String msg) { + return new ParseException(pos, msg); + } + } + + private static final class ParseException extends RuntimeException { + final int position; + + ParseException(int position, String message) { + super(message); + this.position = position; + } + } +} diff --git a/xyz-models/src/test/java/com/here/xyz/JsonPathValidatorTest.java b/xyz-models/src/test/java/com/here/xyz/JsonPathValidatorTest.java new file mode 100644 index 0000000000..81d1970a00 --- /dev/null +++ b/xyz-models/src/test/java/com/here/xyz/JsonPathValidatorTest.java @@ -0,0 +1,287 @@ +/* + * Copyright (C) 2017-2025 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.xyz; + +import com.here.xyz.util.JsonPathValidator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class JsonPathValidatorTest { + + private void assertValid(String expr) { + var res = JsonPathValidator.validate(expr); + assertTrue(res.isValid(), () -> "Expected valid, got error: " + res.errorMessage().orElse("") + + " at " + res.errorPosition().orElse(-1) + " for expr: " + expr); + } + + private void assertInvalid(String expr) { + var res = JsonPathValidator.validate(expr); + assertFalse(res.isValid(), () -> "Expected invalid but was valid: " + expr); + } + + @Test + @DisplayName("Must start with $") + void mustStartWithDollar() { + assertInvalid("a.b"); + assertValid("$.a"); + } + + @Test + @DisplayName("Only root($)") + void rootOnly() { + assertValid("$"); + } + + @Test + @DisplayName("Simple dot and quoted members") + void dotAndQuotedMembers() { + assertValid("$.a"); + assertValid("$.a.b"); + assertValid("$.\"spaced name\""); + assertValid("$['spaced name']"); + assertValid("$['member\\'quote']"); + } + + @Test + @DisplayName("Wildcards and bracket members") + void wildcardsAndBracketMembers() { + assertValid("$.*"); + assertValid("$['*']"); + assertValid("$[*]"); + assertValid("$.a[*]"); + } + + @Test + @DisplayName("Array indices, slices, and unions") + void indicesSlicesUnions() { + assertValid("$.a[0]"); + assertValid("$.a[-1]"); + assertValid("$.a[0:10]"); + assertValid("$.a[:10]"); + assertValid("$.a[1:10:2]"); + assertValid("$.a[0,1,2]"); + assertValid("$.a[0,'x',\"y\",*]"); + assertValid("$.a[:]"); + assertValid("$.a[::2]"); + assertValid("$.a[0:]"); + assertValid("$.a[:10:2]"); + assertInvalid("$.a[0::]"); // step missing + + var r = JsonPathValidator.validate("$.a[1:4:0]"); // step cannot be 0 + assertFalse(r.isValid()); + assertTrue(r.errorMessage().orElse("").toLowerCase().contains("step")); + } + + @Test + @DisplayName("Reject recursive descent operator") + void rejectRecursiveDescent() { + assertInvalid("$.a..b"); + } + + @Test + @DisplayName("Filters: literals, comparisons, existence, grouping") + void filters() { + // comparisons and literals + assertValid("$.store.book[?(@.price >= 0)]"); + assertValid("$.store.book[?(@.author == 'John')]"); + assertValid("$.store.book[?(@.author != 'John')]"); + assertValid("$.store.book[?(!(@.soldOut))]"); + // existence + assertValid("$.items[?(@.name)]"); + // regex + assertValid("$.items[?(@.name =~ '^[A-Z].*')]"); + // invalid single '=' + assertInvalid("$.a[?(@.x = 1)]"); + } + + @Test + @DisplayName("Comparison right-hand side missing") + void comparisonRhsMissing() { + assertInvalid("$.a[?(@.x == )]"); + assertInvalid("$.a[?(@.x >= )]"); + } + + @Test + @DisplayName("Unterminated braces and parans") + void unterminated() { + assertInvalid("$.a["); + assertInvalid("$.a[?( @.x == 1 )"); + assertInvalid("$.a[?(@.x == (1)]"); + } + + @Test + @DisplayName("Relative wildcards are allowed") + void relativeWildcard() { + assertValid("$.a[?(@.*)]"); + assertValid("$.a[?(@[*])]"); + } + + @Test + @DisplayName("Regex must be string") + void regexRightMustBeString() { + assertInvalid("$.items[?(@.name =~ 123)]"); + } + + @Test + @DisplayName("Unicode strings in bracket notation") + void unicodeStrings() { + assertValid("$['café']"); + assertValid("$['\u00E9']"); + assertInvalid("$['\\u00GZ']"); + } + + @Test + @DisplayName("Empty or malformed bracket selectors") + void emptyMalformedBrackets() { + assertInvalid("$[]"); + assertInvalid("$.a[,,]"); + } + + @Test + @DisplayName("Logical operators") + void logicalOperators() { + assertValid("$.a[?(@.x && @.y)]"); + assertValid("$.a[?(!@.x || @.y)]"); + assertInvalid("$.a[?(@.x & @.y)]"); + assertInvalid("$.a[?(@.x | @.y)]"); + } + + @Nested + @DisplayName("Relative @-paths inside filters") + class RelativePaths { + + @Test + void simple() { + assertValid("$.a[?(@.b)]"); + } + @Test void withIndex() { + assertValid("$.a[?(@[0])]"); + } + @Test void withBracketMembers() { + assertValid("$.a[?(@['x'])]"); + } + @Test void withSlice() { + assertValid("$.a[?(@[1:3])]"); + } + } + + @Test + @DisplayName("Null, true, false keywords allowed in filters") + void primitives() { + assertValid("$.a[?(true)]"); + assertValid("$.a[?(false)]"); + assertValid("$.a[?(null)]"); + assertValid("$.a[?(@.a == null)]"); + } + + @Test + @DisplayName("Unexpected trailing input should fail") + void trailingInput() { + assertInvalid("$.a]extra"); + assertInvalid("$.a)extra"); + } + + @Test + @DisplayName("Empty input is not allowed") + void emptyInput() { + assertInvalid(""); + } + + @Test + @DisplayName("Trailing Whitespace is allowed") + void trailingWhitespace() { + assertValid("$.a "); + } + + @Test + @DisplayName("length(): one value-like argument, used in comparisons") + void lengthBasics() { + assertValid("$.a[?( length(@.tags) > 0 )]"); + assertValid("$.a[?( length(@) >= 1 )]"); + assertValid("$.a[?( length('abc') == 3 )]"); + assertInvalid("$.a[?( length() > 0 )]"); + assertInvalid("$.a[?( length(@.x, @.y) > 0 )]"); + } + + @Test + @DisplayName("count(): one value-like argument, can compare with other function") + void countBasics() { + assertValid("$.a[?( count(@[*]) == 0 )]"); + assertValid("$.a[?( count(@.items) == length(@.items) )]"); + assertInvalid("$.a[?( count() == 1 )]"); + assertInvalid("$.a[?( count(@.x, 1) == 2 )]"); + } + + @Test + @DisplayName("value(): one value-like argument, compare numerically") + void valueBasics() { + assertValid("$.a[?( value(@.price) >= 10 )]"); + assertValid("$.a[?( value( ( @.price ) ) < 100 )]"); + assertInvalid("$.a[?( value() >= 0 )]"); + assertInvalid("$.a[?( value(@.x, @.y) >= 0 )]"); + } + + @Test + @DisplayName("match(): boolean regex, 2nd arg must be string") + void matchBasics() { + assertValid("$.a[?( match(@.name, \"^[A-Z].*\") )]"); + assertValid("$.a[?( match(@.name, \"^[A-Z].*\", \"i\") )]"); + assertInvalid("$.a[?( match(@.name, 123) )]"); + assertInvalid("$.a[?( match(@.name) )]"); + assertInvalid("$.a[?( match(@.name, \"re\", 1) )]"); + assertInvalid("$.a[?( match() )]"); + assertInvalid("$.a[?( match(@.a, \"re\", \"i\", \"x\") )]"); + } + + @Test + @DisplayName("search(): boolean regex, 2nd arg must be string") + void searchBasics() { + assertValid("$.a[?( search(@.text, \"foo\") )]"); + assertValid("$.a[?( search(@.text, \"foo\", \"i\") )]"); + assertInvalid("$.a[?( search(@.text, @.pattern) )]"); + assertInvalid("$.a[?( search(@.text) )]"); + assertInvalid("$.a[?( search(@.text, \"foo\", 1) )]"); + } + + @Test + @DisplayName("Functions can appear on either side of comparisons") + void functionsBothSides() { + assertValid("$.a[?( length(@.tags) == count(@.tags) )]"); + assertValid("$.a[?( value(@.price) < length(\"12345\") )]"); + } + + @Test + @DisplayName("Nested function calls") + void nestedFunctions() { + assertValid("$.a[?( length( value(@.nested) ) > 0 )]"); + } + + @Test + @DisplayName("Unknown function names should be rejected") + void unknownFunction() { + assertInvalid("$.a[?( foo(@.x) )]"); + assertInvalid("$.a[?( bar() )]"); + } +} + diff --git a/xyz-util/src/main/java/com/here/xyz/util/db/pg/IndexHelper.java b/xyz-util/src/main/java/com/here/xyz/util/db/pg/IndexHelper.java index 5ad7a906c4..ef04d7fffe 100644 --- a/xyz-util/src/main/java/com/here/xyz/util/db/pg/IndexHelper.java +++ b/xyz-util/src/main/java/com/here/xyz/util/db/pg/IndexHelper.java @@ -205,12 +205,57 @@ public static SQLQuery buildDropIndexQuery(String schema, String indexName) { } public static List getActivatedSearchableProperties(Map searchableProperties) { - return searchableProperties == null ? List.of() : searchableProperties.entrySet().stream() + return searchableProperties == null + ? List.of() + : searchableProperties.entrySet().stream() .filter(Map.Entry::getValue) - .map(entry -> new OnDemandIndex().withPropertyPath(entry.getKey())) + .map(entry -> new OnDemandIndex() + .withPropertyPath(extractLogicalPropertyPath(entry.getKey()))) .collect(Collectors.toList()); } + private static String extractLogicalPropertyPath(String key) { + if (key == null) + return null; + + key = key.trim(); + + // New-style: $alias:[$.jsonPath]::scalar|array + if (key.startsWith("$") && key.contains("::")) { + String[] typeSplit = key.split("::", 2); + String leftPart = typeSplit[0].trim(); + + int colonIdx = leftPart.indexOf(':'); + String exprPart = colonIdx > -1 + ? leftPart.substring(colonIdx + 1).trim() + : leftPart.substring(1).trim(); + + // Strip [] if present + if (exprPart.startsWith("[") && exprPart.endsWith("]") && exprPart.length() > 2) { + exprPart = exprPart.substring(1, exprPart.length() - 1).trim(); + } + + if (exprPart.startsWith("$.properties.") && exprPart.length() > "$.properties.".length()) { + return exprPart.substring("$.properties.".length()); + } + if (exprPart.startsWith("$.") && exprPart.length() > 2) { + return exprPart.substring(2); + } + if (exprPart.startsWith("$") && exprPart.length() > 1) { + return exprPart.substring(1); // fallback + } + return exprPart; + } + + // Legacy keys + int sepIdx = key.lastIndexOf("::"); + if (sepIdx > -1) { + return key.substring(0, sepIdx).trim(); + } + + return key; + } + public static SQLQuery buildOnDemandIndexCreationQuery(String schema, String table, String propertyPath, boolean async){ return buildOnDemandIndexCreationQuery(schema, table, propertyPath, "jsondata", async); }