diff --git a/.github/workflows/analytics-engine-compat.yml b/.github/workflows/analytics-engine-compat.yml index f1e722b0579..e39adb0a311 100644 --- a/.github/workflows/analytics-engine-compat.yml +++ b/.github/workflows/analytics-engine-compat.yml @@ -15,7 +15,7 @@ on: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 55002cf6dc5..9e7c5c2136e 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -1,30 +1,12 @@ +--- name: Backport on: pull_request_target: - types: - - closed - - labeled + types: [closed, labeled] jobs: backport: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - name: Backport - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - - name: Backport - uses: VachaShah/backport@142d3b8a8c70dc54db515e653e5ed3c3fac64100 # v2.2.0 - with: - github_token: ${{ steps.github_app_token.outputs.token }} - head_template: backport/backport-<%= number %>-to-<%= base %> - failure_labels: backport-failed + if: github.repository == 'opensearch-project/sql' + uses: opensearch-project/opensearch-build/.github/workflows/backport-pr.yml@main + secrets: + OPENSEARCH_CI_BOT_TOKEN: ${{ secrets.OPENSEARCH_CI_BOT_TOKEN }} diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml deleted file mode 100644 index 61089ed1334..00000000000 --- a/.github/workflows/delete_backport_branch.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Delete merged branch of the backport PRs -on: - pull_request: - types: - - closed - -jobs: - delete-branch: - runs-on: ubuntu-latest - permissions: - pull-requests: write - if: startsWith(github.event.pull_request.head.ref,'backport/') || startsWith(github.event.pull_request.head.ref,'release-chores/') - steps: - - name: Delete merged branch - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - script: | - github.rest.git.deleteRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: `heads/${context.payload.pull_request.head.ref}`, - }) diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 5ef979e2875..10d66bc30a9 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -13,7 +13,7 @@ on: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.github/workflows/issue-dedupe.yml b/.github/workflows/issue-dedupe.yml index ee0b162e772..609d356eb29 100644 --- a/.github/workflows/issue-dedupe.yml +++ b/.github/workflows/issue-dedupe.yml @@ -23,7 +23,7 @@ on: jobs: detect: if: (github.event_name == 'issues' && github.event.issue.user.type != 'Bot') || (github.event_name == 'workflow_dispatch' && inputs.job == 'detect') - uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-detect.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-detect.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main permissions: contents: read issues: write @@ -36,7 +36,7 @@ jobs: auto-close: if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.job == 'auto-close') - uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-autoclose.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-autoclose.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main permissions: issues: write with: diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml new file mode 100644 index 00000000000..9e9d5e11dc8 --- /dev/null +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -0,0 +1,157 @@ +name: PPL lint rule validation + +# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rule +# `unsupported-window-function-in-eventstats` and the SQL backend must agree. +# +# Frontend half (frontend job): a SQL-owned Node script loads the compiled OSD +# analyzer from an OSD checkout and asserts the rule's diagnostic counts. This +# runs on a bare ubuntu-latest runner because OSD requires a modern Node whose +# prebuilt binary needs a newer glibc than the OpenSearch CI container (Amazon +# Linux 2) provides. +# Backend half (backend job): a Gradle integration test sends the same queries +# to the live `/_plugins/_ppl` endpoint of the SQL plugin built from this +# checkout. This runs inside the OpenSearch CI container because the Gradle +# test cluster needs it. +# +# Both jobs are required; a failure on either side fails the SQL PR check. +# +# The OSD detector is loaded from `main` by default (PR and nightly runs), so a +# removed or changed detector is detected. `workflow_dispatch` can target a +# specific OSD ref to reproduce a run or test an unmerged OSD branch. + +on: + pull_request: + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + inputs: + osd_ref: + description: OSD commit or branch to test instead of main + required: false + type: string + +jobs: + frontend: + name: Frontend contract (OSD analyzer) + runs-on: ubuntu-latest + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve OSD ref + id: osd-ref + env: + REQUESTED_REF: ${{ inputs.osd_ref }} + run: echo "ref=${REQUESTED_REF:-main}" >> "$GITHUB_OUTPUT" + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: opensearch-project/OpenSearch-Dashboards + ref: ${{ steps.osd-ref.outputs.ref }} + path: .ci/OpenSearch-Dashboards + + - name: Record OSD revision + run: | + sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "OSD revision: \`$sha\` (ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" + + # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding + # it, so an OSD toolchain bump does not silently drift this job. + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + # Take the lower bound of the engines.yarn range (e.g. "^1.22.10" -> "1.22.10"). + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: yarn osd bootstrap + + # The Gradle test cluster runs the version from build.gradle's + # `opensearch.version` default (e.g. 3.8.0-SNAPSHOT). Export the release + # portion (3.8.0) as PPL_SQL_VERSION so the frontend applies the same + # version filtering the backend does, without maintaining a second string. + - name: Resolve OpenSearch version + id: os-version + run: | + raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | sed -E 's/.*"opensearch.version", "([^"]+)"/\1/') + version="${raw%%-*}" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "OpenSearch version: \`$version\` (from \`$raw\`)" >> "$GITHUB_STEP_SUMMARY" + + - name: Run frontend contract + working-directory: .ci/OpenSearch-Dashboards + env: + PPL_LINT_CONTRACT_FILE: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json + PPL_SQL_VERSION: ${{ steps.os-version.outputs.version }} + run: | + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + | tee "$GITHUB_WORKSPACE/frontend-contract.log" + + - name: Upload frontend log + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-frontend-contract-log + path: frontend-contract.log + + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + backend: + name: Backend integration test (live /_plugins/_ppl) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # OpenSearch refuses to start as root, so run Gradle as a non-root user. + - name: Run backend integration test + run: | + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + + - name: Upload failure artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-backend-artifacts + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* diff --git a/.github/workflows/pr_review.yml b/.github/workflows/pr_review.yml index ccafed6754f..e351989a11d 100644 --- a/.github/workflows/pr_review.yml +++ b/.github/workflows/pr_review.yml @@ -6,7 +6,7 @@ on: jobs: Code-Diff-Analyzer: - uses: opensearch-project/opensearch-build/.github/workflows/code-diff-analyzer.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/code-diff-analyzer.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main if: github.repository == 'opensearch-project/sql' permissions: id-token: write # github oidc to assume aws roles @@ -18,7 +18,7 @@ jobs: update_pr_comment_with_analyzer_report: true Code-Diff-Reviewer: - uses: opensearch-project/opensearch-build/.github/workflows/code-diff-reviewer.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/code-diff-reviewer.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main needs: Code-Diff-Analyzer if: github.repository == 'opensearch-project/sql' permissions: diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml index 8c0fbd305fe..2085e3afc97 100644 --- a/.github/workflows/sql-pitest.yml +++ b/.github/workflows/sql-pitest.yml @@ -12,7 +12,7 @@ run-name: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 2b8f2e25794..6e24e2838fd 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -20,7 +20,7 @@ on: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index ec23758e126..990cb45010c 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -12,7 +12,7 @@ run-name: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.lycheeignore b/.lycheeignore new file mode 100644 index 00000000000..335800e6047 --- /dev/null +++ b/.lycheeignore @@ -0,0 +1,12 @@ +# example opensearch/dashboards ports +http://localhost:5601 +http://localhost:9200 + +# sites that block scraping +https://hg.openjdk.org/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/ +https://www.linkedin.com/in/*/ + +# we have many pull requests links due to changelogs, skip these as they're unlikely to break and it saves API usage +https://github.com/opendistro-for-elasticsearch/sql/pull/* +https://github.com/opensearch-project/sql/pull/* + diff --git a/README.md b/README.md index 4a7e1e5ec9e..4059fd68ac2 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,6 @@ The following projects are related to the SQL plugin, but stored in the differen | [![untriaged open][untriaged-badge]][untriaged-link] | | [![nolabel open][nolabel-badge]][nolabel-link] | -[dco-badge]: https://github.com/opensearch-project/sql/actions/workflows/dco.yml/badge.svg -[dco-badge-link]: https://github.com/opensearch-project/sql/actions/workflows/dco.yml [link-check-badge]: https://github.com/opensearch-project/sql/actions/workflows/link-checker.yml/badge.svg [link-check-link]: https://github.com/opensearch-project/sql/actions/workflows/link-checker.yml [bwc-tests-badge]: https://img.shields.io/badge/BWC%20tests-in%20progress-yellow diff --git a/api/build.gradle b/api/build.gradle index 638aa8700ce..0ac7ed81880 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -14,13 +14,13 @@ plugins { dependencies { api project(':ppl') api project(':sql') - api group: 'org.apache.calcite', name: 'calcite-babel', version: '1.41.0' + api group: 'org.apache.calcite', name: 'calcite-babel', version: "${calcite_version}" testImplementation testFixtures(project(':api')) testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" - testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0' + testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: "${calcite_version}" testFixturesApi group: 'junit', name: 'junit', version: '4.13.2' testFixturesApi group: 'org.hamcrest', name: 'hamcrest', version: "${hamcrest_version}" diff --git a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java index 82c7b3cf917..27b76e6e5e4 100644 --- a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java +++ b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java @@ -262,7 +262,7 @@ public UnifiedQueryContext build() { private UnifiedQueryParser createParser(CalcitePlanContext planContext, Settings settings) { return switch (queryType) { case PPL -> new PPLQueryParser(settings); - case SQL -> new SqlV2QueryParser(); + case SQL -> new SqlV2QueryParser(settings); }; } diff --git a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java index a84300e65f8..9440833503f 100644 --- a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java +++ b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java @@ -27,6 +27,7 @@ import org.opensearch.sql.calcite.CalciteRelNodeVisitor; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.QueryEngineException; import org.opensearch.sql.exception.SemanticCheckException; @@ -73,6 +74,10 @@ public RelNode plan(String query) { } return plan; }); + } catch (CalciteUnsupportedException e) { + // Unsupported feature (e.g. table functions) is an invalid query, i.e. a client error. + // Must precede the QueryEngineException branch as it is a subclass. + throw new SemanticCheckException(e.getMessage(), e); } catch (SyntaxCheckException | QueryEngineException | UnsupportedOperationException diff --git a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java index d60ef7ed4ec..b1f61e732ff 100644 --- a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java +++ b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java @@ -10,29 +10,47 @@ import static org.opensearch.sql.ast.dsl.AstDSL.join; import static org.opensearch.sql.ast.dsl.AstDSL.union; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.tree.ParseTree; +import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Not; import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.ast.tree.Join.JoinType; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.antlr.AstBuildGuard; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ExistsSubqueryExpressionAtomContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.FromClauseContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.InSubqueryPredicateContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.JoinClauseContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.OrderByClauseContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.QuerySpecificationContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.UnionSelectContext; import org.opensearch.sql.sql.parser.AstBuilder; import org.opensearch.sql.sql.parser.AstExpressionBuilder; +import org.opensearch.sql.sql.parser.AstSortBuilder; import org.opensearch.sql.sql.parser.AstStatementBuilder; +import org.opensearch.sql.sql.parser.context.QuerySpecification; /** SQL query parser that produces {@link UnresolvedPlan} using the V2 ANTLR grammar. */ +@RequiredArgsConstructor public class SqlV2QueryParser implements UnifiedQueryParser { + /** Settings containing execution limits and feature flags used by AST builders. */ + private final Settings settings; + /** Reusable ANTLR-based SQL syntax parser. Stateless and thread-safe. */ private final SQLSyntaxParser syntaxParser = new SQLSyntaxParser(); @@ -41,7 +59,7 @@ public UnresolvedPlan parse(String query) { ParseTree cst = syntaxParser.parse(query); AstStatementBuilder astStmtBuilder = new AstStatementBuilder( - new ExtendedAstBuilder(query), + new ExtendedAstBuilder(query, settings), AstStatementBuilder.StatementBuilderContext.builder().build()); Statement statement = cst.accept(astStmtBuilder); @@ -58,13 +76,60 @@ public UnresolvedPlan parse(String query) { */ private static class ExtendedAstBuilder extends AstBuilder { - ExtendedAstBuilder(String query) { - super(query); + ExtendedAstBuilder(String query, Settings settings) { + super(query, settings); + } + + @Override + public UnresolvedPlan visitQuerySpecification(QuerySpecificationContext queryContext) { + if (!hasWindowFunctionInProjectList(queryContext)) { + return super.visitQuerySpecification(queryContext); + } + + context.push(); + context.peek().collect(queryContext, query); + Project project = (Project) visit(queryContext.selectClause()); + UnresolvedPlan result = project.attach(visit(queryContext.fromClause())); + + // Window output must be computed before ORDER BY/LIMIT, so build Limit(Sort(Project(from))) + OrderByClauseContext orderByClause = queryContext.fromClause().orderByClause(); + if (orderByClause != null) { + result = new ExtendedAstSortBuilder(context.peek()).visit(orderByClause).attach(result); + } + if (queryContext.limitClause() != null) { + result = visit(queryContext.limitClause()).attach(result); + } + + context.pop(); + return result; + } + + @Override + public UnresolvedPlan visitFromClause(FromClauseContext ctx) { + UnresolvedPlan from = super.visitFromClause(ctx); + if (hasWindowFunctionInProjectList(context.peek()) && from instanceof Sort sort) { + // Drop the ORDER BY Sort for window queries; it is re-attached above the Project + return sort.getChild().get(0); + } + return from; + } + + private boolean hasWindowFunctionInProjectList(QuerySpecificationContext queryContext) { + if (queryContext.fromClause() == null) { + return false; + } + QuerySpecification probe = new QuerySpecification(); + probe.collect(queryContext, query); + return hasWindowFunctionInProjectList(probe); + } + + private static boolean hasWindowFunctionInProjectList(QuerySpecification querySpec) { + return querySpec.getSelectItems().stream().anyMatch(item -> item instanceof WindowFunction); } @Override protected AstExpressionBuilder createExpressionBuilder() { - return new ExtendedAstExpressionBuilder(); + return new ExtendedAstExpressionBuilder(guard); } @Override @@ -99,6 +164,10 @@ public UnresolvedPlan visitUnionSelect(UnionSelectContext ctx) { */ private class ExtendedAstExpressionBuilder extends AstExpressionBuilder { + ExtendedAstExpressionBuilder(AstBuildGuard guard) { + super(guard); + } + @Override public UnresolvedExpression visitInSubqueryPredicate(InSubqueryPredicateContext ctx) { UnresolvedPlan subquery = ExtendedAstBuilder.this.visit(ctx.querySpecification()); @@ -114,4 +183,32 @@ public UnresolvedExpression visitExistsSubqueryExpressionAtom( } } } + + /** + * Keeps an ORDER BY window-alias as a column reference (Sort is above the Project) to avoid a + * second RexOver. + */ + private static class ExtendedAstSortBuilder extends AstSortBuilder { + + ExtendedAstSortBuilder(QuerySpecification querySpec) { + super(querySpec); + } + + @Override + public UnresolvedPlan visitOrderByClause(OrderByClauseContext ctx) { + List fields = new ArrayList<>(); + List items = querySpec.getOrderByItems(); + List options = querySpec.getOrderByOptions(); + for (int i = 0; i < items.size(); i++) { + UnresolvedExpression item = items.get(i); + UnresolvedExpression sortKey = + (querySpec.isSelectAlias(item) + && querySpec.getSelectItemByAlias(item) instanceof WindowFunction) + ? item + : querySpec.replaceIfAliasOrOrdinal(item); + fields.add(new Field(sortKey, createSortArguments(options.get(i)))); + } + return new Sort(fields); + } + } } diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index 8538ebbae61..3320391c0d2 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -272,7 +272,7 @@ public void selectExpressionWithoutFrom() { givenQuery("SELECT 1 + 1") .assertPlan( """ - LogicalProject(1 + 1=[+(1, 1)]) + LogicalProject(1 + 1=[+(1:BIGINT, 1:BIGINT)]) LogicalValues(tuples=[[{ 0 }]]) """); } @@ -404,7 +404,7 @@ public void testArithmeticOnAggregates() { givenQuery("SELECT MAX(age) + MIN(age) AS range_sum FROM catalog.employees") .assertPlan( """ - LogicalProject(range_sum=[+($0, $1)]) + LogicalProject(range_sum=[+(CAST($0):BIGINT, CAST($1):BIGINT)]) LogicalAggregate(group=[{}], MAX(age)=[MAX($0)], MIN(age)=[MIN($0)]) LogicalProject(age=[$2]) LogicalTableScan(table=[[catalog, employees]]) @@ -547,4 +547,73 @@ SELECT LENGTH(name) FROM catalog.employees GROUP BY LENGTH(name) ORDER BY LENGTH LogicalTableScan(table=[[catalog, employees]]) """); } + + @Test + public void testWindowOverGroupByWithLimit() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM catalog.employees GROUP BY department LIMIT 3 + """) + .assertPlan( + """ + LogicalSort(fetch=[3]) + LogicalProject(department=[$0], cnt=[$1], rn=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS FIRST)]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testWindowOverGroupByOrderByWindowAlias() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM catalog.employees GROUP BY department ORDER BY rn LIMIT 3 + """) + .assertPlan( + """ + LogicalSort(sort0=[$2], dir0=[ASC-nulls-first], fetch=[3]) + LogicalProject(department=[$0], cnt=[$1], rn=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS FIRST)]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testWindowOverGroupByOrderByWindowAliasWithoutLimit() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM catalog.employees GROUP BY department ORDER BY rn + """) + .assertPlan( + """ + LogicalSort(sort0=[$2], dir0=[ASC-nulls-first]) + LogicalProject(department=[$0], cnt=[$1], rn=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS FIRST)]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testMultipleWindowFunctionsOrderByWindowAlias() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn, + ROW_NUMBER() OVER (ORDER BY department) AS rn2 + FROM catalog.employees GROUP BY department ORDER BY rn LIMIT 3 + """) + .assertPlan( + """ + LogicalSort(sort0=[$2], dir0=[ASC-nulls-first], fetch=[3]) + LogicalProject(department=[$0], cnt=[$1], rn=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS FIRST)], rn2=[ROW_NUMBER() OVER (ORDER BY $0 NULLS FIRST)]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } } diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java index 296e9eb2519..008121e8377 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java @@ -16,6 +16,7 @@ import org.junit.Test; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.executor.QueryType; @@ -147,6 +148,29 @@ public void invalidTableIsRethrownAsSemanticCheckException() { .assertCauseType(CalciteException.class); } + @Test + public void unsupportedFeatureIsRethrownAsSemanticCheckException() { + // A feature unsupported on the analytics engine (here a PPL command that raises + // CalciteUnsupportedException; SQL table functions like vectorSearch() take the same path) is + // an invalid query, normalized to a SemanticCheckException so callers classify it as a 4xx. + givenInvalidQuery("source = catalog.employees | kmeans") + .assertErrorType(SemanticCheckException.class) + .assertCauseType(CalciteUnsupportedException.class) + .assertErrorMessageContains("unsupported in Calcite"); + } + + @Test + public void unsupportedWindowFunctionIsRethrownAsSemanticCheckException() { + // Window functions outside WINDOW_FUNC_MAPPING reach + // CalciteRexNodeVisitor#visitWindowFunction's + // orElseThrow. The throw site emits CalciteUnsupportedException so this path normalizes to a + // 4xx SemanticCheckException rather than escaping as a 500. + givenInvalidQuery("source = catalog.employees | eventstats rank()") + .assertErrorType(SemanticCheckException.class) + .assertCauseType(CalciteUnsupportedException.class) + .assertErrorMessageContains("Unexpected window function: rank"); + } + @Test public void assertionErrorIsWrappedAsSemanticCheckException() { // Remove when the underlying Calcite assertion is fixed. diff --git a/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java b/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java index 1b6b5181aef..bb06e7c7b8b 100644 --- a/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java +++ b/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java @@ -26,9 +26,12 @@ import static org.opensearch.sql.ast.dsl.AstDSL.relation; import org.junit.Test; +import org.opensearch.sql.api.UnifiedQueryContext; import org.opensearch.sql.api.UnifiedQueryTestBase; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.executor.QueryType; public class UnifiedQueryParserTest extends UnifiedQueryTestBase { @@ -80,6 +83,38 @@ public void testSyntaxErrorThrows() { assertThrows(SyntaxCheckException.class, () -> context.getParser().parse("not a valid query")); } + @Test + public void deeplyNestedExpressionShouldBeRejected() throws Exception { + String key = Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(); + try (UnifiedQueryContext ppl = + UnifiedQueryContext.builder() + .language(QueryType.PPL) + .catalog(DEFAULT_CATALOG, testSchema) + .setting(key, 20) + .build(); + UnifiedQueryContext sql = + UnifiedQueryContext.builder() + .language(QueryType.SQL) + .catalog(DEFAULT_CATALOG, testSchema) + .setting(key, 20) + .build()) { + assertThrows( + IllegalArgumentException.class, + () -> ppl.getParser().parse("source = catalog.employees | where " + orChain(30))); + assertThrows( + IllegalArgumentException.class, + () -> sql.getParser().parse("SELECT * FROM catalog.employees WHERE " + orChain(30))); + } + } + + private String orChain(int terms) { + StringBuilder sb = new StringBuilder("age = 1"); + for (int i = 2; i <= terms; i++) { + sb.append(" or age = ").append(i); + } + return sb.toString(); + } + private void assertEqual(String query, UnresolvedPlan expected) { UnresolvedPlan actual = (UnresolvedPlan) context.getParser().parse(query); assertEquals(expected, actual); diff --git a/async-query-core/README.md b/async-query-core/README.md index 08301c024d7..f6d08ad7103 100644 --- a/async-query-core/README.md +++ b/async-query-core/README.md @@ -16,7 +16,7 @@ Following is the list of extension points where the consumer of the library need - Data store interface - [AsyncQueryJobMetadataStorageService](src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java) - - [SessionStorageService](java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java) + - [SessionStorageService](src/main/java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java) - [StatementStorageService](src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java) - [FlintIndexMetadataService](src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java) - [FlintIndexStateModelService](src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java) @@ -39,4 +39,4 @@ To update the grammar files, update `build.gradle` file (in `downloadG4Files` ta ``` ./gradlew async-query-core:downloadG4Files ``` -This will overwrite the files under `src/main/antlr`. \ No newline at end of file +This will overwrite the files under `src/main/antlr`. diff --git a/benchmarks/README.md b/benchmarks/README.md index 6a720c7201c..7467e37f2f0 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,6 @@ # OpenSearch SQL/PPL Microbenchmark Suite -This directory contains the microbenchmark suite of OpenSearch SQL/PPL. It relies on [JMH](http://openjdk.java.net/projects/code-tools/jmh/). +This directory contains the microbenchmark suite of OpenSearch SQL/PPL. It relies on [JMH](https://openjdk.java.net/projects/code-tools/jmh/). ## Purpose @@ -25,6 +25,6 @@ Run specific benchmarks using the `-Pjmh.includes` parameter: ## Adding Microbenchmarks -Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the [JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/). +Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the [JMH samples](https://hg.openjdk.org/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/). -In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`. \ No newline at end of file +In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`. diff --git a/build.gradle b/build.gradle index 7b76532aad0..9047a9c3feb 100644 --- a/build.gradle +++ b/build.gradle @@ -52,6 +52,7 @@ buildscript { // TODO: Migrate following to Gradle version catalog || Read from OpenSearch BOM in the future. // See: https://github.com/opensearch-project/sql/issues/3257 aws_java_sdk_version = "1.12.651" + calcite_version = "1.42.0" guava_version = "33.3.0-jre" resilience4j_version = "1.5.0" hamcrest_version = "2.1" @@ -162,6 +163,9 @@ allprojects { resolutionStrategy.force 'org.apache.commons:commons-text:1.11.0' resolutionStrategy.force 'commons-io:commons-io:2.15.0' resolutionStrategy.force 'org.yaml:snakeyaml:2.2' + // Align Calcite 1.42.0's bumped transitive deps under strict conflict resolution + resolutionStrategy.force 'com.jayway.jsonpath:json-path:2.10.0' + resolutionStrategy.force 'org.jooq:joou-java-6:0.9.5' resolutionStrategy.dependencySubstitution { substitute module('commons-lang:commons-lang') using module('org.apache.commons:commons-lang3:3.18.0') because 'CVE-2025-48924: commons-lang 2.x vulnerable to StackOverflowError' } diff --git a/common/src/main/java/org/opensearch/sql/common/antlr/AstBuildGuard.java b/common/src/main/java/org/opensearch/sql/common/antlr/AstBuildGuard.java new file mode 100644 index 00000000000..7dd6311a150 --- /dev/null +++ b/common/src/main/java/org/opensearch/sql/common/antlr/AstBuildGuard.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.common.antlr; + +import java.util.function.Supplier; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.utils.StringUtils; + +/** + * Guards AST construction against pathological queries by bounding the depth of parse-tree visitor + * recursion, rejecting over-nested expressions with an {@link IllegalArgumentException} instead of + * letting a StackOverflowError crash the node. + * + *

Holds per-traversal state, so an instance must not be shared across threads. + */ +@RequiredArgsConstructor +public final class AstBuildGuard { + + public static final int DEFAULT_MAX_DEPTH = 1000; + + /** Maximum nesting depth allowed; {@code 0} or less means unlimited. */ + private final int maxDepth; + + /** Live nesting depth of the in-progress traversal; resets to 0 between top-level visits. */ + private int depth = 0; + + public AstBuildGuard() { + this(DEFAULT_MAX_DEPTH); + } + + /** Builds a guard from the configured {@code plugins.query.max_expression_depth} setting. */ + public static AstBuildGuard fromSettings(Settings settings) { + Integer configured = + settings == null ? null : settings.getSettingValue(Settings.Key.MAX_EXPRESSION_DEPTH); + return new AstBuildGuard(configured == null ? DEFAULT_MAX_DEPTH : configured); + } + + /** + * Runs a single AST-build descent under the configured guardrails. + * + * @param visit the visitor descent to execute + * @return the result of {@code visit} + * @throws IllegalArgumentException if a positive maximum depth is configured and would be + * exceeded + */ + public T enforce(Supplier visit) { + if (maxDepth > 0 && depth >= maxDepth) { + throw new IllegalArgumentException( + StringUtils.format( + "Expression nesting depth exceeds the maximum allowed [%d]; simplify the query or" + + " reduce the number of chained conditions.", + maxDepth)); + } + depth++; + try { + return visit.get(); + } finally { + depth--; + } + } +} diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 96fe2e04eea..5473aa8812e 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,6 +36,8 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), + PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), + PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), @@ -51,6 +53,7 @@ public enum Key { /** Common Settings for SQL and PPL. */ QUERY_MEMORY_LIMIT("plugins.query.memory_limit"), QUERY_SIZE_LIMIT("plugins.query.size_limit"), + MAX_EXPRESSION_DEPTH("plugins.query.max_expression_depth"), QUERY_BUCKET_SIZE("plugins.query.buckets"), SEARCH_MAX_BUCKETS("search.max_buckets"), ENCYRPTION_MASTER_KEY("plugins.query.datasources.encryption.masterkey"), diff --git a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java index 4b7752a9de5..a3f55fcd231 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java @@ -6,8 +6,10 @@ package org.opensearch.sql.common.utils; import com.google.common.base.Strings; +import java.util.Collection; import java.util.IllegalFormatException; import java.util.Locale; +import java.util.Optional; public class StringUtils { /** @@ -96,4 +98,83 @@ public static String format(final String format, Object... args) { private static boolean isQuoted(String text, String mark) { return !Strings.isNullOrEmpty(text) && text.startsWith(mark) && text.endsWith(mark); } + + /** + * Calculates the Levenshtein distance between two strings. + * + * @param s1 first string + * @param s2 second string + * @return the Levenshtein distance between s1 and s2 + */ + public static int levenshteinDistance(String s1, String s2) { + if (s1 == null || s2 == null) { + return Integer.MAX_VALUE; + } + if (s1.equals(s2)) { + return 0; + } + + int len1 = s1.length(); + int len2 = s2.length(); + + if (len1 == 0) { + return len2; + } + if (len2 == 0) { + return len1; + } + + int[] prev = new int[len2 + 1]; + int[] curr = new int[len2 + 1]; + + for (int j = 0; j <= len2; j++) { + prev[j] = j; + } + + for (int i = 1; i <= len1; i++) { + curr[0] = i; + for (int j = 1; j <= len2; j++) { + int cost = (s1.charAt(i - 1) == s2.charAt(j - 1)) ? 0 : 1; + curr[j] = Math.min(Math.min(curr[j - 1] + 1, prev[j] + 1), prev[j - 1] + cost); + } + int[] temp = prev; + prev = curr; + curr = temp; + } + + return prev[len2]; + } + + /** + * Finds the closest match to a target string from a collection of candidates using Levenshtein + * distance. Returns empty if no candidates are provided or if the best match distance is too + * large. + * + * @param target the string to match against + * @param candidates the collection of candidate strings + * @return the closest match, or empty if no good match is found + */ + public static Optional findClosestMatch(String target, Collection candidates) { + if (target == null || candidates == null || candidates.isEmpty()) { + return Optional.empty(); + } + + String bestMatch = null; + int bestDistance = Integer.MAX_VALUE; + + for (String candidate : candidates) { + int distance = levenshteinDistance(target.toLowerCase(), candidate.toLowerCase()); + if (distance < bestDistance) { + bestDistance = distance; + bestMatch = candidate; + } + } + + // Only return a suggestion if the distance is reasonable + if (bestMatch != null && bestDistance <= Math.max(4, target.length() / 2)) { + return Optional.of(bestMatch); + } + + return Optional.empty(); + } } diff --git a/core/build.gradle b/core/build.gradle index 4d0d98edb29..f4ccaa0d6e1 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -60,10 +60,10 @@ dependencies { api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' api "net.minidev:json-smart:${versions.json_smart}" - api('org.apache.calcite:calcite-core:1.41.0') { + api("org.apache.calcite:calcite-core:${calcite_version}") { exclude group: 'net.minidev', module: 'json-smart' } - api 'org.apache.calcite:calcite-linq4j:1.41.0' + api "org.apache.calcite:calcite-linq4j:${calcite_version}" api project(':common') compileOnly 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT' // Needed because analytics-api's QueryPlanExecutor signature uses @@ -72,7 +72,7 @@ dependencies { testImplementation 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT' testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" implementation "com.github.seancfoley:ipaddress:5.4.2" - implementation "com.jayway.jsonpath:json-path:2.9.0" + implementation "com.jayway.jsonpath:json-path:2.10.0" annotationProcessor('org.immutables:value:2.8.8') compileOnly 'org.immutables:value-annotations:2.8.8' diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index d7b8ceb1bff..8c1dbee006f 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -74,6 +74,7 @@ import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; +import org.opensearch.sql.ast.tree.Foreach; import org.opensearch.sql.ast.tree.GraphLookup; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Join; @@ -571,6 +572,11 @@ public LogicalPlan visitGraphLookup(GraphLookup node, AnalysisContext context) { throw getOnlyForCalciteException("graphlookup"); } + @Override + public LogicalPlan visitForeach(Foreach node, AnalysisContext context) { + throw getOnlyForCalciteException("foreach"); + } + /** Build {@link ParseExpression} to context and skip to child nodes. */ @Override public LogicalPlan visitParse(Parse node, AnalysisContext context) { diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index be02547a2da..a32354883bf 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -18,6 +18,7 @@ import org.opensearch.sql.ast.expression.Compare; import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.ForeachPlaceholder; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; @@ -62,6 +63,7 @@ import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; +import org.opensearch.sql.ast.tree.Foreach; import org.opensearch.sql.ast.tree.GraphLookup; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Join; @@ -91,6 +93,7 @@ import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Timewrap; import org.opensearch.sql.ast.tree.Transpose; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Union; @@ -173,6 +176,10 @@ public T visitProject(Project node, C context) { return visitChildren(node, context); } + public T visitForeach(Foreach node, C context) { + return visitChildren(node, context); + } + public T visitAggregation(Aggregation node, C context) { return visitChildren(node, context); } @@ -253,6 +260,10 @@ public T visitField(Field node, C context) { return visitChildren(node, context); } + public T visitForeachPlaceholder(ForeachPlaceholder node, C context) { + return visitChildren(node, context); + } + public T visitQualifiedName(QualifiedName node, C context) { return visitChildren(node, context); } @@ -301,6 +312,10 @@ public T visitChart(Chart node, C context) { return visitChildren(node, context); } + public T visitTimewrap(Timewrap node, C context) { + return visitChildren(node, context); + } + public T visitRegex(Regex node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/ForeachPlaceholder.java b/core/src/main/java/org/opensearch/sql/ast/expression/ForeachPlaceholder.java new file mode 100644 index 00000000000..9362a70d22f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/expression/ForeachPlaceholder.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.expression; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +/** Placeholder used by the PPL foreach command, such as {@code <>}. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class ForeachPlaceholder extends UnresolvedExpression { + private final String name; + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitForeachPlaceholder(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java b/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java index 84fb486702a..3290c438cd5 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java @@ -20,22 +20,41 @@ import org.opensearch.sql.ast.AbstractNodeVisitor; @Getter -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode( + callSuper = false, + exclude = {"line", "column"}) public class QualifiedName extends UnresolvedExpression { public static final String DELIMITER = "."; private final List parts; + private final Integer line; + + private final Integer column; + public QualifiedName(String name) { - this.parts = Collections.singletonList(name); + this(Collections.singletonList(name), null, null); } /** QualifiedName Constructor. */ public QualifiedName(Iterable parts) { + this(parts, null, null); + } + + /** + * Constructor with source position. + * + * @param parts The parts of the qualified name + * @param line Line number (1-based), null if not available + * @param column Column position (0-based), null if not available + */ + public QualifiedName(Iterable parts, Integer line, Integer column) { List partsList = StreamSupport.stream(parts.spliterator(), false).collect(toList()); if (partsList.isEmpty()) { throw new IllegalArgumentException("parts is empty"); } this.parts = partsList; + this.line = line; + this.column = column; } /** Construct {@link QualifiedName} from list of string. */ diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java index b2fcb2b8179..01ce206ad23 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java @@ -9,6 +9,7 @@ import lombok.Getter; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.protocol.response.format.Format; /** Explain Statement. */ @Getter @@ -18,15 +19,21 @@ public class Explain extends Statement { private final Statement statement; private final QueryType queryType; private final ExplainMode mode; + private final Format format; public Explain(Statement statement, QueryType queryType) { - this(statement, queryType, null); + this(statement, queryType, null, null); } public Explain(Statement statement, QueryType queryType, String mode) { + this(statement, queryType, mode, null); + } + + public Explain(Statement statement, QueryType queryType, String mode, Format format) { this.statement = statement; this.queryType = queryType; this.mode = ExplainMode.of(mode); + this.format = format; } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Foreach.java b/core/src/main/java/org/opensearch/sql/ast/tree/Foreach.java new file mode 100644 index 00000000000..e11ecf541cd --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Foreach.java @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** AST node representing the PPL foreach command. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Foreach extends UnresolvedPlan { + private final Mode mode; + private final Map options; + private final List fieldPatterns; + private final UnresolvedExpression collectionExpression; + private final List evalClauses; + private UnresolvedPlan child; + + @Override + public Foreach attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return child == null ? ImmutableList.of() : ImmutableList.of(child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitForeach(this, context); + } + + /** Iteration mode of the foreach command. */ + public enum Mode { + MULTIFIELD, + MULTIVALUE, + JSON_ARRAY, + AUTO_COLLECTIONS; + + public static Mode of(String name) { + try { + return valueOf(name.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("foreach mode [" + name + "] is not supported"); + } + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + @Getter + @ToString + @EqualsAndHashCode + @RequiredArgsConstructor + public static class ForeachEvalClause { + private final String targetTemplate; + private final UnresolvedExpression expression; + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java new file mode 100644 index 00000000000..d0a0e34d3c9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import java.util.Collections; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * Extend Relation to mark a {@code rest} leading command. The single table name is a reserved, + * encoded token (produced by {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}) that + * carries the validated REST endpoint spec; it resolves through the storage engine to a REST source + * table on the Calcite path, exactly as {@link DescribeRelation} resolves to a system index. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +public class RestRelation extends Relation { + public RestRelation(UnresolvedExpression tableName) { + super(Collections.singletonList(tableName)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Rex.java b/core/src/main/java/org/opensearch/sql/ast/tree/Rex.java index c3b1d975ac1..4b0e2188617 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Rex.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Rex.java @@ -79,7 +79,7 @@ public Rex attach(UnresolvedPlan child) { @Override public List getChild() { - return ImmutableList.of(child); + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Timewrap.java b/core/src/main/java/org/opensearch/sql/ast/tree/Timewrap.java new file mode 100644 index 00000000000..88c151db331 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Timewrap.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.SpanUnit; + +/** AST node representing the timewrap command. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Timewrap extends UnresolvedPlan { + private final SpanUnit unit; + private final int value; + private final String align; // "end" or "now" + private final String series; // "relative", "short", or "exact" + private final String timeFormat; // format string for series=exact, nullable + private final Literal spanLiteral; // original span literal for display + + private UnresolvedPlan child; + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitTimewrap(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 7ca7ab09304..162a4895805 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -17,10 +17,12 @@ import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.FrameworkConfig; +import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.UnresolvedExpression; @@ -44,6 +46,18 @@ public class CalcitePlanContext { /** This thread local variable is only used to skip script encoding in script pushdown. */ public static final ThreadLocal skipEncoding = ThreadLocal.withInitial(() -> false); + /** When true, the execution engine strips all-null columns from the result (used by timewrap). */ + public static final ThreadLocal stripNullColumns = ThreadLocal.withInitial(() -> false); + + /** + * Timewrap span unit name for column renaming in the execution engine. When set, the execution + * engine uses __base_offset__ to compute absolute period names (e.g., "501days_before"). + */ + public static final ThreadLocal timewrapUnitName = new ThreadLocal<>(); + + /** Timewrap series mode: "relative", "short", or "exact". */ + public static final ThreadLocal timewrapSeries = new ThreadLocal<>(); + /** Thread-local switch that tells whether the current query prefers legacy behavior. */ private static final ThreadLocal legacyPreferredFlag = ThreadLocal.withInitial(() -> true); @@ -66,6 +80,26 @@ public class CalcitePlanContext { @Getter public Map rexLambdaRefMap; + /** + * Foreach placeholder bindings active in this context, keyed by upper-cased placeholder name. + * Multifield mode activates bindings directly (placeholders resolve against the current row); + * collection modes stage bindings via {@link #stageForeachLambdaBindings} instead, so they only + * become active inside the lambda context cloned for the generated {@code reduce} call. + */ + @Getter private Map foreachBindings = new HashMap<>(); + + /** Bare identifiers enabled by explicit options such as {@code itemstr=ITEM}. */ + @Getter private Map foreachIdentifierBindings = new HashMap<>(); + + /** Bindings that become active in lambda contexts cloned from this one. */ + private Map stagedForeachLambdaBindings = new HashMap<>(); + + /** Bare identifier bindings staged for a generated foreach lambda. */ + private Map stagedForeachIdentifierBindings = new HashMap<>(); + + /** Expressions computed by earlier assignments in the same foreach eval iteration. */ + @Getter private Map foreachComputedBindings = new HashMap<>(); + /** * Maps AggregateFunction AST nodes to their output field index for HAVING/post-aggregate * resolution. @@ -113,6 +147,12 @@ private CalcitePlanContext(CalcitePlanContext parent) { this.rexLambdaRefMap = new HashMap<>(); // New map for lambda variables this.capturedVariables = new ArrayList<>(); // New list for captured variables this.inLambdaContext = true; // Mark that we're inside a lambda + // Active bindings carry over; staged bindings become active inside the lambda. + this.foreachBindings = new HashMap<>(parent.foreachBindings); + this.foreachBindings.putAll(parent.stagedForeachLambdaBindings); + this.foreachIdentifierBindings = new HashMap<>(parent.foreachIdentifierBindings); + this.foreachIdentifierBindings.putAll(parent.stagedForeachIdentifierBindings); + this.foreachComputedBindings = new HashMap<>(parent.foreachComputedBindings); } public RexNode resolveJoinCondition( @@ -169,6 +209,7 @@ public static void run(Runnable action, Settings settings) { action.run(); } finally { legacyPreferredFlag.remove(); + clearTimewrapSignals(); } } @@ -179,6 +220,59 @@ public static boolean isLegacyPreferred() { return legacyPreferredFlag.get(); } + /** + * Resets the timewrap thread-locals set by {@code CalciteRelNodeVisitor.visitTimewrap}. Called + * from the query lifecycle's {@code finally} on every path (execute, explain, and exceptions) so + * the signals never leak onto the next query that reuses this pooled worker thread. + */ + public static void clearTimewrapSignals() { + stripNullColumns.set(false); + timewrapUnitName.set(null); + timewrapSeries.set(null); + } + + public void pushForeachBindings( + Map bindings, Map identifierBindings) { + foreachBindings = new HashMap<>(bindings); + foreachIdentifierBindings = new HashMap<>(identifierBindings); + } + + public void stageForeachLambdaBindings( + Map bindings, Map identifierBindings) { + stagedForeachLambdaBindings = new HashMap<>(bindings); + stagedForeachIdentifierBindings = new HashMap<>(identifierBindings); + } + + public void putForeachComputedBinding(String name, RexNode expression) { + foreachComputedBindings.put(name.toUpperCase(java.util.Locale.ROOT), expression); + } + + public void clearForeachBindings() { + foreachBindings.clear(); + foreachIdentifierBindings.clear(); + stagedForeachLambdaBindings.clear(); + stagedForeachIdentifierBindings.clear(); + foreachComputedBindings.clear(); + } + + /** + * A foreach placeholder binding. {@code FIELD} resolves to the named row field, {@code LITERAL} + * to a string literal, and {@code PAIR_SLOT} to slot {@code pairIndex} (typed {@code pairType}) + * of the named lambda pair variable. + */ + public record ForeachBinding( + String value, ForeachBindingType type, int pairIndex, @Nullable RelDataType pairType) { + public ForeachBinding(String value, ForeachBindingType type) { + this(value, type, -1, null); + } + } + + public enum ForeachBindingType { + FIELD, + LITERAL, + PAIR_SLOT + } + public void putRexLambdaRefMap(Map candidateMap) { this.rexLambdaRefMap.putAll(candidateMap); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index b07f308f91f..0df3c571c6b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Streams; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -129,6 +130,7 @@ import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; +import org.opensearch.sql.ast.tree.Foreach; import org.opensearch.sql.ast.tree.GraphLookup; import org.opensearch.sql.ast.tree.GraphLookup.Direction; import org.opensearch.sql.ast.tree.Head; @@ -160,6 +162,7 @@ import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Timewrap; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Trendline.TrendlineType; import org.opensearch.sql.ast.tree.Union; @@ -176,6 +179,7 @@ import org.opensearch.sql.calcite.utils.JoinAndLookupUtils; import org.opensearch.sql.calcite.utils.PPLHintUtils; import org.opensearch.sql.calcite.utils.PlanUtils; +import org.opensearch.sql.calcite.utils.TimewrapUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.calcite.utils.WildcardUtils; import org.opensearch.sql.common.error.ErrorCode; @@ -187,6 +191,7 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.PPLFuncImpTable; import org.opensearch.sql.expression.parse.RegexCommonUtils; import org.opensearch.sql.utils.ParseUtils; @@ -210,12 +215,14 @@ public class CalciteRelNodeVisitor extends AbstractNodeVisitor newFields, List newNames, CalcitePlanContext context) { Set originalFieldNameSet = new HashSet<>(context.relBuilder.peek().getRowType().getFieldNames()); @@ -3822,6 +3836,148 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { return relBuilder.peek(); } + @Override + public RelNode visitTimewrap(Timewrap node, CalcitePlanContext context) { + visitChildren(node, context); + + // Signal the execution engine to strip all-null columns and rename with absolute offsets + CalcitePlanContext.stripNullColumns.set(true); + CalcitePlanContext.timewrapUnitName.set( + TimewrapUtils.unitBaseName(node.getUnit(), node.getValue()) + "|_before"); + CalcitePlanContext.timewrapSeries.set(node.getSeries()); + + RelBuilder b = context.relBuilder; + RexBuilder rx = context.rexBuilder; + + List fieldNames = + b.peek().getRowType().getFieldNames().stream().filter(f -> !isMetadataField(f)).toList(); + String tsFieldName = fieldNames.get(0); + List valueFieldNames = fieldNames.subList(1, fieldNames.size()); + + boolean variableLength = TimewrapUtils.isVariableLengthUnit(node.getUnit()); + RelDataType bigintType = rx.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + + RexNode periodNum; + RexNode displayTimestamp; + RexNode baseOffset; + + if (variableLength) { + // --- Variable-length units (month, quarter, year): EXTRACT-based calendar arithmetic --- + RexNode tsField = b.field(tsFieldName); + RexNode tsUnitNum = + TimewrapUtils.calendarUnitNumber(rx, tsField, node.getUnit(), node.getValue()); + + b.projectPlus(b.aggregateCall(SqlStdOperatorTable.MAX, tsField).over().as("__max_ts__")); + RexNode maxTs = b.field("__max_ts__"); + RexNode maxUnitNum = + TimewrapUtils.calendarUnitNumber(rx, maxTs, node.getUnit(), node.getValue()); + + periodNum = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall(SqlStdOperatorTable.MINUS, maxUnitNum, tsUnitNum), + rx.makeExactLiteral(BigDecimal.ONE, bigintType)); + + RexNode tsEpoch = + rx.makeCast(bigintType, rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, tsField), true); + RexNode unitStartEpoch = TimewrapUtils.calendarUnitStartEpoch(rx, tsField, node.getUnit()); + RexNode offsetSec = rx.makeCall(SqlStdOperatorTable.MINUS, tsEpoch, unitStartEpoch); + RexNode maxUnitStartEpoch = TimewrapUtils.calendarUnitStartEpoch(rx, maxTs, node.getUnit()); + RexNode displayEpoch = rx.makeCall(SqlStdOperatorTable.PLUS, maxUnitStartEpoch, offsetSec); + displayTimestamp = rx.makeCall(PPLBuiltinOperators.FROM_UNIXTIME, displayEpoch); + + long nowEpochSec = context.functionProperties.getQueryStartClock().millis() / 1000; + Long referenceEpoch = null; + if ("end".equals(node.getAlign())) { + referenceEpoch = TimewrapUtils.extractTimestampUpperBound(node); + } + if (referenceEpoch == null) { + referenceEpoch = nowEpochSec; + } + long refUnitNum = + TimewrapUtils.calendarUnitNumberFromEpoch( + referenceEpoch, node.getUnit(), node.getValue()); + RexNode refUnitNumLit = rx.makeBigintLiteral(BigDecimal.valueOf(refUnitNum)); + baseOffset = rx.makeCall(SqlStdOperatorTable.MINUS, refUnitNumLit, maxUnitNum); + + } else { + // --- Fixed-length units (sec, min, hr, day, week): epoch-based arithmetic --- + long spanSec = TimewrapUtils.spanToSeconds(node.getUnit(), node.getValue()); + + RexNode tsEpochExpr = + rx.makeCast( + bigintType, + rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, b.field(tsFieldName)), + true); + b.projectPlus( + b.alias(tsEpochExpr, "__ts_epoch__"), + b.aggregateCall(SqlStdOperatorTable.MAX, tsEpochExpr).over().as("__max_epoch__")); + + RexNode tsEpoch = b.field("__ts_epoch__"); + RexNode maxEpoch = b.field("__max_epoch__"); + RexNode spanLit = rx.makeBigintLiteral(BigDecimal.valueOf(spanSec)); + + RexNode diff = rx.makeCall(SqlStdOperatorTable.MINUS, maxEpoch, tsEpoch); + periodNum = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall(SqlStdOperatorTable.DIVIDE, diff, spanLit), + rx.makeExactLiteral(BigDecimal.ONE, bigintType)); + + RexNode offsetSec = rx.makeCall(SqlStdOperatorTable.MOD, tsEpoch, spanLit); + RexNode latestPeriodStart = + rx.makeCall( + SqlStdOperatorTable.MINUS, + maxEpoch, + rx.makeCall(SqlStdOperatorTable.MOD, maxEpoch, spanLit)); + RexNode displayEpoch = rx.makeCall(SqlStdOperatorTable.PLUS, latestPeriodStart, offsetSec); + displayTimestamp = rx.makeCall(PPLBuiltinOperators.FROM_UNIXTIME, displayEpoch); + + long nowEpochSec = context.functionProperties.getQueryStartClock().millis() / 1000; + Long referenceEpoch = null; + if ("end".equals(node.getAlign())) { + referenceEpoch = TimewrapUtils.extractTimestampUpperBound(node); + } + if (referenceEpoch == null) { + referenceEpoch = nowEpochSec; + } + RexNode refLit = rx.makeBigintLiteral(BigDecimal.valueOf(referenceEpoch)); + // Floor-divide (ref - maxEpoch) by span: integer DIVIDE truncates toward zero, which is wrong + // when the reference is below maxEpoch (e.g. align=now over future-dated data) — it would + // shift period labels by one across the latest/before/after boundary. Cast to DOUBLE and + // FLOOR + // to get true floor division, then back to BIGINT. + RelDataType doubleType = rx.getTypeFactory().createSqlType(SqlTypeName.DOUBLE); + RexNode refDiff = rx.makeCall(SqlStdOperatorTable.MINUS, refLit, maxEpoch); + RexNode refDiffDouble = rx.makeCast(doubleType, refDiff, true); + baseOffset = + rx.makeCast( + bigintType, + rx.makeCall( + SqlStdOperatorTable.FLOOR, + rx.makeCall(SqlStdOperatorTable.DIVIDE, refDiffDouble, spanLit)), + true); + } + + // Step 3: Project [display_timestamp, value_columns..., base_offset, period] + // base_offset is included in the group key so it survives the PIVOT + List projections = new ArrayList<>(); + projections.add(b.alias(displayTimestamp, tsFieldName)); + for (String vf : valueFieldNames) { + projections.add(b.field(vf)); + } + projections.add(b.alias(baseOffset, "__base_offset__")); + projections.add(b.alias(periodNum, "__period__")); + b.project(projections); + + // Step 4: Sort by offset, then period (execution engine will pivot) + // No Calcite PIVOT -- the execution engine pivots dynamically after reading all rows. + // Output schema: [display_timestamp, value_columns..., __base_offset__, __period__] + b.sort(b.field(tsFieldName), b.field("__period__")); + + return b.peek(); + } + /** * Aggregate by column split then rank by grand total (summed value of each category). The output * is [col-split, grand-total, row-number] diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 44c6d87da12..1bf1e217b51 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -19,6 +19,8 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.Nullable; @@ -53,6 +55,7 @@ import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; import org.opensearch.sql.ast.expression.EqualTo; +import org.opensearch.sql.ast.expression.ForeachPlaceholder; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; @@ -79,6 +82,8 @@ import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.Sort.SortOrder; import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.calcite.CalcitePlanContext.ForeachBinding; +import org.opensearch.sql.calcite.CalcitePlanContext.ForeachBindingType; import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit; import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; @@ -92,10 +97,13 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.CoercionUtils; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.PPLFuncImpTable; @RequiredArgsConstructor public class CalciteRexNodeVisitor extends AbstractNodeVisitor { + private static final Pattern FOREACH_TEMPLATE = Pattern.compile("<<([A-Za-z0-9_]+)>>"); + private final CalciteRelNodeVisitor planVisitor; public RexNode analyze(UnresolvedExpression unresolved, CalcitePlanContext context) { @@ -134,6 +142,10 @@ public RexNode visitLiteral(Literal node, CalcitePlanContext context) { case NULL: return rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.NULL)); case STRING: + RexNode foreachTemplate = foreachTemplateLiteral(value.toString(), context); + if (foreachTemplate != null) { + return foreachTemplate; + } if (value.toString().length() == 1) { // To align Spark/PostgreSQL, Char(1) is useful, such as cast('1' to boolean) should // return true @@ -399,9 +411,98 @@ private boolean isBooleanLiteral(RexNode node) { /** Resolve qualified name. Note, the name should be case-sensitive. */ @Override public RexNode visitQualifiedName(QualifiedName node, CalcitePlanContext context) { + String name = node.toString(); + RexNode computed = context.getForeachComputedBindings().get(name.toUpperCase(Locale.ROOT)); + if (computed != null) { + return computed; + } + ForeachBinding binding = + context.getForeachIdentifierBindings().get(name.toUpperCase(Locale.ROOT)); + if (binding != null) { + return foreachBindingToRexNode(node.toString(), binding, context); + } return QualifiedNameResolver.resolve(node, context); } + @Override + public RexNode visitForeachPlaceholder(ForeachPlaceholder node, CalcitePlanContext context) { + ForeachBinding binding = foreachBinding(node.getName(), context); + if (binding == null) { + throw new SemanticCheckException("Unresolved foreach placeholder <<" + node.getName() + ">>"); + } + return foreachBindingToRexNode(node.getName(), binding, context); + } + + private ForeachBinding foreachBinding(String name, CalcitePlanContext context) { + return context.getForeachBindings().get(name.toUpperCase(Locale.ROOT)); + } + + private RexNode foreachTemplateLiteral(String value, CalcitePlanContext context) { + Matcher matcher = FOREACH_TEMPLATE.matcher(value); + List parts = new ArrayList<>(); + int start = 0; + boolean replaced = false; + while (matcher.find()) { + ForeachBinding binding = foreachBinding(matcher.group(1), context); + if (binding == null) { + continue; + } + if (matcher.start() > start) { + parts.add(context.rexBuilder.makeLiteral(value.substring(start, matcher.start()))); + } + if (binding.type() == ForeachBindingType.PAIR_SLOT) { + RelDataType varchar = + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + parts.add( + context.rexBuilder.makeCast( + varchar, foreachBindingToRexNode(matcher.group(1), binding, context), true, true)); + } else { + parts.add(context.rexBuilder.makeLiteral(binding.value())); + } + start = matcher.end(); + replaced = true; + } + if (!replaced) { + return null; + } + if (start < value.length()) { + parts.add(context.rexBuilder.makeLiteral(value.substring(start))); + } + if (parts.isEmpty()) { + return context.rexBuilder.makeLiteral(""); + } + RexNode result = parts.getFirst(); + for (int i = 1; i < parts.size(); i++) { + result = context.rexBuilder.makeCall(SqlStdOperatorTable.CONCAT, result, parts.get(i)); + } + return result; + } + + private RexNode foreachBindingToRexNode( + String name, ForeachBinding binding, CalcitePlanContext context) { + switch (binding.type()) { + case FIELD: + return context.relBuilder.field(binding.value()); + case PAIR_SLOT: + RexLambdaRef pair = context.getRexLambdaRefMap().get(binding.value()); + if (pair == null) { + throw new SemanticCheckException("Unresolved foreach lambda placeholder " + name); + } + // The slot's type is known at plan time, so assign it directly to the opaque extraction + // call. LambdaUtils' re-inference preserves it (a CAST would not survive the enumerable + // backend for complex types like arrays). + return context.rexBuilder.makeCall( + binding.pairType(), + PPLBuiltinOperators.FOREACH_PAIR_ITEM, + List.of( + pair, + context.rexBuilder.makeExactLiteral(BigDecimal.valueOf(binding.pairIndex())))); + case LITERAL: + default: + return context.rexBuilder.makeLiteral(binding.value()); + } + } + @Override public RexNode visitAlias(Alias node, CalcitePlanContext context) { RexNode expr = analyze(node.getDelegated(), context); @@ -705,7 +806,7 @@ public RexNode visitWindowFunction(WindowFunction node, CalcitePlanContext conte node.getWindowFrame()); }) .orElseThrow( - () -> new UnsupportedOperationException("Unexpected window function: " + funcName)); + () -> new CalciteUnsupportedException("Unexpected window function: " + funcName)); } private List translateOrderKeys( @@ -835,6 +936,7 @@ public RexNode visitCast(Cast node, CalcitePlanContext context) { @Override public RexNode visitCase(Case node, CalcitePlanContext context) { List caseOperands = new ArrayList<>(); + List resultTypes = new ArrayList<>(); for (When when : node.getWhenClauses()) { RexNode condition = analyze(when.getCondition(), context); if (!SqlTypeUtil.isBoolean(condition.getType())) { @@ -843,11 +945,22 @@ public RexNode visitCase(Case node, CalcitePlanContext context) { "Condition expected a boolean type, but got %s", condition.getType())); } caseOperands.add(condition); - caseOperands.add(analyze(when.getResult(), context)); + RexNode result = analyze(when.getResult(), context); + caseOperands.add(result); + resultTypes.add(result.getType()); } RexNode elseExpr = node.getElseClause().map(e -> analyze(e, context)).orElse(context.relBuilder.literal(null)); caseOperands.add(elseExpr); + resultTypes.add(elseExpr.getType()); + + // Pre-validate the THEN/ELSE result types so an unsupertyped mix surfaces as a clean + // 400 here instead of an opaque NPE deep in Calcite's makeCall return-type inference. + RelDataType commonType = context.rexBuilder.getTypeFactory().leastRestrictive(resultTypes); + if (commonType == null) { + throw new ExpressionEvaluationException( + StringUtils.format("case branches must have a common type, but got %s", resultTypes)); + } return context.rexBuilder.makeCall(SqlStdOperatorTable.CASE, caseOperands); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java b/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java new file mode 100644 index 00000000000..d9ef71b2220 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java @@ -0,0 +1,630 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; + +import com.google.gson.JsonSyntaxException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.ArraySqlType; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.expression.Compare; +import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.ForeachPlaceholder; +import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.LambdaFunction; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.QualifiedName; +import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.ast.tree.Foreach; +import org.opensearch.sql.ast.tree.Foreach.ForeachEvalClause; +import org.opensearch.sql.calcite.CalcitePlanContext.ForeachBinding; +import org.opensearch.sql.calcite.CalcitePlanContext.ForeachBindingType; +import org.opensearch.sql.calcite.utils.WildcardUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; +import org.opensearch.sql.expression.function.PPLFuncImpTable; + +/** + * Plans the PPL {@code foreach} command. + * + *

Multifield mode expands each eval clause once per matching field, binding {@code <>} + * (and match placeholders) so the clause expression resolves against the current field. + * + *

Collection modes (multivalue / json_array / auto_collections) rewrite each eval clause into a + * {@code reduce} over the collection: elements are packed into internal pairs {@code [item, iter, + * captured-field...]} so the lambda can reference the loop item, the loop index, and any row fields + * the clause mentions. Placeholder bindings are staged on the context and only become active inside + * the lambda; the reduce call's other arguments resolve against the row as usual. + */ +class ForeachPlanner { + + private static final String OPTION_FIELDSTR = "fieldstr"; + private static final String OPTION_MATCHSTR = "matchstr"; + private static final String OPTION_MATCHSEG = "matchseg"; + private static final String OPTION_ITEMSTR = "itemstr"; + private static final String OPTION_ITERSTR = "iterstr"; + private static final String PLACEHOLDER_FIELD = "FIELD"; + private static final String PLACEHOLDER_MATCHSTR = "MATCHSTR"; + private static final String PLACEHOLDER_MATCHSEG = "MATCHSEG"; + private static final String PLACEHOLDER_ITEM = "ITEM"; + private static final String PLACEHOLDER_ITER = "ITER"; + + /** Name of the lambda variable holding the internal pair inside generated reduce calls. */ + private static final String PAIR_VAR = "__foreach_pair"; + + private static final String STATE_VAR = "__foreach_state"; + + private static final Set ARITHMETIC_OPERATORS = Set.of("+", "-", "*", "/", "%"); + + private final CalciteRelNodeVisitor relVisitor; + private final CalciteRexNodeVisitor rexVisitor; + + ForeachPlanner(CalciteRelNodeVisitor relVisitor, CalciteRexNodeVisitor rexVisitor) { + this.relVisitor = relVisitor; + this.rexVisitor = rexVisitor; + } + + RelNode plan(Foreach node, CalcitePlanContext context) { + return node.getMode() == Foreach.Mode.MULTIFIELD + ? planMultifield(node, context) + : planCollection(node, context); + } + + // ---------------------------------------------------------------------- multifield + + private RelNode planMultifield(Foreach node, CalcitePlanContext context) { + List currentFields = context.relBuilder.peek().getRowType().getFieldNames(); + Set matchingFields = new LinkedHashSet<>(); + for (String pattern : node.getFieldPatterns()) { + matchingFields.addAll(WildcardUtils.expandWildcardPattern(pattern, currentFields)); + } + + for (String fieldName : matchingFields) { + ForeachBindings bindings = + multifieldBindings(node.getFieldPatterns(), fieldName, node.getOptions()); + context.pushForeachBindings(bindings.values(), bindings.identifiers()); + try { + for (ForeachEvalClause clause : node.getEvalClauses()) { + RexNode expr = rexVisitor.analyze(clause.getExpression(), context); + String alias = substituteTemplate(clause.getTargetTemplate(), bindings.values()); + relVisitor.projectPlusOverriding( + List.of(context.relBuilder.alias(expr, alias)), List.of(alias), context); + } + } finally { + context.clearForeachBindings(); + } + } + return context.relBuilder.peek(); + } + + private ForeachBindings multifieldBindings( + List patterns, String fieldName, Map options) { + Map bindings = new LinkedHashMap<>(); + Map identifiers = new LinkedHashMap<>(); + bindOption( + bindings, + identifiers, + new ForeachBinding(fieldName, ForeachBindingType.FIELD), + PLACEHOLDER_FIELD, + OPTION_FIELDSTR, + options); + List orderedPatterns = + Stream.concat( + patterns.stream() + .filter(pattern -> !WildcardUtils.containsWildcard(pattern)) + .filter(pattern -> WildcardUtils.matchesWildcardPattern(pattern, fieldName)), + patterns.stream().filter(WildcardUtils::containsWildcard)) + .toList(); + for (String pattern : orderedPatterns) { + List segments = wildcardSegments(pattern, fieldName); + if (segments == null) { + continue; + } + bindOption( + bindings, + identifiers, + new ForeachBinding(String.join("", segments), ForeachBindingType.LITERAL), + PLACEHOLDER_MATCHSTR, + OPTION_MATCHSTR, + options); + for (int i = 0; i < segments.size(); i++) { + String defaultName = PLACEHOLDER_MATCHSEG + (i + 1); + bindOption( + bindings, + identifiers, + new ForeachBinding(segments.get(i), ForeachBindingType.LITERAL), + defaultName, + OPTION_MATCHSEG + (i + 1), + options); + } + break; + } + return new ForeachBindings(bindings, identifiers); + } + + /** + * Returns the substrings of {@code fieldName} captured by the wildcards in {@code pattern}, an + * empty list if the pattern matches without wildcards, or null if it does not match. + */ + private List wildcardSegments(String pattern, String fieldName) { + if (!WildcardUtils.matchesWildcardPattern(pattern, fieldName)) { + return null; + } + if (!WildcardUtils.containsWildcard(pattern)) { + return List.of(); + } + Matcher matcher = + Pattern.compile(WildcardUtils.convertWildcardPatternToRegex(pattern)).matcher(fieldName); + if (!matcher.matches()) { + return null; + } + List segments = new ArrayList<>(); + for (int i = 1; i <= matcher.groupCount(); i++) { + segments.add(matcher.group(i)); + } + return segments; + } + + // ---------------------------------------------------------------------- collection modes + + private RelNode planCollection(Foreach node, CalcitePlanContext context) { + Foreach.Mode mode = node.getMode(); + UnresolvedExpression collection = node.getCollectionExpression(); + if (collection == null && mode == Foreach.Mode.AUTO_COLLECTIONS) { + collection = firstArrayField(context); + } + if (collection == null) { + throw new SemanticCheckException("foreach " + mode + " mode requires a field"); + } + RexNode rawCollection = rexVisitor.analyze(collection, context); + boolean nativeArray = rawCollection.getType() instanceof ArraySqlType; + if ((mode == Foreach.Mode.MULTIVALUE && !nativeArray) + || (mode == Foreach.Mode.JSON_ARRAY && nativeArray)) { + return context.relBuilder.peek(); + } + collection = asArrayExpression(collection, nativeArray, context, node); + RexNode collectionRex = rexVisitor.analyze(collection, context); + if (!(collectionRex.getType() instanceof ArraySqlType arrayType)) { + return context.relBuilder.peek(); + } + RelDataType itemType = arrayType.getComponentType(); + RelDataType iterType = context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + List targetAliases = + node.getEvalClauses().stream().map(ForeachEvalClause::getTargetTemplate).toList(); + List initialTargets = + targetAliases.stream() + .map(alias -> rexVisitor.analyze(new Field(new QualifiedName(alias)), context)) + .toList(); + List capturedFields = + capturedFields(node.getEvalClauses(), context, node.getOptions(), targetAliases); + + // Pack [item, iter, captured...] so the reduce lambda can reach everything through one var. + List pairArgs = new ArrayList<>(); + pairArgs.add(collection); + capturedFields.stream() + .map(field -> new Field(new QualifiedName(field.getName()))) + .forEach(pairArgs::add); + Function pairedCollection = + new Function( + BuiltinFunctionName.FOREACH_PAIR_COLLECTION.getName().getFunctionName(), pairArgs); + RexNode pairedCollectionRex = rexVisitor.analyze(pairedCollection, context); + + List targetTypes = initialTargets.stream().map(RexNode::getType).toList(); + List probedExpressions = + analyzeCollectionEval( + node, + context, + pairedCollectionRex, + state(initialTargets, context), + itemType, + iterType, + capturedFields, + targetAliases, + targetTypes); + targetTypes = probedExpressions.stream().map(RexNode::getType).toList(); + List castInitialTargets = new ArrayList<>(); + for (int i = 0; i < initialTargets.size(); i++) { + castInitialTargets.add( + context.rexBuilder.makeCast(targetTypes.get(i), initialTargets.get(i), true, true)); + } + RexNode initialState = state(castInitialTargets, context); + + ForeachBindings bindings = + collectionBindings(node, itemType, iterType, capturedFields, targetAliases, targetTypes); + context.stageForeachLambdaBindings(bindings.values(), bindings.identifiers()); + try { + CalcitePlanContext lambdaContext = + prepareStateLambdaContext(context, pairedCollectionRex, initialState); + List updatedTargets = analyzeClauses(node.getEvalClauses(), lambdaContext); + RexNode updatedState = state(updatedTargets, lambdaContext); + RexNode lambda = + context.rexBuilder.makeLambdaCall( + updatedState, + List.of( + lambdaContext.getRexLambdaRefMap().get(STATE_VAR), + lambdaContext.getRexLambdaRefMap().get(PAIR_VAR))); + RexNode reducedState = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, + BuiltinFunctionName.REDUCE, + pairedCollectionRex, + initialState, + lambda); + List outputs = new ArrayList<>(); + for (int i = 0; i < targetAliases.size(); i++) { + RexNode value = stateSlot(reducedState, i, targetTypes.get(i), context); + outputs.add(context.relBuilder.alias(value, targetAliases.get(i))); + } + relVisitor.projectPlusOverriding(outputs, targetAliases, context); + } finally { + context.clearForeachBindings(); + } + return context.relBuilder.peek(); + } + + private List analyzeCollectionEval( + Foreach node, + CalcitePlanContext context, + RexNode pairedCollection, + RexNode initialState, + RelDataType itemType, + RelDataType iterType, + List capturedFields, + List targetAliases, + List targetTypes) { + ForeachBindings bindings = + collectionBindings(node, itemType, iterType, capturedFields, targetAliases, targetTypes); + context.stageForeachLambdaBindings(bindings.values(), bindings.identifiers()); + try { + return analyzeClauses( + node.getEvalClauses(), + prepareStateLambdaContext(context, pairedCollection, initialState)); + } finally { + context.clearForeachBindings(); + } + } + + private CalcitePlanContext prepareStateLambdaContext( + CalcitePlanContext context, RexNode pairedCollection, RexNode initialState) { + LambdaFunction template = + new LambdaFunction( + new Literal(0, DataType.INTEGER), + List.of(new QualifiedName(STATE_VAR), new QualifiedName(PAIR_VAR))); + return rexVisitor.prepareLambdaContext( + context, + template, + List.of(pairedCollection, initialState), + BuiltinFunctionName.REDUCE.getName().getFunctionName(), + initialState.getType()); + } + + private List analyzeClauses( + List clauses, CalcitePlanContext lambdaContext) { + List expressions = new ArrayList<>(); + for (ForeachEvalClause clause : clauses) { + RexNode expression = rexVisitor.analyze(clause.getExpression(), lambdaContext); + expressions.add(expression); + lambdaContext.putForeachComputedBinding(clause.getTargetTemplate(), expression); + } + return expressions; + } + + private RexNode state(List values, CalcitePlanContext context) { + return PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.FOREACH_STATE, values.toArray(RexNode[]::new)); + } + + private RexNode stateSlot(RexNode state, int slot, RelDataType type, CalcitePlanContext context) { + return context.rexBuilder.makeCall( + type, + PPLBuiltinOperators.FOREACH_PAIR_ITEM, + List.of(state, context.rexBuilder.makeExactLiteral(BigDecimal.valueOf(slot)))); + } + + private ForeachBindings collectionBindings( + Foreach node, + RelDataType itemType, + RelDataType iterType, + List capturedFields, + List targetAliases, + List targetTypes) { + Map bindings = new LinkedHashMap<>(); + Map identifiers = new LinkedHashMap<>(); + bindPairPlaceholder( + bindings, identifiers, 0, itemType, PLACEHOLDER_ITEM, OPTION_ITEMSTR, node.getOptions()); + bindPairPlaceholder( + bindings, identifiers, 1, iterType, PLACEHOLDER_ITER, OPTION_ITERSTR, node.getOptions()); + for (int i = 0; i < capturedFields.size(); i++) { + RelDataTypeField field = capturedFields.get(i); + bindIdentifier(identifiers, field.getName(), PAIR_VAR, i + 2, field.getType()); + } + for (int i = 0; i < targetAliases.size(); i++) { + bindIdentifier(identifiers, targetAliases.get(i), STATE_VAR, i, targetTypes.get(i)); + } + return new ForeachBindings(bindings, identifiers); + } + + private void bindPairPlaceholder( + Map bindings, + Map identifiers, + int slot, + RelDataType type, + String placeholder, + String option, + Map options) { + ForeachBinding binding = new ForeachBinding(PAIR_VAR, ForeachBindingType.PAIR_SLOT, slot, type); + bindings.put(placeholder, binding); + if (options.containsKey(option)) { + String customName = options.get(option).toUpperCase(Locale.ROOT); + bindings.put(customName, binding); + identifiers.put(customName, binding); + } + } + + private void bindIdentifier( + Map identifiers, + String name, + String variable, + int slot, + RelDataType type) { + String key = name.toUpperCase(Locale.ROOT); + identifiers.put(key, new ForeachBinding(variable, ForeachBindingType.PAIR_SLOT, slot, type)); + } + + private UnresolvedExpression firstArrayField(CalcitePlanContext context) { + return context.relBuilder.peek().getRowType().getFieldList().stream() + .filter(field -> field.getType() instanceof ArraySqlType) + .findFirst() + .map(field -> (UnresolvedExpression) new Field(new QualifiedName(field.getName()))) + .orElseThrow( + () -> + new SemanticCheckException( + "foreach auto_collections mode requires a multivalue field or JSON array")); + } + + /** + * Ensures the collection expression evaluates to a Calcite array. Non-array expressions (JSON + * array strings or {@code json_array()} calls) are wrapped in {@code foreach_json_array}, which + * parses the JSON and casts every element to one inferred type. + */ + private UnresolvedExpression asArrayExpression( + UnresolvedExpression collection, + boolean nativeArray, + CalcitePlanContext context, + Foreach node) { + if (nativeArray) { + return collection; + } + return new Function( + BuiltinFunctionName.FOREACH_JSON_ARRAY.getName().getFunctionName(), + List.of( + collection, + new Literal(jsonElementType(collection, context, node).name(), DataType.STRING))); + } + + /** + * Infers the element type a JSON array collection should be read as. JSON only distinguishes + * numbers and strings here, so the answer is DOUBLE (gson parses JSON numbers as doubles) or + * VARCHAR. + * + *

For {@code json_array()} calls and string literals the content is visible at plan time; + * mixed content is rejected. For opaque expressions — typically a field holding JSON text, the + * primary Splunk use of json_array mode — content is unknowable, so infer from usage: an item + * placeholder consumed by arithmetic means numeric elements, anything else means strings. + */ + private SqlTypeName jsonElementType( + UnresolvedExpression collection, CalcitePlanContext context, Foreach node) { + if (collection instanceof Function function + && BuiltinFunctionName.JSON_ARRAY + .getName() + .getFunctionName() + .equalsIgnoreCase(function.getFuncName())) { + return elementTypeOf( + function.getFuncArgs().stream() + .map(arg -> rexVisitor.analyze(arg, context).getType()) + .map(RelDataType::getFamily) + .collect(Collectors.toSet())); + } + if (collection instanceof Literal literal && literal.getType() == DataType.STRING) { + try { + List values = gson.fromJson(String.valueOf(literal.getValue()), List.class); + if (values != null) { + return elementTypeOf( + values.stream() + .filter(Objects::nonNull) + .map(v -> v instanceof Number ? SqlTypeFamily.NUMERIC : SqlTypeFamily.CHARACTER) + .collect(Collectors.toSet())); + } + } catch (JsonSyntaxException ignored) { + // Malformed JSON literals return null at runtime; type choice is irrelevant. + } + return SqlTypeName.VARCHAR; + } + return itemRequiresNumericType(node, context) ? SqlTypeName.DOUBLE : SqlTypeName.VARCHAR; + } + + private boolean itemRequiresNumericType(Foreach node, CalcitePlanContext context) { + String itemName = node.getOptions().getOrDefault(OPTION_ITEMSTR, PLACEHOLDER_ITEM); + boolean customIdentifier = node.getOptions().containsKey(OPTION_ITEMSTR); + return node.getEvalClauses().stream() + .anyMatch( + clause -> + itemRequiresNumericType( + clause.getExpression(), itemName, customIdentifier, context)); + } + + private boolean itemRequiresNumericType( + Node node, String itemName, boolean customIdentifier, CalcitePlanContext context) { + if (node instanceof Compare compare) { + if ((containsItem(compare.getLeft(), itemName, customIdentifier) + && isNumericExpression(compare.getRight(), context)) + || (containsItem(compare.getRight(), itemName, customIdentifier) + && isNumericExpression(compare.getLeft(), context))) { + return true; + } + } + if (node instanceof Function function) { + for (int i = 0; i < function.getFuncArgs().size(); i++) { + if (containsItem(function.getFuncArgs().get(i), itemName, customIdentifier) + && (ARITHMETIC_OPERATORS.contains(function.getFuncName()) + || PPLFuncImpTable.INSTANCE.requiresNumericArgument(function.getFuncName(), i))) { + return true; + } + } + } + return node.getChild().stream() + .anyMatch(child -> itemRequiresNumericType(child, itemName, customIdentifier, context)); + } + + private boolean containsItem(Node node, String itemName, boolean customIdentifier) { + if (isItemReference(node, itemName, customIdentifier)) { + return true; + } + return node.getChild().stream() + .anyMatch(child -> containsItem(child, itemName, customIdentifier)); + } + + private boolean isItemReference(Node node, String itemName, boolean customIdentifier) { + String name; + if (node instanceof ForeachPlaceholder placeholder) { + name = placeholder.getName(); + } else if (node instanceof QualifiedName qualifiedName) { + if (!customIdentifier) { + return false; + } + name = qualifiedName.toString(); + } else { + return false; + } + return PLACEHOLDER_ITEM.equalsIgnoreCase(name) || itemName.equalsIgnoreCase(name); + } + + private boolean isNumericExpression(Node node, CalcitePlanContext context) { + if (node instanceof Literal literal) { + return Set.of( + DataType.SHORT, + DataType.INTEGER, + DataType.LONG, + DataType.FLOAT, + DataType.DOUBLE, + DataType.DECIMAL) + .contains(literal.getType()); + } + try { + return rexVisitor.analyze((UnresolvedExpression) node, context).getType().getFamily() + == SqlTypeFamily.NUMERIC; + } catch (RuntimeException e) { + return false; + } + } + + private SqlTypeName elementTypeOf(Set families) { + boolean numeric = families.contains(SqlTypeFamily.NUMERIC); + boolean character = families.contains(SqlTypeFamily.CHARACTER); + if (numeric && character) { + throw new SemanticCheckException( + "foreach json_array elements must be consistently strings or numbers"); + } + return numeric ? SqlTypeName.DOUBLE : SqlTypeName.VARCHAR; + } + + /** Row fields referenced by the eval clauses that must ride along inside the pairs. */ + private List capturedFields( + List evalClauses, + CalcitePlanContext context, + Map options, + List targetAliases) { + Set excluded = + Stream.concat(Stream.of(PAIR_VAR), targetAliases.stream()) + .map(name -> name.toUpperCase(Locale.ROOT)) + .collect(Collectors.toSet()); + if (options.containsKey(OPTION_ITEMSTR)) { + excluded.add(options.get(OPTION_ITEMSTR).toUpperCase(Locale.ROOT)); + } + if (options.containsKey(OPTION_ITERSTR)) { + excluded.add(options.get(OPTION_ITERSTR).toUpperCase(Locale.ROOT)); + } + Map rowFields = + context.relBuilder.peek().getRowType().getFieldList().stream() + .collect( + Collectors.toMap( + field -> field.getName().toUpperCase(Locale.ROOT), + field -> field, + (left, right) -> left, + LinkedHashMap::new)); + Map captured = new LinkedHashMap<>(); + evalClauses.forEach( + clause -> collectFieldReferences(clause.getExpression(), excluded, rowFields, captured)); + return new ArrayList<>(captured.values()); + } + + private void collectFieldReferences( + Node node, + Set excluded, + Map rowFields, + Map captured) { + if (node instanceof QualifiedName qualifiedName) { + String key = qualifiedName.toString().toUpperCase(Locale.ROOT); + RelDataTypeField field = rowFields.get(key); + if (field != null && !excluded.contains(key)) { + captured.putIfAbsent(key, field); + } + } + node.getChild().forEach(child -> collectFieldReferences(child, excluded, rowFields, captured)); + } + + // ---------------------------------------------------------------------- shared + + private void bindOption( + Map bindings, + Map identifiers, + ForeachBinding binding, + String defaultName, + String option, + Map options) { + bindings.put(defaultName.toUpperCase(Locale.ROOT), binding); + if (options.containsKey(option)) { + String customName = options.get(option).toUpperCase(Locale.ROOT); + bindings.put(customName, binding); + identifiers.put(customName, binding); + } + } + + private String substituteTemplate(String template, Map bindings) { + String result = template; + for (Map.Entry entry : bindings.entrySet()) { + result = + result.replaceAll( + "(?i)<<" + Pattern.quote(entry.getKey()) + ">>", + Matcher.quoteReplacement(entry.getValue().value())); + } + return result; + } + + private record ForeachBindings( + Map values, Map identifiers) {} +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java b/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java index dba881b3fc3..c75c0829b5f 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java +++ b/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -18,6 +19,7 @@ import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.common.error.ErrorCode; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLFuncImpTable; @@ -58,7 +60,7 @@ private static RexNode resolveInJoinCondition( return resolveFieldWithAlias(nameNode, context, 2) .or(() -> resolveFieldWithoutAlias(nameNode, context, 2)) - .orElseThrow(() -> getNotFoundException(nameNode)); + .orElseThrow(() -> getNotFoundException(nameNode, context)); } /** Resolves qualified name in non-join condition context. */ @@ -91,7 +93,7 @@ private static RexNode resolveInNonJoinCondition( return resolveCorrelationField(nameNode, context) .or(() -> replaceWithNullLiteralInCoalesce(context)) - .orElseThrow(() -> getNotFoundException(nameNode)); + .orElseThrow(() -> getNotFoundException(nameNode, context)); } private static String joinParts(List parts, int start, int length) { @@ -327,10 +329,27 @@ private static Optional replaceWithNullLiteralInCoalesce(CalcitePlanCon return Optional.empty(); } - private static ErrorReport getNotFoundException(QualifiedName node) { - return ErrorReport.wrap( - new IllegalArgumentException(String.format("Field [%s] not found.", node.toString()))) - .code(ErrorCode.FIELD_NOT_FOUND) - .build(); + private static ErrorReport getNotFoundException(QualifiedName node, CalcitePlanContext context) { + // Collect all available fields from the current context + List availableFields = context.relBuilder.peek().getRowType().getFieldNames(); + + ErrorReport.Builder builder = + ErrorReport.wrap( + new IllegalArgumentException( + String.format("Field [%s] not found.", node.toString()))) + .code(ErrorCode.FIELD_NOT_FOUND) + .context("requested_field", node.toString()) + .context("available_fields", availableFields); + + // Add a suggestion based on Levenshtein distance + StringUtils.findClosestMatch(node.toString(), availableFields) + .ifPresent(suggestion -> builder.suggestion("Did you mean: " + suggestion)); + + // Add source position if available (populated by PPL parser) + if (node.getLine() != null && node.getColumn() != null) { + builder.context("query_pos", Map.of("line", node.getLine(), "column", node.getColumn())); + } + + return builder.build(); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalSystemLimit.java b/core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalSystemLimit.java index 4f999cf0792..2ba6e8812dc 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalSystemLimit.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalSystemLimit.java @@ -83,8 +83,8 @@ public Sort copy( @Override public RelWriter explainTerms(RelWriter pw) { super.explainTerms(pw); - // Show type in the explain - pw.item("type", type); + // Show type in the explain - convert to string for JSON serialization compatibility + pw.item("type", type.name()); return pw; } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLAggregateConvertRule.java b/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLAggregateConvertRule.java index 81e2c295de7..7c6338fe876 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLAggregateConvertRule.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLAggregateConvertRule.java @@ -135,7 +135,9 @@ public void apply(RelOptRuleCall call, LogicalAggregate aggregate, LogicalProjec final Function> literalConverterProvider; RexCall rexCall = (RexCall) project.getProjects().get(aggCall.getArgList().getFirst()); if (rexCall.getOperator().kind == SqlKind.PLUS - || rexCall.getOperator().kind == SqlKind.MINUS) { + || rexCall.getOperator().kind == SqlKind.MINUS + || rexCall.getOperator().kind == SqlKind.CHECKED_PLUS + || rexCall.getOperator().kind == SqlKind.CHECKED_MINUS) { AggregateCall countCall = AggregateCall.create( aggCall.getParserPosition(), @@ -280,7 +282,12 @@ private static boolean isCallWithLiteral(RexNode node) { List CONVERTABLE_FUNCTIONS = List.of( - SqlKind.PLUS, SqlKind.MINUS, SqlKind.TIMES + SqlKind.PLUS, + SqlKind.MINUS, + SqlKind.TIMES, + SqlKind.CHECKED_PLUS, + SqlKind.CHECKED_MINUS, + SqlKind.CHECKED_TIMES // Don't support division because of the issue of integer division // e.g. (2000 / 3) * 3 = 1998 while 2000 * 3 / 3 = 2000 // SqlKind.DIVIDE diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapPivot.java b/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapPivot.java new file mode 100644 index 00000000000..c4712ff957e --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapPivot.java @@ -0,0 +1,208 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.executor.ExecutionEngine.Schema.Column; + +/** + * Pivots the unpivoted rows produced by {@code CalciteRelNodeVisitor.visitTimewrap} into the + * Splunk-style period columns timewrap returns. + * + *

The timewrap RelNode intentionally does NOT pivot — the set of period columns is only known + * once all rows are read. Instead it emits rows of shape {@code [display_ts, value_col(s)..., + * __base_offset__, __period__]} and signals this post-processing step via the thread-locals on + * {@link CalcitePlanContext}. This helper performs the dynamic pivot so both execution engines (the + * v2 {@code OpenSearchExecutionEngine} and the analytics-route {@code AnalyticsExecutionEngine}) + * produce identical output. + */ +public final class TimewrapPivot { + + private TimewrapPivot() {} + + /** Result of a pivot: the rebuilt columns and rows. */ + public record Result(List columns, List values) {} + + /** + * Returns true when the current query is a timewrap query whose results need pivoting. Reads the + * {@link CalcitePlanContext} thread-locals set by {@code visitTimewrap}. + */ + public static boolean isTimewrap() { + return Boolean.TRUE.equals(CalcitePlanContext.stripNullColumns.get()) + && CalcitePlanContext.timewrapUnitName.get() != null; + } + + /** + * Pivots the unpivoted timewrap rows into period columns. If the input is empty or the + * bookkeeping columns are absent, the input is returned unchanged. + * + * @param columns the unpivoted columns {@code [display_ts, value_col(s)..., __base_offset__, + * __period__]} + * @param values the unpivoted rows + * @param unitInfo the {@code visitTimewrap} unit descriptor + * ("spanValue|singular|plural|_before"); null means this is not a timewrap query and the + * input is returned unchanged + * @param seriesMode the timewrap {@code series} mode (relative / short / exact); may be null + */ + public static Result pivot( + List columns, List values, String unitInfo, String seriesMode) { + if (unitInfo == null || values.isEmpty()) { + return new Result(columns, values); + } + + // Locate the bookkeeping and value columns. visitTimewrap always emits + // [display_ts, value_col(s)..., __base_offset__, __period__], so column 0 is the timestamp, + // __period__/__base_offset__ are bookkeeping, and every other column is a value column. + int tsIdx = 0; + int periodIdx = -1; + int baseOffsetIdx = -1; + List valueIdxs = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + String name = columns.get(i).getName(); + if ("__period__".equals(name)) periodIdx = i; + else if ("__base_offset__".equals(name)) baseOffsetIdx = i; + else if (i > 0) valueIdxs.add(i); + } + if (periodIdx < 0 || baseOffsetIdx < 0) { + return new Result(columns, values); + } + + // Read __base_offset__ (constant across all rows). + long baseOffset = 0; + ExprValue boVal = values.getFirst().tupleValue().get("__base_offset__"); + if (boVal != null && !boVal.isNull()) { + baseOffset = boVal.longValue(); + } + + // Collect distinct periods (sorted descending = oldest first in output) and precompute each + // period's name once. The name depends only on (period, baseOffset, unitInfo, seriesMode), so + // computing it inside the per-row loop would repeat the same split/parse/switch + // O(rows x valueCols) times. + Set periodSet = new TreeSet<>(Collections.reverseOrder()); + for (ExprValue row : values) { + ExprValue pv = row.tupleValue().get("__period__"); + if (pv != null && !pv.isNull()) { + periodSet.add(pv.longValue()); + } + } + List periods = new ArrayList<>(periodSet); + Map periodNames = new HashMap<>(); + for (long period : periods) { + periodNames.put(period, renameTimewrapPeriod(period, baseOffset, unitInfo, seriesMode)); + } + + // Value column names. + List valueColNames = new ArrayList<>(); + for (int vi : valueIdxs) { + valueColNames.add(columns.get(vi).getName()); + } + + // Build output column names: [ts, val1_period1, val1_period2, ..., val2_period1, ...]. + // Splunk order: for each period, all value columns (oldest period first). + List outColNames = new ArrayList<>(); + outColNames.add(columns.get(tsIdx).getName()); + List outColTypes = new ArrayList<>(); + outColTypes.add(columns.get(tsIdx).getExprType()); + + for (long period : periods) { + for (int vi = 0; vi < valueColNames.size(); vi++) { + outColNames.add(valueColNames.get(vi) + "_" + periodNames.get(period)); + outColTypes.add(columns.get(valueIdxs.get(vi)).getExprType()); + } + } + + // Group rows by display_ts, pivot periods into columns. LinkedHashMap preserves the + // ts-sorted insertion order Calcite produced. + Map> pivoted = new LinkedHashMap<>(); + String tsColName = columns.get(tsIdx).getName(); + for (ExprValue row : values) { + Map tuple = row.tupleValue(); + String tsKey = tuple.get(tsColName).toString(); + long period = tuple.get("__period__").longValue(); + + Map outRow = + pivoted.computeIfAbsent( + tsKey, + k -> { + Map r = new LinkedHashMap<>(); + r.put(outColNames.get(0), tuple.get(tsColName)); + // Initialize all period columns to null. + for (int i = 1; i < outColNames.size(); i++) { + r.put(outColNames.get(i), ExprNullValue.of()); + } + return r; + }); + + // Fill in the value for this period. + String periodName = periodNames.get(period); + for (int vi = 0; vi < valueColNames.size(); vi++) { + String colName = valueColNames.get(vi) + "_" + periodName; + ExprValue val = tuple.get(valueColNames.get(vi)); + if (val != null) { + outRow.put(colName, val); + } + } + } + + // Build output. + List outColumns = new ArrayList<>(); + for (int i = 0; i < outColNames.size(); i++) { + outColumns.add(new Column(outColNames.get(i), null, outColTypes.get(i))); + } + List outValues = new ArrayList<>(); + for (Map outRow : pivoted.values()) { + outValues.add(ExprTupleValue.fromExprValueMap(outRow)); + } + return new Result(outColumns, outValues); + } + + /** + * Generates a period name from a relative period number and base offset. Returns the suffix only + * (no value prefix). E.g., "2days_before", "latest_day", "s2". unitInfo format: + * "spanValue|singular|plural|_before". + */ + private static String renameTimewrapPeriod( + long relativePeriod, long baseOffset, String unitInfo, String seriesMode) { + String[] parts = unitInfo.split("\\|", -1); + if (parts.length < 4) return String.valueOf(relativePeriod); + int spanValue = Integer.parseInt(parts[0]); + String singular = parts[1]; + String plural = parts[2]; + + long absolutePeriod = (baseOffset + relativePeriod - 1) * spanValue; + + String mode = seriesMode == null ? "relative" : seriesMode; + + return switch (mode) { + // series=exact (+ time_format) is not yet implemented; it intentionally falls back to the + // short "s" naming. TODO: format the period start date with time_format. + case "short", "exact" -> "s" + absolutePeriod; + default -> { + if (absolutePeriod == 0) { + yield "latest_" + singular; + } else if (absolutePeriod > 0) { + String unit = absolutePeriod == 1 ? singular : plural; + yield absolutePeriod + unit + "_before"; + } else { + long absPeriod = Math.abs(absolutePeriod); + String unit = absPeriod == 1 ? singular : plural; + yield absPeriod + unit + "_after"; + } + } + }; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapUtils.java new file mode 100644 index 00000000000..8c172027314 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapUtils.java @@ -0,0 +1,356 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.expression.And; +import org.opensearch.sql.ast.expression.Compare; +import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.SpanUnit; +import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.ast.tree.Filter; +import org.opensearch.sql.ast.tree.Timewrap; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +/** Utility methods for the timewrap command's Calcite plan construction. */ +public class TimewrapUtils { + + /** Check if the span unit is variable-length (month, quarter, year). */ + public static boolean isVariableLengthUnit(SpanUnit unit) { + return "M".equals(unit.getName()) || "q".equals(unit.getName()) || "y".equals(unit.getName()); + } + + /** + * Convert a fixed-length span unit and value to seconds. Only fixed-length units (second, minute, + * hour, day, week) are supported; variable-length units (month, quarter, year) have no exact + * second count and must use the calendar arithmetic path ({@link #calendarUnitNumber} / {@link + * #calendarUnitStartEpoch}) instead. + */ + public static long spanToSeconds(SpanUnit unit, int value) { + return switch (unit.getName()) { + case "s" -> value; + case "m" -> value * 60L; + case "h" -> value * 3_600L; + case "d" -> value * 86_400L; + case "w" -> value * 7L * 86_400L; + case "M", "q", "y" -> + throw new IllegalArgumentException( + "Variable-length unit '" + + unit.getName() + + "' cannot be converted to a fixed number of seconds; use the calendar" + + " arithmetic path instead"); + default -> + throw new SemanticCheckException("Unsupported time unit in timewrap: " + unit.getName()); + }; + } + + /** + * Get the timescale base name for column naming. Returns "spanValue|singular|plural" e.g., + * "1|day|days". + */ + public static String unitBaseName(SpanUnit unit, int value) { + String singular = + switch (unit.getName()) { + case "s" -> "second"; + case "m" -> "minute"; + case "h" -> "hour"; + case "d" -> "day"; + case "w" -> "week"; + case "M" -> "month"; + case "q" -> "quarter"; + case "y" -> "year"; + default -> "period"; + }; + String plural = singular + "s"; + return value + "|" + singular + "|" + plural; + } + + /** + * Compute a calendar unit number for a timestamp as a Calcite RexNode. For months: year*12 + + * month. For quarters: year*4 + quarter. For years: year. Divided by spanValue. + */ + public static RexNode calendarUnitNumber( + RexBuilder rx, RexNode tsField, SpanUnit unit, int spanValue) { + RexNode year = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("YEAR"), tsField); + RelDataType bigintType = rx.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + + RexNode unitNum; + switch (unit.getName()) { + case "M" -> { + RexNode month = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("MONTH"), tsField); + unitNum = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + year, + rx.makeExactLiteral(BigDecimal.valueOf(12), bigintType)), + month); + } + case "q" -> { + RexNode month = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("MONTH"), tsField); + RexNode quarter = + rx.makeCall( + SqlStdOperatorTable.DIVIDE, + rx.makeCall( + SqlStdOperatorTable.MINUS, + month, + rx.makeExactLiteral(BigDecimal.ONE, bigintType)), + rx.makeExactLiteral(BigDecimal.valueOf(3), bigintType)); + unitNum = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + year, + rx.makeExactLiteral(BigDecimal.valueOf(4), bigintType)), + quarter); + } + case "y" -> unitNum = year; + default -> throw new SemanticCheckException("Not a variable-length unit: " + unit.getName()); + } + + if (spanValue > 1) { + unitNum = + rx.makeCall( + SqlStdOperatorTable.DIVIDE, + unitNum, + rx.makeExactLiteral(BigDecimal.valueOf(spanValue), bigintType)); + } + return unitNum; + } + + /** + * Compute the epoch seconds of the start of the calendar unit containing a timestamp. Month: + * first day of the month. Quarter: first day of the quarter (precise with leap year). Year: Jan + * 1. + */ + public static RexNode calendarUnitStartEpoch(RexBuilder rx, RexNode tsField, SpanUnit unit) { + RelDataType bigintType = rx.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + RexNode tsEpoch = + rx.makeCast(bigintType, rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, tsField), true); + RexNode hour = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("HOUR"), tsField); + RexNode minute = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("MINUTE"), tsField); + RexNode second = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("SECOND"), tsField); + RexNode one = rx.makeExactLiteral(BigDecimal.ONE, bigintType); + RexNode sec86400 = rx.makeExactLiteral(BigDecimal.valueOf(86400), bigintType); + + RexNode timeWithinDay = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + hour, + rx.makeExactLiteral(BigDecimal.valueOf(3600), bigintType)), + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + minute, + rx.makeExactLiteral(BigDecimal.valueOf(60), bigintType))), + second); + + if ("M".equals(unit.getName())) { + RexNode dayOfMonth = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("DAY"), tsField); + return rx.makeCall( + SqlStdOperatorTable.MINUS, + rx.makeCall( + SqlStdOperatorTable.MINUS, + tsEpoch, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + rx.makeCall(SqlStdOperatorTable.MINUS, dayOfMonth, one), + sec86400)), + timeWithinDay); + } else if ("y".equals(unit.getName())) { + RexNode doy = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("DOY"), tsField); + return rx.makeCall( + SqlStdOperatorTable.MINUS, + rx.makeCall( + SqlStdOperatorTable.MINUS, + tsEpoch, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + rx.makeCall(SqlStdOperatorTable.MINUS, doy, one), + sec86400)), + timeWithinDay); + } else { + // Quarter: precise day-within-quarter via cumulative day lookup + leap year + RexNode doy = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("DOY"), tsField); + RexNode month = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("MONTH"), tsField); + RexNode year = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("YEAR"), tsField); + + RexNode monthsIntoQuarter = + rx.makeCall( + SqlStdOperatorTable.MOD, + rx.makeCall(SqlStdOperatorTable.MINUS, month, one), + rx.makeExactLiteral(BigDecimal.valueOf(3), bigintType)); + RexNode quarterStartMonth = rx.makeCall(SqlStdOperatorTable.MINUS, month, monthsIntoQuarter); + + RexNode cumDaysBeforeQS = cumDaysBeforeMonth(rx, quarterStartMonth, year, bigintType); + RexNode quarterStartDOY = rx.makeCall(SqlStdOperatorTable.PLUS, cumDaysBeforeQS, one); + RexNode dayWithinQuarter = rx.makeCall(SqlStdOperatorTable.MINUS, doy, quarterStartDOY); + + return rx.makeCall( + SqlStdOperatorTable.MINUS, + rx.makeCall( + SqlStdOperatorTable.MINUS, + tsEpoch, + rx.makeCall(SqlStdOperatorTable.MULTIPLY, dayWithinQuarter, sec86400)), + timeWithinDay); + } + } + + /** + * Build a CASE expression for cumulative days before a given month, with leap year handling. + * Month 1→0, Month 2→31, Month 3→59+leap, ..., Month 12→334+leap. + */ + public static RexNode cumDaysBeforeMonth( + RexBuilder rx, RexNode month, RexNode year, RelDataType bigintType) { + RexNode mod4 = + rx.makeCall( + SqlStdOperatorTable.MOD, year, rx.makeExactLiteral(BigDecimal.valueOf(4), bigintType)); + RexNode mod100 = + rx.makeCall( + SqlStdOperatorTable.MOD, + year, + rx.makeExactLiteral(BigDecimal.valueOf(100), bigintType)); + RexNode mod400 = + rx.makeCall( + SqlStdOperatorTable.MOD, + year, + rx.makeExactLiteral(BigDecimal.valueOf(400), bigintType)); + RexNode zero = rx.makeExactLiteral(BigDecimal.ZERO, bigintType); + RexNode isLeap = + rx.makeCall( + SqlStdOperatorTable.CASE, + rx.makeCall( + SqlStdOperatorTable.AND, + rx.makeCall(SqlStdOperatorTable.EQUALS, mod4, zero), + rx.makeCall( + SqlStdOperatorTable.OR, + rx.makeCall(SqlStdOperatorTable.NOT_EQUALS, mod100, zero), + rx.makeCall(SqlStdOperatorTable.EQUALS, mod400, zero))), + rx.makeExactLiteral(BigDecimal.ONE, bigintType), + zero); + + int[] cumDays = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}; + + List caseArgs = new ArrayList<>(); + for (int m = 1; m <= 12; m++) { + caseArgs.add( + rx.makeCall( + SqlStdOperatorTable.EQUALS, + month, + rx.makeExactLiteral(BigDecimal.valueOf(m), bigintType))); + RexNode days = rx.makeExactLiteral(BigDecimal.valueOf(cumDays[m - 1]), bigintType); + if (m >= 3) { + days = rx.makeCall(SqlStdOperatorTable.PLUS, days, isLeap); + } + caseArgs.add(days); + } + caseArgs.add(zero); + + return rx.makeCall(SqlStdOperatorTable.CASE, caseArgs.toArray(new RexNode[0])); + } + + /** Compute calendar unit number from an epoch at plan time (Java). */ + public static long calendarUnitNumberFromEpoch(long epochSec, SpanUnit unit, int spanValue) { + java.time.Instant instant = java.time.Instant.ofEpochSecond(epochSec); + java.time.ZonedDateTime zdt = instant.atZone(java.time.ZoneOffset.UTC); + long unitNum; + switch (unit.getName()) { + case "M" -> unitNum = zdt.getYear() * 12L + zdt.getMonthValue(); + case "q" -> unitNum = zdt.getYear() * 4L + (zdt.getMonthValue() - 1) / 3; + case "y" -> unitNum = zdt.getYear(); + default -> throw new SemanticCheckException("Not a variable-length unit: " + unit.getName()); + } + return unitNum / spanValue; + } + + /** + * Walk the AST from a Timewrap node to the deepest Filter node and extract the timestamp upper + * bound. The frontend time picker always appends the timestamp filter as the first pipe (closest + * to source), making it the deepest Filter in the AST chain: + * + * 

+   *   Timewrap -> Chart -> [user filters] -> Filter(@timestamp >= X AND @timestamp <= Y) -> Source
+   *
+ * + * We walk all Filter nodes and return the last (deepest) timestamp upper bound found. This + * ensures user filters like `where age > 30` between timechart and the time picker filter don't + * interfere. + */ + public static Long extractTimestampUpperBound(Timewrap node) { + Node current = node; + Long lastBound = null; + while (current != null && !current.getChild().isEmpty()) { + current = current.getChild().get(0); + if (current instanceof Filter filter) { + Long bound = findUpperBound(filter.getCondition()); + if (bound != null) { + lastBound = bound; + } + } + } + return lastBound; + } + + private static Long findUpperBound(UnresolvedExpression expr) { + if (expr instanceof And and) { + Long left = findUpperBound(and.getLeft()); + Long right = findUpperBound(and.getRight()); + if (left != null && right != null) return Math.min(left, right); + return left != null ? left : right; + } + if (expr instanceof Compare cmp) { + String op = cmp.getOperator(); + if (("<=".equals(op) || "<".equals(op)) && isTimestampField(cmp.getLeft())) { + return parseTimestampLiteral(cmp.getRight()); + } + if ((">=".equals(op) || ">".equals(op)) && isTimestampField(cmp.getRight())) { + return parseTimestampLiteral(cmp.getLeft()); + } + } + return null; + } + + private static boolean isTimestampField(UnresolvedExpression expr) { + if (expr instanceof Field field) { + String name = field.getField().toString(); + return "@timestamp".equals(name) || "timestamp".equals(name); + } + return false; + } + + private static Long parseTimestampLiteral(UnresolvedExpression expr) { + if (expr instanceof Literal lit && lit.getValue() instanceof String s) { + try { + java.time.LocalDateTime ldt = + java.time.LocalDateTime.parse( + s, java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + return ldt.toEpochSecond(java.time.ZoneOffset.UTC); + } catch (java.time.format.DateTimeParseException e) { + try { + return java.time.Instant.parse(s).getEpochSecond(); + } catch (java.time.format.DateTimeParseException ignored) { + return null; + } + } + } + return null; + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java index 2a5d392a149..9b51876c004 100644 --- a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java @@ -21,6 +21,7 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.pagination.Cursor; import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.protocol.response.format.Format; /** Execution engine that encapsulates execution details. */ public interface ExecutionEngine { @@ -75,6 +76,16 @@ default void explain( getClass().getSimpleName() + " does not support RelNode explain")); } + default void explain( + RelNode plan, + ExplainMode mode, + Format format, + CalcitePlanContext context, + ResponseListener listener) { + // Default: ignore format parameter, delegate to old signature for BWC + explain(plan, mode, context, listener); + } + /** Data class that encapsulates ExprValue. */ @Data class QueryResponse { @@ -163,5 +174,8 @@ class ExplainResponseNodeV2 { private final String logical; private final String physical; private final String extended; + // For json_tree format: parsed JSON objects instead of strings + private Object logicalTree; + private Object physicalTree; } } diff --git a/core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java b/core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java index 941f42de46c..7eb1e3cacd4 100644 --- a/core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java +++ b/core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java @@ -27,14 +27,10 @@ public class OpenSearchTypeSystem extends RelDataTypeSystemImpl { private OpenSearchTypeSystem() {} - @Override - public int getMaxNumericPrecision() { - return MAX_PRECISION; - } - @Override public int getMaxPrecision(SqlTypeName typeName) { return switch (typeName) { + case DECIMAL -> MAX_PRECISION; case TIME, TIME_WITH_LOCAL_TIME_ZONE, TIME_TZ, @@ -47,8 +43,11 @@ public int getMaxPrecision(SqlTypeName typeName) { } @Override - public int getMaxNumericScale() { - return MAX_SCALE; + public int getMaxScale(SqlTypeName typeName) { + return switch (typeName) { + case DECIMAL -> MAX_SCALE; + default -> super.getMaxScale(typeName); + }; } @Override diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index fe9d3e55dc1..b97a679cbd3 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -16,10 +16,16 @@ import org.apache.calcite.plan.RelTraitDef; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; @@ -54,6 +60,7 @@ import org.opensearch.sql.planner.logical.LogicalPaginate; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.protocol.response.format.Format; /** The low level interface of core engine. */ @RequiredArgsConstructor @@ -124,8 +131,19 @@ public void explain( HighlightConfig highlightConfig, ResponseListener listener, ExplainMode mode) { + explain(plan, queryType, highlightConfig, listener, mode, null); + } + + /** Explain with optional highlight config and format. */ + public void explain( + UnresolvedPlan plan, + QueryType queryType, + HighlightConfig highlightConfig, + ResponseListener listener, + ExplainMode mode, + Format format) { if (shouldUseCalcite(queryType)) { - explainWithCalcite(plan, queryType, highlightConfig, listener, mode); + explainWithCalcite(plan, queryType, highlightConfig, listener, mode, format); } else { explainWithLegacy(plan, queryType, listener, mode, Optional.empty()); } @@ -162,16 +180,14 @@ public void executeWithCalcite( RelNode calcitePlan = StageErrorHandler.executeStage( QueryProcessingStage.PLAN_CONVERSION, - () -> convertToCalcitePlan(relNode, context), + () -> + withCheckedArithmetic( + convertToCalcitePlan(relNode, context), context), "while converting the query to an executable plan"); analyzeMetric.set(System.nanoTime() - analyzeStart); - // Wrap execution with EXECUTING stage tracking - StageErrorHandler.executeStageVoid( - QueryProcessingStage.EXECUTING, - () -> executionEngine.execute(calcitePlan, context, listener), - "while running the query"); + executeCalcitePlan(calcitePlan, context, listener); }, QueryService.class); } catch (Throwable t) { @@ -186,12 +202,41 @@ public void executeWithCalcite( settings); } + private void executeCalcitePlan( + RelNode calcitePlan, + CalcitePlanContext context, + ResponseListener listener) { + try { + StageErrorHandler.executeStageVoid( + QueryProcessingStage.EXECUTING, + () -> executionEngine.execute(calcitePlan, context, listener), + "while running the query"); + } catch (RuntimeException e) { + ArithmeticException overflow = findArithmeticOverflow(e); + if (overflow != null) { + throw new NonFallbackCalciteException( + "Arithmetic overflow: " + overflow.getMessage(), overflow); + } + throw e; + } + } + public void explainWithCalcite( UnresolvedPlan plan, QueryType queryType, HighlightConfig highlightConfig, ResponseListener listener, ExplainMode mode) { + explainWithCalcite(plan, queryType, highlightConfig, listener, mode, null); + } + + public void explainWithCalcite( + UnresolvedPlan plan, + QueryType queryType, + HighlightConfig highlightConfig, + ResponseListener listener, + ExplainMode mode, + Format format) { CalcitePlanContext.run( () -> { try { @@ -205,8 +250,13 @@ public void explainWithCalcite( context.run( () -> { RelNode relNode = analyze(plan, context); - RelNode calcitePlan = convertToCalcitePlan(relNode, context); - executionEngine.explain(calcitePlan, mode, context, listener); + RelNode calcitePlan = + withCheckedArithmetic(convertToCalcitePlan(relNode, context), context); + if (format != null) { + executionEngine.explain(calcitePlan, mode, format, context, listener); + } else { + executionEngine.explain(calcitePlan, mode, context, listener); + } }, settings); }, @@ -357,6 +407,82 @@ private boolean isCalciteEnabled(Settings settings) { } } + /** + * Rewrite {@code +}/{@code -}/{@code *} to their overflow-checked variants ({@code CHECKED_PLUS} + * / {@code CHECKED_MINUS} / {@code CHECKED_MULTIPLY}) so integer and long arithmetic overflow + * throws {@link ArithmeticException} (via {@code Math.addExact} etc.) instead of silently + * wrapping. Applied before pushdown so both coordinator-executed and pushed-down (script) + * arithmetic are checked. Floating-point arithmetic is unchanged (IEEE 754). + * + *

This does the same rewrite as Calcite's {@code ConvertToChecked} but preserves each call's + * originally inferred type (via {@code makeCall(type, op, operands)}) and touches only the three + * arithmetic operators, so it does not re-derive the types of unrelated calls (e.g. {@code + * CEIL}/{@code DIVIDE}) the way {@code ConvertToChecked} does. + */ + private static RelNode withCheckedArithmetic(RelNode calcitePlan, CalcitePlanContext context) { + RexShuttle checkedShuttle = + new RexShuttle() { + @Override + public RexNode visitCall(RexCall call) { + RexNode visited = super.visitCall(call); + if (!(visited instanceof RexCall rexCall)) { + return visited; + } + SqlOperator checked = + switch (rexCall.getOperator().getKind()) { + case PLUS -> SqlStdOperatorTable.CHECKED_PLUS; + case MINUS -> SqlStdOperatorTable.CHECKED_MINUS; + case TIMES -> SqlStdOperatorTable.CHECKED_MULTIPLY; + default -> null; + }; + // Only integer/long arithmetic can overflow silently and has a checked + // implementation (Math.addExact etc.). Float/double/decimal have no checked variant + // (SqlFunctions.checkedMultiply(double,double) does not exist) and follow IEEE 754, so + // leave them untouched. + if (checked == null || !isCheckableIntegerArithmetic(rexCall)) { + return visited; + } + return context.rexBuilder.makeCall(rexCall.getType(), checked, rexCall.getOperands()); + } + }; + return calcitePlan.accept( + new RelHomogeneousShuttle() { + @Override + public RelNode visit(RelNode other) { + RelNode visited = super.visitChildren(other); + return visited.accept(checkedShuttle); + } + }); + } + + /** Returns whether the result and every operand are BIGINT. */ + private static boolean isCheckableIntegerArithmetic(RexCall call) { + if (!isCheckableLongType(call.getType())) { + return false; + } + return call.getOperands().stream().allMatch(op -> isCheckableLongType(op.getType())); + } + + private static boolean isCheckableLongType(org.apache.calcite.rel.type.RelDataType type) { + return type.getSqlTypeName() == org.apache.calcite.sql.type.SqlTypeName.BIGINT; + } + + /** + * Walk the cause chain to find an {@link ArithmeticException} raised by checked arithmetic. Row- + * level overflow surfaces wrapped (SQLException -> RuntimeException -> ErrorReport), so a + * top-level {@code catch (ArithmeticException)} is insufficient. + */ + private static ArithmeticException findArithmeticOverflow(@Nullable Throwable t) { + for (Throwable cause = t; + cause != null && cause != cause.getCause(); + cause = cause.getCause()) { + if (cause instanceof ArithmeticException arithmeticException) { + return arithmeticException; + } + } + return null; + } + // TODO https://github.com/opensearch-project/sql/issues/3457 // Calcite is not available for SQL query now. Maybe release in 3.1.0? private boolean shouldUseCalcite(QueryType queryType) { diff --git a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java index c4fd2541afb..b98204fb31a 100644 --- a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java @@ -112,6 +112,11 @@ public void execute( ProfileContext profileCtx = QueryProfiling.current(); long execStart = System.nanoTime(); + // Capture the timewrap pivot signals now: they live in CalcitePlanContext thread-locals set by + // visitTimewrap on this (planning) thread, but the result callback below runs on a different + // analytics worker thread. Clearing them here keeps this thread's thread-locals clean. + TimewrapSignals timewrap = TimewrapSignals.captureAndClear(); + planExecutor.execute( plan, queryCtx, @@ -125,10 +130,12 @@ public void onResponse(Iterable rows) { List fields = plan.getRowType().getFieldList(); List results = convertRows(rows, fields); Schema schema = buildSchema(fields, results); + QueryResponse response = + timewrap.pivot(new QueryResponse(schema, results, Cursor.None)); profileCtx .getOrCreateMetric(MetricName.EXECUTE) .set(System.nanoTime() - execStart); - listener.onResponse(new QueryResponse(schema, results, Cursor.None)); + listener.onResponse(response); } catch (Exception e) { listener.onFailure(e); } @@ -169,6 +176,12 @@ public void executeWithProfile( org.opensearch.analytics.QueryRequestContext queryCtx, ResponseListener listener) { + ProfileContext profileCtx = QueryProfiling.current(); + long execStart = System.nanoTime(); + + // See execute(): capture the timewrap pivot signals on this planning thread. + TimewrapSignals timewrap = TimewrapSignals.captureAndClear(); + planExecutor.executeWithProfile( plan, queryCtx, @@ -178,7 +191,8 @@ public void onResponse(ProfiledResult result) { try { // ProfiledResult delivers the profile on BOTH success and failure paths // so users get stage timing visibility even when a query partially fails. - QueryResponse response = buildProfiledResponse(plan, result); + profileCtx.getOrCreateMetric(MetricName.EXECUTE).set(System.nanoTime() - execStart); + QueryResponse response = buildProfiledResponse(plan, result, timewrap); listener.onResponse(response); } catch (Exception e) { listener.onFailure(e); @@ -192,12 +206,13 @@ public void onFailure(Exception e) { }); } - private QueryResponse buildProfiledResponse(RelNode plan, ProfiledResult result) { + private QueryResponse buildProfiledResponse( + RelNode plan, ProfiledResult result, TimewrapSignals timewrap) { List fields = plan.getRowType().getFieldList(); List results = result.rows() != null ? convertRows(result.rows(), fields) : List.of(); Schema schema = buildSchema(fields, results); - QueryResponse response = new QueryResponse(schema, results, Cursor.None); + QueryResponse response = timewrap.pivot(new QueryResponse(schema, results, Cursor.None)); response.setProfile(result.profile()); if (!result.isSuccess()) { response.setError(result.failure()); diff --git a/core/src/main/java/org/opensearch/sql/executor/analytics/TimewrapSignals.java b/core/src/main/java/org/opensearch/sql/executor/analytics/TimewrapSignals.java new file mode 100644 index 00000000000..d7907e93d87 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/analytics/TimewrapSignals.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.analytics; + +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.utils.TimewrapPivot; +import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; +import org.opensearch.sql.executor.ExecutionEngine.Schema; + +/** + * A snapshot of the timewrap pivot signals that {@code CalciteRelNodeVisitor.visitTimewrap} stores + * in {@link CalcitePlanContext} thread-locals. + * + *

On the analytics route, planning and result-conversion run on different threads — the result + * callback fires on an analytics worker pool, not the SQL worker thread that planned the query. + * Capturing the signals at execute() entry (on the planning thread) and carrying them into the + * callback keeps the pivot correct across that thread hop. Capturing also clears the thread-locals + * so they don't leak onto the planning thread's next query. + */ +public final class TimewrapSignals { + + private final boolean active; + private final String unitName; + private final String series; + + private TimewrapSignals(boolean active, String unitName, String series) { + this.active = active; + this.unitName = unitName; + this.series = series; + } + + /** Captures the current thread's timewrap signals and clears the thread-locals. */ + public static TimewrapSignals captureAndClear() { + boolean active = TimewrapPivot.isTimewrap(); + String unitName = CalcitePlanContext.timewrapUnitName.get(); + String series = CalcitePlanContext.timewrapSeries.get(); + CalcitePlanContext.clearTimewrapSignals(); + return new TimewrapSignals(active, unitName, series); + } + + /** + * Applies the timewrap pivot to {@code response} if this snapshot is from a timewrap query; + * otherwise returns it unchanged. The returned response carries over the input's profile/error. + */ + public QueryResponse pivot(QueryResponse response) { + if (!active) { + return response; + } + TimewrapPivot.Result pivoted = + TimewrapPivot.pivot( + response.getSchema().getColumns(), response.getResults(), unitName, series); + QueryResponse out = + new QueryResponse(new Schema(pivoted.columns()), pivoted.values(), response.getCursor()); + out.setProfile(response.getProfile()); + out.setError(response.getError()); + return out; + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java index e470d12507e..fbdabe2fa44 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java @@ -12,6 +12,7 @@ import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.protocol.response.format.Format; /** AbstractPlan represent the execution entity of the Statement. */ @RequiredArgsConstructor @@ -32,4 +33,16 @@ public abstract class AbstractPlan { */ public abstract void explain( ResponseListener listener, ExplainMode mode); + + /** + * Explain query execution with format. + * + * @param listener query explain response listener. + * @param mode explain mode + * @param format output format + */ + public void explain( + ResponseListener listener, ExplainMode mode, Format format) { + explain(listener, mode); + } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java index 27f7a47e504..0a196c6f484 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java @@ -10,12 +10,14 @@ import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.protocol.response.format.Format; /** Explain plan. */ public class ExplainPlan extends AbstractPlan { private final AbstractPlan plan; private final ExplainMode mode; + private final Format format; private final ResponseListener explainListener; @@ -26,15 +28,27 @@ public ExplainPlan( AbstractPlan plan, ExplainMode mode, ResponseListener explainListener) { + this(queryId, queryType, plan, mode, null, explainListener); + } + + /** Constructor with format. */ + public ExplainPlan( + QueryId queryId, + QueryType queryType, + AbstractPlan plan, + ExplainMode mode, + Format format, + ResponseListener explainListener) { super(queryId, queryType); this.plan = plan; this.mode = mode; + this.format = format; this.explainListener = explainListener; } @Override public void execute() { - plan.explain(explainListener, mode); + plan.explain(explainListener, mode, format); } @Override @@ -42,4 +56,10 @@ public void explain( ResponseListener listener, ExplainMode mode) { throw new UnsupportedOperationException("explain query can not been explained."); } + + @Override + public void explain( + ResponseListener listener, ExplainMode mode, Format format) { + throw new UnsupportedOperationException("explain query can not been explained."); + } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java index 3f6407e8873..762997439f4 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java @@ -16,6 +16,7 @@ import org.opensearch.sql.executor.QueryId; import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.protocol.response.format.Format; /** Query plan which includes a select query. */ public class QueryPlan extends AbstractPlan { @@ -86,12 +87,18 @@ public void execute() { @Override public void explain( ResponseListener listener, ExplainMode mode) { + explain(listener, mode, null); + } + + @Override + public void explain( + ResponseListener listener, ExplainMode mode, Format format) { if (pageSize.isPresent()) { listener.onFailure( new NotImplementedException( "`explain` feature for paginated requests is not implemented yet.")); } else { - queryService.explain(plan, getQueryType(), highlightConfig, listener, mode); + queryService.explain(plan, getQueryType(), highlightConfig, listener, mode, format); } } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java index 48e2b3ce5e0..93c73a2315b 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -144,6 +144,7 @@ public AbstractPlan visitExplain( node.getQueryType(), create(node.getStatement(), NO_CONSUMER_RESPONSE_LISTENER, context.getRight()), node.getMode(), + node.getFormat(), context.getRight()); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 5ce02e6e0d3..e30d723ccfc 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -69,6 +69,9 @@ public enum BuiltinFunctionName { /** Collection functions */ ARRAY(FunctionName.of("array")), + FOREACH_JSON_ARRAY(FunctionName.of("foreach_json_array"), true), + FOREACH_PAIR_COLLECTION(FunctionName.of("foreach_pair_collection"), true), + FOREACH_STATE(FunctionName.of("foreach_state"), true), ARRAY_LENGTH(FunctionName.of("array_length")), ARRAY_SLICE(FunctionName.of("array_slice"), true), ARRAY_COMPACT(FunctionName.of("array_compact")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 79ea58b8608..1e19050f6e3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -162,7 +162,7 @@ private Optional resolve( || sourceTypes.equals(targetTypes)) { return Optional.of(funcBuilder); } - return Optional.of(castArguments(sourceTypes, targetTypes, funcBuilder)); + return Optional.of(castArguments(functionName, sourceTypes, targetTypes, funcBuilder)); } else { return Optional.empty(); } @@ -175,8 +175,24 @@ private Optional resolve( * this case, wrap F and return equal(BOOL, cast_to_bool(STRING)). */ private FunctionBuilder castArguments( - List sourceTypes, List targetTypes, FunctionBuilder funcBuilder) { + FunctionName funcName, + List sourceTypes, + List targetTypes, + FunctionBuilder funcBuilder) { return (fp, arguments) -> { + // A resolver may return a fixed-arity resolved signature regardless of how many arguments it + // was called with (e.g. a table function called with a duplicate named argument). When more + // arguments are supplied than the resolved signature declares, the cast loop would index past + // the end of the resolved type list and throw an unchecked IndexOutOfBoundsException that + // surfaces as HTTP 500. Reject that overflow with a clean ExpressionEvaluationException. + // Under-arity is left to the resolver's own validation, which produces a more specific + // message; it cannot overflow the resolved type list. + if (arguments.size() > targetTypes.size()) { + throw new ExpressionEvaluationException( + String.format( + "Expected %d arguments for function %s but received %d", + targetTypes.size(), funcName.getFunctionName(), arguments.size())); + } List argsCasted = new ArrayList<>(); for (int i = 0; i < arguments.size(); i++) { Expression arg = arguments.get(i); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairCollectionFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairCollectionFunctionImpl.java new file mode 100644 index 00000000000..918a4715291 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairCollectionFunctionImpl.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** Builds the internal foreach pair array: [item, iter, captured-field...]. */ +public class ForeachPairCollectionFunctionImpl extends ImplementorUDF { + public ForeachPairCollectionFunctionImpl() { + super(new ForeachPairCollectionImplementor(), NullPolicy.NONE); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataType pair = + opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(SqlTypeName.OTHER), true); + return SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), pair, true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class ForeachPairCollectionImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(ForeachPairCollectionFunctionImpl.class, "eval", Object[].class), + translatedOperands); + } + } + + public static Object eval(Object... args) { + if (args.length == 0 || args[0] == null) { + return null; + } + List source = toList(args[0]); + List pairs = new ArrayList<>(); + for (int i = 0; i < source.size(); i++) { + Object[] pair = new Object[args.length + 1]; + pair[0] = source.get(i); + pair[1] = i; + for (int j = 1; j < args.length; j++) { + pair[j + 1] = args[j]; + } + pairs.add(pair); + } + return pairs; + } + + private static List toList(Object value) { + if (value instanceof List list) { + return list; + } + if (value instanceof Object[] array) { + return Arrays.asList(array); + } + throw new IllegalArgumentException("foreach pair collection requires an array input"); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairItemFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairItemFunctionImpl.java new file mode 100644 index 00000000000..075453f38a6 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairItemFunctionImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * Extracts one slot of an internal foreach pair: {@code foreach_pair_item(pair, index)}. Returns + * OTHER by default; the foreach planner assigns every call its inferred slot type explicitly. + */ +public class ForeachPairItemFunctionImpl extends ImplementorUDF { + public ForeachPairItemFunctionImpl() { + super(new ForeachPairItemImplementor(), NullPolicy.NONE); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> + opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(SqlTypeName.OTHER), true); + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class ForeachPairItemImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(ForeachPairItemFunctionImpl.class, "eval", Object[].class), + translatedOperands); + } + } + + public static Object eval(Object... args) { + if (args.length < 2 || args[0] == null || args[1] == null) { + return null; + } + int index = ((Number) args[1]).intValue(); + List pair = args[0] instanceof Object[] array ? Arrays.asList(array) : (List) args[0]; + return index < 0 || index >= pair.size() ? null : pair.get(index); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachStateFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachStateFunctionImpl.java new file mode 100644 index 00000000000..f71294f05b2 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachStateFunctionImpl.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** Packs the typed accumulator slots used by a collection-mode foreach eval. */ +public class ForeachStateFunctionImpl extends ImplementorUDF { + public ForeachStateFunctionImpl() { + super(new ForeachStateImplementor(), NullPolicy.NONE); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataType slot = + opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(SqlTypeName.OTHER), true); + return SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), slot, true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class ForeachStateImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(ForeachStateFunctionImpl.class, "eval", Object[].class), + translatedOperands); + } + } + + public static Object eval(Object... args) { + return Arrays.asList(args); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java index 314ac3ad945..e8f6d75d7ab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java @@ -16,6 +16,8 @@ import org.apache.calcite.rex.RexLambda; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlCastFunction; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; @@ -68,6 +70,11 @@ public static RelDataType inferReturnTypeFromLambda( public static RexCall reInferReturnTypeForRexCallInsideLambda( RexCall rexCall, Map argTypes, RelDataTypeFactory typeFactory) { + // A CAST's target type lives on the call itself and cannot be re-derived from its operands; + // re-inferring it trips SqlCastFunction's assertion. Keep the call as-is. + if (rexCall.getKind() == SqlKind.CAST || rexCall.getOperator() instanceof SqlCastFunction) { + return rexCall; + } List filledOperands = new ArrayList<>(); List rexCallOperands = rexCall.getOperands(); for (RexNode rexNode : rexCallOperands) { @@ -89,6 +96,14 @@ public static RexCall reInferReturnTypeForRexCallInsideLambda( .getOperator() .inferReturnType( new RexCallBinding(typeFactory, rexCall.getOperator(), filledOperands, List.of())); + // An opaque helper call may carry a more precise type assigned when the call was built (e.g. + // foreach pair slot extraction); re-inference must not erase it. + if ((returnType.getSqlTypeName() == SqlTypeName.ANY + || returnType.getSqlTypeName() == SqlTypeName.OTHER) + && rexCall.getType().getSqlTypeName() != SqlTypeName.ANY + && rexCall.getType().getSqlTypeName() != SqlTypeName.OTHER) { + returnType = rexCall.getType(); + } return rexCall.clone(returnType, filledOperands); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java index f9a67e4d6d8..c37189cb20b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java @@ -5,37 +5,72 @@ package org.opensearch.sql.expression.function.CollectionUDF; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; +import org.apache.calcite.sql.type.SqlTypeName; /** Core logic for `mvappend` command to collect elements from list of args */ public class MVAppendCore { /** * Collect non-null elements from `args`. If an item is a list, it will collect non-null elements - * of the list. See {@ref MVAppendFunctionImplTest} for detailed behavior. + * of the list. Each collected element is coerced to {@code elementType} so a heterogeneously + * boxed input (e.g. an {@code array(int_col)} operand contributing {@code Integer} cells to a + * {@code BIGINT}-typed result) does not throw {@code ClassCastException} when the array is later + * materialized by Avatica's per-type accessor. See {@ref MVAppendFunctionImplTest} for detailed + * behavior. */ + /** Untyped overload — collects without element coercion (used by map-append and unit tests). */ public static List collectElements(Object... args) { + return collectElements((SqlTypeName) null, args); + } + + public static List collectElements(SqlTypeName elementType, Object... args) { List elements = new ArrayList<>(); for (Object arg : args) { if (arg == null) { continue; } else if (arg instanceof List) { - addListElements((List) arg, elements); + addListElements((List) arg, elements, elementType); } else { - elements.add(arg); + elements.add(coerce(arg, elementType)); } } return elements.isEmpty() ? null : elements; } - private static void addListElements(List list, List elements) { + private static void addListElements( + List list, List elements, SqlTypeName elementType) { for (Object item : list) { if (item != null) { - elements.add(item); + elements.add(coerce(item, elementType)); } } } + + /** + * Align a boxed numeric element to the array's target element type. Only numeric widenings that + * arise from operand widening (e.g. INTEGER cells into a BIGINT array) are handled; non-numeric + * or null-typed targets pass the value through unchanged so mixed / ANY-typed arrays keep their + * existing {@code Object[]} runtime semantics. + */ + private static Object coerce(Object value, SqlTypeName elementType) { + if (elementType == null || !(value instanceof Number)) { + return value; + } + Number num = (Number) value; + return switch (elementType) { + case TINYINT -> num.byteValue(); + case SMALLINT -> num.shortValue(); + case INTEGER -> num.intValue(); + case BIGINT -> num.longValue(); + case FLOAT, REAL -> num.floatValue(); + case DOUBLE -> num.doubleValue(); + case DECIMAL -> num instanceof BigDecimal ? num : BigDecimal.valueOf(num.doubleValue()); + default -> value; + }; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java index bafbeb09c43..dc3402e464d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java @@ -127,12 +127,27 @@ public Expression implement( coerced.add(EnumUtils.convert(op, elementClass)); } } + // Pass the target element SqlTypeName so the runtime can align the elements flattened out of + // ARRAY operands. Calcite does not element-wise cast inside an array operand, so + // `mvappend(array(int_col), int_col * 2)` — where operand widening makes the result element + // type BIGINT while `array(int_col)` still yields Integer cells — would otherwise throw + // `Integer cannot be cast to Long` when the array is materialized. Scalars are already + // pre-cast above; the runtime coercion is a no-op for them. + SqlTypeName targetType = elementType == null ? null : elementType.getSqlTypeName(); return Expressions.call( - Types.lookupMethod(MVAppendFunctionImpl.class, "mvappend", Object[].class), + Types.lookupMethod( + MVAppendFunctionImpl.class, "mvappendTyped", SqlTypeName.class, Object[].class), + Expressions.constant(targetType, SqlTypeName.class), Expressions.newArrayInit(Object.class, coerced)); } } + /** Codegen entry point: coerces flattened elements to {@code elementType}. */ + public static Object mvappendTyped(SqlTypeName elementType, Object... args) { + return MVAppendCore.collectElements(elementType, args); + } + + /** Untyped entry point used by unit tests; performs no element coercion. */ public static Object mvappend(Object... args) { return MVAppendCore.collectElements(args); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java index 24e4b489632..a244e290ae1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java @@ -5,17 +5,16 @@ package org.opensearch.sql.expression.function.CollectionUDF; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY_LENGTH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY_SLICE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM; import static org.opensearch.sql.expression.function.BuiltinFunctionName.LESS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACT; import java.math.BigDecimal; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.sql.expression.function.PPLFuncImpTable; /** @@ -37,6 +36,10 @@ *
  • Range access uses Calcite's ARRAY_SLICE operator (0-based indexing with length parameter) *
  • Index conversion handles the difference between PPL's 0-based indexing and Calcite's * conventions + *
  • Index arithmetic uses Calcite's raw {@code PLUS}/{@code MINUS} rather than PPL's widening + * {@code +}/{@code -} operators: array indices are int-domain and {@code ITEM}/{@code + * ARRAY_SLICE} require an INTEGER index, so the deliberate integer-overflow widening applied + * to user arithmetic must not leak into these internal, bounded computations. * */ public class MVIndexFunctionImp implements PPLFuncImpTable.FunctionImp { @@ -59,6 +62,16 @@ public RexNode resolve(RexBuilder builder, RexNode... args) { } } + /** Non-widening integer addition for internal, int-domain array-index math. */ + private static RexNode add(RexBuilder builder, RexNode left, RexNode right) { + return builder.makeCall(SqlStdOperatorTable.PLUS, left, right); + } + + /** Non-widening integer subtraction for internal, int-domain array-index math. */ + private static RexNode subtract(RexBuilder builder, RexNode left, RexNode right) { + return builder.makeCall(SqlStdOperatorTable.MINUS, left, right); + } + /** * Resolves single element access: mvindex(array, index) * @@ -72,11 +85,9 @@ private RexNode resolveSingleElement( RexNode one = builder.makeExactLiteral(BigDecimal.ONE); RexNode isNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, startIdx, zero); - RexNode sumArrayLenStart = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, startIdx); - RexNode negativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, sumArrayLenStart, one); - RexNode positiveCase = PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, startIdx, one); + RexNode sumArrayLenStart = add(builder, arrayLen, startIdx); + RexNode negativeCase = add(builder, sumArrayLenStart, one); + RexNode positiveCase = add(builder, startIdx, one); RexNode normalizedStart = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isNegative, negativeCase, positiveCase); @@ -97,21 +108,18 @@ private RexNode resolveRange( RexNode one = builder.makeExactLiteral(BigDecimal.ONE); RexNode isStartNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, startIdx, zero); - RexNode startNegativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, startIdx); + RexNode startNegativeCase = add(builder, arrayLen, startIdx); RexNode normalizedStart = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isStartNegative, startNegativeCase, startIdx); RexNode isEndNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, endIdx, zero); - RexNode endNegativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, endIdx); + RexNode endNegativeCase = add(builder, arrayLen, endIdx); RexNode normalizedEnd = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isEndNegative, endNegativeCase, endIdx); // Calculate length: (normalizedEnd - normalizedStart) + 1 - RexNode diff = - PPLFuncImpTable.INSTANCE.resolve(builder, SUBTRACT, normalizedEnd, normalizedStart); - RexNode length = PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, diff, one); + RexNode diff = subtract(builder, normalizedEnd, normalizedStart); + RexNode length = add(builder, diff, one); // Call ARRAY_SLICE(array, normalizedStart, length) return PPLFuncImpTable.INSTANCE.resolve(builder, ARRAY_SLICE, array, normalizedStart, length); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index a24015de992..d64f04bb9ad 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -47,6 +47,9 @@ import org.opensearch.sql.expression.function.CollectionUDF.ExistsFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.FilterFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ForallFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.ForeachPairCollectionFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.ForeachPairItemFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.ForeachStateFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.MVAppendFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.MVFindFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.MVZipFunctionImpl; @@ -54,6 +57,7 @@ import org.opensearch.sql.expression.function.CollectionUDF.MapRemoveFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ReduceFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.TransformFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.ForeachJsonArrayFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl; @@ -405,6 +409,14 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("forall"); public static final SqlOperator EXISTS = new ExistsFunctionImpl().toUDF("exists"); public static final SqlOperator ARRAY = new ArrayFunctionImpl().toUDF("array"); + public static final SqlOperator FOREACH_JSON_ARRAY = + new ForeachJsonArrayFunctionImpl().toUDF("foreach_json_array"); + public static final SqlOperator FOREACH_PAIR_COLLECTION = + new ForeachPairCollectionFunctionImpl().toUDF("foreach_pair_collection"); + public static final SqlOperator FOREACH_PAIR_ITEM = + new ForeachPairItemFunctionImpl().toUDF("foreach_pair_item"); + public static final SqlOperator FOREACH_STATE = + new ForeachStateFunctionImpl().toUDF("foreach_state"); public static final SqlOperator MAP_APPEND = new MapAppendFunctionImpl().toUDF("map_append"); public static final SqlOperator MAP_REMOVE = new MapRemoveFunctionImpl().toUDF("MAP_REMOVE"); public static final SqlOperator MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 5750bf6cae8..151c4a96655 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -75,6 +75,9 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.FIRST; import static org.opensearch.sql.expression.function.BuiltinFunctionName.FLOOR; import static org.opensearch.sql.expression.function.BuiltinFunctionName.FORALL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FOREACH_JSON_ARRAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FOREACH_PAIR_COLLECTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FOREACH_STATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_DAYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_UNIXTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.GET_FORMAT; @@ -273,6 +276,7 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -286,9 +290,12 @@ import javax.annotation.Nullable; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexLambda; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlLibraryOperators; @@ -312,6 +319,8 @@ import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.CollectionUDF.MVIndexFunctionImp; @@ -425,6 +434,63 @@ private PPLFuncImpTable(Builder builder, AggBuilder aggBuilder) { this.aggExternalFunctionRegistry = new ConcurrentHashMap<>(); } + /** Returns whether every known signature requires a numeric value at the given argument. */ + public boolean requiresNumericArgument(String functionName, int argumentIndex) { + Optional builtin = BuiltinFunctionName.of(functionName); + if (builtin.isEmpty()) { + return false; + } + List> implementations = + new ArrayList<>(functionRegistry.getOrDefault(builtin.get(), List.of())); + implementations.addAll(externalFunctionRegistry.getOrDefault(builtin.get(), List.of())); + boolean foundArgument = false; + for (Pair implementation : implementations) { + PPLTypeChecker checker = implementation.getKey().typeChecker(); + if (checker == null) { + return false; + } + try { + List> signatures = + checker.getParameterTypes().stream() + .filter(parameters -> argumentIndex < parameters.size()) + .toList(); + if (signatures.isEmpty()) { + return false; + } + foundArgument = true; + List acceptedTypes = + signatures.stream().map(parameters -> parameters.get(argumentIndex)).toList(); + if (acceptedTypes.stream().allMatch(ExprCoreType.numberTypes()::contains)) { + continue; + } + if (acceptedTypes.stream().anyMatch(type -> type != ExprCoreType.UNKNOWN) + || !requiresNumericByValidation(checker, signatures, argumentIndex)) { + return false; + } + } catch (RuntimeException e) { + return false; + } + } + return foundArgument; + } + + private boolean requiresNumericByValidation( + PPLTypeChecker checker, List> signatures, int argumentIndex) { + RelDataType numericType = TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE); + RelDataType stringType = TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR); + return signatures.stream() + .anyMatch( + signature -> { + List numericArguments = + new ArrayList<>(Collections.nCopies(signature.size(), numericType)); + if (!checker.checkOperandTypes(numericArguments)) { + return false; + } + numericArguments.set(argumentIndex, stringType); + return !checker.checkOperandTypes(numericArguments); + }); + } + /** * Register an operator from external services dynamically. * @@ -572,6 +638,10 @@ public RexNode resolve( // return type of the lambda function. compulsoryCast(builder, functionName, args); + // Align integer operand widths when a comparison has a scalar subquery on one side; see + // coerceNumericComparisonOperands for why this is needed for correct decorrelated join keys. + args = coerceNumericComparisonOperands(builder, functionName, args); + List argTypes = Arrays.stream(args).map(RexNode::getType).toList(); try { for (Map.Entry implement : implementList) { @@ -606,6 +676,62 @@ public RexNode resolve( functionName, allowedSignatures, PlanUtils.getActualSignature(argTypes))); } + /** + * Widens the operands of a binary comparison against a scalar subquery to a common integer type + * when the two operands are integers of differing width (e.g. INT vs BIGINT, SMALLINT vs INT). + * Returns the original array unchanged for every other case, so only the ambiguous + * integer-width-against-a-subquery case is touched. + * + *

    Calcite's comparison operators accept mixed integer widths by family, so {@code makeCall(=, + * int, bigint)} type-checks and evaluates correctly as an ordinary scalar predicate (against a + * column or literal), which is why those comparisons are left alone. But when one side is a + * scalar subquery, Calcite's decorrelator turns the comparison into a join whose keys keep their + * original widths; the generated key extractors then box the two sides as different Java types + * (e.g. {@code Integer} vs {@code Long}) that never compare equal, so the join silently drops + * every row. Casting both sides to their least-restrictive common integer type keeps the boolean + * result identical while making the derived join keys share a single type. This surfaces once + * integer arithmetic widens (e.g. {@code min(salary) + 1000} is BIGINT) so a subquery result is + * compared against a narrower column. + */ + private static RexNode[] coerceNumericComparisonOperands( + RexBuilder builder, BuiltinFunctionName functionName, RexNode... args) { + if (args.length != 2 || !BuiltinFunctionName.COMPARATORS.contains(functionName)) { + return args; + } + if (!containsSubQuery(args[0]) && !containsSubQuery(args[1])) { + return args; + } + RelDataType leftType = args[0].getType(); + RelDataType rightType = args[1].getType(); + if (!SqlTypeName.INT_TYPES.contains(leftType.getSqlTypeName()) + || !SqlTypeName.INT_TYPES.contains(rightType.getSqlTypeName()) + || leftType.getSqlTypeName() == rightType.getSqlTypeName()) { + return args; + } + RelDataType commonType = + builder.getTypeFactory().leastRestrictive(List.of(leftType, rightType)); + if (commonType == null) { + return args; + } + return new RexNode[] { + builder.makeCast( + TYPE_FACTORY.createTypeWithNullability(commonType, leftType.isNullable()), args[0]), + builder.makeCast( + TYPE_FACTORY.createTypeWithNullability(commonType, rightType.isNullable()), args[1]) + }; + } + + /** Whether {@code node} is, or transitively contains, a {@link RexSubQuery}. */ + private static boolean containsSubQuery(RexNode node) { + if (node instanceof RexSubQuery) { + return true; + } + if (node instanceof RexCall call) { + return call.getOperands().stream().anyMatch(PPLFuncImpTable::containsSubQuery); + } + return false; + } + /** * Ad-hoc coercion for some functions that require specific casting of arguments. Now it only * applies to the REDUCE function. @@ -727,6 +853,89 @@ protected void registerDivideFunction(BuiltinFunctionName functionName) { PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); } + /** + * Register an arithmetic operator ({@code +}, {@code -}, {@code *}) that widens narrow integer + * operands before applying the operation, deriving the type checker from the operator. + * + *

    Calcite infers {@code SMALLINT op SMALLINT -> SMALLINT} (via {@code ReturnTypes.PLUS} / + * {@code PRODUCT_NULLABLE}, which fall through to {@code LEAST_RESTRICTIVE}), so the product of + * two {@code short} columns overflows: DataFusion silently wraps the {@code i16} result while + * the Calcite Enumerable engine throws {@code ArithmeticException: value out of range}. Casting + * the operands up (byte/short -> int, int -> long) makes the arithmetic compute at a width that + * cannot overflow for the widened tier, matching v2 arithmetic intent across all backends. + */ + protected void registerWideningIntegerOperator( + BuiltinFunctionName functionName, SqlOperator operator) { + PPLTypeChecker typeChecker = + wrapSqlOperandTypeChecker(operator.getOperandTypeChecker(), operator.getName(), false); + register(functionName, wideningIntegerArithmetic(operator), typeChecker); + } + + /** Same as above but with an explicit {@link PPLTypeChecker}. */ + protected void registerWideningIntegerOperator( + BuiltinFunctionName functionName, SqlOperator operator, PPLTypeChecker typeChecker) { + register(functionName, wideningIntegerArithmetic(operator), typeChecker); + } + + private static FunctionImp wideningIntegerArithmetic(SqlOperator operator) { + // +, -, * are strictly binary; FunctionImp2 documents and enforces the two-operand contract. + return (FunctionImp2) + (builder, left, right) -> + builder.makeCall(operator, widenIntegerOperands(builder, left, right)); + } + + private static RexNode[] widenIntegerOperands(RexBuilder builder, RexNode... args) { + SqlTypeName promoted = promotedIntegerType(args); + if (promoted == null) { + return args; + } + // Skip widening arithmetic inside higher-order-function lambda bodies (reduce/mvmap/...). + // Those operands reference lambda parameters whose types are placeholders resolved later by + // the HOF's own return-type inference (see LambdaUtils.inferReturnTypeFromLambda); wrapping a + // RexLambdaRef in a CAST breaks that index-based resolution. They also execute JVM-side via + // linq4j (operands already promote to int), so they never hit the backend wrap path. + for (RexNode arg : args) { + if (referencesLambdaParameter(arg)) { + return args; + } + } + RexNode[] widened = new RexNode[args.length]; + for (int i = 0; i < args.length; i++) { + RelDataType target = TYPE_FACTORY.createSqlType(promoted, args[i].getType().isNullable()); + widened[i] = builder.makeCast(target, args[i]); + } + return widened; + } + + private static boolean referencesLambdaParameter(RexNode node) { + if (node instanceof RexLambdaRef) { + return true; + } + if (node instanceof RexCall call) { + return call.getOperands().stream().anyMatch(AbstractBuilder::referencesLambdaParameter); + } + return false; + } + + /** + * Target integer type that all operands should widen to, or {@code null} to leave the call + * untouched (any non-integral operand, e.g. FLOAT/DOUBLE/DECIMAL/DATETIME, defers to Calcite's + * default inference). byte/short -> INTEGER; anything involving int/long -> BIGINT. + */ + private static SqlTypeName promotedIntegerType(RexNode... args) { + boolean needsLong = false; + for (RexNode arg : args) { + switch (arg.getType().getSqlTypeName()) { + case TINYINT, SMALLINT -> {} + case INTEGER, BIGINT -> needsLong = true; + default -> { + return null; + } + } + } + return needsLong ? SqlTypeName.BIGINT : SqlTypeName.INTEGER; + } + void populate() { // register operators for comparison registerOperator(NOTEQUAL, PPLBuiltinOperators.NOT_EQUALS_IP, SqlStdOperatorTable.NOT_EQUALS); @@ -741,23 +950,25 @@ void populate() { registerOperator(OR, SqlStdOperatorTable.OR); registerOperator(NOT, SqlStdOperatorTable.NOT); - // Register ADDFUNCTION for numeric addition only - registerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); - registerOperator( + // Register ADDFUNCTION for numeric addition only. Widen narrow integer operands so the sum + // cannot overflow the (mis-)inferred SMALLINT/TINYINT result type; see + // registerWideningIntegerOperator. + registerWideningIntegerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); + registerWideningIntegerOperator( SUBTRACTFUNCTION, SqlStdOperatorTable.MINUS, PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - registerOperator( + registerWideningIntegerOperator( SUBTRACT, SqlStdOperatorTable.MINUS, PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - // Add DATETIME-DATETIME variant for timestamp binning support + // Add DATETIME-DATETIME variant for timestamp binning support (no integer widening) registerOperator( SUBTRACT, SqlStdOperatorTable.MINUS, PPLTypeChecker.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)); - registerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); - registerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); + registerWideningIntegerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); + registerWideningIntegerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); registerOperator(TRUNCATE, SqlStdOperatorTable.TRUNCATE); registerOperator(ASCII, SqlStdOperatorTable.ASCII); registerOperator(LENGTH, SqlStdOperatorTable.CHAR_LENGTH); @@ -808,11 +1019,17 @@ void populate() { wrapSqlOperandTypeChecker( SqlLibraryOperators.REGEXP_REPLACE_3.getOperandTypeChecker(), REPLACE.name(), false)); registerOperator(UPPER, SqlStdOperatorTable.UPPER); - registerOperator(ABS, SqlStdOperatorTable.ABS); - registerOperator(ACOS, SqlStdOperatorTable.ACOS); - registerOperator(ASIN, SqlStdOperatorTable.ASIN); - registerOperator(ATAN, SqlStdOperatorTable.ATAN); - registerOperator(ATAN2, SqlStdOperatorTable.ATAN2); + registerOperator(ABS, SqlStdOperatorTable.ABS, PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + registerOperator( + ACOS, SqlStdOperatorTable.ACOS, PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + registerOperator( + ASIN, SqlStdOperatorTable.ASIN, PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + registerOperator( + ATAN, SqlStdOperatorTable.ATAN, PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + registerOperator( + ATAN2, + SqlStdOperatorTable.ATAN2, + PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); // TODO, workaround to support sequence CompositeOperandTypeChecker. registerOperator( CEIL, @@ -1087,6 +1304,9 @@ void populate() { registerOperator(FILTER, PPLBuiltinOperators.FILTER); registerOperator(TRANSFORM, PPLBuiltinOperators.TRANSFORM); registerOperator(REDUCE, PPLBuiltinOperators.REDUCE); + registerOperator(FOREACH_JSON_ARRAY, PPLBuiltinOperators.FOREACH_JSON_ARRAY); + registerOperator(FOREACH_PAIR_COLLECTION, PPLBuiltinOperators.FOREACH_PAIR_COLLECTION); + registerOperator(FOREACH_STATE, PPLBuiltinOperators.FOREACH_STATE); // Register Json function register( @@ -1125,8 +1345,9 @@ void populate() { SqlStdOperatorTable.CONCAT, PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); // Register ADD (+ symbol) for numeric addition - // Replace type checker since PLUS also supports binary addition - registerOperator( + // Replace type checker since PLUS also supports binary addition. Widen narrow integer + // operands so the sum cannot overflow the (mis-)inferred SMALLINT/TINYINT result type. + registerWideningIntegerOperator( ADD, SqlStdOperatorTable.PLUS, PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java new file mode 100644 index 00000000000..28dff66a8ae --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonSyntaxException; +import java.math.BigDecimal; +import java.util.List; +import java.util.stream.StreamSupport; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** Converts a JSON array string to a Calcite enumerable array for foreach collection modes. */ +public class ForeachJsonArrayFunctionImpl extends ImplementorUDF { + public ForeachJsonArrayFunctionImpl() { + super(new ForeachJsonArrayImplementor(), NullPolicy.NONE); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + SqlTypeName elementTypeName = + SqlTypeName.valueOf(opBinding.getOperandLiteralValue(1, String.class)); + return SqlTypeUtil.createArrayType( + opBinding.getTypeFactory(), + opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(elementTypeName), true), + true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class ForeachJsonArrayImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(ForeachJsonArrayFunctionImpl.class, "eval", Object[].class), + translatedOperands); + } + } + + public static Object eval(Object... args) { + if (args.length != 2 || args[0] == null) { + return null; + } + SqlTypeName elementType = SqlTypeName.valueOf(String.valueOf(args[1])); + try { + if (args[0] instanceof List values) { + return values.stream().map(value -> cast(value, elementType)).toList(); + } + JsonArray values = gson.fromJson(String.valueOf(args[0]), JsonArray.class); + if (values == null) { + return List.of(); + } + return StreamSupport.stream(values.spliterator(), false) + .map(value -> cast(value, elementType)) + .toList(); + } catch (JsonSyntaxException e) { + return List.of(); + } + } + + private static Object cast(Object value, SqlTypeName elementType) { + if (value instanceof JsonElement element) { + if (element.isJsonNull()) { + return elementType == SqlTypeName.VARCHAR ? "null" : null; + } + if (element.isJsonArray() || element.isJsonObject()) { + return element.toString(); + } + value = + element.getAsJsonPrimitive().isNumber() ? element.getAsNumber() : element.getAsString(); + } + if (value == null) { + return null; + } + return switch (elementType) { + case DOUBLE -> ((Number) value).doubleValue(); + case VARCHAR -> + value instanceof List || value instanceof java.util.Map + ? gson.toJson(value) + : String.valueOf(value); + case DECIMAL -> BigDecimal.valueOf(((Number) value).doubleValue()); + default -> value; + }; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index e6e8dd01df0..6be0576bada 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -20,6 +20,8 @@ import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.*; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.calcite.utils.MathUtils; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -67,6 +69,11 @@ public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { Expression fieldValue = translatedOperands.get(0); Expression format = translatedOperands.get(1); + // Box numeric operands and pass them as Number so method resolution succeeds whether the + // upstream expression yields a primitive or a boxed value (e.g. a nullable long/double). + if (SqlTypeUtil.isNumeric(call.getOperands().get(0).getType())) { + fieldValue = Expressions.convert_(Expressions.box(fieldValue), Number.class); + } return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); } } @@ -97,13 +104,16 @@ public static String toString(BigDecimal num, String format) { } @Strict - public static String toString(double num, String format) { - return toString(BigDecimal.valueOf(num), format); - } - - @Strict - public static String toString(int num, String format) { - return toString(BigDecimal.valueOf(num), format); + public static String toString(Number num, String format) { + BigDecimal bd; + if (num instanceof BigDecimal decimal) { + bd = decimal; + } else if (MathUtils.isIntegral(num)) { + bd = BigDecimal.valueOf(num.longValue()); + } else { + bd = BigDecimal.valueOf(num.doubleValue()); + } + return toString(bd, format); } @Strict diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java index 00fa9f690da..36fb718dbd3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java @@ -10,7 +10,7 @@ import static org.opensearch.sql.expression.datetime.DateTimeFunctions.exprFromUnixTime; import static org.opensearch.sql.expression.datetime.DateTimeFunctions.exprFromUnixTimeFormat; -import java.math.BigDecimal; +import java.util.ArrayList; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -61,18 +61,18 @@ public static class FromUnixTimeImplementor implements NotNullImplementor { @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { - return Expressions.call(FromUnixTimeImplementor.class, "fromUnixTime", translatedOperands); + // Box the numeric operand and pass it as Number so method resolution succeeds whether the + // upstream expression yields a primitive or a boxed value (e.g. a nullable double). + List operands = new ArrayList<>(translatedOperands); + operands.set(0, Expressions.convert_(Expressions.box(operands.getFirst()), Number.class)); + return Expressions.call(FromUnixTimeImplementor.class, "fromUnixTime", operands); } - public static String fromUnixTime(double unixTime) { + public static String fromUnixTime(Number unixTime) { return (String) exprFromUnixTime(new ExprDoubleValue(unixTime)).valueForCalcite(); } - public static String fromUnixTime(BigDecimal unixTime) { - return (String) exprFromUnixTime(new ExprDoubleValue(unixTime)).valueForCalcite(); - } - - public static String fromUnixTime(double unixTime, String format) { + public static String fromUnixTime(Number unixTime, String format) { return (String) exprFromUnixTimeFormat(new ExprDoubleValue(unixTime), new ExprStringValue(format)) .valueForCalcite(); diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java b/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java index acc5b8521be..63327c2d6dd 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java @@ -17,6 +17,7 @@ public class DefaultProfileContext implements ProfileContext { private boolean finished; private final Map metrics = new ConcurrentHashMap<>(); private ProfilePlanNode planRoot; + private Object enginePlan; private QueryProfile profile; public DefaultProfileContext() {} @@ -40,6 +41,11 @@ public synchronized void setPlanRoot(ProfilePlanNode planRoot) { } } + @Override + public synchronized void setEnginePlan(Object enginePlan) { + this.enginePlan = enginePlan; + } + /** {@inheritDoc} */ @Override public synchronized QueryProfile finish() { @@ -55,7 +61,8 @@ public synchronized QueryProfile finish() { snapshot.put(metricName, millis); } double totalMillis = ProfileUtils.roundToMillis(endNanos - startNanos); - QueryProfile.PlanNode planSnapshot = planRoot == null ? null : planRoot.snapshot(); + Object planSnapshot = + enginePlan != null ? enginePlan : (planRoot == null ? null : planRoot.snapshot()); profile = new QueryProfile(totalMillis, snapshot, planSnapshot); return profile; } diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/ProfileContext.java b/core/src/main/java/org/opensearch/sql/monitor/profile/ProfileContext.java index 0c8048cef3f..be17f162de7 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/ProfileContext.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/ProfileContext.java @@ -27,6 +27,9 @@ public interface ProfileContext { */ void setPlanRoot(ProfilePlanNode planRoot); + /** TODO: merge with planRoot into one generic execution-engine-specific plan profile field. */ + default void setEnginePlan(Object plan) {} + /** * Finalize profiling and return a snapshot. * diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java index 15177ead1ca..d9d2a785868 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java @@ -21,7 +21,8 @@ public final class QueryProfile { private final Map phases; - private final PlanNode plan; + /** Execution-engine-specific plan profile: a {@link PlanNode} tree, or a pre-rendered object. */ + private final Object plan; /** * Create a new query profile snapshot. @@ -40,7 +41,7 @@ public QueryProfile(double totalTimeMillis, Map phases) { * @param phases metric values keyed by {@link MetricName} * @param plan plan tree profiling output */ - public QueryProfile(double totalTimeMillis, Map phases, PlanNode plan) { + public QueryProfile(double totalTimeMillis, Map phases, Object plan) { this.summary = new Summary(totalTimeMillis); this.phases = buildPhases(phases); this.plan = plan; diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java b/core/src/main/java/org/opensearch/sql/protocol/response/format/Format.java similarity index 93% rename from protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java rename to core/src/main/java/org/opensearch/sql/protocol/response/format/Format.java index 6db28e4d4ab..b8daf3bbd85 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java +++ b/core/src/main/java/org/opensearch/sql/protocol/response/format/Format.java @@ -25,6 +25,8 @@ public enum Format { JSON("json"), /** Returns explain output in yaml format */ YAML("yaml"), + /** Returns explain output as structured JSON tree using RelJsonWriter */ + JSON_TREE("json_tree"), /*---- backward compatible format of explain response -----*/ SIMPLE("simple"), @@ -52,6 +54,7 @@ public enum Format { builder = new ImmutableMap.Builder<>(); builder.put(JSON.formatName, JSON); builder.put(YAML.formatName, YAML); + builder.put(JSON_TREE.formatName, JSON_TREE); builder.put(SIMPLE.formatName, SIMPLE); builder.put(STANDARD.formatName, STANDARD); builder.put(EXTENDED.formatName, EXTENDED); diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index c9fa35d7068..7589cf522f6 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -5,6 +5,9 @@ package org.opensearch.sql.utils; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; @@ -34,6 +37,121 @@ public static Boolean isSystemIndex(String indexName) { return indexName.endsWith(SYS_TABLES_SUFFIX); } + /** + * Reserved suffix marking a {@code rest} source table. Distinct from {@link #SYS_TABLES_SUFFIX} + * so {@link #isSystemIndex} and {@link #isRestSource} never overlap. The whole reserved name is a + * single Calcite identifier (REST + lowercase hex + this suffix), so it survives name resolution + * the same way the uppercase system-mapping names do. + */ + private static final String REST_SOURCE_SUFFIX = "__REST_SOURCE"; + + private static final String REST_SOURCE_PREFIX = "REST"; + + /** True if the resolved table name is a {@code rest} source token. */ + public static boolean isRestSource(String indexName) { + return indexName.endsWith(REST_SOURCE_SUFFIX); + } + + /** + * Encode a validated {@link RestSpec} into a single reserved table name. Mirrors {@link + * #mappingTable}: structured metadata travels inside a reserved name rather than a side channel. + * The endpoint/args have already been allow-list-validated before this is called. + */ + public static String restTable(RestSpec spec) { + StringBuilder sb = new StringBuilder(); + sb.append("endpoint=").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append('\n').append("count=").append(spec.getCount()); + } + if (spec.getTimeout() != null) { + sb.append('\n').append("timeout=").append(spec.getTimeout()); + } + if (spec.getArgs() != null) { + for (Map.Entry e : spec.getArgs().entrySet()) { + sb.append('\n').append("arg.").append(e.getKey()).append('=').append(e.getValue()); + } + } + return REST_SOURCE_PREFIX + toHex(sb.toString()) + REST_SOURCE_SUFFIX; + } + + /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ + public static RestSpec decodeRestSpec(String indexName) { + // Validate the token shape before slicing it: callers gate on isRestSource today, but a + // public decoder must not assume its precondition, otherwise a malformed token would throw an + // opaque StringIndexOutOfBoundsException from substring rather than a clear input error. + if (!isRestSource(indexName)) { + throw new IllegalArgumentException("not a valid rest source token: " + indexName); + } + String body = + indexName.substring( + REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); + String decoded = fromHex(body); + String endpoint = null; + Integer count = null; + String timeout = null; + LinkedHashMap args = new LinkedHashMap<>(); + for (String line : decoded.split("\n")) { + if (line.isEmpty()) { + continue; + } + int eq = line.indexOf('='); + if (eq < 0) { + continue; + } + String k = line.substring(0, eq); + String v = line.substring(eq + 1); + if (k.equals("endpoint")) { + endpoint = v; + } else if (k.equals("count")) { + count = Integer.parseInt(v); + } else if (k.equals("timeout")) { + timeout = v; + } else if (k.startsWith("arg.")) { + args.put(k.substring("arg.".length()), v); + } + } + if (endpoint == null) { + throw new IllegalArgumentException("rest source token is missing the endpoint: " + indexName); + } + return new RestSpec(endpoint, args, count, timeout); + } + + private static String toHex(String s) { + StringBuilder h = new StringBuilder(); + for (byte b : s.getBytes(StandardCharsets.UTF_8)) { + h.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return h.toString(); + } + + private static String fromHex(String h) { + if (h.length() % 2 != 0) { + throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); + } + byte[] bytes = new byte[h.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = + (byte) + ((Character.digit(h.charAt(2 * i), 16) << 4) + + Character.digit(h.charAt(2 * i + 1), 16)); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + /** + * The validated spec for a {@code rest} command: an allow-listed read-only endpoint plus optional + * count/timeout/query args. Lives in core so the parser (encode) and the storage engine (decode) + * share it without a cross-module dependency. + */ + @Getter + @RequiredArgsConstructor + public static class RestSpec { + private final String endpoint; + private final Map args; + private final Integer count; + private final String timeout; + } + /** * Compose system mapping table. * diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index fee5bfcaa66..d073a6802aa 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -1943,6 +1943,24 @@ public void regex_command_throws_unsupported_exception_with_legacy_engine() { "Regex is supported only when plugins.calcite.enabled=true", exception.getMessage()); } + @Test + public void foreach_command_throws_unsupported_exception_with_legacy_engine() { + UnsupportedOperationException exception = + assertThrows( + UnsupportedOperationException.class, + () -> + analyze( + new org.opensearch.sql.ast.tree.Foreach( + org.opensearch.sql.ast.tree.Foreach.Mode.MULTIFIELD, + ImmutableMap.of(), + ImmutableList.of("integer_value"), + null, + ImmutableList.of()) + .attach(relation("schema")))); + assertEquals( + "foreach is supported only when plugins.calcite.enabled=true", exception.getMessage()); + } + @Test public void rex_command_throws_unsupported_operation_exception_in_legacy_engine() { UnsupportedOperationException exception = diff --git a/core/src/test/java/org/opensearch/sql/calcite/utils/TimewrapSignalsLeakTest.java b/core/src/test/java/org/opensearch/sql/calcite/utils/TimewrapSignalsLeakTest.java new file mode 100644 index 00000000000..eef98cdc4bf --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/utils/TimewrapSignalsLeakTest.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.setting.Settings.Key; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.executor.ExecutionEngine.Schema.Column; + +/** + * Leak guard for the timewrap pivot signals. {@code visitTimewrap} stashes its pivot state in the + * static {@link CalcitePlanContext} thread-locals read by {@link TimewrapPivot#isTimewrap()}. These + * are set on a pooled worker thread, so if they are not cleared at the end of the query lifecycle + * they would leak onto the next, non-timewrap query that reuses the same thread and wrongly pivot + * its result. This test asserts the lifecycle guard ({@code CalcitePlanContext.run}'s finally) + * clears them. + */ +public class TimewrapSignalsLeakTest { + + @AfterEach + public void cleanUp() { + CalcitePlanContext.clearTimewrapSignals(); + } + + @Test + public void timewrapSignalsDoNotLeakOntoNextQueryOnSameThread() { + Settings settings = mock(Settings.class); + lenient().when(settings.getSettingValue(Key.PPL_SYNTAX_LEGACY_PREFERRED)).thenReturn(false); + + // First "query" behaves like a timewrap query: it sets the pivot signals mid-flight, exactly as + // visitTimewrap does. run()'s finally must clear them before returning. + CalcitePlanContext.run( + () -> { + CalcitePlanContext.stripNullColumns.set(true); + CalcitePlanContext.timewrapUnitName.set("1|day|days|_before"); + CalcitePlanContext.timewrapSeries.set("relative"); + }, + settings); + + // Same thread, next query: the pivot gate must be closed. + assertFalse( + TimewrapPivot.isTimewrap(), + "timewrap signals leaked onto the next query on the same pooled thread"); + + // And a plain (non-timewrap) result must pass through the pivot untouched — no __base_offset__ + // or __period__ artifact columns, same row. + List columns = + List.of( + new Column("host", null, ExprCoreType.STRING), + new Column("count", null, ExprCoreType.LONG)); + List rows = + List.of(ExprValueUtils.tupleValue(ImmutableMap.of("host", "h1", "count", 42L))); + + TimewrapPivot.Result result = + TimewrapPivot.pivot( + columns, + rows, + CalcitePlanContext.timewrapUnitName.get(), + CalcitePlanContext.timewrapSeries.get()); + + assertEquals( + List.of("host", "count"), + result.columns().stream().map(Column::getName).toList(), + "non-timewrap result gained artifact columns after a prior timewrap query"); + assertEquals(rows, result.values()); + } +} diff --git a/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java b/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java index 2a2c9de63ab..b47ec1861fe 100644 --- a/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java @@ -6,8 +6,11 @@ package org.opensearch.sql.common.utils; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.common.utils.StringUtils.unquoteText; +import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.Test; class StringUtilsTest { @@ -46,4 +49,47 @@ void unquoteTest() { assertEquals("hel\"lo", unquoteText("\"hel\"lo\"")); assertEquals("hel\\'\\lo", unquoteText("'hel\\\\''\\\\lo'")); } + + @Test + void levenshteinDistanceTest() { + assertEquals(0, StringUtils.levenshteinDistance("test", "test")); + assertEquals(1, StringUtils.levenshteinDistance("test", "text")); + assertEquals(1, StringUtils.levenshteinDistance("test", "tst")); + assertEquals(3, StringUtils.levenshteinDistance("kitten", "sitting")); + assertEquals(4, StringUtils.levenshteinDistance("hello", "world")); + assertEquals(4, StringUtils.levenshteinDistance("test", "")); + assertEquals(4, StringUtils.levenshteinDistance("", "test")); + assertEquals(0, StringUtils.levenshteinDistance("", "")); + } + + @Test + void findClosestMatchTest() { + List fields = List.of("name", "age", "email", "address", "phone"); + + // Exact match or close typo + Optional match = StringUtils.findClosestMatch("nam", fields); + assertTrue(match.isPresent()); + assertEquals("name", match.get()); + + match = StringUtils.findClosestMatch("emal", fields); + assertTrue(match.isPresent()); + assertEquals("email", match.get()); + + match = StringUtils.findClosestMatch("addres", fields); + assertTrue(match.isPresent()); + assertEquals("address", match.get()); + + // Case insensitive + match = StringUtils.findClosestMatch("NAME", fields); + assertTrue(match.isPresent()); + assertEquals("name", match.get()); + + // Too far off - should not match (longer string with many edits) + match = StringUtils.findClosestMatch("xyzabc", fields); + assertTrue(match.isEmpty()); + + // Empty candidates + match = StringUtils.findClosestMatch("test", List.of()); + assertTrue(match.isEmpty()); + } } diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java index 4cb5c755d14..977c1cb4729 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java @@ -36,12 +36,12 @@ public class ExplainPlanTest { @Test public void execute() { - doNothing().when(queryPlan).explain(any(), any()); + doNothing().when(queryPlan).explain(any(), any(), any()); ExplainPlan explainPlan = new ExplainPlan(queryId, queryType, queryPlan, mode, explainListener); explainPlan.execute(); - verify(queryPlan, times(1)).explain(explainListener, mode); + verify(queryPlan, times(1)).explain(explainListener, mode, null); } @Test diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java index 128df14ff8e..6e05c8c1258 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java @@ -58,9 +58,9 @@ public void execute_no_page_size() { @Test public void explain_no_page_size() { QueryPlan query = new QueryPlan(queryId, queryType, plan, queryService, queryListener); - query.explain(explainListener, mode); + query.explain(explainListener, mode, null); - verify(queryService, times(1)).explain(plan, queryType, null, explainListener, mode); + verify(queryService, times(1)).explain(plan, queryType, null, explainListener, mode, null); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java index 237477050dc..7d8f2d2ac61 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java @@ -181,6 +181,63 @@ void resolve_should_throw_exception_for_unsupported_conversion() { assertEquals(error.getMessage(), "Type conversion to type STRUCT is not supported"); } + @Test + @DisplayName("resolve should not crash when resolved signature is shorter than the arguments") + void resolve_should_not_index_past_resolved_signature() { + // A custom FunctionResolver (e.g. vectorSearch) may return a fixed-arity resolved signature + // regardless of the number of arguments it was called with. When more arguments are supplied + // than the resolved signature declares (e.g. a duplicate named argument), castArguments must + // not index past the end of the resolved type list. Before the fix this threw an unchecked + // IndexOutOfBoundsException, surfacing as an HTTP 500 instead of the resolver's clean 400. + when(mockFunctionName.getFunctionName()).thenReturn("mock"); + + FunctionName funcName = mockFunctionName; + FunctionSignature sourceSignature = + new FunctionSignature(funcName, ImmutableList.of(STRING, STRING)); + FunctionSignature resolvedSignature = new FunctionSignature(funcName, ImmutableList.of(STRING)); + + DefaultFunctionResolver funcResolver = mock(DefaultFunctionResolver.class); + FunctionBuilder funcBuilder = mock(FunctionBuilder.class); + when(mockMap.containsKey(eq(funcName))).thenReturn(true); + when(mockMap.get(eq(funcName))).thenReturn(funcResolver); + when(funcResolver.resolve(any())).thenReturn(Pair.of(resolvedSignature, funcBuilder)); + + ExpressionEvaluationException error = + assertThrows( + ExpressionEvaluationException.class, + () -> + repo.resolve(Collections.emptyList(), sourceSignature) + .apply(functionProperties, ImmutableList.of(mockExpression, mockExpression))); + assertEquals("Expected 1 arguments for function mock but received 2", error.getMessage()); + } + + @Test + @DisplayName("resolve should defer under-arity to the builder rather than the cast guard") + void resolve_should_not_intercept_under_arity() { + // When fewer arguments are supplied than the resolved signature declares, the cast loop cannot + // index past the resolved type list, so there is no crash to guard against. The overflow guard + // must not fire here; the call must reach the resolved builder so a resolver's own (more + // specific) arity validation still runs. + FunctionName funcName = mockFunctionName; + FunctionSignature sourceSignature = new FunctionSignature(funcName, ImmutableList.of(STRING)); + FunctionSignature resolvedSignature = + new FunctionSignature(funcName, ImmutableList.of(STRING, STRING)); + + DefaultFunctionResolver funcResolver = mock(DefaultFunctionResolver.class); + FunctionBuilder funcBuilder = mock(FunctionBuilder.class); + when(mockMap.containsKey(eq(funcName))).thenReturn(true); + when(mockMap.get(eq(funcName))).thenReturn(funcResolver); + when(funcResolver.resolve(any())).thenReturn(Pair.of(resolvedSignature, funcBuilder)); + when(funcBuilder.apply(eq(functionProperties), any())) + .thenReturn(new FakeFunctionExpression(funcName, ImmutableList.of(mockExpression))); + + repo.resolve(Collections.emptyList(), sourceSignature) + .apply(functionProperties, ImmutableList.of(mockExpression)); + + // The builder is reached (overflow guard did not short-circuit under-arity). + verify(funcBuilder).apply(eq(functionProperties), any()); + } + @Test @DisplayName("resolve unregistered function should throw exception") void resolve_unregistered() { diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java new file mode 100644 index 00000000000..e3c3724e0aa --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.jsonUDF.ForeachJsonArrayFunctionImpl; + +public class ForeachFunctionImplTest { + + @Test + public void testPairCollectionPreservesNullElements() { + List pairs = + (List) + ForeachPairCollectionFunctionImpl.eval( + new Object[] {new Object[] {1, null, 3}, "captured"}); + + assertEquals(3, pairs.size()); + assertArrayEquals(new Object[] {1, 0, "captured"}, (Object[]) pairs.get(0)); + assertArrayEquals(new Object[] {null, 1, "captured"}, (Object[]) pairs.get(1)); + assertArrayEquals(new Object[] {3, 2, "captured"}, (Object[]) pairs.get(2)); + } + + @Test + public void testJsonArrayPreservesTopLevelNestedValuesAndNull() { + Object values = ForeachJsonArrayFunctionImpl.eval("[[1,2],{\"a\":1},null]", "VARCHAR"); + + assertEquals(List.of("[1,2]", "{\"a\":1}", "null"), values); + } + + @Test + public void testMalformedJsonArrayIsEmpty() { + assertEquals(List.of(), ForeachJsonArrayFunctionImpl.eval("not-json", "VARCHAR")); + } + + @Test + public void testStatePreservesHeterogeneousAndNullSlots() { + assertEquals( + Arrays.asList(3, null, "seen"), + ForeachStateFunctionImpl.eval(new Object[] {3, null, "seen"})); + } +} diff --git a/docs/attributions.md b/docs/attributions.md index cc16c35bae0..b5eb089181b 100644 --- a/docs/attributions.md +++ b/docs/attributions.md @@ -22,7 +22,7 @@ Apart from the problems we identified earlier, we made significant improvement i 1. *Integration Test*: We migrated all integrate tests to standard OpenSearch IT framework which spins up in-memory cluster for testing. Now all test cases treat plugin code as blackbox and verify functionality from externally. 2. *New JDBC Driver*: We developed our own JDBC driver without any dependency on Elasticsearch proprietary code. - [sql-jdbc](https://github.com/opensearch-project/sql/tree/main/sql-jdbc) + [sql-jdbc](https://github.com/opensearch-project/sql-jdbc) 3. *Better Hash JOIN*: Block Hash Join implementation with circuit break mechanism protect your OpenSearch memory. Performance testing showed our implementation is 1.5 ~ 2x better than old hash join in terms of throughput and latency and much lower error rate under heavy pressure. 4. *Query Planner*: Logical and physical planner was added to support JOIN query in efficient and extendible way. 5. *PartiQL Compatibility*: we are partially compatible with PartiQL specification which allows for query involved in nested JSON documents. diff --git a/docs/category.json b/docs/category.json index ac1dda966d2..a8665b82eac 100644 --- a/docs/category.json +++ b/docs/category.json @@ -20,6 +20,7 @@ "user/ppl/cmd/fieldformat.md", "user/ppl/cmd/fields.md", "user/ppl/cmd/fillnull.md", + "user/ppl/cmd/foreach.md", "user/ppl/cmd/grok.md", "user/ppl/cmd/head.md", "user/ppl/cmd/join.md", @@ -34,6 +35,7 @@ "user/ppl/cmd/rename.md", "user/ppl/cmd/multisearch.md", "user/ppl/cmd/replace.md", + "user/ppl/cmd/rest.md", "user/ppl/cmd/rex.md", "user/ppl/cmd/search.md", "user/ppl/cmd/showdatasources.md", @@ -45,6 +47,7 @@ "user/ppl/cmd/syntax.md", "user/ppl/cmd/chart.md", "user/ppl/cmd/timechart.md", + "user/ppl/cmd/timewrap.md", "user/ppl/cmd/top.md", "user/ppl/cmd/trendline.md", "user/ppl/cmd/transpose.md", diff --git a/docs/dev/intro-v3-architecture.md b/docs/dev/intro-v3-architecture.md index fc5bf3237d1..4f51fd1016b 100644 --- a/docs/dev/intro-v3-architecture.md +++ b/docs/dev/intro-v3-architecture.md @@ -24,7 +24,7 @@ PPL is specifically designed to simplify tasks in observability and security ana The current PPL engine (shared with SQL v2 engine) is built with custom components, including a parser, analyzer, optimizer, and relies heavily on OpenSearch DSL capabilities to execute query plans. By aligning its syntax and concepts with familiar languages like Splunk SPL and SQL, we aim to streamline migration for users from these backgrounds, allowing them to adopt PPL with minimal effort. The lack of comprehensive ability is a critical blocker for Splunk-to-OpenSearch migrations. We added ~20 new commands in PPL-on-Spark, but there are still dozens of command gaps to be filled. Not to mention that there are still a large number of functions to be implemented. ### 2.2 Lack of Unified PPL Experience -The PPL language is currently inconsistent across [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/opensearch-spark/blob/main/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4). There are a lot of new commands added in PPL-on-Spark, such as `join`, `lookup` and `subsearch` are not yet supported in PPL-on-OpenSearch. As more and more new commands and functions are implemented in PPL-on-Spark, this gap will continue to widen. +The PPL language is currently inconsistent across [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/sql/blob/main/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4). There are a lot of new commands added in PPL-on-Spark, such as `join`, `lookup` and `subsearch` are not yet supported in PPL-on-OpenSearch. As more and more new commands and functions are implemented in PPL-on-Spark, this gap will continue to widen. ### 2.3 Lack of mature query optimizer Although the v2 engine framework comes with an optimizer class, it only has a few pushdown optimization rules and lacks of mature optimization rules and cost-based optimizer like those found in traditional databases. Query performance and scalability are core to PPL's design, enabling it to efficiently handle high-performance queries and scale to support large datasets and complex queries. diff --git a/docs/dev/intro-v3-engine.md b/docs/dev/intro-v3-engine.md index fd73cd5c8e1..2d11f30aeaa 100644 --- a/docs/dev/intro-v3-engine.md +++ b/docs/dev/intro-v3-engine.md @@ -89,5 +89,5 @@ If you're interested in the new query engine, please find more details in [V3 Ar The following items are on our roadmap with high priority: - Resolve the [V3 limitation](#33-limitations). - Advancing pushdown optimization and benchmarking -- Unified the PPL syntax between [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/opensearch-spark/blob/main/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4) +- Unified the PPL syntax between [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/sql/blob/main/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4) - Support more DSL aggregation diff --git a/docs/dev/opensearch-relevancy-search.md b/docs/dev/opensearch-relevancy-search.md index 9b25cc757dd..94a63e9ed41 100644 --- a/docs/dev/opensearch-relevancy-search.md +++ b/docs/dev/opensearch-relevancy-search.md @@ -119,7 +119,7 @@ Besides, one of the query engine architecture tenets is to keep the job of every #### Option B: Create a new query plan node dedicated for the search features. -The diagram below is simplified with only the logical plan and physical plan sections and leaves out others. Please check out [OpenSearch SQL Engine Architecture](https://github.com/opensearch-project/sql/blob/main/docs/dev/Architecture.md) for the complete architecture of the query engine. +The diagram below is simplified with only the logical plan and physical plan sections and leaves out others. Please check out [OpenSearch SQL Engine Architecture](https://github.com/opensearch-project/sql/blob/main/docs/dev/intro-architecture.md) for the complete architecture of the query engine. ![relevance-Page-2 (4)](https://user-images.githubusercontent.com/33583073/129938534-28fa4845-4246-4707-9519-e68c9e86d174.png) @@ -291,4 +291,4 @@ All the code changes should be test driven. The pull requests should include uni ### A2. Search flow in search engine -![relevance](https://user-images.githubusercontent.com/33583073/129938659-5b49f43d-a83f-47d5-be5b-937b1c96e5bc.png) \ No newline at end of file +![relevance](https://user-images.githubusercontent.com/33583073/129938659-5b49f43d-a83f-47d5-be5b-937b1c96e5bc.png) diff --git a/docs/dev/testing-doctest.md b/docs/dev/testing-doctest.md index e8b88f19b0f..86fee86bd64 100644 --- a/docs/dev/testing-doctest.md +++ b/docs/dev/testing-doctest.md @@ -192,7 +192,7 @@ Doctest is relying on the console/command line to run code examples in documenta * https://github.com/crate/crate/blob/master/docs/general/dql/selects.rst -Similar to CrateDB using it’s CLI “crash”, we can make use of our own [SQL-CLI](https://github.com/opensearch-project/sql/tree/main/sql-cli) +Similar to CrateDB using it’s CLI “crash”, we can make use of our own [SQL-CLI](https://github.com/opensearch-project/sql-cli) To support PPL, we need to add PPL support to SQL-CLI. Since PPL and SQL expose similar http endpoint for query and share similar response format. The update won’t be much of work. diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index d8402605a4d..dafa8c84172 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -204,6 +204,38 @@ Result set:: } } +plugins.query.max_expression_depth +================================== + +Version +------- +3.8 + +Description +----------- + +This setting bounds the maximum nesting depth of an expression while a query is parsed into its abstract syntax tree, keeping parsing bounded for very large or deeply nested expressions. A query exceeding the limit is rejected with a 400 error. The default value is 1000. Set it to 0 to disable the limit (unlimited). Here is an example:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.max_expression_depth" : 500 + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "query" : { + "max_expression_depth" : "500" + } + } + } + } + plugins.query.buckets ===================== diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index e4f55ef1b3e..92aaa0c7db0 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -35,7 +35,7 @@ Example 1: Show All Indices Information SQL query:: os> SHOW TABLES LIKE '%' - fetched rows / total rows = 24/24 + fetched rows / total rows = 25/25 +----------------+-------------+-------------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | |----------------+-------------+-------------------+------------+---------+----------+------------+-----------+---------------------------+----------------| @@ -48,7 +48,7 @@ SQL query:: | docTestCluster | null | events_many_hosts | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | events_null | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | json_test | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | mvcombine_data | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | mvcombine_data | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | occupation | BASE TABLE | null | null | null | null | null | null | @@ -59,6 +59,7 @@ SQL query:: | docTestCluster | null | time_data | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | time_data2 | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | time_test | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | timewrap_test | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | weblogs | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | work_information | BASE TABLE | null | null | null | null | null | null | diff --git a/docs/user/ppl/cmd/fields.md b/docs/user/ppl/cmd/fields.md index b4bf110f7f2..1eb7694a19e 100644 --- a/docs/user/ppl/cmd/fields.md +++ b/docs/user/ppl/cmd/fields.md @@ -198,8 +198,8 @@ fetched rows / total rows = 3/3 ## Example 8: Selecting all fields -The following query selects all fields defined in the index schema using `` `*` ``. Fields with null values are included in the result set: - +The following query selects all fields defined in the index schema using `` `*` ``. Struct fields are returned alongside their flattened sub-fields, and fields with null values are included in the result set: + ```ppl source=otellogs | where severityText = 'WARN' @@ -211,11 +211,11 @@ The query returns the following results: ```text fetched rows / total rows = 1/1 -+----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+ -| spanId | traceId | @timestamp | instrumentationScope | severityText | resource | flags | attributes | droppedAttributesCount | severityNumber | time | body | -|----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------| -| span0003 | abcd1234efgh5678 | 2024-02-01 09:12:00 | {'name': 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc', 'droppedAttributesCount': 0, 'version': '0.49.0'} | WARN | {'attributes': {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}}, 'droppedAttributesCount': 0} | 0 | {} | 0 | 13 | 2024-02-01 09:12:00 | Slow query detected: SELECT * FROM products WHERE category = 'electronics' took 3200ms | -+----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+ ++----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+ +| spanId | traceId | @timestamp | instrumentationScope | instrumentationScope.droppedAttributesCount | instrumentationScope.name | instrumentationScope.version | severityText | resource | resource.attributes | resource.attributes.host | resource.attributes.host.name | resource.attributes.service | resource.attributes.service.name | resource.droppedAttributesCount | flags | attributes | droppedAttributesCount | severityNumber | time | body | +|----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------| +| span0003 | abcd1234efgh5678 | 2024-02-01 09:12:00 | {'name': 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc', 'droppedAttributesCount': 0, 'version': '0.49.0'} | 0 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc | 0.49.0 | WARN | {'attributes': {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}}, 'droppedAttributesCount': 0} | {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}} | {'name': 'productcatalog-7c9d-zn4p2'} | productcatalog-7c9d-zn4p2 | {'name': 'product-catalog'} | product-catalog | 0 | 0 | {} | 0 | 13 | 2024-02-01 09:12:00 | Slow query detected: SELECT * FROM products WHERE category = 'electronics' took 3200ms | ++----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+ ``` diff --git a/docs/user/ppl/cmd/foreach.md b/docs/user/ppl/cmd/foreach.md new file mode 100644 index 00000000000..45dcb939f8a --- /dev/null +++ b/docs/user/ppl/cmd/foreach.md @@ -0,0 +1,191 @@ +# foreach + +The `foreach` command runs a templated `eval` expression for each field in a field list, each element of a multivalue (array) field, or each element of a JSON array. It eliminates repetitive `eval` statements when the same computation applies to many fields or to every element of a collection. + +## Syntax + +The `foreach` command has the following syntax: + +```syntax +foreach [mode=] [

    ClassCastException: RelCompositeTrait cannot be cast to RelCollation
    + * + *

    Root cause: {@code reIndexCollations()} and {@code pushDownSort()} both used {@code + * RelTraitSet.plus()} which composes collation traits into a {@link + * org.apache.calcite.rel.RelCompositeTrait} when a collation is already present. Calcite's {@code + * RelTraitSet.getCollation()} then fails with a ClassCastException. Fixed by using {@code + * RelTraitSet.replace()} instead to always replace the collation trait. + */ + @Test + public void testMultisearchWithTimestampAliasFieldDoesNotThrow() throws IOException { + // alias-typed fields are stripped when loading indices in analytics-engine parquet mode, + // so @timestamp does not exist in TEST_INDEX_ALIAS on that route. + assumeFalse( + "alias-typed fields are stripped in analytics-engine parquet mode;" + + " @timestamp won't exist in TEST_INDEX_ALIAS on that route.", + isAnalyticsParquetIndicesEnabled()); + // TEST_INDEX_ALIAS has @timestamp defined as an alias field pointing to original_date. + // Running multisearch on such an index used to crash with ClassCastException. + JSONObject result = + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | where original_col > 1 | fields original_col," + + " @timestamp] " + + "[search source=%s | where original_col = 1 | fields original_col," + + " @timestamp]", + TEST_INDEX_ALIAS, TEST_INDEX_ALIAS)); + + verifySchemaInOrder( + result, schema("original_col", null, "int"), schema("@timestamp", null, "timestamp")); + // 2 rows from original_col > 1, 1 row from original_col = 1 + assertEquals(3, result.getInt("total")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index a5937d06f31..a2ab93b6599 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TELEMETRY; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -28,6 +29,7 @@ import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLAggregationIT extends PPLIntegTestCase { @@ -967,6 +969,9 @@ public void testTake() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testPercentile() throws IOException { JSONObject actual = executeQuery( @@ -1184,6 +1189,9 @@ public void testPercentileShortcutsWithDecimals() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testPercentileShortcutsFloatingPoint() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java index 6372b818b2b..56eece17ff4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java @@ -8,6 +8,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.Capability.HEAD_WITHOUT_STABLE_SORT; +import static org.opensearch.sql.util.Capability.IP_UDT_BINARY_REPRESENTATION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -22,6 +24,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLAppendCommandIT extends PPLIntegTestCase { @Override @@ -195,6 +198,11 @@ public void testAppendDifferentIndex() throws IOException { } @Test + @RequiresCapability( + value = HEAD_WITHOUT_STABLE_SORT, + note = + "head 5 over the two-branch append has no globally-unique sort key, so the" + + " surviving/ordered rows diverge on the AE route (HEAD_WITHOUT_STABLE_SORT).") public void testAppendWithMergedColumn() throws IOException { JSONObject actual = executeQuery( @@ -258,6 +266,11 @@ public void testAppendSchemaMergeWithTimestampUDT() throws IOException { } @Test + @RequiresCapability( + value = IP_UDT_BINARY_REPRESENTATION, + note = + "cidrmatch over an appended IP column hits the IP-UDT-as-byte[] gap on the AE route" + + " (IP_UDT_BINARY_REPRESENTATION).") public void testAppendSchemaMergeWithIpUDT() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendPipeCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendPipeCommandIT.java index 6ae37a027ba..8414571812a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendPipeCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendPipeCommandIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.Capability.APPENDPIPE_MAIN_RESULT_DROPPED; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,6 +18,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLAppendPipeCommandIT extends PPLIntegTestCase { @Override @@ -145,6 +147,9 @@ public void testTripleAppendPipe() throws IOException { /** Regression test: double appendpipe with non-aggregation (filter) subpipeline. */ @Test + @RequiresCapability( + value = APPENDPIPE_MAIN_RESULT_DROPPED, + note = "appendpipe drops the main pipeline's rows on the AE route (filter applied in place).") public void testDoubleAppendPipeWithFilter() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java index 877c10947b8..c6fc62fae74 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java @@ -81,4 +81,18 @@ public void testAppendColOverride() throws IOException { rows("F", "DE", 101, null), rows("F", "FL", 310, null)); } + + /** Verifies that rex can be used as the first command of an appendcol subsearch. */ + @Test + public void testAppendColWithRexInSubsearch() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats count() as cnt by gender" + + " | appendcol [ rex field=email '^(?[^@]+)@.*' | fields user ]" + + " | head 2", + TEST_INDEX_ACCOUNT)); + verifySchema( + actual, schema("gender", "string"), schema("cnt", "bigint"), schema("user", "string")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java index dbf7f32ca96..d17d151d7ff 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java @@ -8,6 +8,7 @@ import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestUtils.isIndexExist; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.REGEXP_FILTER; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -21,6 +22,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBasicIT extends PPLIntegTestCase { @@ -124,6 +126,20 @@ public void testFieldsShouldBeCaseSensitive() { verifyErrorMessageContains(e, "Field [NAME] not found."); } + @Test + public void testFieldNotFoundWithSuggestion() { + // Typo: "nam" instead of "name" + Throwable e = + assertThrowsWithReplace( + IllegalStateException.class, () -> executeQuery("source=test | fields nam")); + String stack = org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace(e); + verifyErrorMessageContains(e, "Field [nam] not found."); + // Verify suggestion based on Levenshtein distance + verifyErrorMessageContains(e, "Did you mean: name"); + // Verify available fields are listed + verifyErrorMessageContains(e, "available_fields"); + } + @Test public void testFilterQuery1() throws IOException { JSONObject actual = executeQuery("source=test | where age = 30 | fields name, age"); @@ -154,6 +170,9 @@ public void testFilterQuery4() throws IOException { } @Test + @RequiresCapability( + value = REGEXP_FILTER, + note = "REGEXP filter throws a backend NullPointerException on the AE route.") public void testRegexpFilter() throws IOException { JSONObject actual = executeQuery("source=test | where name REGEXP 'he.*' | fields name, age"); verifySchema(actual, schema("name", "string"), schema("age", "bigint")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinDatetimeFunctionInvalidIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinDatetimeFunctionInvalidIT.java index 9c94d1e025d..aa096690dde 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinDatetimeFunctionInvalidIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinDatetimeFunctionInvalidIT.java @@ -6,12 +6,14 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS_WITH_NULL; +import static org.opensearch.sql.util.Capability.INVALID_DATETIME_ERROR_SHAPE; import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; import org.junit.jupiter.api.Test; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBuiltinDatetimeFunctionInvalidIT extends PPLIntegTestCase { @Override @@ -216,6 +218,11 @@ public void testDAYInvalid() { } @Test + @RequiresCapability( + value = INVALID_DATETIME_ERROR_SHAPE, + note = + "dayname/monthname over an invalid datetime literal throws a different error-message" + + " shape on the AE route (INVALID_DATETIME_ERROR_SHAPE).") public void testDAYNAMEInvalid() { Throwable e1 = @@ -760,6 +767,11 @@ public void testMONTH_OF_YEARInvalid() { } @Test + @RequiresCapability( + value = INVALID_DATETIME_ERROR_SHAPE, + note = + "dayname/monthname over an invalid datetime literal throws a different error-message" + + " shape on the AE route (INVALID_DATETIME_ERROR_SHAPE).") public void testMONTHNAMEInvalid() { Throwable e1 = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java index 58f4cb849b3..82debdb0310 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NULL_MISSING; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; +import static org.opensearch.sql.util.Capability.FLOAT_ARITHMETIC_PRECISION; import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -21,6 +22,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBuiltinFunctionIT extends PPLIntegTestCase { @Override @@ -255,6 +257,9 @@ public void testModWithSortAndFields() throws IOException { } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float modulo keeps 32-bit precision on AE; v2 widens to double.") public void testModFloatAndNegative() throws IOException { JSONObject actual = executeQuery( @@ -262,11 +267,15 @@ public void testModFloatAndNegative() throws IOException { "source=%s | eval f = mod(float_number, 2), n = -1 * short_number %% 2, nd = -1 *" + " double_number %% 2 | fields f, n, nd", TEST_INDEX_DATATYPE_NUMERIC)); - verifySchema(actual, schema("f", "float"), schema("n", "int"), schema("nd", "double")); + // -1 * short_number widens the integer operands to bigint (overflow-safe widening). + verifySchema(actual, schema("f", "float"), schema("n", "bigint"), schema("nd", "double")); verifyDataRows(actual, closeTo(0.2, -1, -1.1)); } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float modulo keeps 32-bit precision on AE; v2 widens to double.") public void testModShouldReturnWiderTypes() throws IOException { JSONObject actual = executeQuery( @@ -347,6 +356,9 @@ public void testSignAndRound() throws IOException { } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float/half_float division keeps 32-bit precision on AE; v2 widens to double.") public void testDivide() throws IOException { JSONObject actual = executeQuery( @@ -403,4 +415,66 @@ public void testDivideShouldReturnNull() throws IOException { schema("r6", "double")); verifyDataRows(actual, rows(null, null, null, null, null)); } + + /** + * Integer arithmetic must widen narrow operands so the result cannot overflow the (mis-)inferred + * SMALLINT/TINYINT result type. Historically {@code short * short} stayed SHORT, silently + * wrapping on the analytics-engine backend and throwing "value out of range" on the Calcite path. + * byte/short now widen to INT and int/long widen to BIGINT for {@code +}, {@code -}, {@code *}. + */ + @Test + public void testIntegerArithmeticWidensResultType() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval ss = short_number * short_number, bb = byte_number *" + + " byte_number, ii = integer_number * integer_number, sp = short_number +" + + " short_number, sm = short_number - byte_number | fields ss, bb, ii, sp, sm", + TEST_INDEX_DATATYPE_NUMERIC)); + // short/byte products widen to int; int product widens to bigint; +/- widen likewise. + verifySchema( + actual, + schema("ss", "int"), + schema("bb", "int"), + schema("ii", "bigint"), + schema("sp", "int"), + schema("sm", "int")); + // short_number=3, byte_number=4, integer_number=2. + verifyDataRows(actual, rows(9, 16, 4, 6, -1)); + } + + /** + * Regression for the reported overflow: {@code short * } whose product exceeds the 16-bit + * SHORT range (32767) must produce the correct widened value rather than a wrapped one. + */ + @Test + public void testShortArithmeticDoesNotOverflow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval area = short_number * 20000 | where area > 32767 | fields area", + TEST_INDEX_DATATYPE_NUMERIC)); + // 3 * 20000 = 60000, which overflows SHORT (max 32767) but is exact once widened. The INTEGER + // literal 20000 promotes the product to BIGINT (any int operand -> long). + verifySchema(actual, schema("area", "bigint")); + verifyDataRows(actual, rows(60000)); + } + + /** + * The INTEGER->BIGINT tier: {@code integer_number * } whose product exceeds + * the 32-bit INT range must widen to BIGINT and stay exact rather than wrapping i32. + */ + @Test + public void testIntegerArithmeticDoesNotOverflow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval big = integer_number * 2000000000 | where big > 2147483647 |" + + " fields big", + TEST_INDEX_DATATYPE_NUMERIC)); + // integer_number=2; 2 * 2,000,000,000 = 4,000,000,000 overflows INT (max 2,147,483,647) but is + // exact once widened to BIGINT. + verifySchema(actual, schema("big", "bigint")); + verifyDataRows(actual, rows(4000000000L)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionsNullIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionsNullIT.java index c7651014ea6..4bb83582ca4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionsNullIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionsNullIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS_WITH_NULL; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NULL_MISSING; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; +import static org.opensearch.sql.util.Capability.TIME_TYPE_WIDENED_TO_TIMESTAMP; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -17,6 +18,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBuiltinFunctionsNullIT extends PPLIntegTestCase { @Override @@ -887,6 +889,11 @@ public void testTimeToSecNull() throws IOException { } @Test + @RequiresCapability( + value = TIME_TYPE_WIDENED_TO_TIMESTAMP, + note = + "the TIME field reads back as TIMESTAMP on the AE route, defeating TIMEDIFF's [TIME,TIME]" + + " signature (TIME_TYPE_WIDENED_TO_TIMESTAMP).") public void testTimediffNull() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java index 84159c1bb96..f40ca7b4f4e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java @@ -5,10 +5,12 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestUtils.isIndexExist; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.Capability.BIN_TIME_FIELD_BUCKETING; import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -23,6 +25,7 @@ import org.opensearch.client.Request; import org.opensearch.sql.legacy.TestsConstants; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLCaseFunctionIT extends PPLIntegTestCase { @@ -31,12 +34,20 @@ public void init() throws Exception { super.init(); enableCalcite(); + // Seed only on first creation: the AE parquet store is append-only on same-_id PUT. + boolean weblogsExisted = isIndexExist(client(), TEST_INDEX_WEBLOGS); loadIndex(Index.WEBLOG); loadIndex(Index.TIME_TEST_DATA); loadIndex(Index.STATE_COUNTRY_WITH_NULL); loadIndex(Index.BANK); - loadIndex(Index.OTELLOGS); - appendDataForBadResponse(); + // otel_logs has a multi-value keyword the AE store rejects at load; only the (AE-skipped) + // testNestedCaseAggWithAutoDateHistogram needs it. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.OTELLOGS); + } + if (!weblogsExisted) { + appendDataForBadResponse(); + } } private void appendDataForBadResponse() throws IOException { @@ -478,6 +489,9 @@ public void testCaseAggWithNullValues() throws IOException { } @Test + @RequiresCapability( + value = BIN_TIME_FIELD_BUCKETING, + note = "bin @timestamp then group by it: bucket column typed string (not timestamp) on AE.") public void testNestedCaseAggWithAutoDateHistogram() throws IOException { // TODO: Remove after resolving: https://github.com/opensearch-project/sql/issues/4578 Assume.assumeFalse( @@ -523,4 +537,22 @@ public void testNestedCaseAggWithAutoDateHistogram() throws IOException { schema("flags", "bigint")); verifyNumOfRows(actual2, 32); } + + /** Case with no common branch supertype must return a clean 4xx, not a 500. */ + @Test + public void testCaseWithIncompatibleBranchTypesRejectsCleanly() { + org.opensearch.client.ResponseException e = + org.junit.Assert.assertThrows( + org.opensearch.client.ResponseException.class, + () -> + executeQuery( + String.format( + "source=%s | eval x = case(age > 30, 'old', age > 20, 1 else 0.0) | fields" + + " x", + TEST_INDEX_BANK))); + org.junit.Assert.assertEquals( + "expected 400 status, got: " + e.getMessage(), + 400, + e.getResponse().getStatusLine().getStatusCode()); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java index 71e9e69e3ae..4177d108440 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java @@ -7,12 +7,14 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DUPLICATION_NULLABLE; +import static org.opensearch.sql.util.Capability.DEDUP_NONDETERMINISTIC; import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLDedupIT extends PPLIntegTestCase { @@ -95,6 +97,9 @@ public void testDedupMultipleFieldsKeepEmpty() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = "dedup CONSECUTIVE behavior diverges on the AE route.") public void testConsecutiveImplicitFallbackV2() throws IOException { JSONObject actual = executeQuery( @@ -252,6 +257,10 @@ public void testReorderDedupFieldsShouldNotAffectResult() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = + "dedup surviving-duplicate selection diverges on the AE route (no stable merge order).") public void testDedupComplex() throws IOException { JSONObject actual = executeQuery(String.format("source=%s | dedup 1 name", TEST_INDEX_DUPLICATION_NULLABLE)); @@ -339,6 +348,45 @@ public void testMultiColumnSortThenDedup() throws IOException { verifyDataRows(actual, rows("M", "AK", 20, 23), rows("F", "AK", 21, 334)); } + /** + * Regression test for https://github.com/opensearch-project/sql/issues/5150 + * + *

    A renamed field that is not the dedup key must retain its value after dedup aggregation + * pushdown. Previously the top_hits response returned the original index field name ({@code + * name}), which the enumerator could not resolve to the renamed output name ({@code nm}), + * yielding null. + */ + @Test + public void testDedupWithRenamedField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | rename name as nm | dedup 1 category | fields category, nm", + TEST_INDEX_DUPLICATION_NULLABLE)); + // One representative row per category; nm must not be null + verifyDataRows(actual, rows("X", "A"), rows("Z", "B"), rows("Y", "A")); + } + + /** + * Regression test for https://github.com/opensearch-project/sql/issues/5197 + * + *

    When both a {@code rename} and an {@code eval} column-reference resolve to the same original + * index field, the old {@code Map<String,String>} mapping silently dropped one alias on + * collision. With {@code Map<String,List<String>>} both aliases must appear in the + * result with correct values. + */ + @Test + public void testDedupWithRenamedFieldMappingCollision() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval nm2 = name | rename name as nm | dedup 1 category" + + " | fields category, nm, nm2", + TEST_INDEX_DUPLICATION_NULLABLE)); + // Both nm (from rename) and nm2 (from eval col-ref) must carry the same non-null name value + verifyDataRows(actual, rows("X", "A", "A"), rows("Z", "B", "B"), rows("Y", "A", "A")); + } + /** Regression test for https://github.com/opensearch-project/sql/issues/3922 */ @Test public void testSortThenDedupKeepEmpty() throws IOException { @@ -364,6 +412,10 @@ public void testSortThenDedupKeepEmpty() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = + "dedup surviving-duplicate selection diverges on the AE route (no stable merge order).") public void testDedupExpr() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java index 674a7d96f8d..78ff6fc0401 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java @@ -7,10 +7,14 @@ import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.ppl.PPLIntegTestCase.GlobalPushdownConfig; +import org.opensearch.sql.protocol.response.format.Format; public class CalcitePPLExplainIT extends PPLIntegTestCase { @@ -74,6 +78,51 @@ public void testExplainCommandSimple() throws IOException { assertJsonEquals(expected, result); } + @Test + public void testJsonTreeFormat() throws IOException { + var resultStr = + explainQuery( + "source=test | where age > 20 | fields name", Format.JSON_TREE, ExplainMode.STANDARD); + + // Parse JSON + var mapper = new ObjectMapper(); + var result = mapper.readTree(resultStr); + + // Verify tree structure exists + assertTrue(result.has("calcite")); + assertTrue(result.get("calcite").has("logical")); + assertTrue(result.get("calcite").has("physical")); + + // Verify logical and physical are parsed JSON objects, not strings + assertTrue(result.get("calcite").get("logical").isObject()); + assertTrue(result.get("calcite").get("physical").isObject()); + + // Verify sourceBuilder exists in physical plan rels + var physical = result.get("calcite").get("physical"); + assertTrue(physical.has("rels")); + var rels = physical.get("rels"); + assertTrue(rels.isArray()); + + // Find a rel with sourceBuilder (only present when pushdown is enabled) + boolean foundSourceBuilder = false; + for (int i = 0; i < rels.size(); i++) { + var rel = rels.get(i); + if (rel.has("sourceBuilder")) { + foundSourceBuilder = true; + // Verify sourceBuilder is a parsed JSON object, not a string + assertTrue(rel.get("sourceBuilder").isObject()); + // Verify it has expected OpenSearch DSL fields + assertTrue(rel.get("sourceBuilder").has("from")); + assertTrue(rel.get("sourceBuilder").has("size")); + break; + } + } + // Only assert sourceBuilder exists when pushdown is enabled + if (GlobalPushdownConfig.enabled) { + assertTrue("sourceBuilder not found in physical plan rels", foundSourceBuilder); + } + } + /** * Executes the PPL query and returns the result as a string with windows-style line breaks * replaced with Unix-style ones. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java index 99af10302ae..02ebae95e8e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.JSON_DOLLAR_PATH; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -15,6 +16,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLJsonBuiltinFunctionIT extends PPLIntegTestCase { @Override @@ -30,7 +32,12 @@ public void init() throws Exception { loadIndex(Index.PEOPLE2); loadIndex(Index.BANK); loadIndex(Index.JSON_TEST); - loadIndex(Index.GAME_OF_THRONES); + // game_of_thrones has a multi-value array for the scalar-mapped `titles` field, which the + // parquet store rejects at bulk load; skip it on the AE route so it doesn't abort init() for + // the rest of the suite. No test in this class queries game_of_thrones. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.GAME_OF_THRONES); + } } @Test @@ -297,6 +304,9 @@ public void testJsonSetPartialSet() throws IOException { } @Test + @RequiresCapability( + value = JSON_DOLLAR_PATH, + note = "json_set with a $-prefixed path is a no-op on the AE route (JSON_DOLLAR_PATH).") public void testJsonSetWithDollarPrefixedPath() throws IOException { // Issue #5167: json_set with $.key path should not double-prefix JSONObject actual = @@ -313,6 +323,9 @@ public void testJsonSetWithDollarPrefixedPath() throws IOException { } @Test + @RequiresCapability( + value = JSON_DOLLAR_PATH, + note = "json_delete with a $-prefixed path is a no-op on the AE route (JSON_DOLLAR_PATH).") public void testJsonDeleteWithDollarPrefixedPath() throws IOException { // Issue #5167: json_delete with $.key path should remove the key JSONObject actual = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java index 0e24bc33d38..04ccba25b6c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.util.Capability.ADDTOTALS_JOIN_PANIC; +import static org.opensearch.sql.util.Capability.MVCOMBINE_ARRAY_AGG; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -22,6 +24,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** * Integration tests for PPL queries that reference MAP dotted paths (e.g. {@code doc.user.name}). @@ -141,6 +144,9 @@ public void testFieldsExclusionOnMapPath() throws IOException { } @Test + @RequiresCapability( + value = ADDTOTALS_JOIN_PANIC, + note = "addtotals crashes the DataFusion backend with a join panic on the AE route.") public void testAddtotalsOnMapPath() throws IOException { JSONObject result = ppl( @@ -163,6 +169,9 @@ public void testAddtotalsOnMapPath() throws IOException { } @Test + @RequiresCapability( + value = MVCOMBINE_ARRAY_AGG, + note = "mvcombine lowers to ARRAY_AGG, unregistered on the analytics backend.") public void testMvcombineOnMapPath() throws IOException { JSONObject result = ppl( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 3503d7c533c..6bac3032bd2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; +import static org.opensearch.sql.util.Capability.WILDCARD_COLUMN_ORDER; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -21,6 +22,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLRenameIT extends PPLIntegTestCase { @@ -204,6 +206,9 @@ public void testRenameFullWildcard() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = "rename * returns columns in a different order on the AE route.") public void testRenameFullWildcardExcludesMetadataFields() throws IOException { JSONObject result = executeQuery(String.format("source = %s | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java new file mode 100644 index 00000000000..70ee4ef0de6 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -0,0 +1,210 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code + * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also + * verifies that a non-allow-listed / mutating endpoint is refused. + */ +public class CalcitePPLRestIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + } + + @Test + public void testRestClusterHealthSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); + } + + @Test + public void testRestClusterHealthDataRows() throws IOException { + // Single-node test cluster: exactly one node, status is green or yellow. + JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestRejectsNonAllowListedEndpoint() throws IOException { + assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); + } + + @Test + public void testRestRejectsEmptyEndpoint() throws IOException { + assertRestBadRequest("| rest ''", "non-empty path"); + } + + @Test + public void testRestRejectsDisallowedArg() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); + } + + @Test + public void testRestRejectsNegativeCount() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); + } + + @Test + public void testRestRejectsTimeoutArg() throws IOException { + assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); + } + + /** + * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) + * with the given substring in the response body. Covers allow-list and bad-argument rejection. + */ + private void assertRestBadRequest(String query, String expectedSubstring) { + ResponseException e = + org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); + org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertTrue( + "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), + e.getMessage().contains(expectedSubstring)); + } + + @Test + public void testRestCatIndicesSchema() throws IOException { + // Schema is fixed by the registry, independent of how many indices exist. + JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); + verifySchema(result, schema("index", "string"), schema("health", "string")); + } + + @Test + public void testRestCatIndicesReturnsCreatedIndex() throws IOException { + // Create a known index, then confirm rest surfaces it and downstream where/fields compose. + client().performRequest(new Request("PUT", "/rest_cat_test")); + JSONObject result = + executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); + verifyDataRows(result, rows("rest_cat_test")); + } + + @Test + public void testRestCatNodesSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); + verifySchema(result, schema("name", "string"), schema("cpu", "int")); + } + + @Test + public void testRestCatNodesSingleNode() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatClusterManagerSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); + verifySchema(result, schema("node", "string"), schema("id", "string")); + } + + @Test + public void testRestCatClusterManagerSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatPluginsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); + verifySchema(result, schema("component", "string"), schema("version", "string")); + } + + @Test + public void testRestCatShardsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); + verifySchema( + result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); + } + + @Test + public void testRestClusterStateSchema() throws IOException { + // Assert the string columns; version is the LONG epoch column. + JSONObject result = + executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); + verifySchema( + result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); + } + + @Test + public void testRestClusterStateSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestClusterSettingsSchema() throws IOException { + // Schema is registry-fixed regardless of how many settings are configured. + JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); + verifySchema( + result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); + } + + @Test + public void testRestResolveIndexSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { + client().performRequest(new Request("PUT", "/rest_resolve_test")); + JSONObject result = + executeQuery( + "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); + verifyDataRows(result, rows("rest_resolve_test", "index")); + } + + // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- + + @Test + public void testRestClusterHealthLocalArg() throws IOException { + // local=true reads health from the local node; on a single-node cluster the row is unchanged. + JSONObject result = + executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { + // health filters rows server-side; a healthy cluster has no red indices, so count is 0. + JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); + verifyDataRows(result, rows(0)); + } + + @Test + public void testRestResolveIndexExpandWildcardsArg() throws IOException { + // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. + JSONObject result = + executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestRejectsDroppedLevelArg() throws IOException { + // level was dropped from the allow-list (no-op against the fixed health schema). + assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); + } + + @Test + public void testRestRejectsBadArgValue() throws IOException { + assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java index 1b090580d94..2641fe6b73c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java @@ -166,15 +166,18 @@ public void testSortAgeAndFieldsAge() throws IOException { @Test public void testSortAgeAndFieldsNameAge() throws IOException { + // firstname is a unique secondary key so the age=36 tie has a deterministic, + // engine-independent order (sort is otherwise free to order ties differently per engine). JSONObject actual = executeQuery( - String.format("source=%s | sort - age | fields firstname, age", TEST_INDEX_BANK)); + String.format( + "source=%s | sort - age, firstname | fields firstname, age", TEST_INDEX_BANK)); verifySchema(actual, schema("firstname", "string"), schema("age", "int")); verifyDataRowsInOrder( actual, rows("Virginia", 39), - rows("Hattie", 36), rows("Elinor", 36), + rows("Hattie", 36), rows("Dillard", 34), rows("Dale", 33), rows("Amber JOHnny", 32), @@ -201,15 +204,17 @@ public void testSortAgeNameAndFieldsNameAge() throws IOException { @Test public void testSortWithNullValue() throws IOException { + // firstname is a unique secondary key so the three null-balance rows have a deterministic, + // engine-independent order (sort is otherwise free to order ties differently per engine). JSONObject result = executeQuery( String.format( - "source=%s | sort balance | fields firstname, balance", + "source=%s | sort balance, firstname | fields firstname, balance", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifyDataRowsInOrder( result, - rows("Hattie", null), rows("Elinor", null), + rows("Hattie", null), rows("Virginia", null), rows("Dale", 4180), rows("Nanette", 32838), @@ -316,9 +321,12 @@ public void testSortWithStrCast() throws IOException { @Test public void testSortWithAutoCast() throws IOException { + // firstname is a unique secondary key so the age=36 tie has a deterministic, + // engine-independent order (sort is otherwise free to order ties differently per engine). JSONObject result = executeQuery( - String.format("source=%s | sort AUTO(age) | fields firstname, age", TEST_INDEX_BANK)); + String.format( + "source=%s | sort AUTO(age), firstname | fields firstname, age", TEST_INDEX_BANK)); verifySchema(result, schema("firstname", "string"), schema("age", "int")); verifyDataRowsInOrder( result, @@ -326,8 +334,8 @@ public void testSortWithAutoCast() throws IOException { rows("Amber JOHnny", 32), rows("Dale", 33), rows("Dillard", 34), - rows("Hattie", 36), rows("Elinor", 36), + rows("Hattie", 36), rows("Virginia", 39)); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java index f3d3852f938..e703e0fc366 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; +import static org.opensearch.sql.util.Capability.DOC_MUTATION; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -17,6 +18,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLStringBuiltinFunctionIT extends PPLIntegTestCase { @Override @@ -60,6 +62,9 @@ public void testConcat() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testConcatWithField() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); @@ -78,6 +83,9 @@ public void testConcatWithField() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testConcatWs() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); @@ -212,6 +220,9 @@ public void testPosition() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testTrim() throws IOException { prepareTrim(); JSONObject actual = @@ -226,6 +237,9 @@ public void testTrim() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testRTrim() throws IOException { prepareTrim(); JSONObject actual = @@ -240,6 +254,9 @@ public void testRTrim() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testLTrim() throws IOException { prepareTrim(); JSONObject actual = @@ -254,6 +271,9 @@ public void testLTrim() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testReverse() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); @@ -272,6 +292,9 @@ public void testReverse() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testRight() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePartialFilterPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePartialFilterPushdownIT.java new file mode 100644 index 00000000000..7921ecf6d55 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePartialFilterPushdownIT.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Tests partial filter pushdown when some predicates can be pushed natively and others require + * script evaluation. + * + *

    Regression test for issue where LIKE on text fields (without .keyword subfield) caused entire + * AND filter to fall back to script, preventing timestamp range pushdown. + */ +public class CalcitePartialFilterPushdownIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.LOGS); + } + + @Test + public void testTimestampRangePushesWithUnpushableLike() throws IOException { + // message is text field without .keyword — LIKE on it requires script evaluation + // @timestamp is date field — range should push natively despite LIKE failing + String query = + String.format( + "source=%s | where `@timestamp` >= '2023-01-01' and `@timestamp` < '2023-01-04' " + + "and LIKE(message, '%%failed%%') | stats count()", + TEST_INDEX_LOGS); + + JSONObject result = executeQuery(query); + + // Just verify query executes and returns reasonable results + // The key regression is that this doesn't do a full table scan + verifySchema(result, schema("count()", "bigint")); + // Should find "Database connection failed" in the date range + verifyDataRows(result, rows(1L)); + } + + @Test + public void testMultipleUnpushablePredicatesInAnd() throws IOException { + // Both LIKE conditions are on text field, but timestamp should still push + String query = + String.format( + "source=%s | where `@timestamp` >= '2023-01-01' and LIKE(message, '%%space%%') " + + "and LIKE(message, '%%low%%') | stats count()", + TEST_INDEX_LOGS); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("count()", "bigint")); + // Should find "Disk space low" + verifyDataRows(result, rows(1L)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java index 5c381bb5346..fb88e9be73b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; +import static org.opensearch.sql.util.Capability.WILDCARD_COLUMN_ORDER; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -18,6 +19,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteReverseCommandIT extends PPLIntegTestCase { @@ -230,6 +232,11 @@ public void testReverseWithTimestampAndExplicitSort() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = + "streamstats carries all source columns through; the AE route returns them in a different" + + " order (WILDCARD_COLUMN_ORDER).") public void testStreamstatsWithReverse() throws IOException { // Test that reverse is ignored when used directly after streamstats // streamstats maintains order via __stream_seq__, but this field is projected out @@ -259,6 +266,11 @@ public void testStreamstatsWithReverse() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = + "streamstats carries all source columns through; the AE route returns them in a different" + + " order (WILDCARD_COLUMN_ORDER).") public void testStreamstatsWindowWithReverse() throws IOException { // Test that reverse is ignored after streamstats with window JSONObject result = @@ -286,6 +298,11 @@ public void testStreamstatsWindowWithReverse() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = + "streamstats carries all source columns through; the AE route returns them in a different" + + " order (WILDCARD_COLUMN_ORDER).") public void testStreamstatsByWithReverse() throws IOException { // Test that reverse is effective after streamstats with partitioning (by clause). // Backtracking finds the __stream_seq__ sort from streamstats and reverses its order. @@ -314,6 +331,11 @@ public void testStreamstatsByWithReverse() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = + "streamstats carries all source columns through; the AE route returns them in a different" + + " order (WILDCARD_COLUMN_ORDER).") public void testStreamstatsWithSortThenReverse() throws IOException { // Test that reverse works when there's an explicit sort after streamstats // The explicit sort creates a collation that reverse can detect and reverse diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java index 3e01ba9f9d7..0c6f0250807 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.Capability.LUCENE_PUSHDOWN_EXPLAIN; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyOrder; @@ -16,6 +17,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.ppl.SortCommandIT; +import org.opensearch.sql.util.RequiresCapability; public class CalciteSortCommandIT extends SortCommandIT { @Override @@ -80,6 +82,11 @@ public void testPushdownSortCastExpression() throws IOException { } @Test + @RequiresCapability( + value = LUCENE_PUSHDOWN_EXPLAIN, + note = + "asserts a Lucene SORT-> pushdown fragment in the explain plan, absent on the AE route" + + " (LUCENE_PUSHDOWN_EXPLAIN).") public void testPushdownSortCastToDoubleExpression() throws IOException { // Similar to query: 'source=%s | sort num(age)'. But left query doesn't output casted column. String ppl = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java index 43ede1606bc..e70812e3c3b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java @@ -6,7 +6,9 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.CHAINED_STREAMSTATS_BY; import static org.opensearch.sql.util.Capability.DOC_MUTATION; +import static org.opensearch.sql.util.Capability.STREAMSTATS_SORT_NOT_HONORED; import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; @@ -558,6 +560,7 @@ public void testStreamstatsGlobal() throws IOException { } @Test + @RequiresCapability(DOC_MUTATION) public void testStreamstatsGlobalWithNull() throws IOException { final int docId = 7; Request insertRequest = @@ -613,6 +616,7 @@ public void testStreamstatsGlobalWithNull() throws IOException { } @Test + @RequiresCapability(DOC_MUTATION) public void testStreamstatsGlobalWithNullBucket() throws IOException { final int docId = 7; Request insertRequest = @@ -718,6 +722,7 @@ public void testStreamstatsReset() throws IOException { } @Test + @RequiresCapability(DOC_MUTATION) public void testStreamstatsResetWithNull() throws IOException { final int docId = 7; Request insertRequest = @@ -773,6 +778,7 @@ public void testStreamstatsResetWithNull() throws IOException { } @Test + @RequiresCapability(DOC_MUTATION) public void testStreamstatsResetWithNullBucket() throws IOException { final int docId = 7; Request insertRequest = @@ -845,6 +851,7 @@ public void testUnsupportedWindowFunctions() { } @Test + @RequiresCapability(CHAINED_STREAMSTATS_BY) public void testMultipleStreamstats() throws IOException { JSONObject actual = executeQuery( @@ -863,6 +870,7 @@ public void testMultipleStreamstats() throws IOException { } @Test + @RequiresCapability(CHAINED_STREAMSTATS_BY) public void testMultipleStreamstatsWithWindow() throws IOException { // Test case from GitHub issue #4800: chained streamstats with window=2 JSONObject actual = @@ -899,6 +907,7 @@ public void testMultipleStreamstatsWithWindow() throws IOException { // causing Calcite's RelDecorrelator to fail on duplicate correlate references. @Test + @RequiresCapability(CHAINED_STREAMSTATS_BY) public void testMultipleStreamstatsWithNull1() throws IOException { JSONObject actual = executeQuery( @@ -1008,6 +1017,7 @@ public void testStreamstatsAndEventstats() throws IOException { } @Test + @RequiresCapability(STREAMSTATS_SORT_NOT_HONORED) public void testStreamstatsAndSort() throws IOException { JSONObject actual = executeQuery( @@ -1074,6 +1084,7 @@ public void testWhereInWithStreamstatsSubquery() throws IOException { } @Test + @RequiresCapability(CHAINED_STREAMSTATS_BY) public void testMultipleStreamstatsWithEval() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimewrapCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimewrapCommandIT.java new file mode 100644 index 00000000000..9383e56f490 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimewrapCommandIT.java @@ -0,0 +1,762 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteTimewrapCommandIT extends PPLIntegTestCase { + + // Standard WHERE clause covering all test data — simulates frontend time picker + private static final String WHERE_ALL = + " | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-04 06:00:00'"; + private static final String WHERE_JUL1_TO_JUL3 = + " | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-03 18:00:00'"; + private static final String WHERE_JUL2_TO_JUL3 = + " | where @timestamp >= '2024-07-02 00:00:00' and @timestamp <= '2024-07-03 18:00:00'"; + private static final String WHERE_JUL1_ONLY = + " | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-01 18:00:00'"; + private static final String WHERE_JUL1_TO_JUL2 = + " | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-02 18:00:00'"; + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + disallowCalciteFallback(); + loadIndex(Index.TIMEWRAP_TEST); + } + + // --- Day-over-day with different aggregations --- + + @Test + public void testTimewrapDayOverDayWithSum() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_3days_before", "bigint"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + @Test + public void testTimewrapDayOverDayWithAvg() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h avg(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("avg(requests)_3days_before", "double"), + schema("avg(requests)_2days_before", "double"), + schema("avg(requests)_1day_before", "double"), + schema("avg(requests)_latest_day", "double")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 90.0, 102.5, 82.5, 40.0), + rows("2024-07-04 06:00:00", 120.0, 130.0, 112.5, 50.0), + rows("2024-07-04 12:00:00", null, 155.0, 165.0, 142.5), + rows("2024-07-04 18:00:00", null, 95.0, 107.5, 82.5)); + } + + @Test + public void testTimewrapDayOverDayWithCount() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + WHERE_ALL + " | timechart span=6h count() | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_3days_before", "bigint"), + schema("count()_2days_before", "bigint"), + schema("count()_1day_before", "bigint"), + schema("count()_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 2, 2, 2, 2), + rows("2024-07-04 06:00:00", 2, 2, 2, 2), + rows("2024-07-04 12:00:00", null, 2, 2, 2), + rows("2024-07-04 18:00:00", null, 2, 2, 2)); + } + + @Test + public void testTimewrapWithDifferentAggField() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(errors) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(errors)_3days_before", "bigint"), + schema("sum(errors)_2days_before", "bigint"), + schema("sum(errors)_1day_before", "bigint"), + schema("sum(errors)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 2, 4, 1, 0), + rows("2024-07-04 06:00:00", 6, 6, 3, 1), + rows("2024-07-04 12:00:00", null, 5, 9, 6), + rows("2024-07-04 18:00:00", null, 1, 3, 1)); + } + + // --- Incomplete period / null fill --- + + @Test + public void testTimewrapIncompletePeriodNullFill() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_3days_before", "bigint"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + // --- Different timescales --- + + @Test + public void testTimewrapWeekSpanSinglePeriod() throws IOException { + // 3 days of daily data in 1 week -> single period, 3 rows + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=1day sum(requests) | timewrap 1week"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("sum(requests)_latest_week", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-01 00:00:00", 920), + rows("2024-07-02 00:00:00", 1010), + rows("2024-07-03 00:00:00", 840)); + } + + @Test + public void testTimewrapTwelveHourSpan() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 12h"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_72hours_before", "bigint"), + schema("sum(requests)_60hours_before", "bigint"), + schema("sum(requests)_48hours_before", "bigint"), + schema("sum(requests)_36hours_before", "bigint"), + schema("sum(requests)_24hours_before", "bigint"), + schema("sum(requests)_12hours_before", "bigint"), + schema("sum(requests)_latest_hour", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 310, 205, 330, 165, 285, 80), + rows("2024-07-04 06:00:00", 240, 190, 260, 215, 225, 165, 100)); + } + + @Test + public void testTimewrapWithMinuteSpan() throws IOException { + loadIndex(Index.EVENTS); + JSONObject result = + executeQuery( + "source=events | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-01 00:04:00' | timechart span=1m count() | timewrap 1min"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_4minutes_before", "bigint"), + schema("count()_3minutes_before", "bigint"), + schema("count()_2minutes_before", "bigint"), + schema("count()_1minute_before", "bigint"), + schema("count()_latest_minute", "bigint")); + verifyDataRows(result, rows("2024-07-01 00:04:00", 1, 1, 1, 1, 1)); + } + + // --- WHERE clause with different time ranges --- + + @Test + public void testTimewrapWithWhereThreeDays() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + } + + @Test + public void testTimewrapWithWhereTwoDays() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL2_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 205, 165), + rows("2024-07-03 06:00:00", 260, 225), + rows("2024-07-03 12:00:00", 330, 285), + rows("2024-07-03 18:00:00", 215, 165)); + } + + @Test + public void testTimewrapWithWhereSingleDay() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_ONLY + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-01 00:00:00", 180), + rows("2024-07-01 06:00:00", 240), + rows("2024-07-01 12:00:00", 310), + rows("2024-07-01 18:00:00", 190)); + } + + @Test + public void testTimewrapWithWhereAvg() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=6h avg(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("avg(requests)_1day_before", "double"), + schema("avg(requests)_latest_day", "double")); + verifyDataRowsInOrder( + result, + rows("2024-07-02 00:00:00", 90.0, 102.5), + rows("2024-07-02 06:00:00", 120.0, 130.0), + rows("2024-07-02 12:00:00", 155.0, 165.0), + rows("2024-07-02 18:00:00", 95.0, 107.5)); + } + + @Test + public void testTimewrapWithWhere12hSpan() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=6h sum(requests) | timewrap 12h"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_36hours_before", "bigint"), + schema("sum(requests)_24hours_before", "bigint"), + schema("sum(requests)_12hours_before", "bigint"), + schema("sum(requests)_latest_hour", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-02 12:00:00", 180, 310, 205, 330), + rows("2024-07-02 18:00:00", 240, 190, 260, 215)); + } + + @Test + public void testTimewrapWithWhereCount() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h count() | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_2days_before", "bigint"), + schema("count()_1day_before", "bigint"), + schema("count()_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 2, 2, 2), + rows("2024-07-03 06:00:00", 2, 2, 2), + rows("2024-07-03 12:00:00", 2, 2, 2), + rows("2024-07-03 18:00:00", 2, 2, 2)); + } + + @Test + public void testTimewrapWithWhereErrors() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL2_TO_JUL3 + + " | timechart span=6h sum(errors) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(errors)_1day_before", "bigint"), + schema("sum(errors)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 4, 1), + rows("2024-07-03 06:00:00", 6, 3), + rows("2024-07-03 12:00:00", 9, 6), + rows("2024-07-03 18:00:00", 3, 1)); + } + + // --- WHERE upper bound above data: shifts column numbers --- + + @Test + public void testTimewrapWithWhereUpperBoundAboveData() throws IOException { + // WHERE upper bound = July 10 (~5.75 days after max data July 4 06:00) + // baseOffset = floor(Jul10/86400) - floor(Jul4_06/86400) = 5 + // periodFromNow for oldest(rel=4): (5+4-1)*1=8, newest(rel=1): (5+1-1)*1=5 + JSONObject result = + executeQuery( + "source=timewrap_test | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-10 00:00:00' | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_8days_before", "bigint"), + schema("sum(requests)_7days_before", "bigint"), + schema("sum(requests)_6days_before", "bigint"), + schema("sum(requests)_5days_before", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + // --- align=end vs align=now --- + + @Test + public void testTimewrapAlignEndIsDefault() throws IOException { + JSONObject resultDefault = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day"); + JSONObject resultEnd = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day align=end"); + + verifySchema( + resultEnd, + schema("@timestamp", "timestamp"), + schema("sum(requests)_3days_before", "bigint"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + resultEnd, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + verifyDataRowsInOrder( + resultDefault, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + @Test + public void testTimewrapAlignNow() throws IOException { + // align=now uses current time — column names are dynamic + // Extract actual column names from the result for verification + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day align=now"); + + // Get actual column names from result schema + org.json.JSONArray schemaArr = result.getJSONArray("schema"); + String c1 = schemaArr.getJSONObject(1).getString("name"); + String c2 = schemaArr.getJSONObject(2).getString("name"); + String c3 = schemaArr.getJSONObject(3).getString("name"); + String c4 = schemaArr.getJSONObject(4).getString("name"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema(c1, "bigint"), + schema(c2, "bigint"), + schema(c3, "bigint"), + schema(c4, "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + // --- Every timescale --- + + @Test + public void testTimewrapSecondSpan() throws IOException { + // 5 events at minute-level, wrap by 1 minute (60sec) + // timechart span=1m gives 3 buckets (00:00, 01:00, 02:00) + // timewrap 1min: each bucket is in a different 1-minute period → 1 offset row, 3 periods + loadIndex(Index.EVENTS); + JSONObject result = + executeQuery( + "source=events | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-01 00:02:00' | timechart span=1m count() | timewrap 1min"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_2minutes_before", "bigint"), + schema("count()_1minute_before", "bigint"), + schema("count()_latest_minute", "bigint")); + verifyDataRows(result, rows("2024-07-01 00:02:00", 1, 1, 1)); + } + + @Test + public void testTimewrapMinuteSpan() throws IOException { + loadIndex(Index.EVENTS); + JSONObject result = + executeQuery( + "source=events | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-01 00:04:00' | timechart span=1m count() | timewrap 1min"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_4minutes_before", "bigint"), + schema("count()_3minutes_before", "bigint"), + schema("count()_2minutes_before", "bigint"), + schema("count()_1minute_before", "bigint"), + schema("count()_latest_minute", "bigint")); + verifyDataRows(result, rows("2024-07-01 00:04:00", 1, 1, 1, 1, 1)); + } + + @Test + public void testTimewrapHourSpan() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=6h sum(requests) | timewrap 12h"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_36hours_before", "bigint"), + schema("sum(requests)_24hours_before", "bigint"), + schema("sum(requests)_12hours_before", "bigint"), + schema("sum(requests)_latest_hour", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-02 12:00:00", 180, 310, 205, 330), + rows("2024-07-02 18:00:00", 240, 190, 260, 215)); + } + + @Test + public void testTimewrapDaySpan() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + } + + @Test + public void testTimewrapWeekSpan() throws IOException { + // 2 days of daily data in 1 week -> single period, 2 rows + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=1day sum(requests) | timewrap 1week"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("sum(requests)_latest_week", "bigint")); + verifyDataRowsInOrder( + result, rows("2024-07-01 00:00:00", 920), rows("2024-07-02 00:00:00", 1010)); + } + + @Test + public void testTimewrapMonthSpan() throws IOException { + // Jul 1-4 only: all data within same month → single month period + JSONObject result = + executeQuery( + "source=timewrap_test | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-04 06:00:00' | timechart span=1day sum(requests) | timewrap 1month"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("sum(requests)_latest_month", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-01 00:00:00", 920), + rows("2024-07-02 00:00:00", 1010), + rows("2024-07-03 00:00:00", 840), + rows("2024-07-04 00:00:00", 180)); + } + + @Test + public void testTimewrapQuarterSpan() throws IOException { + // Jan 15 (Q1) and Apr 15 (Q2) → 2 quarter periods + // With precise day-within-quarter offset: Jan 15 = day 15 of Q1, Apr 15 = day 15 of Q2 + // Both are at the same offset (day 15) → they align on the same row + JSONObject result = + executeQuery( + "source=timewrap_test | where @timestamp >= '2024-01-15 00:00:00' and @timestamp <=" + + " '2024-04-15 12:00:00' | timechart span=1day sum(requests) | timewrap 1quarter"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_1quarter_before", "bigint"), + schema("sum(requests)_latest_quarter", "bigint")); + // Day 15 of each quarter aligns -- both values on the same row + verifyDataRows(result, rows("2024-04-15 00:00:00", 300, 350)); + } + + @Test + public void testTimewrapYearSpan() throws IOException { + // Jan 15 2024 (300) and Jan 15 2025 (400) -- 2 data points in 2 different years + // timechart span=1year: 2 yearly buckets + // timewrap 1year: 2 periods, 1 offset row + JSONObject result = + executeQuery( + "source=timewrap_test | where @timestamp >= '2024-01-15 12:00:00' and @timestamp <=" + + " '2025-01-15 12:00:00' | timechart span=1year sum(requests) | timewrap 1year"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_1year_before", "bigint"), + schema("sum(requests)_latest_year", "bigint")); + // 2024 yearly sum = all 2024 data in WHERE range; 2025 = Jan 15 only (400) + verifyDataRows(result, rows("2025-01-01 00:00:00", 4050, 400)); + } + + // --- series parameter --- + + @Test + public void testTimewrapSeriesRelativeIsDefault() throws IOException { + // series=relative is the default — same as no series parameter + JSONObject resultDefault = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day"); + JSONObject resultRelative = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day series=relative"); + + verifySchema( + resultRelative, + schema("@timestamp", "timestamp"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + resultRelative, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + verifySchema( + resultDefault, + schema("@timestamp", "timestamp"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + resultDefault, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + } + + @Test + public void testTimewrapSeriesShort() throws IOException { + // series=short: columns named _s + // With align=end and WHERE upper bound = Jul 3 18:00, baseOffset=0 + // Periods: oldest=2, middle=1, newest=0 + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day series=short"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_s2", "bigint"), + schema("sum(requests)_s1", "bigint"), + schema("sum(requests)_s0", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + } + + @Test + public void testTimewrapSeriesShortWithCount() throws IOException { + // series=short with count aggregation + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL2_TO_JUL3 + + " | timechart span=6h count() | timewrap 1day series=short"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_s1", "bigint"), + schema("count()_s0", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 2, 2), + rows("2024-07-03 06:00:00", 2, 2), + rows("2024-07-03 12:00:00", 2, 2), + rows("2024-07-03 18:00:00", 2, 2)); + } + + @Test + public void testTimewrapSeriesShortWeekSpan() throws IOException { + // series=short with week span, single period = s0, daily buckets + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=1day sum(requests) | timewrap 1week series=short"); + + verifySchema(result, schema("@timestamp", "timestamp"), schema("sum(requests)_s0", "bigint")); + verifyDataRowsInOrder( + result, rows("2024-07-01 00:00:00", 920), rows("2024-07-02 00:00:00", 1010)); + } + + @Test + public void testTimewrapSeriesShortWithAvg() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=6h avg(requests) | timewrap 1day series=short"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("avg(requests)_s1", "double"), + schema("avg(requests)_s0", "double")); + verifyDataRowsInOrder( + result, + rows("2024-07-02 00:00:00", 90.0, 102.5), + rows("2024-07-02 06:00:00", 120.0, 130.0), + rows("2024-07-02 12:00:00", 155.0, 165.0), + rows("2024-07-02 18:00:00", 95.0, 107.5)); + } + + @Test + public void testTimewrapSeriesShortWithErrors() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL2_TO_JUL3 + + " | timechart span=6h sum(errors) | timewrap 1day series=short"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(errors)_s1", "bigint"), + schema("sum(errors)_s0", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 4, 1), + rows("2024-07-03 06:00:00", 6, 3), + rows("2024-07-03 12:00:00", 9, 6), + rows("2024-07-03 18:00:00", 3, 1)); + } + + // BY clause tests are pending -- blocked by timechart BY output format gap. + // See docs/dev/ppl-timewrap-command.md for design options. +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java index 1dbd34357ab..c0151a3b232 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOCATIONS_TYPE_CONFLICT; +import static org.opensearch.sql.util.Capability.SAME_INDEX_UNION_CONFLATION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -18,6 +19,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteUnionCommandIT extends PPLIntegTestCase { @@ -48,6 +50,9 @@ public void testBasicUnionTwoSubsearches() throws IOException { } @Test + @RequiresCapability( + value = SAME_INDEX_UNION_CONFLATION, + note = "same-index union conflates on the AE route (delegated predicate leak).") public void testUnionThreeSubsearches() throws IOException { JSONObject result = executeQuery( @@ -155,6 +160,9 @@ public void testUnionAllDatasetsDifferentSchemas() throws IOException { } @Test + @RequiresCapability( + value = SAME_INDEX_UNION_CONFLATION, + note = "same-index union conflates on the AE route (delegated predicate leak).") public void testUnionMidPipeline_SingleExplicitDataset() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index 593e56b467c..225d5f259fb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -31,7 +31,12 @@ public void init() throws Exception { super.init(); enableCalcite(); loadIndex(Index.NESTED_SIMPLE); - loadIndex(Index.DEEP_NESTED); + // deep_nested has a multi-value array for the scalar-mapped `accounts.id` field, which the + // parquet store rejects at bulk load; skip it on the AE route so it doesn't abort init(). The + // only test that queries deep_nested is already excluded on the AE route (nested fields). + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.DEEP_NESTED); + } loadIndex(Index.CASCADED_NESTED); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java new file mode 100644 index 00000000000..2fea0858abd --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java @@ -0,0 +1,135 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.sql.legacy.TestUtils; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** Foreach collection modes over index fields (Splunk-parity scenarios). */ +public class ForeachFieldJsonIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + + if (!TestUtils.isIndexExist(client(), "test_foreach_field2")) { + String mapping = + "{\"mappings\":{\"properties\":{\"jsonfield\":{\"type\":\"keyword\"}," + + "\"jsonstrs\":{\"type\":\"keyword\"},\"nativenums\":{\"type\":\"long\"}," + + "\"nested_objs\":{\"type\":\"nested\",\"properties\":{\"a\":{\"type\":\"long\"}}}}}}"; + TestUtils.createIndexByRestClient(client(), "test_foreach_field2", mapping); + + Request r = new Request("PUT", "/test_foreach_field2/_doc/1?refresh=true"); + r.setJsonEntity( + "{\"jsonfield\": \"[10,20,30]\", \"jsonstrs\": \"[\\\"a\\\",\\\"b\\\"]\"," + + " \"nativenums\": [1, 2, 3], \"nested_objs\": [{\"a\": 1}, {\"a\": 2}]}"); + client().performRequest(r); + } + } + + @Test + public void testJsonArrayModeOnFieldWithNumericContent() throws IOException { + // Splunk: field holding "[10,20,30]" with foreach mode=json_array sums to 60. + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval total = 0 | foreach mode=json_array jsonfield [" + + " eval total = total + <> ] | fields total"); + verifySchema(result, schema("total", "double")); + verifyDataRows(result, rows(60.0)); + } + + @Test + public void testJsonArrayFieldInfersNumericItemForAbs() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval result = 0 | foreach mode=json_array jsonfield [" + + " eval result = abs(<>) ] | fields result"); + verifySchema(result, schema("result", "double")); + verifyDataRows(result, rows(30.0)); + } + + @Test + public void testJsonArrayFieldInfersNumericItemForComparison() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval count = 0 | foreach mode=json_array jsonfield [" + + " eval count = count + if(<> > 9, 1, 0) ] | fields count"); + verifySchema(result, schema("count", "int")); + verifyDataRows(result, rows(3)); + } + + @Test + public void testJsonArrayModeOnFieldWithStringContent() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval r = '' | foreach mode=json_array jsonstrs [" + + " eval r = concat(r, <>) ] | fields r"); + verifySchema(result, schema("r", "string")); + verifyDataRows(result, rows("ab")); + } + + /** + * Native OpenSearch array fields (a long field holding [1,2,3]) are typed as scalar BIGINT at + * plan time because OpenSearch mappings do not distinguish scalars from arrays. foreach + * multivalue mode therefore rejects them; documents the current known limitation rather than the + * desired behavior. + */ + @Test + public void testNativeArrayFieldMappingMismatchIsNoOp() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval total = 0 | foreach mode=multivalue nativenums" + + " [ eval total = total + <> ] | fields total"); + verifySchema(result, schema("total", "int")); + verifyDataRows(result, rows(0)); + } + + /** + * Nested-typed fields map to ARRAY<ANY> at plan time, so multivalue mode iterates them. The + * lambda here only counts elements; it does not dereference the object item. + */ + @Test + public void testNestedFieldMultivalueIterates() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval total = 0 | foreach mode=multivalue nested_objs [" + + " eval total = total + 1 ] | fields total"); + verifySchema(result, schema("total", "int")); + verifyDataRows(result, rows(2)); + } + + /** Splunk silently no-ops when a mode is fed the wrong collection shape. */ + @Test + public void testJsonArrayModeOnRealArrayIsNoOp() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval nums = array(1, 2, 3), total = 0 | foreach" + + " mode=json_array nums [ eval total = total + <> ] | fields total | head" + + " 1"); + verifySchema(result, schema("total", "int")); + verifyDataRows(result, rows(0)); + } + + @Test + public void testMultivalueModeOnJsonTextFieldIsNoOp() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval total = 0 | foreach mode=multivalue jsonfield" + + " [ eval total = total + <> ] | fields total"); + verifySchema(result, schema("total", "int")); + verifyDataRows(result, rows(0)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java new file mode 100644 index 00000000000..ac92f979595 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Backend half of the PPL lint rule validation contract. + * + *

    This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from + * the current checkout and asserts, per contract case, that: + * + *

      + *
    • a rejected query returns the contracted HTTP status and structured error body ({@code + * status}, {@code error.type}, {@code error.reason}); and + *
    • a valid control query returns HTTP 200 with data. + *
    + * + *

    The contract file is shared verbatim with the SQL-owned OSD frontend adapter ({@code + * scripts/ppl-lint/run-frontend-contract.mjs}) so the same reviewed cases pin both the OSD analyzer + * diagnostic and the SQL backend behavior. The rejection-body parsing mirrors the existing {@link + * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT} pattern; the live fixture and + * Calcite setup follow {@link org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. + */ +public class PplLintRuleValidationIT extends PPLIntegTestCase { + + private static final String CONTRACT_RESOURCE = + "src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json"; + + @Override + public void init() throws Exception { + super.init(); + // eventstats via the Calcite path. Disallow fallback so an unsupported window function is + // rejected rather than silently degrading to the V2 engine. + enableCalcite(); + disallowCalciteFallback(); + loadIndex(Index.ACCOUNT); + } + + @Test + public void testValidatesUnsupportedWindowFunctionContract() throws IOException { + JSONObject contract = loadContract(); + String index = contract.getString("index"); + JSONArray cases = contract.getJSONArray("cases"); + + for (int i = 0; i < cases.length(); i++) { + JSONObject testCase = cases.getJSONObject(i); + String caseId = testCase.getString("id"); + String query = testCase.getString("query").replace("{{index}}", index); + JSONObject backendExpected = testCase.getJSONObject("backendExpected"); + int expectedStatus = backendExpected.getInt("httpStatus"); + + if (expectedStatus == 200) { + verifyAcceptedCase(caseId, query); + } else { + verifyRejectedCase(caseId, query, expectedStatus, backendExpected.getJSONObject("body")); + } + } + } + + /** A valid control query must return HTTP 200. executeQuery already asserts the 200 status. */ + private void verifyAcceptedCase(String caseId, String query) throws IOException { + JSONObject response = executeQuery(query); + assertTrue( + "case \"" + + caseId + + "\": expected a datarows array in the 200 response for query: " + + query, + response.has("datarows")); + } + + /** + * A rejected query must throw a {@link ResponseException} whose response carries the contracted + * HTTP status and structured error fields. executeQuery internally asserts 200, so a non-200 + * response surfaces as a ResponseException before it can return. + */ + private void verifyRejectedCase( + String caseId, String query, int expectedStatus, JSONObject expectedBody) { + ResponseException exception = assertThrows(ResponseException.class, () -> executeQuery(query)); + + int actualStatus = exception.getResponse().getStatusLine().getStatusCode(); + assertEquals( + "case \"" + caseId + "\": unexpected HTTP status for query: " + query, + expectedStatus, + actualStatus); + + JSONObject body; + try { + body = new JSONObject(getResponseBody(exception.getResponse(), true)); + } catch (IOException e) { + throw new RuntimeException( + "case \"" + caseId + "\": failed to read rejection response body for query: " + query, e); + } + + assertEquals( + "case \"" + caseId + "\": unexpected top-level status field for query: " + query, + expectedBody.getInt("status"), + body.getInt("status")); + + JSONObject expectedError = expectedBody.getJSONObject("error"); + JSONObject actualError = body.getJSONObject("error"); + + assertEquals( + "case \"" + caseId + "\": unexpected error.type for query: " + query, + expectedError.getString("type"), + actualError.getString("type")); + assertEquals( + "case \"" + caseId + "\": unexpected error.reason for query: " + query, + expectedError.getString("reason"), + actualError.getString("reason")); + } + + private JSONObject loadContract() throws IOException { + String path = TestUtils.getResourceFilePath(CONTRACT_RESOURCE); + return new JSONObject(new String(Files.readAllBytes(Paths.get(path)))); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java index d83140e0dd5..0642b8f5651 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java @@ -314,7 +314,7 @@ public void testQ15() throws IOException { schema("total_revenue", "double")); verifyDataRows( actual, - rows(10, "Supplier#000000010", "Saygah3gYWMp72i PY", "34-852-489-8585", 797313.3838)); + closeTo(10, "Supplier#000000010", "Saygah3gYWMp72i PY", "34-852-489-8585", 797313.3838)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java index b75da57c571..bd5b928aebe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java @@ -20,6 +20,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static org.opensearch.sql.util.Capability.RESPONSE_FORMAT; import java.io.IOException; import java.util.ArrayList; @@ -39,8 +40,10 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.executor.csv.CSVResult; +import org.opensearch.sql.util.RequiresCapability; /** Tests to cover requests with "?format=csv" parameter */ +@RequiresCapability(RESPONSE_FORMAT) public class CsvFormatResponseIT extends SQLIntegTestCase { private boolean flatOption = false; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index 5dea06b7634..0b0a510cb3d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import static org.opensearch.sql.util.TestUtils.verifyIsV2Cursor; import java.io.IOException; @@ -27,7 +28,9 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(PAGINATION_CURSOR) public class CursorIT extends SQLIntegTestCase { private static final String CURSOR = "cursor"; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java index 4ad88c632ba..a0d74cd5977 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java @@ -7,6 +7,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import java.io.IOException; import org.json.JSONArray; @@ -14,6 +16,7 @@ import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class JdbcTestIT extends SQLIntegTestCase { @@ -25,6 +28,7 @@ protected void init() throws Exception { loadIndex(Index.WEBLOG); } + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentilesQuery() { JSONObject response = executeJdbcRequest( @@ -43,6 +47,7 @@ public void testPercentilesQuery() { // https://github.com/opensearch-project/sql/issues/537 @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testSlowQuery() throws IOException { // set slow log threshold = 0s updateClusterSettings(new ClusterSetting(PERSISTENT, "plugins.sql.slowlog", "0")); @@ -89,6 +94,7 @@ public void testDivisionInQuery() { assertThat(response.getJSONArray("datarows").getJSONArray(0).getDouble(0), equalTo(16827.0)); } + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testGroupByInQuery() { JSONObject response = executeJdbcRequest( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java index 3933338f0a6..63d37b3ed6a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java @@ -6,6 +6,8 @@ package org.opensearch.sql.legacy; import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.sql.util.Capability.EXPLAIN_FORMAT; +import static org.opensearch.sql.util.Capability.QUERY_ERROR_MESSAGE; import java.io.IOException; import org.junit.Ignore; @@ -13,6 +15,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.opensearch.client.ResponseException; +import org.opensearch.sql.util.RequiresCapability; /** Test cases for writing missing join table aliases. */ public class JoinAliasWriterRuleIT extends SQLIntegTestCase { @@ -26,6 +29,7 @@ protected void init() throws Exception { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void noTableAliasNoCommonColumns() throws IOException { sameExplain( query( @@ -45,6 +49,7 @@ public void noTableAliasNoCommonColumns() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void oneTableAliasNoCommonColumns() throws IOException { sameExplain( query( @@ -61,6 +66,7 @@ public void oneTableAliasNoCommonColumns() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void bothTableAliasNoCommonColumns() throws IOException { sameExplain( query( @@ -78,6 +84,7 @@ public void bothTableAliasNoCommonColumns() throws IOException { @Test @Ignore + @RequiresCapability(EXPLAIN_FORMAT) public void tableNamesWithTypeName() throws IOException { sameExplain( query( @@ -98,6 +105,7 @@ public void tableNamesWithTypeName() throws IOException { @Ignore @Test + @RequiresCapability(EXPLAIN_FORMAT) public void tableNamesWithTypeNameExplicitTableAlias() throws IOException { sameExplain( query( @@ -114,6 +122,7 @@ public void tableNamesWithTypeNameExplicitTableAlias() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void actualTableNameAsAliasOnColumnFields() throws IOException { sameExplain( query( @@ -130,6 +139,7 @@ public void actualTableNameAsAliasOnColumnFields() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void actualTableNameAsAliasOnColumnFieldsTwo() throws IOException { sameExplain( query( @@ -150,6 +160,7 @@ public void actualTableNameAsAliasOnColumnFieldsTwo() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void columnsWithTableAliasNotAffected() throws IOException { sameExplain( query( @@ -166,6 +177,7 @@ public void columnsWithTableAliasNotAffected() throws IOException { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void commonColumnWithoutTableAliasDifferentTables() throws IOException { exception.expect(ResponseException.class); exception.expectMessage("Field name [firstname] is ambiguous"); @@ -179,6 +191,7 @@ public void commonColumnWithoutTableAliasDifferentTables() throws IOException { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void sameTablesNoAliasAndNoAliasOnColumns() throws IOException { exception.expect(ResponseException.class); exception.expectMessage("Not unique table/alias: [opensearch-sql_test_index_bank]"); @@ -192,6 +205,7 @@ public void sameTablesNoAliasAndNoAliasOnColumns() throws IOException { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void sameTablesNoAliasWithTableNameAsAliasOnColumns() throws IOException { exception.expect(ResponseException.class); exception.expectMessage("Not unique table/alias: [opensearch-sql_test_index_bank]"); @@ -206,6 +220,7 @@ public void sameTablesNoAliasWithTableNameAsAliasOnColumns() throws IOException } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void sameTablesWithExplicitAliasOnFirst() throws IOException { sameExplain( query( @@ -221,6 +236,7 @@ public void sameTablesWithExplicitAliasOnFirst() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void sameTablesWithExplicitAliasOnSecond() throws IOException { sameExplain( query( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java index 84b60fdabd7..bf49f239465 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy; +import static org.opensearch.sql.util.Capability.QUERY_ERROR_MESSAGE; + import java.io.IOException; import java.util.Locale; import org.apache.hc.core5.http.ParseException; @@ -12,8 +14,10 @@ import org.json.JSONObject; import org.junit.Assert; import org.opensearch.client.ResponseException; +import org.opensearch.sql.util.RequiresCapability; /** Tests for clean handling of various types of invalid queries */ +@RequiresCapability(QUERY_ERROR_MESSAGE) public class MalformedQueryIT extends SQLIntegTestCase { @Override protected void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java index 7589304af0a..4f39b8d4efa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java @@ -7,12 +7,15 @@ import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.sql.util.Capability.EXPLAIN_FORMAT; +import static org.opensearch.sql.util.Capability.LEGACY_METHOD_QUERY; import java.io.IOException; import java.util.Locale; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; /** * 定製方法查詢． @@ -41,6 +44,7 @@ protected void init() throws Exception { * @throws IOException */ @Test + @RequiresCapability(EXPLAIN_FORMAT) public void queryTest() throws IOException { final String result = explainQuery( @@ -70,6 +74,7 @@ public void queryTest() throws IOException { * @throws IOException */ @Test + @RequiresCapability(EXPLAIN_FORMAT) public void matchQueryTest() throws IOException { final String result = explainQuery( @@ -153,6 +158,7 @@ public void scoreQueryTest() throws IOException { } @Test + @RequiresCapability(LEGACY_METHOD_QUERY) public void regexpQueryTest() throws IOException { final String result = explainQuery( @@ -168,6 +174,7 @@ public void regexpQueryTest() throws IOException { } @Test + @RequiresCapability(LEGACY_METHOD_QUERY) public void negativeRegexpQueryTest() throws IOException { final String result = explainQuery( @@ -198,6 +205,7 @@ public void negativeRegexpQueryTest() throws IOException { * @throws IOException */ @Test + @RequiresCapability(LEGACY_METHOD_QUERY) public void wildcardQueryTest() throws IOException { final String result = explainQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java index aadd79469db..4f64cdb704d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED; +import static org.opensearch.sql.util.Capability.STRUCT_PARENT_FIELD; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -16,11 +17,13 @@ import org.junit.Test; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; /** * Integration test for OpenSearch object field (and nested field). This class is focused on simple * SELECT-FROM query to ensure right column number and value is returned. */ +@RequiresCapability(STRUCT_PARENT_FIELD) public class ObjectFieldSelectIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OrdinalAliasRewriterIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OrdinalAliasRewriterIT.java index caea2aa7c66..92bbe365e84 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OrdinalAliasRewriterIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OrdinalAliasRewriterIT.java @@ -25,13 +25,13 @@ public void simpleGroupByOrdinal() { String expected = executeQuery( StringUtils.format( - "SELECT lastname FROM %s AS b GROUP BY lastname LIMIT 3", + "SELECT lastname FROM %s AS b GROUP BY lastname ORDER BY lastname LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT lastname FROM %s AS b GROUP BY 1 LIMIT 3", + "SELECT lastname FROM %s AS b GROUP BY 1 ORDER BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); @@ -43,13 +43,14 @@ public void multipleGroupByOrdinal() { executeQuery( StringUtils.format( "SELECT lastname, firstname, age FROM %s AS b GROUP BY firstname, age, lastname" - + " LIMIT 3", + + " ORDER BY lastname, firstname, age LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT lastname, firstname, age FROM %s AS b GROUP BY 2, 3, 1 LIMIT 3", + "SELECT lastname, firstname, age FROM %s AS b GROUP BY 2, 3, 1" + + " ORDER BY 1, 2, 3 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); @@ -60,13 +61,13 @@ public void selectFieldiWithBacticksGroupByOrdinal() { String expected = executeQuery( StringUtils.format( - "SELECT `lastname` FROM %s AS b GROUP BY `lastname` LIMIT 3", + "SELECT `lastname` FROM %s AS b GROUP BY `lastname` ORDER BY `lastname` LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT `lastname` FROM %s AS b GROUP BY 1 LIMIT 3", + "SELECT `lastname` FROM %s AS b GROUP BY 1 ORDER BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); @@ -78,13 +79,14 @@ public void selectFieldiWithBacticksAndTableAliasGroupByOrdinal() { executeQuery( StringUtils.format( "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY `age`," - + " `b`.`lastname` , firstname LIMIT 10", + + " `b`.`lastname` , firstname ORDER BY `b`.`lastname`, `age` LIMIT 10", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY 2, 1, 3 LIMIT 10", + "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY 2, 1, 3 ORDER BY 1," + + " 2 LIMIT 10", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); @@ -166,14 +168,14 @@ public void selectFieldiWithBacticksAndTableAliasOrderByOrdinalAndNull() { executeQuery( StringUtils.format( "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY `b`.`lastname` IS NOT NULL DESC," - + " age is NULL LIMIT 3", + + " age is NULL, `b`.`lastname` LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY 1 IS NOT NULL DESC, 2 IS NULL" - + " LIMIT 3", + "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY 1 IS NOT NULL DESC, 2 IS NULL," + + " 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PointInTimeLeakIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PointInTimeLeakIT.java index ab0f196ce33..9c4513a973a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PointInTimeLeakIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PointInTimeLeakIT.java @@ -7,6 +7,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import java.io.IOException; import org.json.JSONArray; @@ -17,12 +18,14 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; /** * Integration test verifying PIT contexts are created only when needed and properly cleaned up. * * @see Issue #5002 */ +@RequiresCapability(PAGINATION_CURSOR) public class PointInTimeLeakIT extends SQLIntegTestCase { private static final String TEST_INDEX = "test-logs-2025.01.01"; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java index 6745d90d50b..7e4950a5fab 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java @@ -10,6 +10,9 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.opensearch.sql.util.Capability.DYNAMIC_STRING_NO_KEYWORD; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; +import static org.opensearch.sql.util.Capability.RESPONSE_FORMAT; import com.google.common.collect.Sets; import java.io.IOException; @@ -27,6 +30,7 @@ import org.junit.Ignore; import org.junit.Test; import org.opensearch.client.Request; +import org.opensearch.sql.util.RequiresCapability; /** * @@ -80,7 +84,11 @@ protected void init() throws Exception { loadIndex(Index.ACCOUNT); loadIndex(Index.PHRASE); loadIndex(Index.GAME_OF_THRONES); - loadIndex(Index.NESTED); + // Skip on the analytics-engine route, where the parquet store rejects nested_objects' + // multi-value array in the scalar-mapped myNum field at bulk load. + if (!TestUtils.AnalyticsIndexConfig.isEnabled()) { + loadIndex(Index.NESTED); + } } @Override @@ -93,6 +101,7 @@ protected Request getSqlRequest(String request, boolean explain) { // This is testing a deprecated feature @Test + @RequiresCapability(RESPONSE_FORMAT) public void wrongIndexType() throws IOException { String type = "wrongType"; try { @@ -163,18 +172,21 @@ public void selectScore() throws IOException { } @Test + @RequiresCapability(NESTED_FIELDS) public void selectAllFromNestedWithoutFieldInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT * FROM %s", regularFields, fields("message", "comment")); } @Test + @RequiresCapability(NESTED_FIELDS) public void selectAllFromNestedWithFieldInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT * FROM %s e, e.message m", regularFields, messageFields); } @Test + @RequiresCapability(NESTED_FIELDS) public void selectAllFromNestedWithMultipleFieldsInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT * FROM %s e, e.message m, e.comment c", @@ -182,12 +194,14 @@ public void selectAllFromNestedWithMultipleFieldsInFrom() throws IOException { } @Test + @RequiresCapability(NESTED_FIELDS) public void selectAllNestedFromNestedWithFieldInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT m.* FROM %s e, e.message m", messageFields); } @Test + @RequiresCapability(NESTED_FIELDS) public void selectSpecificRegularFieldAndAllFromNestedWithFieldInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT e.someField, m.* FROM %s e, e.message m", fields("someField"), messageFields); @@ -214,6 +228,7 @@ private Set fields(String... fieldNames) { } @Test + @RequiresCapability(NESTED_FIELDS) public void selectNestedFields() throws IOException { JSONObject response = executeQuery( @@ -233,6 +248,7 @@ public void selectNestedFields() throws IOException { } @Test + @RequiresCapability(NESTED_FIELDS) public void selectNestedFieldWithWildcard() throws IOException { JSONObject response = executeQuery( @@ -452,6 +468,7 @@ public void aggregationFunctionInHaving() throws IOException { // TEST_INDEX_ACCOUNT); // } @Test + @RequiresCapability(RESPONSE_FORMAT) public void fieldsWithAlias() throws IOException { JSONObject response = executeQuery( @@ -502,6 +519,7 @@ public void joinQuery() throws IOException { } @Test + @RequiresCapability(RESPONSE_FORMAT) public void joinQueryWithAlias() throws IOException { JSONObject response = executeQuery( @@ -561,6 +579,7 @@ public void joinQuerySelectOnlyOnOneTable() throws Exception { } @Test + @RequiresCapability(DYNAMIC_STRING_NO_KEYWORD) public void fieldOrder() throws IOException { final String[] expectedFields = {"age", "firstname", "address", "gender", "email"}; @@ -570,6 +589,7 @@ public void fieldOrder() throws IOException { } @Test + @RequiresCapability(DYNAMIC_STRING_NO_KEYWORD) public void fieldOrderOther() throws IOException { final String[] expectedFields = {"email", "firstname", "age", "gender", "address"}; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index ef05923b060..fc15c908c63 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -139,6 +139,11 @@ public static void dumpCoverage() { */ @AfterClass public static void cleanUpIndices() throws IOException { + // No client when every test in the class was skipped (e.g. @RequiresCapability on the AE + // route). + if (client() == null) { + return; + } if (System.getProperty("tests.rest.bwcsuite") == null) { wipeAllOpenSearchIndices(); wipeAllClusterSettings(); @@ -984,7 +989,12 @@ public enum Index { "events_traffic", "events_traffic", getMappingFile("events_traffic_index_mapping.json"), - "src/test/resources/events_traffic.json"); + "src/test/resources/events_traffic.json"), + TIMEWRAP_TEST( + "timewrap_test", + "timewrap_test", + "{\"mappings\":{\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"host\":{\"type\":\"keyword\"},\"requests\":{\"type\":\"integer\"},\"errors\":{\"type\":\"integer\"}}}}", + "src/test/resources/timewrap_test.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SqlLegacyEngineSanityIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SqlLegacyEngineSanityIT.java index b2d22808a49..1c92cee74a9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SqlLegacyEngineSanityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SqlLegacyEngineSanityIT.java @@ -7,12 +7,14 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE; +import static org.opensearch.sql.util.Capability.TEXT_FIELD_EXACT_MATCH; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; /** * Sanity tests for the legacy SQL engine. Many legacy integration tests (JoinIT, SubqueryIT, @@ -38,6 +40,7 @@ public void testInnerJoinFallback() throws IOException { } @Test + @RequiresCapability(TEXT_FIELD_EXACT_MATCH) public void testLeftJoinFallback() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java index 421aae9622b..d6f238fbff1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java @@ -96,11 +96,11 @@ public void testLengthWithTextFieldReturnsInt() { public void testLengthWithGroupByExpr() { JSONObject response = executeJdbcRequest( - "SELECT Length(firstname) FROM " + "SELECT LENGTH(firstname) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY LENGTH(firstname) LIMIT 5"); - verifySchema(response, schema("Length(firstname)", null, "integer")); + verifySchema(response, schema("LENGTH(firstname)", null, "integer")); } /* diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index f6ec903c395..b5c08cc8ad0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,9 +5,13 @@ package org.opensearch.sql.plugin; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import org.opensearch.client.Request; @@ -56,4 +60,31 @@ public void testClusterStarted() { // If the cluster booted with analytics-engine present, all plugins loaded without classloader // errors. The assumption above guarantees we only assert this where it is meaningful. } + + /** + * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it + * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its + * fixed schema and correct data unchanged when the analytics-engine plugin is present. + */ + @Test + public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { + Request request = new Request("POST", "/_plugins/_ppl"); + request.setJsonEntity( + "{\"query\": \"| rest '/_cluster/health' | fields status, number_of_nodes\"}"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + + JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); + + JSONArray schema = result.getJSONArray("schema"); + assertEquals(2, schema.length()); + assertEquals("status", schema.getJSONObject(0).getString("name")); + assertEquals("string", schema.getJSONObject(0).getString("type")); + assertEquals("number_of_nodes", schema.getJSONObject(1).getString("name")); + assertEquals("int", schema.getJSONObject(1).getString("type")); + + JSONArray datarows = result.getJSONArray("datarows"); + assertEquals(1, datarows.length()); + assertTrue(datarows.getJSONArray(0).getInt(1) >= 1); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/CastFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/CastFunctionIT.java index 56e52610aba..2d87fe536fc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/CastFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/CastFunctionIT.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.Capability.IP_UDT_BINARY_REPRESENTATION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -24,6 +25,7 @@ import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.util.RequiresCapability; public class CastFunctionIT extends PPLIntegTestCase { @Override @@ -403,6 +405,11 @@ public void testCastTimestamp() throws IOException { } @Test + @RequiresCapability( + value = IP_UDT_BINARY_REPRESENTATION, + note = + "cast(... as IP) sees the IP UDT as BINARY on the AE route" + + " (IP_UDT_BINARY_REPRESENTATION).") public void testCastToIP() throws IOException { // Test casting IP to IP type JSONObject actual = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 25e7c12ffff..1af872a8ab6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -11,6 +11,11 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ALIAS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.Capability.CROSS_INDEX_INCOMPATIBLE_TYPES; +import static org.opensearch.sql.util.Capability.DOC_MUTATION; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; +import static org.opensearch.sql.util.Capability.SCALED_FLOAT_TYPE; +import static org.opensearch.sql.util.Capability.STRING_TO_NUMERIC_COERCION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -23,6 +28,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.client.Request; +import org.opensearch.sql.util.RequiresCapability; public class DataTypeIT extends PPLIntegTestCase { @@ -35,6 +41,9 @@ public void init() throws Exception { } @Test + @RequiresCapability( + value = SCALED_FLOAT_TYPE, + note = "scaled_float reports bigint (not double) on the AE route.") public void test_numeric_data_types() throws IOException { JSONObject result = executeQuery(String.format("source=%s", TEST_INDEX_DATATYPE_NUMERIC)); verifySchema( @@ -50,6 +59,9 @@ public void test_numeric_data_types() throws IOException { } @Test + @RequiresCapability( + value = NESTED_FIELDS, + note = "nested_value/object_value/geo_point_value: stripped/unsupported on the AE route.") public void test_nonnumeric_data_types() throws IOException { JSONObject result = executeQuery(String.format("source=%s", TEST_INDEX_DATATYPE_NONNUMERIC)); verifySchemaInOrder( @@ -102,6 +114,9 @@ public void test_long_integer_data_type() throws IOException { } @Test + @RequiresCapability( + value = NESTED_FIELDS, + note = "alias_col is type=alias, stripped from the mapping on the AE route.") public void test_alias_data_type() throws IOException { JSONObject result = executeQuery( @@ -113,6 +128,9 @@ public void test_alias_data_type() throws IOException { } @Test + @RequiresCapability( + value = STRING_TO_NUMERIC_COERCION, + note = "empty-string -> numeric coerces to 0 on v2/Calcite but null on the AE route.") public void testNumericFieldFromString() throws Exception { final int docId = 2; Request insertRequest = @@ -146,6 +164,9 @@ public void testNumericFieldFromString() throws Exception { } @Test + @RequiresCapability( + value = CROSS_INDEX_INCOMPATIBLE_TYPES, + note = "AE route rejects incompatible cross-index field types instead of coercing.") public void testBooleanFieldFromNumberAcrossWildcardIndices() throws Exception { // Reproduce issue #5269: querying across indices where same field has conflicting types // (boolean vs text) and the text-typed index stores a numeric value like 0. @@ -187,6 +208,9 @@ public void testBooleanFieldFromNumberAcrossWildcardIndices() throws Exception { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Seeds + deletes a doc; the AE store doesn't support DELETE.") public void testBooleanFieldFromString() throws Exception { final int docId = 2; Request insertRequest = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 1a2911794f9..97b1d9601a2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -7,7 +7,9 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; import static org.opensearch.sql.util.Capability.DOC_MUTATION; +import static org.opensearch.sql.util.Capability.UNIX_TIMESTAMP_SUBSECOND; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -1219,6 +1221,9 @@ void verifyDateFormat(String date, String type, String format, String formatted) } @Test + @RequiresCapability( + value = DATETIME_FORMAT_RENDERING, + note = "date_format renders some tokens differently on the AE route.") public void testDateFormat() throws IOException { String timestamp = "1998-01-31 13:14:15.012345"; String timestampFormat = @@ -1369,6 +1374,11 @@ public void testUnixTimeStamp() throws IOException { } @Test + @RequiresCapability( + value = UNIX_TIMESTAMP_SUBSECOND, + note = + "unix_timestamp drops the sub-second fraction on the AE route" + + " (UNIX_TIMESTAMP_SUBSECOND).") public void testUnixTimestampWithTimestampString() throws IOException { var result = executeQuery( @@ -1557,7 +1567,14 @@ public void testTimestampDiff() throws IOException { "source=%s | eval f = timestampdiff(YEAR, '1997-01-01 00:00:00', '2001-03-06" + " 00:00:00') | fields f", TEST_INDEX_DATE)); - verifySchema(result, schema("f", null, isCalciteEnabled() ? "bigint" : "timestamp")); + // The AE route runs the Calcite path, returning bigint even though the cluster's calcite + // setting reads false. + verifySchema( + result, + schema( + "f", + null, + isCalciteEnabled() || isAnalyticsParquetIndicesEnabled() ? "bigint" : "timestamp")); verifySome(result.getJSONArray("datarows"), rows(4)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java index 4d755c1ab77..2ab3ce6dad6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java @@ -6,6 +6,10 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.CROSS_INDEX_OBJECT_LEAF_MERGE; +import static org.opensearch.sql.util.Capability.ID_METADATA; +import static org.opensearch.sql.util.Capability.INDEX_METADATA; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; import static org.opensearch.sql.util.MatcherUtils.columnName; import static org.opensearch.sql.util.MatcherUtils.columnPattern; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -22,6 +26,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.util.RequiresCapability; public class FieldsCommandIT extends PPLIntegTestCase { @@ -89,6 +94,9 @@ public void testSpecialDataTypes(String querySource) throws IOException { } @Test + @RequiresCapability( + value = {ID_METADATA, INDEX_METADATA}, + note = "queries _id and _index, which parquet-backed scans don't expose on the AE route.") public void testMetadataFields() throws IOException { // Test basic metadata fields JSONObject basicResult = @@ -128,6 +136,11 @@ public void testMetadataFieldsWithEvalError() { } @Test + @RequiresCapability( + value = {CROSS_INDEX_OBJECT_LEAF_MERGE, NESTED_FIELDS}, + note = + "an object leaf present in only one merge_test_* index is FIELD_NOT_FOUND on the AE" + + " route; also reads a nested array field.") public void testMergedObjectFields() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java index bc98c312fb6..a4410095e4a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; +import static org.opensearch.sql.util.Capability.TEXT_KEYWORD_PUSHDOWN_REWRITE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; @@ -13,6 +14,7 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class LikeQueryIT extends PPLIntegTestCase { @@ -97,6 +99,11 @@ public void test_like_on_text_field_with_greater_than_one_word() throws IOExcept } @Test + @RequiresCapability( + value = TEXT_KEYWORD_PUSHDOWN_REWRITE, + note = + "like() over a text+keyword field doesn't rewrite to .keyword in the explain plan on the" + + " AE route (TEXT_KEYWORD_PUSHDOWN_REWRITE).") public void test_convert_field_text_to_keyword() throws IOException { String query = "source=" diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java index 6cd5063b5a0..0a81ed92ad6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java @@ -7,6 +7,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; +import static org.opensearch.sql.util.Capability.RAND_SEED_UNSUPPORTED; import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -17,6 +19,7 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.sql.util.RequiresCapability; public class MathematicalFunctionIT extends PPLIntegTestCase { @@ -69,6 +72,7 @@ public void testDivideFunction() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testCeil() throws IOException { JSONObject result = executeQuery(String.format("source=%s | eval f = ceil(age) | fields f", TEST_INDEX_BANK)); @@ -429,6 +433,9 @@ public void testPi() throws IOException { } @Test + @RequiresCapability( + value = RAND_SEED_UNSUPPORTED, + note = "rand(seed) is rejected on the AE route (RAND_SEED_UNSUPPORTED).") public void testRand() throws IOException { JSONObject result = executeQuery(String.format("source=%s | eval f = rand() | fields f", TEST_INDEX_BANK)); @@ -564,7 +571,12 @@ public void testEvalSumMultipleIntegers() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(1, 2, 3) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + // Calcite widens integer arithmetic operands to avoid overflow, so the result is bigint. + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(6), rows(6), rows(6), rows(6), rows(6)); } @@ -583,7 +595,11 @@ public void testEvalSumWithFields() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(age, 10) | fields f | head 7", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(42), rows(46), rows(38), rows(43), rows(46), rows(49), rows(44)); } @@ -593,7 +609,11 @@ public void testEvalSumMultipleNumericArguments() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(1, 2, 3, 4, 5) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(15), rows(15), rows(15), rows(15), rows(15)); } @@ -674,7 +694,11 @@ public void testEvalSumAndAvgComparison() throws IOException { "source=%s | eval sum_val = sum(10, 20, 30), avg_val = avg(10, 20, 30) | fields" + " sum_val, avg_val | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("sum_val", null, "int"), schema("avg_val", null, "double")); + if (isCalciteEnabled()) { + verifySchema(result, schema("sum_val", null, "bigint"), schema("avg_val", null, "double")); + } else { + verifySchema(result, schema("sum_val", null, "int"), schema("avg_val", null, "double")); + } verifyDataRows( result, rows(60, 20.0), rows(60, 20.0), rows(60, 20.0), rows(60, 20.0), rows(60, 20.0)); } @@ -686,7 +710,11 @@ public void testEvalSumInWhereClause() throws IOException { String.format( "source=%s | where sum(age, 10) > 40 | eval f = sum(age, 10) | fields f | head 6", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } // Should return rows where age + 10 > 40, so age > 30 verifyDataRows(result, rows(42), rows(46), rows(43), rows(46), rows(49), rows(44)); } @@ -733,7 +761,11 @@ public void testEvalSumWithMultipleFields() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(age, age, 10) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } // sum(age, age, 10) = age + age + 10 = 2*age + 10 verifyDataRows(result, rows(74), rows(82), rows(66), rows(76), rows(82)); } @@ -755,7 +787,11 @@ public void testEvalSumWithNegativeNumbers() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(-5, 10, -3) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(2), rows(2), rows(2), rows(2), rows(2)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java index 36e9e2c4c56..45b8a6c0e68 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java @@ -6,10 +6,12 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class MultiMatchIT extends PPLIntegTestCase { @@ -45,6 +47,9 @@ public void test_multi_match_all_params() throws IOException { } @Test + @RequiresCapability( + value = FULLTEXT_RELEVANCE_FUNC, + note = "multi_match/query_string/simple_query_string are full-text relevance funcs.") public void test_wildcard_multi_match() throws IOException { String query1 = "SOURCE=" + TEST_INDEX_BEER + " | WHERE multi_match(['Tags'], 'taste') | fields Id"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 837865a3585..2ccda31eea7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -32,6 +32,19 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + @Test + public void testRest() throws IOException { + JSONObject result; + try { + result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + if (isCalciteEnabled()) { + assertFalse(result.getJSONArray("datarows").isEmpty()); + } + } + @Test public void testJoin() throws IOException { JSONObject result; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 3db2142cffe..f22cea74734 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -108,7 +108,7 @@ protected String explainQueryToString(String query, ExplainMode mode) throws IOE return explainQuery(query, Format.JSON, mode).replace("\\r\\n", "\\n"); } - private String explainQuery(String query, Format format, ExplainMode mode) throws IOException { + protected String explainQuery(String query, Format format, ExplainMode mode) throws IOException { Response response = client() .performRequest(buildRequest(query, String.format(EXPLAIN_API_ENDPOINT, format, mode))); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java index d800de51c80..44778c40839 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java @@ -6,10 +6,12 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class QueryStringIT extends PPLIntegTestCase { @@ -56,6 +58,9 @@ public void all_params_test() throws IOException { } @Test + @RequiresCapability( + value = FULLTEXT_RELEVANCE_FUNC, + note = "multi_match/query_string/simple_query_string are full-text relevance funcs.") public void wildcard_test() throws IOException { String query1 = "source=" + TEST_INDEX_BEER + " | where query_string(['Tags'], 'taste')"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryValidationIT.java new file mode 100644 index 00000000000..4e4083ad5ef --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryValidationIT.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import org.junit.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.common.setting.Settings; + +/** PPL counterpart of the SQL {@code QueryValidationIT} for query-level rejection cases. */ +public class QueryValidationIT extends PPLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void deeplyNestedPredicateIsRejectedInsteadOfCrashingNode() throws IOException { + // Lower the limit so a small, safe-to-parse query triggers the guard (a query large enough + // to exhaust the default limit could overflow the ANTLR parser itself before the guard runs). + updateClusterSettings( + new ClusterSetting(TRANSIENT, Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(), "20")); + try { + StringBuilder predicate = new StringBuilder("age = 1"); + for (int i = 2; i <= 30; i++) { + predicate.append(" or age = ").append(i); + } + executeQuery("source=opensearch-sql_test_index_account | where " + predicate); + fail("Expected ResponseException for an over-nested predicate"); + } catch (ResponseException e) { + assertTrue(e.getMessage().contains("Expression nesting depth exceeds the maximum allowed")); + } finally { + updateClusterSettings( + new ClusterSetting(TRANSIENT, Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(), null)); + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java index d450ba46c51..d6be3a6d765 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java @@ -6,10 +6,12 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class SimpleQueryStringIT extends PPLIntegTestCase { @Override @@ -45,6 +47,9 @@ public void test_simple_query_string_all_params() throws IOException { } @Test + @RequiresCapability( + value = FULLTEXT_RELEVANCE_FUNC, + note = "multi_match/query_string/simple_query_string are full-text relevance funcs.") public void test_wildcard_simple_query_string() throws IOException { String query1 = "SOURCE=" + TEST_INDEX_BEER + " | WHERE simple_query_string(['Tags'], 'taste') | fields Id"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java index a9001f5c995..76275b3ed72 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.Capability.HEAD_WITHOUT_STABLE_SORT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyOrder; @@ -23,6 +24,7 @@ import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class SortCommandIT extends PPLIntegTestCase { @@ -313,6 +315,11 @@ public void testSortAllDefaultFields() throws IOException { } @Test + @RequiresCapability( + value = HEAD_WITHOUT_STABLE_SORT, + note = + "head 2 without a stable sort returns a non-deterministic row set on the AE route" + + " (HEAD_WITHOUT_STABLE_SORT).") public void testHeadThenSort() throws IOException { JSONObject result = executeQuery(String.format("source=%s | head 2 | sort age | fields age", TEST_INDEX_BANK)); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index adbe04bfd8a..7417fd112ec 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -9,6 +9,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATE_NULL; +import static org.opensearch.sql.util.Capability.BIN_TIME_FIELD_BUCKETING; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -19,6 +21,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.util.RequiresCapability; public class StatsCommandIT extends PPLIntegTestCase { @@ -622,8 +625,10 @@ public void testStatsPercentileWithMin() throws IOException { "source=%s | eval decimal=ceil(balance/100000.0) | stats percentile(decimal, 50)," + " min(decimal)", TEST_INDEX_BANK)); + // The AE route runs the Calcite path, so it returns the Calcite (double) type even though the + // cluster's calcite setting reads false; treat it like the Calcite branch. String returnType = "bigint"; - if (isCalciteEnabled()) { + if (isCalciteEnabled() || isAnalyticsParquetIndicesEnabled()) { returnType = "double"; } @@ -635,6 +640,9 @@ public void testStatsPercentileWithMin() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileWithNull() throws IOException { JSONObject response = executeQuery( @@ -673,6 +681,9 @@ public void testStatsPercentileWhere() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileByNullValue() throws IOException { JSONObject response = executeQuery( @@ -691,6 +702,9 @@ public void testStatsPercentileByNullValue() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileByNullValueNonNullBucket() throws IOException { JSONObject response = executeQuery( @@ -708,6 +722,9 @@ public void testStatsPercentileByNullValueNonNullBucket() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileBySpan() throws IOException { JSONObject response = executeQuery( @@ -755,6 +772,9 @@ public void testDisableLegacyPreferred() throws IOException { } @Test + @RequiresCapability( + value = BIN_TIME_FIELD_BUCKETING, + note = "span() time bucketing differs on the AE route (bucket set/null bucket).") public void testStatsBySpanTimeWithNullBucket() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index b0e119bffb1..00eb26b43dd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -9,12 +9,15 @@ import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.Capability.SCALED_FLOAT_TYPE; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class SystemFunctionIT extends PPLIntegTestCase { @@ -26,6 +29,7 @@ public void init() throws Exception { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void typeof_sql_types() throws IOException { JSONObject response = executeQuery( @@ -53,6 +57,9 @@ public void typeof_sql_types() throws IOException { } @Test + @RequiresCapability( + value = SCALED_FLOAT_TYPE, + note = "typeof(scaled_float) is bigint on the AE route, not double.") public void typeof_opensearch_types() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java index 7fe360d8844..bc1b60eebac 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -14,6 +15,7 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.sql.util.RequiresCapability; public class TextFunctionIT extends PPLIntegTestCase { @Override @@ -78,6 +80,7 @@ void verifyRegexQuery(String pattern, Boolean outputRow1, Boolean outputRow2, Bo } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testRegexp() throws IOException { if (isCalciteEnabled()) { verifyRegexQuery("hello", true, false, true); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java index dbb54505453..65b9e26a851 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java @@ -31,7 +31,12 @@ public void init() throws Exception { super.init(); loadIndex(Index.ACCOUNT); loadIndex(Index.BANK_WITH_NULL_VALUES); - loadIndex(Index.GAME_OF_THRONES); + // game_of_thrones has a multi-value array for the scalar-mapped `titles` field, which the + // parquet store rejects at bulk load; skip it on the AE route so it doesn't abort init() for + // the rest of the suite. No test in this class hierarchy queries game_of_thrones. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.GAME_OF_THRONES); + } loadIndex(Index.DATETIME); } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java index 626a7658dab..2b4d346fa0d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java @@ -393,6 +393,94 @@ public void testPPLQueryWithWildcardIndexPartialAccessDenied() throws IOExceptio assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); } + // --- Multi-index comma-separated source tests (FGAC bypass regression) --- + + @Test + public void testPPLMultiIndexDeniedWhenSecondIndexUnauthorized() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser( + "source = " + TEST_INDEX + ", " + FORBIDDEN_INDEX + " | fields name, age", + ALLOWED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLMultiIndexDeniedWithBackticksAuthorizedFirst() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser( + "source = `" + TEST_INDEX + "`, `" + FORBIDDEN_INDEX + "` | fields name, age", + ALLOWED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLMultiIndexDeniedWithUnauthorizedFirst() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser( + "source = " + FORBIDDEN_INDEX + ", " + TEST_INDEX + " | fields name, age", + ALLOWED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLMultiIndexAllowedWhenAllAuthorized() throws IOException { + try { + JSONObject result = + executePPLAsUser( + "source = " + TEST_INDEX + ", " + TEST_INDEX_2 + " | fields name, age", + WILDCARD_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) when all indices are authorized", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } + } + + // --- Edge cases: malformed comma-separated source patterns --- + + @Test + public void testPPLDoubleCommaRejected() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser( + "source = " + TEST_INDEX + ",," + FORBIDDEN_INDEX + " | fields name, age", + ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLLeadingCommaRejected() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser("source = ," + TEST_INDEX + " | fields name, age", ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLTrailingCommaRejected() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser("source = " + TEST_INDEX + ", | fields name, age", ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + @Test public void testSQLQueryAllowedForAuthorizedUser() throws IOException { try { @@ -441,6 +529,51 @@ public void testSQLQueryDeniedWithSearchPermissionOnly() throws IOException { assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); } + // --- SQL multi-index syntax validation --- + + @Test + public void testSQLMultiIndexCommaInFromRejected() throws IOException { + // SQL FROM "idx1,idx2" — comma inside identifier, should be rejected as syntax error + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executeSQLAsUser( + "SELECT name, age FROM " + TEST_INDEX + "," + FORBIDDEN_INDEX + " LIMIT 3", + ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testSQLMultiIndexCrossJoinRejected() throws IOException { + // SQL cross join syntax — should not bypass FGAC + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executeSQLAsUser( + "SELECT a.name FROM " + TEST_INDEX + " a, " + FORBIDDEN_INDEX + " b LIMIT 3", + ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testSQLMultiIndexJoinRejected() throws IOException { + // Explicit JOIN — should not bypass FGAC + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executeSQLAsUser( + "SELECT a.name FROM " + + TEST_INDEX + + " a JOIN " + + FORBIDDEN_INDEX + + " b ON a.name = b.name LIMIT 3", + ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + /** Executes a PPL query via the production SQL plugin endpoint (/_plugins/_ppl). */ private JSONObject executePPLAsUser(String query, String username) throws IOException { Request request = new Request("POST", "/_plugins/_ppl"); diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java new file mode 100644 index 00000000000..042a3aefbf7 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.security; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; + +/** + * Integration tests that verify the rest command is subject to the security plugin fine grained + * access control. The command dispatches standard transport actions under the caller identity, so + * the security ActionFilter authorizes each one by action name. A caller without the required + * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege + * can run them, and the resolve index endpoint requires the resolve index privilege because the + * command resolves all indices. + */ +public class RestCommandSecurityIT extends SecurityTestBase { + + private static final String ALPHA_INDEX = "rest_sec_alpha"; + private static final String BETA_INDEX = "rest_sec_beta"; + + private static final String MONITOR_USER = "rest_monitor_user"; + private static final String MONITOR_ROLE = "rest_monitor_role"; + + private static final String NO_MONITOR_USER = "rest_no_monitor_user"; + private static final String NO_MONITOR_ROLE = "rest_no_monitor_role"; + + @Override + protected void init() throws Exception { + super.init(); + setupRolesUsersAndIndices(); + enableCalcite(); + // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported + // command error. Disable fallback so the denial reason surfaces to the caller. + disallowCalciteFallback(); + } + + private void setupRolesUsersAndIndices() throws IOException { + createIndexIfAbsent(ALPHA_INDEX); + createIndexIfAbsent(BETA_INDEX); + + createRoleWithPermissions( + MONITOR_ROLE, + "*", + new String[] { + "cluster:admin/opensearch/ppl", + "cluster:monitor/health", + "cluster:monitor/state", + "cluster:monitor/nodes/stats", + "cluster:monitor/nodes/info" + }, + new String[] {"indices:admin/resolve/index"}); + createUser(MONITOR_USER, MONITOR_ROLE); + + createRoleWithPermissions( + NO_MONITOR_ROLE, + ALPHA_INDEX, + new String[] {"cluster:admin/opensearch/ppl"}, + new String[] {"indices:data/read/search*"}); + createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); + } + + @Test + public void monitorUserCanRunCatNodes() throws IOException { + JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); + verifyColumn(result, columnName("name")); + } + + @Test + public void monitorUserCanResolveIndex() throws IOException { + JSONObject result = + executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); + Set names = resolvedNames(result); + assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); + assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); + } + + @Test + public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { + assertDenied( + "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); + } + + @Test + public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { + assertDenied( + "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); + } + + @Test + public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { + assertDenied( + "| rest '/_resolve/index' | fields name, type", + NO_MONITOR_USER, + "indices:admin/resolve/index"); + } + + /** + * Asserts the query is rejected for a caller lacking the privilege. A denied transport action on + * the Calcite only rest path surfaces as a client or server error whose body carries the security + * denial reason, so this checks the denial signal rather than a fixed status code. + */ + private void assertDenied(String query, String user, String deniedAction) throws IOException { + try { + executeQueryAsUser(query, user); + fail("Expected a permission denial for user without privilege: " + user); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + String body = TestUtils.getResponseBody(e.getResponse(), false); + assertTrue("Expected an error status, got " + status, status >= 400); + assertTrue( + "Response should indicate a permission denial. Status " + status + ", body: " + body, + body.contains("no permissions") + || body.contains("Forbidden") + || body.contains("security_exception") + || body.contains(deniedAction)); + } + } + + private void createIndexIfAbsent(String name) throws IOException { + Request request = new Request("PUT", "/" + name); + request.setJsonEntity( + "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); + try { + client().performRequest(request); + } catch (ResponseException e) { + String body = TestUtils.getResponseBody(e.getResponse(), false); + if (!body.contains("resource_already_exists_exception")) { + throw e; + } + } + } + + private Set resolvedNames(JSONObject result) { + Set names = new HashSet<>(); + JSONArray datarows = result.getJSONArray("datarows"); + for (int i = 0; i < datarows.length(); i++) { + names.add(datarows.getJSONArray(i).getString(0)); + } + return names; + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java index 0966c63e0d6..66122db4aa9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java @@ -7,6 +7,9 @@ import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.FILTERED_AGGREGATE; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verify; @@ -26,6 +29,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class AggregationIT extends SQLIntegTestCase { @Override @@ -37,6 +41,7 @@ protected void init() throws Exception { } @Test + @RequiresCapability(FILTERED_AGGREGATE) public void testFilteredAggregatePushDown() throws IOException { JSONObject response = executeQuery("SELECT COUNT(*) FILTER(WHERE age > 35) FROM " + TEST_INDEX_BANK); @@ -45,6 +50,7 @@ public void testFilteredAggregatePushDown() throws IOException { } @Test + @RequiresCapability(FILTERED_AGGREGATE) public void testFilteredAggregateNotPushDown() throws IOException { JSONObject response = executeQuery( @@ -221,6 +227,7 @@ public void testPushDownAggregationOnNullNumericValuesReturnsNull() throws IOExc } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullDateTimeValuesFromTableReturnsNull() throws IOException { var response = executeQuery( @@ -236,6 +243,7 @@ public void testPushDownAggregationOnNullDateTimeValuesFromTableReturnsNull() th } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullDateValuesReturnsNull() throws IOException { var response = executeQuery( @@ -252,6 +260,7 @@ public void testPushDownAggregationOnNullDateValuesReturnsNull() throws IOExcept } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullTimeValuesReturnsNull() throws IOException { var response = executeQuery( @@ -268,6 +277,7 @@ public void testPushDownAggregationOnNullTimeValuesReturnsNull() throws IOExcept } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullTimeStampValuesReturnsNull() throws IOException { var response = executeQuery( @@ -284,6 +294,7 @@ public void testPushDownAggregationOnNullTimeStampValuesReturnsNull() throws IOE } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullDateTimeValuesReturnsNull() throws IOException { var response = executeQuery( @@ -473,6 +484,7 @@ public void testMaxDatePushedDown() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgDatePushedDown() throws IOException { var response = executeQuery(String.format("SELECT avg(date0)" + " from %s", TEST_INDEX_CALCS)); verifySchema(response, schema("avg(date0)", null, "date")); @@ -500,6 +512,7 @@ public void testMaxDateTimePushedDown() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgDateTimePushedDown() throws IOException { var response = executeQuery( @@ -524,6 +537,7 @@ public void testMaxTimePushedDown() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgTimePushedDown() throws IOException { var response = executeQuery(String.format("SELECT avg(time1)" + " from %s", TEST_INDEX_CALCS)); verifySchema(response, schema("avg(time1)", null, "time")); @@ -551,6 +565,7 @@ public void testMaxTimeStampPushedDown() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgTimeStampPushedDown() throws IOException { var response = executeQuery( @@ -581,6 +596,7 @@ public void testMaxDateInMemory() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgDateInMemory() throws IOException { var response = executeQuery( @@ -625,6 +641,7 @@ public void testMaxDateTimeInMemory() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgDateTimeInMemory() throws IOException { var response = executeQuery( @@ -662,6 +679,7 @@ public void testMaxTimeInMemory() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgTimeInMemory() throws IOException { var response = executeQuery( @@ -702,6 +720,7 @@ public void testMaxTimeStampInMemory() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgTimeStampInMemory() throws IOException { var response = executeQuery( @@ -725,6 +744,7 @@ public void testPercentilePushedDown() throws IOException { } @Test + @RequiresCapability(FILTERED_AGGREGATE) public void testFilteredPercentilePushDown() throws IOException { JSONObject response = executeQuery( @@ -735,6 +755,7 @@ public void testFilteredPercentilePushDown() throws IOException { } @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentileGroupByPushDown() throws IOException { var response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java index c0eb800c103..bb33ab891a6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java @@ -7,6 +7,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME_NESTED; +import static org.opensearch.sql.util.Capability.ID_METADATA; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; import java.io.IOException; import java.util.Locale; @@ -16,6 +18,7 @@ import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class ComplexTimestampQueryIT extends SQLIntegTestCase { @Override @@ -26,6 +29,7 @@ protected void init() throws Exception { /** See: 3159 */ @Test + @RequiresCapability(ID_METADATA) public void joinWithTimestampFieldsSchema() throws IOException { String query = String.format( @@ -100,6 +104,7 @@ public void nonJoinTimestampComparison() throws IOException { /** See: 1545 */ @Test + @RequiresCapability(NESTED_FIELDS) public void selectDatetimeWithNested() throws IOException { String query = String.format( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java index 1f8aba29dea..88c1bd54ce0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; import static org.opensearch.sql.util.MatcherUtils.hitAny; import static org.opensearch.sql.util.MatcherUtils.kvInt; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -32,6 +33,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.SearchHits; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class ConditionalIT extends SQLIntegTestCase { @@ -46,15 +48,12 @@ public void init() throws Exception { public void ifnullShouldPassJDBC() throws IOException { JSONObject response = executeJdbcRequest( - "SELECT IFNULL(lastname, 'unknown') AS name FROM " - + TEST_INDEX_ACCOUNT - + " GROUP BY name"); - assertEquals("IFNULL(lastname, 'unknown')", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("keyword", response.query("/schema/0/type")); + "SELECT IFNULL(lastname, 'unknown') FROM " + TEST_INDEX_ACCOUNT + " GROUP BY 1"); + verifySchema(response, schema("IFNULL(lastname, 'unknown')", null, "keyword")); } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void ifnullWithNullInputTest() { JSONObject response = new JSONObject( @@ -109,9 +108,7 @@ public void ifnullWithMissingInputTest() { public void nullifShouldPassJDBC() throws IOException { JSONObject response = executeJdbcRequest("SELECT NULLIF(lastname, 'unknown') AS name FROM " + TEST_INDEX_ACCOUNT); - assertEquals("NULLIF(lastname, 'unknown')", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("keyword", response.query("/schema/0/type")); + verifySchema(response, schema("NULLIF(lastname, 'unknown')", "name", "keyword")); } @Test @@ -152,9 +149,7 @@ public void nullifWithNullInputTest() { public void isnullShouldPassJDBC() throws IOException { JSONObject response = executeJdbcRequest("SELECT ISNULL(lastname) AS name FROM " + TEST_INDEX_ACCOUNT); - assertEquals("ISNULL(lastname)", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("boolean", response.query("/schema/0/type")); + verifySchema(response, schema("ISNULL(lastname)", "name", "boolean")); } @Ignore( @@ -208,9 +203,7 @@ public void isnullWithMathExpr() throws IOException { public void ifShouldPassJDBC() throws IOException { JSONObject response = executeJdbcRequest("SELECT IF(2 > 0, 'hello', 'world') AS name FROM " + TEST_INDEX_ACCOUNT); - assertEquals("IF(2 > 0, 'hello', 'world')", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("keyword", response.query("/schema/0/type")); + verifySchema(response, schema("IF(2 > 0, 'hello', 'world')", "name", "keyword")); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java index d400ad646fc..34850e7e5bc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_CSV_SANITIZE; import static org.opensearch.sql.protocol.response.format.CsvResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.Capability.RESPONSE_FORMAT; import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; @@ -16,7 +17,9 @@ import org.opensearch.client.Response; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(RESPONSE_FORMAT) public class CsvFormatIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index a24775a9755..1e44a62fa24 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -23,6 +24,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class DateTimeFormatsIT extends SQLIntegTestCase { @@ -34,6 +36,7 @@ public void init() throws Exception { } @Test + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testReadingDateFormats() throws IOException { String query = String.format( @@ -50,6 +53,7 @@ public void testReadingDateFormats() throws IOException { } @Test + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testDateFormatsWithOr() throws IOException { String query = String.format("SELECT yyyy-MM-dd_OR_epoch_millis FROM %s", TEST_INDEX_DATE_FORMATS); @@ -153,6 +157,7 @@ public void testDateNanosWithFormats() { @Test @SneakyThrows + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testDateNanosWithFunctions() { // in memory funcs String query = diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index e5132eb02f1..2435256fcb5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -8,6 +8,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -31,6 +33,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class DateTimeFunctionIT extends SQLIntegTestCase { @@ -1170,6 +1173,7 @@ void verifyDateFormat(String date, String type, String format, String formatted) } @Test + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testDateFormat() throws IOException { String timestamp = "1998-01-31 13:14:15.012345"; String timestampFormat = @@ -1397,6 +1401,7 @@ protected JSONObject executeQuery(String query) throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testDateStringAsTimestamp() throws IOException { JSONObject result = executeQuery("select {timestamp '2025-07-10'} as t"); verifySchema(result, schema("{timestamp '2025-07-10'}", "t", "timestamp")); @@ -1404,6 +1409,7 @@ public void testDateStringAsTimestamp() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testTimestampBracket() throws IOException { JSONObject result = executeQuery("select {timestamp '2020-09-16 17:30:00'}"); verifySchema(result, schema("{timestamp '2020-09-16 17:30:00'}", null, "timestamp")); @@ -1423,6 +1429,7 @@ public void testTimestampBracket() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testTimeBracket() throws IOException { JSONObject result = executeQuery("select {time '17:30:00'}"); verifySchema(result, schema("{time '17:30:00'}", null, "time")); @@ -1461,6 +1468,7 @@ private void compareBrackets(String query1, String query2, String timestamp) thr } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testBracketedEquivalent() throws IOException { compareBrackets("timestamp", "timestamp", "2020-09-16 17:30:00"); compareBrackets("timestamp", "ts", "2020-09-16 17:30:00"); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java index 08ceb8c35f9..c3a6a4c5ea3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java @@ -5,12 +5,15 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.LUCENE_PUSHDOWN_EXPLAIN; + import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Explain-plan integration tests asserting that {@code IS NOT NULL} / {@code IS NULL} predicates @@ -22,6 +25,7 @@ * with a single {@code must_not[exists]} child for {@code IS NULL}. This matches what downstream * tooling, serverless / AOSS, and the Calcite path already produce. */ +@RequiresCapability(LUCENE_PUSHDOWN_EXPLAIN) public class ExistsPushdownIT extends SQLIntegTestCase { // Anchored on the surrounding `sourceBuilder=...`, `pitId=` tokens in OpenSearchRequest's diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java index 68dc9a18f34..fc79618fc07 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.GEOPOINT_TYPE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,6 +18,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class GeopointFormatsIT extends SQLIntegTestCase { @@ -26,6 +28,7 @@ public void init() throws Exception { } @Test + @RequiresCapability(GEOPOINT_TYPE) public void testReadingGeopoints() throws IOException { String query = String.format("SELECT point FROM %s LIMIT 5", Index.GEOPOINTS.getName()); JSONObject result = executeJdbcRequest(query); @@ -41,6 +44,7 @@ public void testReadingGeopoints() throws IOException { public static final double TOLERANCE = 1E-5; + @RequiresCapability(GEOPOINT_TYPE) public void testReadingGeoHash() throws IOException { String query = String.format("SELECT point FROM %s WHERE _id='6'", Index.GEOPOINTS.getName()); JSONObject result = executeJdbcRequest(query); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java index d0f890526b7..245b87f0d41 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,7 +18,9 @@ import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public class HighlightFunctionIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java index ce866dc3bdf..50ddc0324d2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.IDENTIFIER_RESOLUTION; +import static org.opensearch.sql.util.Capability.ID_METADATA; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,6 +19,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** Integration tests for identifiers including index and field name symbol. */ public class IdentifierIT extends SQLIntegTestCase { @@ -56,6 +59,7 @@ public void testSpecialFieldName() throws IOException { } @Test + @RequiresCapability(IDENTIFIER_RESOLUTION) public void testMultipleQueriesWithSpecialIndexNames() throws IOException { createIndexWithOneDoc("test.one", "test.two"); queryAndAssertTheDoc("SELECT * FROM test.one"); @@ -63,6 +67,7 @@ public void testMultipleQueriesWithSpecialIndexNames() throws IOException { } @Test + @RequiresCapability(IDENTIFIER_RESOLUTION) public void testDoubleUnderscoreIdentifierTest() throws IOException { new Index("test.twounderscores").addDoc("{\"__age\": 30}"); final JSONObject result = @@ -73,6 +78,7 @@ public void testDoubleUnderscoreIdentifierTest() throws IOException { } @Test + @RequiresCapability(ID_METADATA) public void testMetafieldIdentifierTest() throws IOException { // create an index, but the contents doesn't matter String id = "12345"; @@ -94,6 +100,7 @@ public void testMetafieldIdentifierTest() throws IOException { } @Test + @RequiresCapability(ID_METADATA) public void testMetafieldIdentifierRoutingSelectTest() throws IOException { // create an index, but the contents doesn't really matter String index = "test.routing_select"; @@ -132,6 +139,7 @@ public void testMetafieldIdentifierRoutingSelectTest() throws IOException { } @Test + @RequiresCapability(ID_METADATA) public void testMetafieldIdentifierRoutingFilterTest() throws IOException { // create an index, but the contents doesn't really matter String index = "test.routing_filter"; @@ -172,6 +180,7 @@ public void testMetafieldIdentifierRoutingFilterTest() throws IOException { } @Test + @RequiresCapability(ID_METADATA) public void testMetafieldIdentifierWithAliasTest() throws IOException { // create an index, but the contents doesn't matter String id = "99999"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/LegacyAPICompatibilityIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/LegacyAPICompatibilityIT.java index 155d9002aed..de804555587 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/LegacyAPICompatibilityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/LegacyAPICompatibilityIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.plugin.rest.RestQuerySettingsAction.SETTINGS_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import java.io.IOException; import org.json.JSONObject; @@ -17,6 +18,7 @@ import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; /** For backward compatibility, check if legacy API endpoints are accessible. */ public class LegacyAPICompatibilityIT extends SQLIntegTestCase { @@ -50,6 +52,7 @@ public void explain() { } @Test + @RequiresCapability(PAGINATION_CURSOR) public void closeCursor() throws IOException { String sql = StringUtils.format("SELECT firstname FROM %s WHERE balance > 100", TEST_INDEX_ACCOUNT); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java index 118dd9849b4..6e5451192d8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; +import static org.opensearch.sql.util.Capability.TEXT_KEYWORD_PUSHDOWN_REWRITE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -13,6 +14,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class LikeQueryIT extends SQLIntegTestCase { @Override @@ -145,6 +147,7 @@ public void test_like_on_text_field_with_greater_than_one_word() throws IOExcept } @Test + @RequiresCapability(TEXT_KEYWORD_PUSHDOWN_REWRITE) public void test_convert_field_text_to_keyword() throws IOException { String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextKeywordBody LIKE '*'"; String result = explainQuery(query); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java index 5bde838e190..68f6a0070f7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java @@ -7,6 +7,8 @@ import static org.hamcrest.Matchers.containsString; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; +import static org.opensearch.sql.util.Capability.QUERY_ERROR_MESSAGE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -20,6 +22,7 @@ import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; public class MatchIT extends SQLIntegTestCase { @Override @@ -46,6 +49,7 @@ public void match_in_having() throws IOException { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void missing_field_test() { String query = StringUtils.format("SELECT * FROM %s WHERE match(invalid, 'Bates')", TEST_INDEX_ACCOUNT); @@ -61,6 +65,7 @@ public void missing_field_test() { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void missing_quoted_field_test() { String query = StringUtils.format("SELECT * FROM %s WHERE match('invalid', 'Bates')", TEST_INDEX_ACCOUNT); @@ -76,6 +81,7 @@ public void missing_quoted_field_test() { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void missing_backtick_field_test() { String query = StringUtils.format("SELECT * FROM %s WHERE match(`invalid`, 'Bates')", TEST_INDEX_ACCOUNT); @@ -182,6 +188,7 @@ public void match_alternate_syntaxes_return_the_same_results() throws IOExceptio } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void matchPhraseQueryTest() throws IOException { final String result = explainQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java index b7f2ced5fbd..d345643a89e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -22,6 +23,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class MathematicalFunctionIT extends SQLIntegTestCase { @@ -40,6 +42,7 @@ public void testPI() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testCeil() throws IOException { JSONObject result = executeQuery("select ceil(0)"); verifySchema(result, schema("ceil(0)", null, "long")); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java index 0bc091b0d20..1bb40b41c8b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -13,6 +14,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class MultiMatchIT extends SQLIntegTestCase { @Override @@ -53,6 +55,7 @@ public void test_all_params() { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void verify_wildcard_test() { String query1 = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE multi_match(['Tags'], 'taste')"; JSONObject result1 = executeJdbcRequest(query1); @@ -141,6 +144,7 @@ public void test_all_params_multimatchquery_alternate_parameter_syntax() { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void multi_match_alternate_syntax() throws IOException { String query = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE CreationDate = multi_match('2014-01-22');"; @@ -149,6 +153,7 @@ public void multi_match_alternate_syntax() throws IOException { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void multimatch_alternate_syntax() throws IOException { String query = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE CreationDate = multimatch('2014-01-22');"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index 18d93dbb2a0..3bf6ba72a78 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -10,6 +10,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_WITH_NULLS; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -27,7 +28,9 @@ import org.junit.Test; import org.junit.jupiter.api.Disabled; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(NESTED_FIELDS) public class NestedIT extends SQLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NullLiteralIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NullLiteralIT.java index f885b6d4e0d..854cae2837d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NullLiteralIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NullLiteralIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -13,6 +14,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** * This manual IT for NULL literal cannot be replaced with comparison test because other database @@ -22,6 +24,7 @@ public class NullLiteralIT extends SQLIntegTestCase { @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void testNullLiteralSchema() { verifySchema( query("SELECT NULL, ABS(NULL), 1 + NULL, NULL + 1.0"), @@ -32,6 +35,7 @@ public void testNullLiteralSchema() { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void testNullLiteralInOperator() { verifyDataRows(query("SELECT NULL = NULL, NULL AND TRUE"), rows(null, null)); } @@ -42,6 +46,7 @@ public void testNullLiteralInFunction() { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void testNullLiteralInInterval() { verifyDataRows( query("SELECT INTERVAL NULL DAY, INTERVAL 60 * 60 * 24 * (NULL - FLOOR(NULL)) SECOND"), diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationBlackboxIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationBlackboxIT.java index 84289d8f57e..02a627c72c4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationBlackboxIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationBlackboxIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; + import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import java.io.IOException; @@ -17,10 +19,12 @@ import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; import org.opensearch.sql.util.TestUtils; // This class has only one test case, because it is parametrized and takes significant time @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@RequiresCapability(PAGINATION_CURSOR) public class PaginationBlackboxIT extends SQLIntegTestCase { private final Index index; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java index dfb0bb2080e..675bbde2c7a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java @@ -7,14 +7,17 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import static org.opensearch.sql.util.TestUtils.verifyIsV1Cursor; import static org.opensearch.sql.util.TestUtils.verifyIsV2Cursor; import java.io.IOException; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; import org.opensearch.sql.util.TestUtils; +@RequiresCapability(PAGINATION_CURSOR) public class PaginationFallbackIT extends SQLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java index 9a945ec86f5..221e67c910b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; + import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import java.io.IOException; @@ -19,6 +21,7 @@ import org.junit.jupiter.api.DisplayNameGenerator; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Test pagination with `WHERE` clause using a parametrized test. See constructor {@link @@ -26,6 +29,7 @@ * #STATEMENT_TO_NUM_OF_PAGES} to see how these parameters are generated. */ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@RequiresCapability(PAGINATION_CURSOR) public class PaginationFilterIT extends SQLIntegTestCase { /** diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java index 938a5d1fcd5..c94a81cd50a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import java.io.IOException; import lombok.SneakyThrows; @@ -21,8 +22,10 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; import org.opensearch.sql.util.TestUtils; +@RequiresCapability(PAGINATION_CURSOR) public class PaginationIT extends SQLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java index 4c387e2c171..a0d6ee5b318 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import java.io.IOException; import java.util.ArrayList; @@ -15,7 +16,9 @@ import org.junit.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(PAGINATION_CURSOR) public class PaginationWindowIT extends SQLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PreparedStatementIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PreparedStatementIT.java index 8200f64b668..0a156c9dba5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PreparedStatementIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PreparedStatementIT.java @@ -5,11 +5,15 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.PREPARED_STATEMENT; + import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(PREPARED_STATEMENT) public class PreparedStatementIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java index 3d4e08b4cdc..2e42413b170 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java @@ -6,11 +6,13 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class QueryStringIT extends SQLIntegTestCase { @Override @@ -70,6 +72,7 @@ public void all_params_test() throws IOException { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void wildcard_test() throws IOException { String query1 = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE query_string(['Tags'], 'taste')"; JSONObject result1 = executeJdbcRequest(query1); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java index 2cdfc67d228..e814252cfe7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java @@ -8,6 +8,7 @@ import static org.hamcrest.Matchers.is; import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.QUERY_ERROR_MESSAGE; import static org.opensearch.sql.util.MatcherUtils.featureValueOf; import java.io.IOException; @@ -21,12 +22,15 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.ResponseException; import org.opensearch.core.rest.RestStatus; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** * The query validation IT only covers test for error cases that not doable in comparison test. For * all other tests, comparison test should be favored over manual written test like this. */ +@RequiresCapability(QUERY_ERROR_MESSAGE) public class QueryValidationIT extends SQLIntegTestCase { @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @@ -36,6 +40,27 @@ protected void init() throws Exception { loadIndex(Index.ACCOUNT); } + @Test + public void testDeeplyNestedPredicateIsRejectedInsteadOfCrashingNode() throws IOException { + // Lower the limit so a small, safe-to-parse query triggers the guard (a query large enough + // to exhaust the default limit could overflow the ANTLR parser itself before the guard runs). + updateClusterSettings( + new ClusterSetting(TRANSIENT, Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(), "20")); + try { + StringBuilder predicate = new StringBuilder("age = 1"); + for (int i = 2; i <= 30; i++) { + predicate.append(" OR age = ").append(i); + } + expectResponseException() + .hasStatusCode(BAD_REQUEST) + .containsMessage("Expression nesting depth exceeds the maximum allowed") + .whenExecute("SELECT * FROM opensearch-sql_test_index_account WHERE " + predicate); + } finally { + updateClusterSettings( + new ClusterSetting(TRANSIENT, Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(), null)); + } + } + @Ignore( "Will add this validation in analyzer later. This test should be enabled once " + "https://github.com/opensearch-project/sql/issues/910 has been resolved") diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java index 0f085a1cdeb..33f891b22e6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_RAW_SANITIZE; import static org.opensearch.sql.protocol.response.format.RawResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.Capability.RESPONSE_FORMAT; import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; @@ -17,7 +18,9 @@ import org.opensearch.client.Response; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(RESPONSE_FORMAT) public class RawFormatIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SQLCorrectnessIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SQLCorrectnessIT.java index 6056a1c4168..72941ee9e09 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SQLCorrectnessIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SQLCorrectnessIT.java @@ -5,6 +5,9 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.BackendCapabilities.requireCapability; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; + import com.google.common.io.Resources; import java.io.IOException; import java.nio.file.Files; @@ -27,6 +30,7 @@ protected void init() throws Exception { @Test public void runAllTests() throws Exception { + requireCapability(UNTYPED_NULL_LITERAL); verifyQueries(EXPR_TEST_DIR, expr -> "SELECT " + expr); verifyQueries(QUERY_TEST_DIR, Function.identity()); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java index a1f71dcf6c4..4244daf9f38 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataAddressRows; @@ -18,7 +19,9 @@ import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public class ScoreQueryIT extends SQLIntegTestCase { @Override protected void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java index 8742dedbc70..05bfa483b6f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; @@ -14,6 +15,7 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class SimpleQueryStringIT extends SQLIntegTestCase { @Override @@ -63,6 +65,7 @@ public void test_all_params() throws IOException { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void verify_wildcard_test() throws IOException { String query1 = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE simple_query_string(['Tags'], 'taste')"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java index 01daded897d..4dbc348cb8a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.executor.QueryType.SQL; import static org.opensearch.sql.ppl.StandaloneIT.getDataSourceMetadataStorage; import static org.opensearch.sql.ppl.StandaloneIT.getDataSourceUserRoleHelper; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -51,9 +52,11 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.sql.util.InternalRestHighLevelClient; +import org.opensearch.sql.util.RequiresCapability; import org.opensearch.sql.util.StandaloneModule; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@RequiresCapability(PAGINATION_CURSOR) public class StandalonePaginationIT extends SQLIntegTestCase { private QueryService queryService; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 7129d058c00..b5504b9ecf1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -7,12 +7,15 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class SystemFunctionIT extends SQLIntegTestCase { @@ -23,6 +26,7 @@ protected void init() throws Exception { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void typeof_sql_types() { JSONObject response = executeJdbcRequest( @@ -40,6 +44,7 @@ public void typeof_sql_types() { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void typeof_opensearch_types() { JSONObject response = executeJdbcRequest( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java index 314132fed0a..a793be2debf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -20,6 +21,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class TextFunctionIT extends SQLIntegTestCase { @@ -47,6 +49,7 @@ void verifyQueryWithNullOutput(String query, String type) throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testRegexp() throws IOException { verifyQuery("'a' regexp 'b'", "integer", 0); verifyQuery("'a' regexp '.*'", "integer", 1); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExecutionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExecutionIT.java index 36e78567d54..401732fff56 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExecutionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExecutionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.VECTOR_SEARCH; import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; import static org.opensearch.sql.util.TestUtils.isIndexExist; import static org.opensearch.sql.util.TestUtils.performRequest; @@ -20,6 +21,7 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** * Happy-path execution tests for the vectorSearch() SQL table function. These tests run an actual @@ -31,6 +33,7 @@ * is absent. Run locally against a cluster that has opensearch-knn installed. Provisioning k-NN in * CI is a separate follow-up. */ +@RequiresCapability(VECTOR_SEARCH) public class VectorSearchExecutionIT extends SQLIntegTestCase { private static final String TEST_INDEX = "vector_exec_test"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExplainIT.java index 8719189b13a..b9efa7e0fb4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExplainIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.VECTOR_SEARCH; + import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -15,12 +17,14 @@ import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Explain-plan integration tests for vectorSearch SQL table function. These tests verify DSL * push-down shape via _explain. They do NOT require the k-NN plugin since _explain only parses and * plans the query without executing it against a knn index. */ +@RequiresCapability(VECTOR_SEARCH) public class VectorSearchExplainIT extends SQLIntegTestCase { // Matches WrapperQueryBuilder's base64 payload in explain JSON. The explain output escapes diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index 8ae3167b40b..c10bc474bae 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.sql.util.Capability.VECTOR_SEARCH; import java.io.IOException; import org.junit.Assume; @@ -15,16 +16,19 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Integration tests for vectorSearch SQL table function — validation and error paths. These tests * verify that invalid inputs are rejected with clear error messages. Explain-plan DSL shape tests * live in {@link VectorSearchExplainIT}. */ +@RequiresCapability(VECTOR_SEARCH) public class VectorSearchIT extends SQLIntegTestCase { @Override protected void init() throws Exception { + super.init(); loadIndex(Index.ACCOUNT); } @@ -110,6 +114,26 @@ public void testEmptyVectorRejects() throws IOException { assertThat(ex.getMessage(), containsString("must not be empty")); } + @Test + public void testDuplicateNamedArgumentRejectsWith400() throws IOException { + // A duplicate named argument on top of all four distinct names sends five arguments to the + // resolver, exceeding its fixed four-arg signature. This previously crashed with an unchecked + // IndexOutOfBoundsException surfaced as HTTP 500; it must now be a clean 400 with a + // user-facing message. + ResponseException ex = + expectThrows( + ResponseException.class, + () -> + executeQuery( + "SELECT v._id FROM vectorSearch(table='t', table='t', field='f', " + + "vector='[1.0]', option='k=5') AS v")); + + assertEquals(400, ex.getResponse().getStatusLine().getStatusCode()); + // The five-argument case is caught by the repository arity guard before the resolver's + // duplicate-name check, so the message reports the argument-count mismatch. + assertThat(ex.getMessage(), containsString("Expected 4 arguments")); + } + @Test public void testInvalidFieldNameRejects() throws IOException { ResponseException ex = diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchSubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchSubqueryIT.java index 04346f87a76..2f894dbaed2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchSubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchSubqueryIT.java @@ -6,12 +6,14 @@ package org.opensearch.sql.sql; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.sql.util.Capability.VECTOR_SEARCH; import java.io.IOException; import org.junit.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Integration tests for vectorSearch() used inside subqueries. Locks in the rejection of outer @@ -22,6 +24,7 @@ *

    Uses _explain-only plus error-path queries, so the k-NN plugin is not required — the planner * validation fires during planning, before any k-NN execution. */ +@RequiresCapability(VECTOR_SEARCH) public class VectorSearchSubqueryIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java index 95c1f7433dc..ad248b98d62 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; @@ -13,6 +14,7 @@ import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; public class WindowFunctionIT extends SQLIntegTestCase { @@ -125,6 +127,7 @@ public void testDistinctCountPartition() { } @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentileOverNull() { JSONObject response = new JSONObject( @@ -145,6 +148,7 @@ public void testPercentileOverNull() { } @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentileOver() { JSONObject response = new JSONObject( @@ -165,6 +169,7 @@ public void testPercentileOver() { } @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentilePartition() { JSONObject response = new JSONObject( diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index e0041e57c8c..1d4067f1414 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -192,7 +192,386 @@ public enum Capability { EVAL_MAX_MIN_INT_WIDENING( "eval max()/min() over integer operands reports the result column as bigint on the" + " analytics-engine route (DataFusion widens integers to Int64), whereas the v2/Calcite" - + " path reports int."); + + " path reports int."), + + /** + * Lucene full-text relevance functions ({@code match}, {@code multi_match}, {@code query_string}, + * {@code simple_query_string}, …) are unsupported on the analytics-engine route — DataFusion has + * no relevance scorer, so a query that filters on one returns no rows. + */ + FULLTEXT_RELEVANCE_FUNC( + "Full-text relevance functions (match/multi_match/query_string/simple_query_string) are" + + " unsupported on the analytics-engine route: DataFusion has no relevance scorer, so the" + + " filter returns no rows."), + + /** + * LIKE is case-insensitive on the analytics-engine route (DataFusion), whereas the v2/Calcite + * path treats {@code LIKE} as case-sensitive (only {@code ILIKE} is case-insensitive). + */ + LIKE_CASE_SENSITIVITY( + "LIKE is case-insensitive on the analytics-engine route (DataFusion), whereas the v2/Calcite" + + " path treats LIKE as case-sensitive."), + + /** + * {@code appendpipe [subpipe]} drops the main pipeline's rows on the analytics-engine route: the + * subpipe's filter is applied to the main result instead of its output being appended to it, so + * the original rows are lost (e.g. {@code stats ... | appendpipe [where gender='F']} returns only + * the filtered F rows, not the originals plus the filtered copy). + */ + APPENDPIPE_MAIN_RESULT_DROPPED( + "appendpipe drops the main pipeline's rows on the analytics-engine route: the subpipe filter" + + " is applied to the main result instead of appended, so the originals are lost."), + + /** + * Higher-order array functions that take a lambda ({@code transform}/{@code mvmap}, {@code + * reduce}, {@code filter}, {@code exists}, {@code forall}) are unsupported on the + * analytics-engine route: the capability registry rejects them ({@code No backend supports scalar + * function [...] among [lucene, datafusion]}) since the backends can't execute a PPL lambda. + */ + ARRAY_HIGHER_ORDER_FUNC( + "Higher-order array functions (transform/mvmap, reduce, filter, exists, forall) are" + + " unsupported on the analytics-engine route: the backends can't execute a PPL lambda."), + + /** + * {@code scaled_float} fields are reported as {@code bigint} on the analytics-engine route + * (DataFusion stores the underlying scaled long) rather than {@code double} as on v2/Calcite. + */ + SCALED_FLOAT_TYPE( + "scaled_float is reported as bigint on the analytics-engine route (DataFusion stores the" + + " scaled long), whereas the v2/Calcite path reports double."), + + /** + * Coercing an empty string to a numeric field yields {@code null} on the analytics-engine route, + * whereas the v2/Calcite path coerces it to {@code 0}. + */ + STRING_TO_NUMERIC_COERCION( + "Coercing an empty/invalid string to a numeric field yields null on the analytics-engine" + + " route, whereas the v2/Calcite path coerces it to 0."), + + /** + * A wildcard/alias source whose member indices map the same field to incompatible types (e.g. + * {@code text} in one, {@code boolean} in another) is rejected on the analytics-engine route + * ({@code resolves to indices with incompatible field types}); the v2/Calcite path coerces. + */ + CROSS_INDEX_INCOMPATIBLE_TYPES( + "A wildcard/alias source with incompatible field types across member indices is rejected on" + + " the analytics-engine route, whereas the v2/Calcite path coerces."), + + /** + * The {@code REGEXP} filter operator throws a backend NullPointerException on the + * analytics-engine route. + */ + REGEXP_FILTER( + "The REGEXP filter operator throws a backend NullPointerException on the analytics-engine" + + " route."), + + /** + * {@code mvcombine} lowers to an {@code ARRAY_AGG} aggregate the analytics-engine backend doesn't + * register ({@code No enum constant ... AggregateFunction.ARRAY_AGG}). + */ + MVCOMBINE_ARRAY_AGG( + "mvcombine lowers to ARRAY_AGG, which the analytics-engine backend does not support (no" + + " AggregateFunction.ARRAY_AGG enum constant)."), + + /** + * {@code addtotals} crashes the DataFusion backend with a join panic (out-of-range slice index) + * on the analytics-engine route. + */ + ADDTOTALS_JOIN_PANIC( + "addtotals crashes the DataFusion backend with a join panic (out-of-range slice index) on the" + + " analytics-engine route."), + + /** + * {@code percentile}/{@code median} is approximate on the analytics-engine route (DataFusion's + * approx percentile) but exact on the v2/Calcite path, so percentile values, null-bucket rows, + * and by-span groupings diverge. + */ + PERCENTILE_APPROXIMATE( + "percentile/median is approximate on the analytics-engine route (DataFusion) but exact on the" + + " v2/Calcite path, so the values diverge."), + + /** + * Arithmetic over {@code float}/{@code half_float}-typed fields keeps 32-bit float precision on + * the analytics-engine route (DataFusion), whereas the v2/Calcite path widens to double, so the + * least-significant digits diverge (e.g. 0.2 vs 0.19999981). + */ + FLOAT_ARITHMETIC_PRECISION( + "Arithmetic over float/half_float fields keeps 32-bit precision on the analytics-engine route" + + " (DataFusion) but widens to double on the v2/Calcite path, so the values diverge in" + + " the least-significant digits."), + + /** + * Datetime formatting functions ({@code date_format}, {@code strftime}) render some tokens / + * sub-second precision differently on the analytics-engine route than on the v2/Calcite path. + */ + DATETIME_FORMAT_RENDERING( + "date_format/strftime render some format tokens and sub-second precision differently on the" + + " analytics-engine route than the v2/Calcite path."), + + /** + * {@code json_set}/{@code json_delete} with a {@code $}-prefixed path ({@code $.key}) is a no-op + * on the analytics-engine route (the JSON UDF doesn't strip the {@code $} prefix), whereas the + * v2/Calcite path applies the modification. + */ + JSON_DOLLAR_PATH( + "json_set/json_delete with a $-prefixed path is a no-op on the analytics-engine route (the" + + " JSON UDF doesn't handle the $ prefix), whereas the v2/Calcite path applies it."), + + /** + * A dataset whose document has a multi-value array for a scalar-mapped field can't be bulk-loaded + * into the parquet/composite store ({@code Cannot accept multiple values for field ...}), so + * tests reading that dataset fail at setup on the analytics-engine route. + */ + MULTI_VALUE_FIELD_LOAD( + "A multi-value array for a scalar-mapped field can't be bulk-loaded into the parquet store on" + + " the analytics-engine route, so the dataset fails to load."), + + /** + * {@code dedup} returns a different/non-deterministic row set on the analytics-engine route — the + * engine merges per-fragment batches without a stable tiebreaker, so which duplicate survives + * (and {@code CONSECUTIVE=true} behavior) diverges from the v2/Calcite path. + */ + DEDUP_NONDETERMINISTIC( + "dedup returns a different row set on the analytics-engine route: per-fragment merge order" + + " has no stable tiebreaker, so the surviving duplicate (and CONSECUTIVE behavior)" + + " diverges."), + + /** + * {@code union}/{@code multisearch} over subsearches that read the same index conflates on the + * analytics-engine route: a delegated predicate from one branch leaks onto the co-located shard + * fragment and is applied to all branches, so counts/rows are wrong. Same root cause as {@link + * #MULTISEARCH_SAME_INDEX_CONFLATION} / {@link #APPENDPIPE_MAIN_RESULT_DROPPED}. + */ + SAME_INDEX_UNION_CONFLATION( + "union over same-index subsearches conflates on the analytics-engine route: a delegated" + + " predicate from one branch leaks across the co-located shard fragment, so counts/rows" + + " are wrong."), + + /** + * A wildcard projection/rename ({@code rename * as ...}, {@code fields *}) returns columns in a + * different order on the analytics-engine route (e.g. not mapping order) than the v2/Calcite + * path, so row-position-sensitive assertions diverge even though the values are correct. + */ + WILDCARD_COLUMN_ORDER( + "A wildcard projection/rename returns columns in a different order on the analytics-engine" + + " route than the v2/Calcite path."), + + /** + * {@code unix_timestamp()} over a timestamp string with sub-second precision drops the fractional + * seconds on the analytics-engine route (e.g. {@code unix_timestamp('1984-06-06 + * 12:00:00.123456')} returns {@code 455371200} instead of {@code 455371200.123456}), whereas the + * v2/Calcite path preserves them. + */ + UNIX_TIMESTAMP_SUBSECOND( + "unix_timestamp() drops sub-second precision on the analytics-engine route (returns whole" + + " seconds), whereas the v2/Calcite path preserves the fractional seconds."), + + /** + * The {@code _index} (and {@code _id}) metadata field is not exposed on the analytics-engine + * route — parquet-backed scans surface only mapped document fields, so a query referencing {@code + * _index} fails with {@code FIELD_NOT_FOUND}. Sibling of {@link #ID_METADATA}. + */ + INDEX_METADATA( + "The analytics-engine route doesn't expose the _index metadata field (parquet-backed scans" + + " surface only mapped document fields)."), + + /** + * An {@code object} leaf sub-field present in only some member indices of a wildcard/alias source + * resolves to {@code FIELD_NOT_FOUND} on the analytics-engine route (e.g. {@code machine.os2} + * mapped only in one of two {@code merge_test_*} indices), whereas the v2/Calcite path merges the + * schemas and returns the leaf with nulls for the indices that lack it. + */ + CROSS_INDEX_OBJECT_LEAF_MERGE( + "An object leaf field present in only some wildcard member indices resolves to" + + " FIELD_NOT_FOUND on the analytics-engine route, whereas the v2/Calcite path merges the" + + " schemas."), + + /** + * {@code dayname}/{@code monthname} (and similar) over an invalid datetime literal throw a + * different error-message shape on the analytics-engine route ({@code timestamp:... yyyy-MM-dd + * HH:mm:ss[.SSSSSSSSS]}) than the v2/Calcite path ({@code date:... yyyy-MM-dd}): the route parses + * the literal through the TIMESTAMP path, so a different parser produces the message. Both + * engines correctly reject the input; only the message text diverges. + */ + INVALID_DATETIME_ERROR_SHAPE( + "An invalid datetime literal throws a different error-message shape on the analytics-engine" + + " route (timestamp/yyyy-MM-dd HH:mm:ss[...]) than the v2/Calcite path" + + " (date/yyyy-MM-dd); both engines reject the input, only the message differs."), + + /** + * Seeded {@code RAND(seed)} is unsupported on the analytics-engine route (rejected with {@code + * Seeded RAND(seed) is not supported on the analytics-engine route}); DataFusion has no + * deterministic seeded RAND equivalent. {@code RAND()} without a seed works. + */ + RAND_SEED_UNSUPPORTED( + "Seeded RAND(seed) is unsupported on the analytics-engine route (DataFusion has no" + + " deterministic seeded RAND); RAND() without a seed works."), + + /** + * The IP user-defined type is materialized as a raw {@code BINARY}/{@code byte[]} column on the + * analytics-engine route, so operations that need its IP type — {@code cast(... as IP)}, {@code + * cidrmatch} over an appended/merged IP column — fail ({@code Cannot convert BINARY to IP} / + * {@code unsupported object class [B}). The v2/Calcite path keeps the column typed IP. + */ + IP_UDT_BINARY_REPRESENTATION( + "The IP user-defined type is materialized as a raw BINARY/byte[] column on the" + + " analytics-engine route, so cast(... as IP) and cidrmatch over an IP column fail; the" + + " v2/Calcite path keeps the column typed IP."), + + /** + * A {@code TIME}-typed field is presented as {@code TIMESTAMP} on the analytics-engine route, so + * a function with a {@code TIME}-only signature (e.g. {@code TIMEDIFF} expects {@code [TIME, + * TIME]}) rejects it with a type error ({@code expects {[TIME,TIME]}, but got + * [TIMESTAMP,TIMESTAMP]}). The v2/Calcite path preserves the {@code TIME} type and the function + * accepts it. + */ + TIME_TYPE_WIDENED_TO_TIMESTAMP( + "A TIME-typed field is presented as TIMESTAMP on the analytics-engine route, so functions" + + " with a TIME-only signature (e.g. TIMEDIFF) reject it with a type error; the" + + " v2/Calcite path preserves the TIME type."), + + /** + * {@code binary}-typed fields are stripped from the dataset at load on the analytics-engine route + * (the parquet/composite store can't hold them), so a query referencing a binary field resolves + * to {@code FIELD_NOT_FOUND}. + */ + BINARY_FIELD_STRIPPED( + "binary-typed fields are stripped from the dataset at load on the analytics-engine route, so" + + " queries referencing a binary field resolve to FIELD_NOT_FOUND."), + + /** + * The {@code plugins.ppl.values.max.limit} cap on {@code values()}/{@code list()} is not honored + * on the analytics-engine route: {@code PplAggregateCallRewriter} emits no sort/limit for these + * aggregates, so all unique values are returned regardless of the configured limit. + */ + VALUES_LIMIT_NOT_HONORED( + "The plugins.ppl.values.max.limit cap on values()/list() is not honored on the" + + " analytics-engine route (the aggregate rewriter emits no limit), so all unique values" + + " are returned."), + + /** + * {@code like()} over a {@code text}+{@code keyword} field does not rewrite the filter to the + * {@code .keyword} sub-field in the explain plan on the analytics-engine route (the DataFusion + * scan has no Lucene term-pushdown to rewrite to), so a test asserting the plan contains {@code + * .keyword} fails. The v2/Calcite-over-Lucene path performs the pushdown rewrite. + */ + TEXT_KEYWORD_PUSHDOWN_REWRITE( + "like() over a text+keyword field doesn't rewrite to the .keyword sub-field in the explain" + + " plan on the analytics-engine route (no Lucene term-pushdown), whereas the v2/Calcite" + + " path does."), + + /** + * A test that asserts a Lucene-specific pushdown fragment in the explain plan (e.g. the {@code + * SORT->[...]} sort-pushdown JSON) can't pass on the analytics-engine route: the DataFusion scan + * produces a different plan shape with no Lucene pushdown fragment. The query results are + * correct; only the plan-text assertion diverges. + */ + LUCENE_PUSHDOWN_EXPLAIN( + "A test asserting a Lucene-specific pushdown fragment in the explain plan (e.g. SORT->[...])" + + " can't pass on the analytics-engine route: the DataFusion scan produces a different" + + " plan with no Lucene pushdown fragment."), + + /** + * Chaining two {@code streamstats} commands where an upstream stage partitions {@code by} a group + * fails on the analytics-engine route. Each {@code streamstats ... by} stage projects a {@code + * ROW_NUMBER() OVER ()} sequence column to order its window; the Calcite plan aliases these + * distinctly ({@code __stream_seq__}), but the Substrait converter names both physical columns + * after the operator ({@code "row_number() ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"}), + * so the stacked schema has a duplicate/ambiguous field name. Verified: it surfaces as a 500 + * ({@code Schema contains duplicate unqualified field name ...} / streaming-fragment failure) or, + * for chained {@code window} streamstats, non-deterministic window values. A single {@code + * streamstats by} (or a chain where only the final stage has {@code by}) works. + */ + CHAINED_STREAMSTATS_BY( + "Chaining two streamstats where an upstream stage partitions by a group fails on the" + + " analytics-engine route: both stages emit a ROW_NUMBER() sequence column the Substrait" + + " converter names identically, producing a duplicate/ambiguous field name (500) or" + + " non-deterministic window values."), + + /** + * {@code streamstats} computes its running/window aggregate over the backend scan order on the + * analytics-engine route, ignoring a preceding {@code | sort}. The {@code OVER} clause carries no + * explicit {@code ORDER BY} (streamstats orders by encounter order by design), so DataFusion + * evaluates the window in scan order rather than the sorted order the v2/Calcite path honors. + * Verified: {@code sort age | streamstats window=2 avg(age)} yields window values computed in + * insertion order, not age order, so the per-row aggregates diverge. + */ + STREAMSTATS_SORT_NOT_HONORED( + "streamstats computes its window over the backend scan order on the analytics-engine route," + + " ignoring a preceding | sort (the OVER clause has no explicit ORDER BY), so the window" + + " values diverge from the v2/Calcite path which honors the sort."), + + // Capabilities below were migrated from build.gradle analytics-engine excludes (commit 1ccf431). + // BACKEND layer: divergence rooted in the AE/DataFusion execution + composite/parquet storage. + + /** BACKEND: kNN/vector search has no analytics-engine (DataFusion) backend. */ + VECTOR_SEARCH( + "Vector/kNN search is unsupported on the analytics-engine route: DataFusion has no kNN" + + " backend."), + + /** BACKEND: geo_point fields are not held by the composite/parquet store. */ + GEOPOINT_TYPE( + "geo_point fields are unsupported on the analytics-engine route: the composite/parquet store" + + " does not hold the geo_point type."), + + /** + * BACKEND: dotted index names and underscore-prefixed identifiers don't resolve on the AE route. + */ + IDENTIFIER_RESOLUTION( + "Dotted index names and underscore-prefixed field identifiers don't resolve on the" + + " analytics-engine route."), + + // FRONTEND layer: divergence rooted in the Calcite parser/planner replacing the V2 engine. + + /** FRONTEND: CSV/raw/pretty response formatters are not wired in the Calcite path. */ + RESPONSE_FORMAT( + "CSV/raw/pretty response formats are not produced by the Calcite path used by the" + + " analytics-engine route."), + + /** FRONTEND: stateful pagination/cursor/PIT is not implemented in the Calcite path. */ + PAGINATION_CURSOR( + "Pagination, cursor, and point-in-time are unsupported on the analytics-engine route: the" + + " Calcite path has no stateful cursor."), + + /** FRONTEND: JDBC prepared statements are not implemented in the Calcite path. */ + PREPARED_STATEMENT( + "Prepared statements are unsupported on the analytics-engine route (Calcite path)."), + + /** + * FRONTEND: legacy method-query syntax (regexp_query/wildcard_query) is not in the Calcite + * grammar. + */ + LEGACY_METHOD_QUERY( + "Legacy method-query syntax (regexp_query/wildcard_query/query/matchquery) is not in the" + + " Calcite grammar used by the analytics-engine route."), + + /** FRONTEND: error/validation message text differs under the Calcite path. */ + QUERY_ERROR_MESSAGE( + "Query validation and error-message text differ on the analytics-engine route (Calcite" + + " produces different wording for the same semantic error)."), + + /** FRONTEND: explain output is a Calcite plan, not the V2 OpenSearch DSL text. */ + EXPLAIN_FORMAT( + "Explain output differs on the analytics-engine route: the Calcite path emits a different" + + " plan shape than the V2 OpenSearch DSL text the test asserts."), + + /** + * FRONTEND: Calcite function return types/signatures differ from V2 (CEIL, REGEXP, typeof, AVG). + */ + FUNCTION_TYPE_COMPAT( + "Function return types and signatures differ on the analytics-engine route: Calcite uses" + + " standard SQL types (e.g. CEIL->double, REGEXP->boolean, typeof ANSI names, AVG" + + " rejects temporal) where V2 used OpenSearch-specific behavior."), + + /** BACKEND: untyped NULL literal in a no-FROM query can't be serialized to Substrait. */ + UNTYPED_NULL_LITERAL( + "An untyped NULL literal in a no-FROM query (SELECT NULL, NULL in operators/intervals," + + " typeof(NULL)) can't be serialized to Substrait on the analytics-engine route."), + + /** BACKEND: FILTER(WHERE) on aggregates can't be executed via Substrait streaming. */ + FILTERED_AGGREGATE( + "FILTER(WHERE) on aggregates can't be executed on the analytics-engine route: the Substrait" + + " streaming path doesn't support filtered aggregates."); private final String reason; diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index dc9ebc30de5..4fb8bd28129 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -27,6 +27,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.logging.log4j.LogManager; @@ -39,6 +40,7 @@ import org.json.JSONObject; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.sql.legacy.TestUtils; import org.opensearch.sql.utils.YamlFormatter; public class MatcherUtils { @@ -275,6 +277,9 @@ public static TypeSafeMatcher schema(String expectedName, String exp public static TypeSafeMatcher schema( String expectedName, String expectedAlias, String expectedType) { return new TypeSafeMatcher() { + private static final Set NUMERIC_TYPES = Set.of("integer", "long", "float", "double"); + private static final Set STRING_TYPES = Set.of("keyword", "text", "string"); + @Override public void describeTo(Description description) { description.appendText( @@ -287,10 +292,32 @@ protected boolean matchesSafely(JSONObject jsonObject) { String actualName = (String) jsonObject.query("/name"); String actualAlias = (String) jsonObject.query("/alias"); String actualType = (String) jsonObject.query("/type"); + + if (TestUtils.AnalyticsIndexConfig.isEnabled()) { + // The analytics-engine route promotes the alias to the name (SQL-standard) and unifies + // keyword/text/string and the numeric types, so relax name and type matching for it only. + boolean nameMatches = + expectedName.equals(actualName) + || (!Strings.isNullOrEmpty(expectedAlias) && expectedAlias.equals(actualName)); + boolean typeMatches = + expectedType.equals(actualType) || isCompatibleType(expectedType, actualType); + return nameMatches && typeMatches; + } + return expectedName.equals(actualName) && (Strings.isNullOrEmpty(expectedAlias) || expectedAlias.equals(actualAlias)) && expectedType.equals(actualType); } + + private boolean isCompatibleType(String expected, String actual) { + if (expected == null || actual == null) { + return false; + } + String e = expected.toLowerCase(); + String a = actual.toLowerCase(); + return (NUMERIC_TYPES.contains(e) && NUMERIC_TYPES.contains(a)) + || (STRING_TYPES.contains(e) && STRING_TYPES.contains(a)); + } }; } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml index 2f539057b66..63c7ea798df 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml @@ -28,8 +28,8 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index 656118ca892..726eeedc429 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -30,10 +30,10 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml index 9267e6faab1..895d30e06e0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml @@ -25,7 +25,7 @@ calcite: EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml index 8b9f72596b1..aec30183545 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml @@ -24,7 +24,7 @@ calcite: EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml index ea3d4d6863e..fdaf8ad1db8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], count()=[COUNT()]) - LogicalFilter(condition=[LIKE($26, '%google%', '\')]) + LogicalFilter(condition=[LIKE($26, '%google%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->LIKE($0, '%google%', '\'), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->LIKE($0, '%google%':VARCHAR, '\'), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml index edd3dabd8d8..a1ae1ea9ab6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml @@ -6,7 +6,7 @@ calcite: LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(SearchPhrase=[$63]) LogicalFilter(condition=[IS NOT NULL($63)]) - LogicalFilter(condition=[AND(LIKE($26, '%google%', '\'), <>($63, ''))]) + LogicalFilter(condition=[AND(LIKE($26, '%google%':VARCHAR, '\'), <>($63, ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(LIKE($0, '%google%', '\'), <>($1, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT()), PROJECT->[c, SearchPhrase], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(LIKE($0, '%google%':VARCHAR, '\'), <>($1, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT()), PROJECT->[c, SearchPhrase], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml index 6f6b5056a9f..ada01f0ebef 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml @@ -6,7 +6,7 @@ calcite: LogicalAggregate(group=[{0}], c=[COUNT()], dc(UserID)=[COUNT(DISTINCT $1)]) LogicalProject(SearchPhrase=[$63], UserID=[$84]) LogicalFilter(condition=[IS NOT NULL($63)]) - LogicalFilter(condition=[AND(LIKE($97, '%Google%', '\'), <>($63, ''), NOT(LIKE($26, '%.google.%', '\')))]) + LogicalFilter(condition=[AND(LIKE($97, '%Google%':VARCHAR, '\'), <>($63, ''), NOT(LIKE($26, '%.google.%':VARCHAR, '\')))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(LIKE($3, '%Google%', '\'), <>($1, ''), NOT(LIKE($0, '%.google.%', '\'))), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),dc(UserID)=COUNT(DISTINCT $2)), PROJECT->[c, dc(UserID), SearchPhrase], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"Title":{"wildcard":"*Google*","boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"wildcard":{"URL":{"wildcard":"*.google.*","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(LIKE($3, '%Google%':VARCHAR, '\'), <>($1, ''), NOT(LIKE($0, '%.google.%':VARCHAR, '\'))), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),dc(UserID)=COUNT(DISTINCT $2)), PROJECT->[c, dc(UserID), SearchPhrase], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"Title":{"wildcard":"*Google*","boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"wildcard":{"URL":{"wildcard":"*.google.*","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml index f24aabcab1a..0c83653d72d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml @@ -3,12 +3,12 @@ calcite: LogicalSystemLimit(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104]) LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[LIKE($26, '%google%', '\')]) + LogicalFilter(condition=[LIKE($26, '%google%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, URLRegionID, HasGCLID, Income, Interests, Robotness, BrowserLanguage, CounterClass, BrowserCountry, OriginalURL, ClientTimeZone, RefererHash, TraficSourceID, HitColor, RefererRegionID, URLCategoryID, LocalEventTime, EventTime, UTMTerm, AdvEngineID, UserAgentMinor, UserAgentMajor, RemoteIP, Sex, JavaEnable, URLHash, URL, ParamOrderID, OpenstatSourceID, HTTPError, SilverlightVersion3, MobilePhoneModel, SilverlightVersion4, SilverlightVersion1, SilverlightVersion2, IsDownload, IsParameter, CLID, FlashMajor, FlashMinor, UTMMedium, WatchID, DontCountHits, CookieEnable, HID, SocialAction, WindowName, ConnectTiming, PageCharset, IsLink, IsArtifical, JavascriptEnable, ClientEventTime, DNSTiming, CodeVersion, ResponseEndTiming, FUniqID, WindowClientHeight, OpenstatServiceName, UTMContent, HistoryLength, IsOldCounter, MobilePhone, SearchPhrase, FlashMinor2, SearchEngineID, IsEvent, UTMSource, RegionID, OpenstatAdID, UTMCampaign, GoodEvent, IsRefresh, ParamCurrency, Params, ResolutionHeight, ClientIP, FromTag, ParamCurrencyID, ResponseStartTiming, ResolutionWidth, SendTiming, RefererCategoryID, OpenstatCampaignID, UserID, WithHash, UserAgent, ParamPrice, ResolutionDepth, IsMobile, Age, SocialSourceNetworkID, OpenerName, OS, IsNotBounce, Referer, NetMinor, Title, NetMajor, IPNetworkID, FetchTiming, SocialNetwork, SocialSourcePage, CounterID, WindowClientWidth], FILTER->LIKE($26, '%google%', '\'), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, URLRegionID, HasGCLID, Income, Interests, Robotness, BrowserLanguage, CounterClass, BrowserCountry, OriginalURL, ClientTimeZone, RefererHash, TraficSourceID, HitColor, RefererRegionID, URLCategoryID, LocalEventTime, EventTime, UTMTerm, AdvEngineID, UserAgentMinor, UserAgentMajor, RemoteIP, Sex, JavaEnable, URLHash, URL, ParamOrderID, OpenstatSourceID, HTTPError, SilverlightVersion3, MobilePhoneModel, SilverlightVersion4, SilverlightVersion1, SilverlightVersion2, IsDownload, IsParameter, CLID, FlashMajor, FlashMinor, UTMMedium, WatchID, DontCountHits, CookieEnable, HID, SocialAction, WindowName, ConnectTiming, PageCharset, IsLink, IsArtifical, JavascriptEnable, ClientEventTime, DNSTiming, CodeVersion, ResponseEndTiming, FUniqID, WindowClientHeight, OpenstatServiceName, UTMContent, HistoryLength, IsOldCounter, MobilePhone, SearchPhrase, FlashMinor2, SearchEngineID, IsEvent, UTMSource, RegionID, OpenstatAdID, UTMCampaign, GoodEvent, IsRefresh, ParamCurrency, Params, ResolutionHeight, ClientIP, FromTag, ParamCurrencyID, ResponseStartTiming, ResolutionWidth, SendTiming, RefererCategoryID, OpenstatCampaignID, UserID, WithHash, UserAgent, ParamPrice, ResolutionDepth, IsMobile, Age, SocialSourceNetworkID, OpenerName, OS, IsNotBounce, Referer, NetMinor, Title, NetMajor, IPNetworkID, FetchTiming, SocialNetwork, SocialSourcePage, CounterID, WindowClientWidth], FILTER->LIKE($26, '%google%':VARCHAR, '\'), SORT->[{ "EventTime" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"_source":{"includes":["EventDate","URLRegionID","HasGCLID","Income","Interests","Robotness","BrowserLanguage","CounterClass","BrowserCountry","OriginalURL","ClientTimeZone","RefererHash","TraficSourceID","HitColor","RefererRegionID","URLCategoryID","LocalEventTime","EventTime","UTMTerm","AdvEngineID","UserAgentMinor","UserAgentMajor","RemoteIP","Sex","JavaEnable","URLHash","URL","ParamOrderID","OpenstatSourceID","HTTPError","SilverlightVersion3","MobilePhoneModel","SilverlightVersion4","SilverlightVersion1","SilverlightVersion2","IsDownload","IsParameter","CLID","FlashMajor","FlashMinor","UTMMedium","WatchID","DontCountHits","CookieEnable","HID","SocialAction","WindowName","ConnectTiming","PageCharset","IsLink","IsArtifical","JavascriptEnable","ClientEventTime","DNSTiming","CodeVersion","ResponseEndTiming","FUniqID","WindowClientHeight","OpenstatServiceName","UTMContent","HistoryLength","IsOldCounter","MobilePhone","SearchPhrase","FlashMinor2","SearchEngineID","IsEvent","UTMSource","RegionID","OpenstatAdID","UTMCampaign","GoodEvent","IsRefresh","ParamCurrency","Params","ResolutionHeight","ClientIP","FromTag","ParamCurrencyID","ResponseStartTiming","ResolutionWidth","SendTiming","RefererCategoryID","OpenstatCampaignID","UserID","WithHash","UserAgent","ParamPrice","ResolutionDepth","IsMobile","Age","SocialSourceNetworkID","OpenerName","OS","IsNotBounce","Referer","NetMinor","Title","NetMajor","IPNetworkID","FetchTiming","SocialNetwork","SocialSourcePage","CounterID","WindowClientWidth"]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"_source":{"includes":["EventDate","URLRegionID","HasGCLID","Income","Interests","Robotness","BrowserLanguage","CounterClass","BrowserCountry","OriginalURL","ClientTimeZone","RefererHash","TraficSourceID","HitColor","RefererRegionID","URLCategoryID","LocalEventTime","EventTime","UTMTerm","AdvEngineID","UserAgentMinor","UserAgentMajor","RemoteIP","Sex","JavaEnable","URLHash","URL","ParamOrderID","OpenstatSourceID","HTTPError","SilverlightVersion3","MobilePhoneModel","SilverlightVersion4","SilverlightVersion1","SilverlightVersion2","IsDownload","IsParameter","CLID","FlashMajor","FlashMinor","UTMMedium","WatchID","DontCountHits","CookieEnable","HID","SocialAction","WindowName","ConnectTiming","PageCharset","IsLink","IsArtifical","JavascriptEnable","ClientEventTime","DNSTiming","CodeVersion","ResponseEndTiming","FUniqID","WindowClientHeight","OpenstatServiceName","UTMContent","HistoryLength","IsOldCounter","MobilePhone","SearchPhrase","FlashMinor2","SearchEngineID","IsEvent","UTMSource","RegionID","OpenstatAdID","UTMCampaign","GoodEvent","IsRefresh","ParamCurrency","Params","ResolutionHeight","ClientIP","FromTag","ParamCurrencyID","ResponseStartTiming","ResolutionWidth","SendTiming","RefererCategoryID","OpenstatCampaignID","UserID","WithHash","UserAgent","ParamPrice","ResolutionDepth","IsMobile","Age","SocialSourceNetworkID","OpenerName","OS","IsNotBounce","Referer","NetMinor","Title","NetMajor","IPNetworkID","FetchTiming","SocialNetwork","SocialSourcePage","CounterID","WindowClientWidth"]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml index de1f6d31f0d..d50a9ec47ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml @@ -2,8 +2,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) - LogicalProject(ResolutionWidth=[$80], $f90=[+($80, 1)], $f91=[+($80, 2)], $f92=[+($80, 3)], $f93=[+($80, 4)], $f94=[+($80, 5)], $f95=[+($80, 6)], $f96=[+($80, 7)], $f97=[+($80, 8)], $f98=[+($80, 9)], $f99=[+($80, 10)], $f100=[+($80, 11)], $f101=[+($80, 12)], $f102=[+($80, 13)], $f103=[+($80, 14)], $f104=[+($80, 15)], $f105=[+($80, 16)], $f106=[+($80, 17)], $f107=[+($80, 18)], $f108=[+($80, 19)], $f109=[+($80, 20)], $f110=[+($80, 21)], $f111=[+($80, 22)], $f112=[+($80, 23)], $f113=[+($80, 24)], $f114=[+($80, 25)], $f115=[+($80, 26)], $f116=[+($80, 27)], $f117=[+($80, 28)], $f118=[+($80, 29)], $f119=[+($80, 30)], $f120=[+($80, 31)], $f121=[+($80, 32)], $f122=[+($80, 33)], $f123=[+($80, 34)], $f124=[+($80, 35)], $f125=[+($80, 36)], $f126=[+($80, 37)], $f127=[+($80, 38)], $f128=[+($80, 39)], $f129=[+($80, 40)], $f130=[+($80, 41)], $f131=[+($80, 42)], $f132=[+($80, 43)], $f133=[+($80, 44)], $f134=[+($80, 45)], $f135=[+($80, 46)], $f136=[+($80, 47)], $f137=[+($80, 48)], $f138=[+($80, 49)], $f139=[+($80, 50)], $f140=[+($80, 51)], $f141=[+($80, 52)], $f142=[+($80, 53)], $f143=[+($80, 54)], $f144=[+($80, 55)], $f145=[+($80, 56)], $f146=[+($80, 57)], $f147=[+($80, 58)], $f148=[+($80, 59)], $f149=[+($80, 60)], $f150=[+($80, 61)], $f151=[+($80, 62)], $f152=[+($80, 63)], $f153=[+($80, 64)], $f154=[+($80, 65)], $f155=[+($80, 66)], $f156=[+($80, 67)], $f157=[+($80, 68)], $f158=[+($80, 69)], $f159=[+($80, 70)], $f160=[+($80, 71)], $f161=[+($80, 72)], $f162=[+($80, 73)], $f163=[+($80, 74)], $f164=[+($80, 75)], $f165=[+($80, 76)], $f166=[+($80, 77)], $f167=[+($80, 78)], $f168=[+($80, 79)], $f169=[+($80, 80)], $f170=[+($80, 81)], $f171=[+($80, 82)], $f172=[+($80, 83)], $f173=[+($80, 84)], $f174=[+($80, 85)], $f175=[+($80, 86)], $f176=[+($80, 87)], $f177=[+($80, 88)], $f178=[+($80, 89)]) + LogicalProject(ResolutionWidth=[$80], $f90=[+(CAST($80):BIGINT, 1)], $f91=[+(CAST($80):BIGINT, 2)], $f92=[+(CAST($80):BIGINT, 3)], $f93=[+(CAST($80):BIGINT, 4)], $f94=[+(CAST($80):BIGINT, 5)], $f95=[+(CAST($80):BIGINT, 6)], $f96=[+(CAST($80):BIGINT, 7)], $f97=[+(CAST($80):BIGINT, 8)], $f98=[+(CAST($80):BIGINT, 9)], $f99=[+(CAST($80):BIGINT, 10)], $f100=[+(CAST($80):BIGINT, 11)], $f101=[+(CAST($80):BIGINT, 12)], $f102=[+(CAST($80):BIGINT, 13)], $f103=[+(CAST($80):BIGINT, 14)], $f104=[+(CAST($80):BIGINT, 15)], $f105=[+(CAST($80):BIGINT, 16)], $f106=[+(CAST($80):BIGINT, 17)], $f107=[+(CAST($80):BIGINT, 18)], $f108=[+(CAST($80):BIGINT, 19)], $f109=[+(CAST($80):BIGINT, 20)], $f110=[+(CAST($80):BIGINT, 21)], $f111=[+(CAST($80):BIGINT, 22)], $f112=[+(CAST($80):BIGINT, 23)], $f113=[+(CAST($80):BIGINT, 24)], $f114=[+(CAST($80):BIGINT, 25)], $f115=[+(CAST($80):BIGINT, 26)], $f116=[+(CAST($80):BIGINT, 27)], $f117=[+(CAST($80):BIGINT, 28)], $f118=[+(CAST($80):BIGINT, 29)], $f119=[+(CAST($80):BIGINT, 30)], $f120=[+(CAST($80):BIGINT, 31)], $f121=[+(CAST($80):BIGINT, 32)], $f122=[+(CAST($80):BIGINT, 33)], $f123=[+(CAST($80):BIGINT, 34)], $f124=[+(CAST($80):BIGINT, 35)], $f125=[+(CAST($80):BIGINT, 36)], $f126=[+(CAST($80):BIGINT, 37)], $f127=[+(CAST($80):BIGINT, 38)], $f128=[+(CAST($80):BIGINT, 39)], $f129=[+(CAST($80):BIGINT, 40)], $f130=[+(CAST($80):BIGINT, 41)], $f131=[+(CAST($80):BIGINT, 42)], $f132=[+(CAST($80):BIGINT, 43)], $f133=[+(CAST($80):BIGINT, 44)], $f134=[+(CAST($80):BIGINT, 45)], $f135=[+(CAST($80):BIGINT, 46)], $f136=[+(CAST($80):BIGINT, 47)], $f137=[+(CAST($80):BIGINT, 48)], $f138=[+(CAST($80):BIGINT, 49)], $f139=[+(CAST($80):BIGINT, 50)], $f140=[+(CAST($80):BIGINT, 51)], $f141=[+(CAST($80):BIGINT, 52)], $f142=[+(CAST($80):BIGINT, 53)], $f143=[+(CAST($80):BIGINT, 54)], $f144=[+(CAST($80):BIGINT, 55)], $f145=[+(CAST($80):BIGINT, 56)], $f146=[+(CAST($80):BIGINT, 57)], $f147=[+(CAST($80):BIGINT, 58)], $f148=[+(CAST($80):BIGINT, 59)], $f149=[+(CAST($80):BIGINT, 60)], $f150=[+(CAST($80):BIGINT, 61)], $f151=[+(CAST($80):BIGINT, 62)], $f152=[+(CAST($80):BIGINT, 63)], $f153=[+(CAST($80):BIGINT, 64)], $f154=[+(CAST($80):BIGINT, 65)], $f155=[+(CAST($80):BIGINT, 66)], $f156=[+(CAST($80):BIGINT, 67)], $f157=[+(CAST($80):BIGINT, 68)], $f158=[+(CAST($80):BIGINT, 69)], $f159=[+(CAST($80):BIGINT, 70)], $f160=[+(CAST($80):BIGINT, 71)], $f161=[+(CAST($80):BIGINT, 72)], $f162=[+(CAST($80):BIGINT, 73)], $f163=[+(CAST($80):BIGINT, 74)], $f164=[+(CAST($80):BIGINT, 75)], $f165=[+(CAST($80):BIGINT, 76)], $f166=[+(CAST($80):BIGINT, 77)], $f167=[+(CAST($80):BIGINT, 78)], $f168=[+(CAST($80):BIGINT, 79)], $f169=[+(CAST($80):BIGINT, 80)], $f170=[+(CAST($80):BIGINT, 81)], $f171=[+(CAST($80):BIGINT, 82)], $f172=[+(CAST($80):BIGINT, 83)], $f173=[+(CAST($80):BIGINT, 84)], $f174=[+(CAST($80):BIGINT, 85)], $f175=[+(CAST($80):BIGINT, 86)], $f176=[+(CAST($80):BIGINT, 87)], $f177=[+(CAST($80):BIGINT, 88)], $f178=[+(CAST($80):BIGINT, 89)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t1):BIGINT], expr#3=[+($t0, $t2)], expr#4=[2], expr#5=[*($t1, $t4)], expr#6=[+($t0, $t5)], expr#7=[3], expr#8=[*($t1, $t7)], expr#9=[+($t0, $t8)], expr#10=[4], expr#11=[*($t1, $t10)], expr#12=[+($t0, $t11)], expr#13=[5], expr#14=[*($t1, $t13)], expr#15=[+($t0, $t14)], expr#16=[6], expr#17=[*($t1, $t16)], expr#18=[+($t0, $t17)], expr#19=[7], expr#20=[*($t1, $t19)], expr#21=[+($t0, $t20)], expr#22=[8], expr#23=[*($t1, $t22)], expr#24=[+($t0, $t23)], expr#25=[9], expr#26=[*($t1, $t25)], expr#27=[+($t0, $t26)], expr#28=[10], expr#29=[*($t1, $t28)], expr#30=[+($t0, $t29)], expr#31=[11], expr#32=[*($t1, $t31)], expr#33=[+($t0, $t32)], expr#34=[12], expr#35=[*($t1, $t34)], expr#36=[+($t0, $t35)], expr#37=[13], expr#38=[*($t1, $t37)], expr#39=[+($t0, $t38)], expr#40=[14], expr#41=[*($t1, $t40)], expr#42=[+($t0, $t41)], expr#43=[15], expr#44=[*($t1, $t43)], expr#45=[+($t0, $t44)], expr#46=[16], expr#47=[*($t1, $t46)], expr#48=[+($t0, $t47)], expr#49=[17], expr#50=[*($t1, $t49)], expr#51=[+($t0, $t50)], expr#52=[18], expr#53=[*($t1, $t52)], expr#54=[+($t0, $t53)], expr#55=[19], expr#56=[*($t1, $t55)], expr#57=[+($t0, $t56)], expr#58=[20], expr#59=[*($t1, $t58)], expr#60=[+($t0, $t59)], expr#61=[21], expr#62=[*($t1, $t61)], expr#63=[+($t0, $t62)], expr#64=[22], expr#65=[*($t1, $t64)], expr#66=[+($t0, $t65)], expr#67=[23], expr#68=[*($t1, $t67)], expr#69=[+($t0, $t68)], expr#70=[24], expr#71=[*($t1, $t70)], expr#72=[+($t0, $t71)], expr#73=[25], expr#74=[*($t1, $t73)], expr#75=[+($t0, $t74)], expr#76=[26], expr#77=[*($t1, $t76)], expr#78=[+($t0, $t77)], expr#79=[27], expr#80=[*($t1, $t79)], expr#81=[+($t0, $t80)], expr#82=[28], expr#83=[*($t1, $t82)], expr#84=[+($t0, $t83)], expr#85=[29], expr#86=[*($t1, $t85)], expr#87=[+($t0, $t86)], expr#88=[30], expr#89=[*($t1, $t88)], expr#90=[+($t0, $t89)], expr#91=[31], expr#92=[*($t1, $t91)], expr#93=[+($t0, $t92)], expr#94=[32], expr#95=[*($t1, $t94)], expr#96=[+($t0, $t95)], expr#97=[33], expr#98=[*($t1, $t97)], expr#99=[+($t0, $t98)], expr#100=[34], expr#101=[*($t1, $t100)], expr#102=[+($t0, $t101)], expr#103=[35], expr#104=[*($t1, $t103)], expr#105=[+($t0, $t104)], expr#106=[36], expr#107=[*($t1, $t106)], expr#108=[+($t0, $t107)], expr#109=[37], expr#110=[*($t1, $t109)], expr#111=[+($t0, $t110)], expr#112=[38], expr#113=[*($t1, $t112)], expr#114=[+($t0, $t113)], expr#115=[39], expr#116=[*($t1, $t115)], expr#117=[+($t0, $t116)], expr#118=[40], expr#119=[*($t1, $t118)], expr#120=[+($t0, $t119)], expr#121=[41], expr#122=[*($t1, $t121)], expr#123=[+($t0, $t122)], expr#124=[42], expr#125=[*($t1, $t124)], expr#126=[+($t0, $t125)], expr#127=[43], expr#128=[*($t1, $t127)], expr#129=[+($t0, $t128)], expr#130=[44], expr#131=[*($t1, $t130)], expr#132=[+($t0, $t131)], expr#133=[45], expr#134=[*($t1, $t133)], expr#135=[+($t0, $t134)], expr#136=[46], expr#137=[*($t1, $t136)], expr#138=[+($t0, $t137)], expr#139=[47], expr#140=[*($t1, $t139)], expr#141=[+($t0, $t140)], expr#142=[48], expr#143=[*($t1, $t142)], expr#144=[+($t0, $t143)], expr#145=[49], expr#146=[*($t1, $t145)], expr#147=[+($t0, $t146)], expr#148=[50], expr#149=[*($t1, $t148)], expr#150=[+($t0, $t149)], expr#151=[51], expr#152=[*($t1, $t151)], expr#153=[+($t0, $t152)], expr#154=[52], expr#155=[*($t1, $t154)], expr#156=[+($t0, $t155)], expr#157=[53], expr#158=[*($t1, $t157)], expr#159=[+($t0, $t158)], expr#160=[54], expr#161=[*($t1, $t160)], expr#162=[+($t0, $t161)], expr#163=[55], expr#164=[*($t1, $t163)], expr#165=[+($t0, $t164)], expr#166=[56], expr#167=[*($t1, $t166)], expr#168=[+($t0, $t167)], expr#169=[57], expr#170=[*($t1, $t169)], expr#171=[+($t0, $t170)], expr#172=[58], expr#173=[*($t1, $t172)], expr#174=[+($t0, $t173)], expr#175=[59], expr#176=[*($t1, $t175)], expr#177=[+($t0, $t176)], expr#178=[60], expr#179=[*($t1, $t178)], expr#180=[+($t0, $t179)], expr#181=[61], expr#182=[*($t1, $t181)], expr#183=[+($t0, $t182)], expr#184=[62], expr#185=[*($t1, $t184)], expr#186=[+($t0, $t185)], expr#187=[63], expr#188=[*($t1, $t187)], expr#189=[+($t0, $t188)], expr#190=[64], expr#191=[*($t1, $t190)], expr#192=[+($t0, $t191)], expr#193=[65], expr#194=[*($t1, $t193)], expr#195=[+($t0, $t194)], expr#196=[66], expr#197=[*($t1, $t196)], expr#198=[+($t0, $t197)], expr#199=[67], expr#200=[*($t1, $t199)], expr#201=[+($t0, $t200)], expr#202=[68], expr#203=[*($t1, $t202)], expr#204=[+($t0, $t203)], expr#205=[69], expr#206=[*($t1, $t205)], expr#207=[+($t0, $t206)], expr#208=[70], expr#209=[*($t1, $t208)], expr#210=[+($t0, $t209)], expr#211=[71], expr#212=[*($t1, $t211)], expr#213=[+($t0, $t212)], expr#214=[72], expr#215=[*($t1, $t214)], expr#216=[+($t0, $t215)], expr#217=[73], expr#218=[*($t1, $t217)], expr#219=[+($t0, $t218)], expr#220=[74], expr#221=[*($t1, $t220)], expr#222=[+($t0, $t221)], expr#223=[75], expr#224=[*($t1, $t223)], expr#225=[+($t0, $t224)], expr#226=[76], expr#227=[*($t1, $t226)], expr#228=[+($t0, $t227)], expr#229=[77], expr#230=[*($t1, $t229)], expr#231=[+($t0, $t230)], expr#232=[78], expr#233=[*($t1, $t232)], expr#234=[+($t0, $t233)], expr#235=[79], expr#236=[*($t1, $t235)], expr#237=[+($t0, $t236)], expr#238=[80], expr#239=[*($t1, $t238)], expr#240=[+($t0, $t239)], expr#241=[81], expr#242=[*($t1, $t241)], expr#243=[+($t0, $t242)], expr#244=[82], expr#245=[*($t1, $t244)], expr#246=[+($t0, $t245)], expr#247=[83], expr#248=[*($t1, $t247)], expr#249=[+($t0, $t248)], expr#250=[84], expr#251=[*($t1, $t250)], expr#252=[+($t0, $t251)], expr#253=[85], expr#254=[*($t1, $t253)], expr#255=[+($t0, $t254)], expr#256=[86], expr#257=[*($t1, $t256)], expr#258=[+($t0, $t257)], expr#259=[87], expr#260=[*($t1, $t259)], expr#261=[+($t0, $t260)], expr#262=[88], expr#263=[*($t1, $t262)], expr#264=[+($t0, $t263)], expr#265=[89], expr#266=[*($t1, $t265)], expr#267=[+($t0, $t266)], sum(ResolutionWidth)=[$t0], sum(ResolutionWidth+1)=[$t3], sum(ResolutionWidth+2)=[$t6], sum(ResolutionWidth+3)=[$t9], sum(ResolutionWidth+4)=[$t12], sum(ResolutionWidth+5)=[$t15], sum(ResolutionWidth+6)=[$t18], sum(ResolutionWidth+7)=[$t21], sum(ResolutionWidth+8)=[$t24], sum(ResolutionWidth+9)=[$t27], sum(ResolutionWidth+10)=[$t30], sum(ResolutionWidth+11)=[$t33], sum(ResolutionWidth+12)=[$t36], sum(ResolutionWidth+13)=[$t39], sum(ResolutionWidth+14)=[$t42], sum(ResolutionWidth+15)=[$t45], sum(ResolutionWidth+16)=[$t48], sum(ResolutionWidth+17)=[$t51], sum(ResolutionWidth+18)=[$t54], sum(ResolutionWidth+19)=[$t57], sum(ResolutionWidth+20)=[$t60], sum(ResolutionWidth+21)=[$t63], sum(ResolutionWidth+22)=[$t66], sum(ResolutionWidth+23)=[$t69], sum(ResolutionWidth+24)=[$t72], sum(ResolutionWidth+25)=[$t75], sum(ResolutionWidth+26)=[$t78], sum(ResolutionWidth+27)=[$t81], sum(ResolutionWidth+28)=[$t84], sum(ResolutionWidth+29)=[$t87], sum(ResolutionWidth+30)=[$t90], sum(ResolutionWidth+31)=[$t93], sum(ResolutionWidth+32)=[$t96], sum(ResolutionWidth+33)=[$t99], sum(ResolutionWidth+34)=[$t102], sum(ResolutionWidth+35)=[$t105], sum(ResolutionWidth+36)=[$t108], sum(ResolutionWidth+37)=[$t111], sum(ResolutionWidth+38)=[$t114], sum(ResolutionWidth+39)=[$t117], sum(ResolutionWidth+40)=[$t120], sum(ResolutionWidth+41)=[$t123], sum(ResolutionWidth+42)=[$t126], sum(ResolutionWidth+43)=[$t129], sum(ResolutionWidth+44)=[$t132], sum(ResolutionWidth+45)=[$t135], sum(ResolutionWidth+46)=[$t138], sum(ResolutionWidth+47)=[$t141], sum(ResolutionWidth+48)=[$t144], sum(ResolutionWidth+49)=[$t147], sum(ResolutionWidth+50)=[$t150], sum(ResolutionWidth+51)=[$t153], sum(ResolutionWidth+52)=[$t156], sum(ResolutionWidth+53)=[$t159], sum(ResolutionWidth+54)=[$t162], sum(ResolutionWidth+55)=[$t165], sum(ResolutionWidth+56)=[$t168], sum(ResolutionWidth+57)=[$t171], sum(ResolutionWidth+58)=[$t174], sum(ResolutionWidth+59)=[$t177], sum(ResolutionWidth+60)=[$t180], sum(ResolutionWidth+61)=[$t183], sum(ResolutionWidth+62)=[$t186], sum(ResolutionWidth+63)=[$t189], sum(ResolutionWidth+64)=[$t192], sum(ResolutionWidth+65)=[$t195], sum(ResolutionWidth+66)=[$t198], sum(ResolutionWidth+67)=[$t201], sum(ResolutionWidth+68)=[$t204], sum(ResolutionWidth+69)=[$t207], sum(ResolutionWidth+70)=[$t210], sum(ResolutionWidth+71)=[$t213], sum(ResolutionWidth+72)=[$t216], sum(ResolutionWidth+73)=[$t219], sum(ResolutionWidth+74)=[$t222], sum(ResolutionWidth+75)=[$t225], sum(ResolutionWidth+76)=[$t228], sum(ResolutionWidth+77)=[$t231], sum(ResolutionWidth+78)=[$t234], sum(ResolutionWidth+79)=[$t237], sum(ResolutionWidth+80)=[$t240], sum(ResolutionWidth+81)=[$t243], sum(ResolutionWidth+82)=[$t246], sum(ResolutionWidth+83)=[$t249], sum(ResolutionWidth+84)=[$t252], sum(ResolutionWidth+85)=[$t255], sum(ResolutionWidth+86)=[$t258], sum(ResolutionWidth+87)=[$t261], sum(ResolutionWidth+88)=[$t264], sum(ResolutionWidth+89)=[$t267]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(ResolutionWidth)=SUM($0),sum(ResolutionWidth+1)_COUNT=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(ResolutionWidth)":{"sum":{"field":"ResolutionWidth"}},"sum(ResolutionWidth+1)_COUNT":{"value_count":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):BIGINT], expr#2=[1:BIGINT], expr#3=[+($t1, $t2)], expr#4=[2:BIGINT], expr#5=[+($t1, $t4)], expr#6=[3:BIGINT], expr#7=[+($t1, $t6)], expr#8=[4:BIGINT], expr#9=[+($t1, $t8)], expr#10=[5:BIGINT], expr#11=[+($t1, $t10)], expr#12=[6:BIGINT], expr#13=[+($t1, $t12)], expr#14=[7:BIGINT], expr#15=[+($t1, $t14)], expr#16=[8:BIGINT], expr#17=[+($t1, $t16)], expr#18=[9:BIGINT], expr#19=[+($t1, $t18)], expr#20=[10:BIGINT], expr#21=[+($t1, $t20)], expr#22=[11:BIGINT], expr#23=[+($t1, $t22)], expr#24=[12:BIGINT], expr#25=[+($t1, $t24)], expr#26=[13:BIGINT], expr#27=[+($t1, $t26)], expr#28=[14:BIGINT], expr#29=[+($t1, $t28)], expr#30=[15:BIGINT], expr#31=[+($t1, $t30)], expr#32=[16:BIGINT], expr#33=[+($t1, $t32)], expr#34=[17:BIGINT], expr#35=[+($t1, $t34)], expr#36=[18:BIGINT], expr#37=[+($t1, $t36)], expr#38=[19:BIGINT], expr#39=[+($t1, $t38)], expr#40=[20:BIGINT], expr#41=[+($t1, $t40)], expr#42=[21:BIGINT], expr#43=[+($t1, $t42)], expr#44=[22:BIGINT], expr#45=[+($t1, $t44)], expr#46=[23:BIGINT], expr#47=[+($t1, $t46)], expr#48=[24:BIGINT], expr#49=[+($t1, $t48)], expr#50=[25:BIGINT], expr#51=[+($t1, $t50)], expr#52=[26:BIGINT], expr#53=[+($t1, $t52)], expr#54=[27:BIGINT], expr#55=[+($t1, $t54)], expr#56=[28:BIGINT], expr#57=[+($t1, $t56)], expr#58=[29:BIGINT], expr#59=[+($t1, $t58)], expr#60=[30:BIGINT], expr#61=[+($t1, $t60)], expr#62=[31:BIGINT], expr#63=[+($t1, $t62)], expr#64=[32:BIGINT], expr#65=[+($t1, $t64)], expr#66=[33:BIGINT], expr#67=[+($t1, $t66)], expr#68=[34:BIGINT], expr#69=[+($t1, $t68)], expr#70=[35:BIGINT], expr#71=[+($t1, $t70)], expr#72=[36:BIGINT], expr#73=[+($t1, $t72)], expr#74=[37:BIGINT], expr#75=[+($t1, $t74)], expr#76=[38:BIGINT], expr#77=[+($t1, $t76)], expr#78=[39:BIGINT], expr#79=[+($t1, $t78)], expr#80=[40:BIGINT], expr#81=[+($t1, $t80)], expr#82=[41:BIGINT], expr#83=[+($t1, $t82)], expr#84=[42:BIGINT], expr#85=[+($t1, $t84)], expr#86=[43:BIGINT], expr#87=[+($t1, $t86)], expr#88=[44:BIGINT], expr#89=[+($t1, $t88)], expr#90=[45:BIGINT], expr#91=[+($t1, $t90)], expr#92=[46:BIGINT], expr#93=[+($t1, $t92)], expr#94=[47:BIGINT], expr#95=[+($t1, $t94)], expr#96=[48:BIGINT], expr#97=[+($t1, $t96)], expr#98=[49:BIGINT], expr#99=[+($t1, $t98)], expr#100=[50:BIGINT], expr#101=[+($t1, $t100)], expr#102=[51:BIGINT], expr#103=[+($t1, $t102)], expr#104=[52:BIGINT], expr#105=[+($t1, $t104)], expr#106=[53:BIGINT], expr#107=[+($t1, $t106)], expr#108=[54:BIGINT], expr#109=[+($t1, $t108)], expr#110=[55:BIGINT], expr#111=[+($t1, $t110)], expr#112=[56:BIGINT], expr#113=[+($t1, $t112)], expr#114=[57:BIGINT], expr#115=[+($t1, $t114)], expr#116=[58:BIGINT], expr#117=[+($t1, $t116)], expr#118=[59:BIGINT], expr#119=[+($t1, $t118)], expr#120=[60:BIGINT], expr#121=[+($t1, $t120)], expr#122=[61:BIGINT], expr#123=[+($t1, $t122)], expr#124=[62:BIGINT], expr#125=[+($t1, $t124)], expr#126=[63:BIGINT], expr#127=[+($t1, $t126)], expr#128=[64:BIGINT], expr#129=[+($t1, $t128)], expr#130=[65:BIGINT], expr#131=[+($t1, $t130)], expr#132=[66:BIGINT], expr#133=[+($t1, $t132)], expr#134=[67:BIGINT], expr#135=[+($t1, $t134)], expr#136=[68:BIGINT], expr#137=[+($t1, $t136)], expr#138=[69:BIGINT], expr#139=[+($t1, $t138)], expr#140=[70:BIGINT], expr#141=[+($t1, $t140)], expr#142=[71:BIGINT], expr#143=[+($t1, $t142)], expr#144=[72:BIGINT], expr#145=[+($t1, $t144)], expr#146=[73:BIGINT], expr#147=[+($t1, $t146)], expr#148=[74:BIGINT], expr#149=[+($t1, $t148)], expr#150=[75:BIGINT], expr#151=[+($t1, $t150)], expr#152=[76:BIGINT], expr#153=[+($t1, $t152)], expr#154=[77:BIGINT], expr#155=[+($t1, $t154)], expr#156=[78:BIGINT], expr#157=[+($t1, $t156)], expr#158=[79:BIGINT], expr#159=[+($t1, $t158)], expr#160=[80:BIGINT], expr#161=[+($t1, $t160)], expr#162=[81:BIGINT], expr#163=[+($t1, $t162)], expr#164=[82:BIGINT], expr#165=[+($t1, $t164)], expr#166=[83:BIGINT], expr#167=[+($t1, $t166)], expr#168=[84:BIGINT], expr#169=[+($t1, $t168)], expr#170=[85:BIGINT], expr#171=[+($t1, $t170)], expr#172=[86:BIGINT], expr#173=[+($t1, $t172)], expr#174=[87:BIGINT], expr#175=[+($t1, $t174)], expr#176=[88:BIGINT], expr#177=[+($t1, $t176)], expr#178=[89:BIGINT], expr#179=[+($t1, $t178)], ResolutionWidth=[$t0], $f90=[$t3], $f91=[$t5], $f92=[$t7], $f93=[$t9], $f94=[$t11], $f95=[$t13], $f96=[$t15], $f97=[$t17], $f98=[$t19], $f99=[$t21], $f100=[$t23], $f101=[$t25], $f102=[$t27], $f103=[$t29], $f104=[$t31], $f105=[$t33], $f106=[$t35], $f107=[$t37], $f108=[$t39], $f109=[$t41], $f110=[$t43], $f111=[$t45], $f112=[$t47], $f113=[$t49], $f114=[$t51], $f115=[$t53], $f116=[$t55], $f117=[$t57], $f118=[$t59], $f119=[$t61], $f120=[$t63], $f121=[$t65], $f122=[$t67], $f123=[$t69], $f124=[$t71], $f125=[$t73], $f126=[$t75], $f127=[$t77], $f128=[$t79], $f129=[$t81], $f130=[$t83], $f131=[$t85], $f132=[$t87], $f133=[$t89], $f134=[$t91], $f135=[$t93], $f136=[$t95], $f137=[$t97], $f138=[$t99], $f139=[$t101], $f140=[$t103], $f141=[$t105], $f142=[$t107], $f143=[$t109], $f144=[$t111], $f145=[$t113], $f146=[$t115], $f147=[$t117], $f148=[$t119], $f149=[$t121], $f150=[$t123], $f151=[$t125], $f152=[$t127], $f153=[$t129], $f154=[$t131], $f155=[$t133], $f156=[$t135], $f157=[$t137], $f158=[$t139], $f159=[$t141], $f160=[$t143], $f161=[$t145], $f162=[$t147], $f163=[$t149], $f164=[$t151], $f165=[$t153], $f166=[$t155], $f167=[$t157], $f168=[$t159], $f169=[$t161], $f170=[$t163], $f171=[$t165], $f172=[$t167], $f173=[$t169], $f174=[$t171], $f175=[$t173], $f176=[$t175], $f177=[$t177], $f178=[$t179]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[ResolutionWidth]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["ResolutionWidth"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml index 5f1b457eaa1..d964a1422a1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml @@ -6,8 +6,8 @@ calcite: LogicalAggregate(group=[{0, 1, 2, 3}], c=[COUNT()]) LogicalProject(ClientIP=[$76], ClientIP - 1=[$111], ClientIP - 2=[$112], ClientIP - 3=[$113]) LogicalFilter(condition=[AND(IS NOT NULL($76), IS NOT NULL($111), IS NOT NULL($112), IS NOT NULL($113))]) - LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], ClientIP - 1=[-($76, 1)], ClientIP - 2=[-($76, 2)], ClientIP - 3=[-($76, 3)]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], ClientIP - 1=[-(CAST($76):BIGINT, 1)], ClientIP - 2=[-(CAST($76):BIGINT, 2)], ClientIP - 3=[-(CAST($76):BIGINT, 3)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[-($t0, $t2)], expr#4=[2], expr#5=[-($t0, $t4)], expr#6=[3], expr#7=[-($t0, $t6)], c=[$t1], ClientIP=[$t0], ClientIP - 1=[$t3], ClientIP - 2=[$t5], ClientIP - 3=[$t7]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[1:BIGINT], expr#4=[-($t2, $t3)], expr#5=[2:BIGINT], expr#6=[-($t2, $t5)], expr#7=[3:BIGINT], expr#8=[-($t2, $t7)], c=[$t1], ClientIP=[$t0], ClientIP - 1=[$t4], ClientIP - 2=[$t6], ClientIP - 3=[$t8]) CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"ClientIP":{"terms":{"field":"ClientIP","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml index f349523ec56..b6adbc58332 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBS3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml index acd95f0ec63..841e131df29 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(age1=[*($8, 10)], age2=[+($8, 10)], age3=[10], age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10:BIGINT], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], expr#5=[10], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t5], age=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml index e7589d8109d..4f57bbf4cd4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml @@ -8,5 +8,5 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[>($t6, $t5)], expr#9=[OR($t7, $t8)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t9]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1:BIGINT], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[1], expr#9=[>($t6, $t8)], expr#10=[OR($t7, $t9)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index c543431c519..bc65d5c4c29 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index c17bd10e18a..1d664d5cd43 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml index 4a9a143cba3..efd325acf52 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml index e8e9ac1f4f2..101238a2a7b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml @@ -2,8 +2,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(age=[$10], age2=[+($10, $7)]) + LogicalProject(age=[$10], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml index 5aa37ee3296..90ec92f0b2f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10], age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml index d80ebc5735b..5186752243b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml index 31efd3c688c..9ac70f8b5c7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$14], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], age3=[$20]) LogicalSort(sort0=[$20], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], age3=[-(+($10, $7), $10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], age3=[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[-($t13, $t10)], proj#0..14=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+($10, $7), $10) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCsHsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIk1JTlVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiKyIsCiAgICAgICAgImtpbmQiOiAiUExVUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[-($t14, $t13)], proj#0..12=[{exprs}], age2=[$t14], age3=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQE3HsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIkNIRUNLRURfTUlOVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICIrIiwKICAgICAgICAia2luZCI6ICJDSEVDS0VEX1BMVVMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAgICAgImtpbmQiOiAiQ0FTVCIsCiAgICAgICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiQ0FTVCIsCiAgICAgICAgInN5bnRheCI6ICJTUEVDSUFMIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml index 3fd07a9682e..5aef351b780 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml @@ -5,10 +5,10 @@ calcite: LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[$19], balance2=[ABS($7)]) LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[ABS($t7)], proj#0..14=[{exprs}]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[ABS($t7)], proj#0..12=[{exprs}], age2=[$t14], balance2=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ "age" : { "order" : "asc", diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml index e1328084f77..1009e374b1f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject($f1=[+($3, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBS3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml index 3830e23a87b..1026165ed7e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml @@ -12,10 +12,10 @@ calcite: })], variablesSet=[[$cor0]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableHashJoin(condition=[=($1, $3)], joinType=[semi]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[true], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t0)], i=[$t1], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000, PROJECT->[uid]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10000], expr#3=[<=($t1, $t2)], expr#4=[IS NOT NULL($t0)], expr#5=[AND($t3, $t4)], proj#0..1=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), PROJECT->[uid]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml index fbec63d8c6f..88bae9d4d59 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCwnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCynsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIkNIRUNLRURfTUlOVVMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml index 23bd0d6df69..421d36454a2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml @@ -12,11 +12,12 @@ calcite: })], variablesSet=[[$cor0]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[=($t0, $t3)], proj#0..3=[{exprs}], $condition=[$t4]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableHashJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[semi]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['Tom':VARCHAR], expr#2=[CAST($t1):VARCHAR], name=[$t2], uid=[$t0]) EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["name","uid"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10000], expr#3=[<=($t1, $t2)], expr#4=[IS NOT NULL($t0)], expr#5=[AND($t3, $t4)], proj#0..1=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), PROJECT->[uid]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_ilike_function.yaml index 0651ff30dbd..97fd56f0c94 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_ilike_function.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml index 98b5bbb2f34..fb337d54483 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[LIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[LIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->LIKE($1, '%mbe%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->LIKE($1, '%mbe%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function_case_insensitive.yaml index 0651ff30dbd..97fd56f0c94 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function_case_insensitive.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml index 690c3ce24e7..27110780ce4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[-($t0, $t1)], ageMinus=[$t2]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[30:BIGINT], expr#2=[-($t0, $t1)], ageMinus=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml index 7fde9c0d05e..a704f1588d9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml @@ -12,9 +12,9 @@ calcite: physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], id=[$t1], name=[$t0], count_dept=[$t6]) EnumerableLimit(fetch=[10000]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["name","id"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid=[$t0], count(name)=[$t5]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0})], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"id":{"terms":{"field":"id","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"uid","boost":1.0}},{"exists":{"field":"name","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"uid":{"terms":{"field":"uid","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(name)":{"value_count":{"field":"name"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml index 042787a458b..e98a407d378 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0]) - LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalFilter(condition=[>(CAST($2):BIGINT, +($SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) @@ -12,6 +12,6 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) - EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + EnumerableNestedLoopJoin(condition=[>(CAST($1):BIGINT, +($2, 999))], joinType=[inner]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json index 066b678a960..2c5f37e6813 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):BIGINT], expr#2=[2:BIGINT], expr#3=[+($t1, $t2)], age=[$t0], age2=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml index d0e66b1d14f..e4274781967 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) - LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) + LogicalJoin(condition=[=(+(CAST($10):BIGINT, 1), -($20, 20))], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) @@ -12,14 +12,14 @@ calcite: EnumerableCalc(expr#0..27=[{inputs}], proj#0..12=[{exprs}], b.account_number=[$t14], b.firstname=[$t15], b.address=[$t16], b.birthdate=[$t17], b.gender=[$t18], b.city=[$t19], b.lastname=[$t20], b.balance=[$t21], b.employer=[$t22], b.state=[$t23], b.age=[$t24], b.email=[$t25], b.male=[$t26]) EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[1], expr#14=[+($t10, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[1:BIGINT], expr#15=[+($t13, $t14)], proj#0..12=[{exprs}], $f13=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ "age" : { "order" : "asc", "missing" : "_last" } }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"age":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[20], expr#14=[-($t7, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[20:BIGINT], expr#14=[-($t7, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ "balance" : { "order" : "asc", diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json index 0a36ec4648d..7494f2453c3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(b=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], b=[+($7, 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[+($t0, $t1)], b=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance], SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\"]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[1:BIGINT], expr#2=[+($t0, $t1)], b=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance], SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\"]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml index de78240cbea..9be9d4ea119 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"-\\\",\\n \\\"kind\\\": \\\"MINUS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"-\\\",\\n \\\"kind\\\": \\\"CHECKED_MINUS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml index 08e75fbbdeb..f1b1e2b4524 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[$20]) LogicalSort(sort0=[$19], sort1=[$20], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], balance2=[+($7, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], balance2=[+($7, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[1], expr#15=[+($t7, $t14)], proj#0..13=[{exprs}], balance2=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+($10, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[1:BIGINT], expr#16=[+($t7, $t15)], proj#0..12=[{exprs}], age2=[$t14], balance2=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+(CAST($10):BIGINT, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml index 124539b9d4c..0478b24369c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml @@ -13,7 +13,7 @@ calcite: EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) - EnumerableNestedLoopJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), IS NOT DISTINCT FROM($4, $19))], joinType=[left]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) EnumerableCalc(expr#0..2=[{inputs}], __r_seq__=[$t2], __r_gender__=[$t0], __r_age__=[$t1]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml index 72f8f4d6ca7..324960f28dd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml @@ -15,26 +15,24 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) - EnumerableLimit(fetch=[10000]) - EnumerableMergeJoin(condition=[AND(=($11, $15), =($12, $16), =($13, $17), IS NOT DISTINCT FROM($4, $14))], joinType=[left]) - EnumerableSort(sort0=[$11], sort1=[$12], sort2=[$13], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t16, $t17)], expr#19=[+($t15, $t18)], proj#0..11=[{exprs}], __seg_id__=[$t19], $f16=[$t14]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) - EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + CalciteEnumerableTopK(sort0=[$11], dir0=[ASC], fetch=[10000]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $14), =($11, $15), =($12, $16), =($13, $17))], joinType=[left]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t16, $t17)], expr#19=[+($t15, $t18)], proj#0..11=[{exprs}], __seg_id__=[$t19], $f16=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) + EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_ilike_function.yaml index 0b13885f73f..3c8bd027e33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_ilike_function.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml index 9ca9c104e89..ce83c3e9cc6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[LIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[LIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->LIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCBnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMSUtFIiwKICAgICJraW5kIjogIkxJS0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->LIKE($2, '%Holmes%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCBnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMSUtFIiwKICAgICJraW5kIjogIkxJS0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function_case_insensitive.yaml index 0b13885f73f..3c8bd027e33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function_case_insensitive.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml index 4f14b591721..0818c18eabb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml @@ -30,7 +30,7 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t0, $t3, $t4)], host=[$t1], cpu_usage=[$t2], @timestamp0=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp","host","cpu_usage"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], avg(cpu_usage)=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml index 6e68f5335d1..f26cb9e5822 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml @@ -29,7 +29,7 @@ calcite: EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t0, $t2, $t3)], host=[$t1], @timestamp0=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"_source":{"includes":["@timestamp","host"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0), PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0), PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap.yaml new file mode 100644 index 00000000000..62b38aafcfe --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), MOD(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), 86400:BIGINT)), MOD(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, 86400:BIGINT)))], avg(cpu_usage)=[$1], __base_offset__=[CAST(FLOOR(/(CAST(-(1720029600, MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER ())):DOUBLE NOT NULL, 86400:BIGINT))):BIGINT NOT NULL], __period__=[+(/(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], avg(cpu_usage)=[$1]) + LogicalAggregate(group=[{1}], avg(cpu_usage)=[AVG($0)]) + LogicalProject(cpu_usage=[$7], @timestamp0=[SPAN($1, 6, 'h')]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + LogicalFilter(condition=[<=($1, TIMESTAMP('2024-07-03 18:00:00':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[86400:BIGINT], expr#5=[MOD($t3, $t4)], expr#6=[-($t3, $t5)], expr#7=[+($t6, $t2)], expr#8=[FROM_UNIXTIME($t7)], expr#9=[1720029600:BIGINT], expr#10=[-($t9, $t3)], expr#11=[CAST($t10):DOUBLE NOT NULL], expr#12=[/($t11, $t4)], expr#13=[FLOOR($t12)], expr#14=[CAST($t13):BIGINT NOT NULL], expr#15=[-($t3, $t1)], expr#16=[/($t15, $t4)], expr#17=[1:BIGINT], expr#18=[+($t16, $t17)], @timestamp=[$t8], avg(cpu_usage)=[$t0], __base_offset__=[$t14], __period__=[$t18]) + EnumerableWindow(window#0=[window(aggs [MAX($1)])]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[UNIX_TIMESTAMP($t0)], expr#3=[CAST($t2):BIGINT NOT NULL], expr#4=[86400:BIGINT], expr#5=[MOD($t3, $t4)], avg(cpu_usage)=[$t1], $1=[$t3], $2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->AND(<=($0, '2024-07-03 18:00:00'), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},avg(cpu_usage)=AVG($0)), SORT->[0]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":null,"to":"2024-07-03T18:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"6h"}}}]},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap_month.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap_month.yaml new file mode 100644 index 00000000000..dc9b7eb1426 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap_month.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(-(CAST(UNIX_TIMESTAMP(MAX($0) OVER ())):BIGINT NOT NULL, *(-(EXTRACT('DAY', MAX($0) OVER ()), 1), 86400)), +(+(*(EXTRACT('HOUR', MAX($0) OVER ()), 3600), *(EXTRACT('MINUTE', MAX($0) OVER ()), 60)), EXTRACT('SECOND', MAX($0) OVER ()))), -(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, -(-(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, *(-(EXTRACT('DAY', $0), 1), 86400)), +(+(*(EXTRACT('HOUR', $0), 3600), *(EXTRACT('MINUTE', $0), 60)), EXTRACT('SECOND', $0))))))], avg(cpu_usage)=[$1], __base_offset__=[-(24295, +(*(EXTRACT('YEAR', MAX($0) OVER ()), 12), EXTRACT('MONTH', MAX($0) OVER ())))], __period__=[+(-(+(*(EXTRACT('YEAR', MAX($0) OVER ()), 12), EXTRACT('MONTH', MAX($0) OVER ())), +(*(EXTRACT('YEAR', $0), 12), EXTRACT('MONTH', $0))), 1)]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], avg(cpu_usage)=[$1]) + LogicalAggregate(group=[{1}], avg(cpu_usage)=[AVG($0)]) + LogicalProject(cpu_usage=[$7], @timestamp0=[SPAN($1, 1, 'd')]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + LogicalFilter(condition=[<=($1, TIMESTAMP('2024-07-03 18:00:00':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[UNIX_TIMESTAMP($t4)], expr#6=[CAST($t5):BIGINT NOT NULL], expr#7=['DAY'], expr#8=[EXTRACT($t7, $t4)], expr#9=[1:BIGINT], expr#10=[-($t8, $t9)], expr#11=[86400:BIGINT], expr#12=[*($t10, $t11)], expr#13=[-($t6, $t12)], expr#14=['HOUR'], expr#15=[EXTRACT($t14, $t4)], expr#16=[3600:BIGINT], expr#17=[*($t15, $t16)], expr#18=['MINUTE'], expr#19=[EXTRACT($t18, $t4)], expr#20=[60:BIGINT], expr#21=[*($t19, $t20)], expr#22=[+($t17, $t21)], expr#23=['SECOND'], expr#24=[EXTRACT($t23, $t4)], expr#25=[+($t22, $t24)], expr#26=[-($t13, $t25)], expr#27=[+($t26, $t2)], expr#28=[FROM_UNIXTIME($t27)], expr#29=[24295:BIGINT], expr#30=['YEAR'], expr#31=[EXTRACT($t30, $t4)], expr#32=[12:BIGINT], expr#33=[*($t31, $t32)], expr#34=['MONTH'], expr#35=[EXTRACT($t34, $t4)], expr#36=[+($t33, $t35)], expr#37=[-($t29, $t36)], expr#38=[-($t36, $t3)], expr#39=[+($t38, $t9)], @timestamp=[$t28], avg(cpu_usage)=[$t1], __base_offset__=[$t37], __period__=[$t39]) + EnumerableWindow(window#0=[window(aggs [MAX($0)])]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[UNIX_TIMESTAMP($t0)], expr#3=[CAST($t2):BIGINT NOT NULL], expr#4=['DAY'], expr#5=[EXTRACT($t4, $t0)], expr#6=[1:BIGINT], expr#7=[-($t5, $t6)], expr#8=[86400:BIGINT], expr#9=[*($t7, $t8)], expr#10=[-($t3, $t9)], expr#11=['HOUR'], expr#12=[EXTRACT($t11, $t0)], expr#13=[3600:BIGINT], expr#14=[*($t12, $t13)], expr#15=['MINUTE'], expr#16=[EXTRACT($t15, $t0)], expr#17=[60:BIGINT], expr#18=[*($t16, $t17)], expr#19=[+($t14, $t18)], expr#20=['SECOND'], expr#21=[EXTRACT($t20, $t0)], expr#22=[+($t19, $t21)], expr#23=[-($t10, $t22)], expr#24=[-($t3, $t23)], expr#25=['YEAR'], expr#26=[EXTRACT($t25, $t0)], expr#27=[12:BIGINT], expr#28=[*($t26, $t27)], expr#29=['MONTH'], expr#30=[EXTRACT($t29, $t0)], expr#31=[+($t28, $t30)], proj#0..1=[{exprs}], $2=[$t24], $3=[$t31]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->AND(<=($0, '2024-07-03 18:00:00'), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},avg(cpu_usage)=AVG($0)), SORT->[0]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":null,"to":"2024-07-03T18:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml index 95e83cdcd19..fe925e0a80a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml @@ -31,7 +31,7 @@ calcite: EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml index 274186e377e..beb3275a6c6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml @@ -31,7 +31,7 @@ calcite: EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml index 76b833ce3f1..f47e62e81d0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml @@ -30,7 +30,7 @@ calcite: EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t2)], expr#15=[AND($t13, $t14)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml index 027d0e30124..2d716443c5a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml @@ -29,7 +29,7 @@ calcite: EnumerableCalc(expr#0..171=[{inputs}], expr#172=[IS NOT NULL($t23)], expr#173=[IS NOT NULL($t163)], expr#174=[AND($t172, $t173)], proj#0..171=[{exprs}], $condition=[$t174]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml index a694c63b2ca..28bee6245d4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10:BIGINT], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], expr#5=[10], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t5], age=[$t0]) EnumerableAggregate(group=[{8}], count()=[COUNT()]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml index 285d0b221e1..1db12fc013f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100:BIGINT], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) EnumerableAggregate(group=[{4}], sum_SUM=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index bf861c337b9..655e16839ed 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml index 5c479c6867e..575e40c0f77 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], age=[$t0]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml index a95c277b40e..2eb653bfbb6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(age=[$10], age2=[+($10, $7)]) + LogicalProject(age=[$10], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml index ef4ea5fc43e..723ca37b3e5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10], age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml index 7df4a4d7f4e..87baf3c8399 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml index 711608264eb..1a9c720ee8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$14], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], age3=[$20]) LogicalSort(sort0=[$20], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], age3=[-(+($10, $7), $10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], age3=[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$14], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], expr#20=[-($t19, $t10)], proj#0..12=[{exprs}], age2=[$t19], age3=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], expr#21=[-($t20, $t19)], proj#0..12=[{exprs}], age2=[$t20], age3=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml index 362f847ae6e..156ec314c53 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml @@ -5,11 +5,11 @@ calcite: LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[$19], balance2=[ABS($7)]) LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..13=[{inputs}], expr#14=[ABS($t7)], proj#0..14=[{exprs}]) EnumerableSort(sort0=[$10], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], proj#0..12=[{exprs}], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], proj#0..12=[{exprs}], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest.json index e2ea6b3ddb5..f17a86f9a0c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $2) OVER (PARTITION BY $1)], latest_message=[ARG_MAX($3, $2) OVER (PARTITION BY $1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12])\n EnumerableWindow(window#0=[window(partition {1} aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t11], latest_message=[$t12])\n EnumerableWindow(window#0=[window(partition {1} aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_custom_time.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_custom_time.json index 27849cce681..d5626e8129c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_custom_time.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_custom_time.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $0) OVER (PARTITION BY $4)], latest_message=[ARG_MAX($3, $0) OVER (PARTITION BY $4)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12])\n EnumerableWindow(window#0=[window(partition {4} aggs [ARG_MIN($3, $0), ARG_MAX($3, $0)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t11], latest_message=[$t12])\n EnumerableWindow(window#0=[window(partition {4} aggs [ARG_MIN($3, $0), ARG_MAX($3, $0)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_no_group.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_no_group.json index 034699c80e9..181d34a5474 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_no_group.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_no_group.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $2) OVER ()], latest_message=[ARG_MAX($3, $2) OVER ()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12])\n EnumerableWindow(window#0=[window(aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t11], latest_message=[$t12])\n EnumerableWindow(window#0=[window(aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml index 400bd549ee8..18146bd8e9c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml @@ -13,13 +13,12 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableHashJoin(condition=[=($1, $4)], joinType=[semi]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[$cor0], expr#4=[$t3.id], expr#5=[=($t4, $t1)], i=[$t2], $condition=[$t5]) - EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..1=[{exprs}], $condition=[$t11]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[10000], expr#12=[<=($t10, $t11)], expr#13=[IS NOT NULL($t1)], expr#14=[AND($t12, $t13)], proj#0..10=[{exprs}], $condition=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml index 90492abbaf8..a1b5e3d3ed5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml @@ -6,5 +6,5 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2:BIGINT], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml index cb17a67d1cd..f64f0d94dbe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml @@ -12,15 +12,15 @@ calcite: })], variablesSet=[[$cor0]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[=($t0, $t3)], proj#0..3=[{exprs}], $condition=[$t4]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) - EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) - EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..1=[{exprs}], $condition=[$t11]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + EnumerableHashJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[semi]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['Tom':VARCHAR], expr#2=[CAST($t1):VARCHAR], name=[$t2], uid=[$t0]) + EnumerableAggregate(group=[{1}]) + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[10000], expr#12=[<=($t10, $t11)], expr#13=[IS NOT NULL($t1)], expr#14=[AND($t12, $t13)], proj#0..10=[{exprs}], $condition=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml index f8b576cb814..13eb24eadfe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%':VARCHAR], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml index 2d164b50d29..68a1125b16b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[LIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[LIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[LIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%':VARCHAR], expr#18=['\'], expr#19=[LIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml index f8b576cb814..13eb24eadfe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%':VARCHAR], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml index fb3daa06769..2ee55737ac2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[-($t8, $t17)], ageMinus=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[-($t8, $t17)], ageMinus=[$t18]) EnumerableLimit(fetch=[5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml index 42e82eca514..f781995261c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -19,7 +19,7 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t8)], state=[$t1], age2=[$t11], $condition=[$t12]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t11)], state=[$t1], age2=[$t11], $condition=[$t12]) EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml index 5e76c380ff2..87249bc2d06 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml @@ -12,13 +12,13 @@ calcite: physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], id=[$t1], name=[$t0], count_dept=[$t6]) EnumerableLimit(fetch=[10000]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid=[$t0], count(name)=[$t5]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{2}]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{1}], count(name)=[COUNT($0)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], expr#11=[IS NOT NULL($t0)], expr#12=[AND($t10, $t11)], proj#0..9=[{exprs}], $condition=[$t12]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml index ba13359c44d..0c4b5d3f606 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0]) - LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalFilter(condition=[>(CAST($2):BIGINT, +($SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) @@ -12,9 +12,9 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) - EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + EnumerableNestedLoopJoin(condition=[>(CAST($1):BIGINT, +($2, 999))], joinType=[inner]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{}], count(name)=[COUNT($0)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t0)], proj#0..9=[{exprs}], $condition=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json index adb4cb6244d..b48850334a2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[2], expr#20=[+($t10, $t19)], age=[$t10], age2=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[2:BIGINT], expr#21=[+($t19, $t20)], age=[$t10], age2=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml index 8897a1023cc..30be0be9ca7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) - LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) + LogicalJoin(condition=[=(+(CAST($10):BIGINT, 1), -($20, 20))], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) @@ -13,9 +13,9 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) EnumerableSort(sort0=[$13], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t10, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[1:BIGINT], expr#21=[+($t19, $t20)], proj#0..12=[{exprs}], $f13=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$13], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[20], expr#20=[-($t7, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[20:BIGINT], expr#20=[-($t7, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) EnumerableLimit(fetch=[50000]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json index 67cf82580e9..2fffb64bef5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(b=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], b=[+($7, 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t7, $t19)], b=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1:BIGINT], expr#20=[+($t7, $t19)], b=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml index 873a778f979..df4e941dfbe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[$20]) LogicalSort(sort0=[$19], sort1=[$20], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], balance2=[+($7, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], balance2=[+($7, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], expr#20=[1], expr#21=[+($t7, $t20)], proj#0..12=[{exprs}], age2=[$t19], balance2=[$t21]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], expr#21=[1:BIGINT], expr#22=[+($t7, $t21)], proj#0..12=[{exprs}], age2=[$t20], balance2=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml index 6ffa5ad304c..95539d91aa7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml @@ -5,6 +5,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..17=[{inputs}], proj#0..10=[{exprs}], $11=[$t17]) + EnumerableCalc(expr#0..17=[{inputs}], proj#0..10=[{exprs}], distinct_states=[$t17]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [DISTINCT_COUNT_APPROX($7)])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml index 79dcbca7555..5ba8abf2038 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml @@ -5,6 +5,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12]) + EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t11], latest_message=[$t12]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml index c56cd5d1bce..0bf9a2c50ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml @@ -14,7 +14,7 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) - EnumerableNestedLoopJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), IS NOT DISTINCT FROM($4, $19))], joinType=[left]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) EnumerableCalc(expr#0..17=[{inputs}], __r_seq__=[$t17], __r_gender__=[$t4], __r_age__=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml index 5664cc6aa87..3ec98ba9382 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml @@ -16,25 +16,24 @@ calcite: physical: | EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) - EnumerableMergeJoin(condition=[AND(=($11, $15), =($12, $16), =($13, $17), IS NOT DISTINCT FROM($4, $14))], joinType=[left]) - EnumerableSort(sort0=[$11], sort1=[$12], sort2=[$13], dir0=[ASC], dir1=[ASC], dir2=[ASC]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $14), =($11, $15), =($12, $16), =($13, $17))], joinType=[left]) + EnumerableSort(sort0=[$11], dir0=[ASC]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t16, $t17)], expr#19=[+($t15, $t18)], proj#0..11=[{exprs}], __seg_id__=[$t19], $f16=[$t14]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $14=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) + EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml index 41638cd1b16..a79db7b3383 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%':VARCHAR], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml index 6be02086bb0..1a450917474 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[LIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[LIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[LIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%':VARCHAR], expr#18=['\'], expr#19=[LIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml index 41638cd1b16..a79db7b3383 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%':VARCHAR], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml index e982ce038e2..da0b6d37d16 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml @@ -31,7 +31,7 @@ calcite: EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], host=[$t4], cpu_usage=[$t7], @timestamp0=[$t18], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml index 2979778506a..6556ef9022f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml @@ -30,7 +30,7 @@ calcite: EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], host=[$t4], @timestamp0=[$t18], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{4}], __grand_total__=[COUNT()]) EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t1)], expr#17=[IS NOT NULL($t4)], expr#18=[AND($t16, $t17)], proj#0..15=[{exprs}], $condition=[$t18]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap.yaml new file mode 100644 index 00000000000..1775322b2c6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap.yaml @@ -0,0 +1,22 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), MOD(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), 86400:BIGINT)), MOD(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, 86400:BIGINT)))], avg(cpu_usage)=[$1], __base_offset__=[CAST(FLOOR(/(CAST(-(1720029600, MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER ())):DOUBLE NOT NULL, 86400:BIGINT))):BIGINT NOT NULL], __period__=[+(/(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], avg(cpu_usage)=[$1]) + LogicalAggregate(group=[{1}], avg(cpu_usage)=[AVG($0)]) + LogicalProject(cpu_usage=[$7], @timestamp0=[SPAN($1, 6, 'h')]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + LogicalFilter(condition=[<=($1, TIMESTAMP('2024-07-03 18:00:00':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[86400:BIGINT], expr#5=[MOD($t3, $t4)], expr#6=[-($t3, $t5)], expr#7=[+($t6, $t2)], expr#8=[FROM_UNIXTIME($t7)], expr#9=[1720029600:BIGINT], expr#10=[-($t9, $t3)], expr#11=[CAST($t10):DOUBLE NOT NULL], expr#12=[/($t11, $t4)], expr#13=[FLOOR($t12)], expr#14=[CAST($t13):BIGINT NOT NULL], expr#15=[-($t3, $t1)], expr#16=[/($t15, $t4)], expr#17=[1:BIGINT], expr#18=[+($t16, $t17)], @timestamp=[$t8], avg(cpu_usage)=[$t0], __base_offset__=[$t14], __period__=[$t18]) + EnumerableWindow(window#0=[window(aggs [MAX($1)])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], expr#8=[UNIX_TIMESTAMP($t0)], expr#9=[CAST($t8):BIGINT NOT NULL], expr#10=[86400:BIGINT], expr#11=[MOD($t9, $t10)], avg(cpu_usage)=[$t7], $1=[$t9], $2=[$t11]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[6], expr#17=['h'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=['2024-07-03 18:00:00':EXPR_TIMESTAMP VARCHAR], expr#20=[<=($t1, $t19)], expr#21=[IS NOT NULL($t7)], expr#22=[AND($t20, $t21)], cpu_usage=[$t7], @timestamp0=[$t18], $condition=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap_month.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap_month.yaml new file mode 100644 index 00000000000..df3e91e740f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap_month.yaml @@ -0,0 +1,22 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(-(CAST(UNIX_TIMESTAMP(MAX($0) OVER ())):BIGINT NOT NULL, *(-(EXTRACT('DAY', MAX($0) OVER ()), 1), 86400)), +(+(*(EXTRACT('HOUR', MAX($0) OVER ()), 3600), *(EXTRACT('MINUTE', MAX($0) OVER ()), 60)), EXTRACT('SECOND', MAX($0) OVER ()))), -(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, -(-(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, *(-(EXTRACT('DAY', $0), 1), 86400)), +(+(*(EXTRACT('HOUR', $0), 3600), *(EXTRACT('MINUTE', $0), 60)), EXTRACT('SECOND', $0))))))], avg(cpu_usage)=[$1], __base_offset__=[-(24295, +(*(EXTRACT('YEAR', MAX($0) OVER ()), 12), EXTRACT('MONTH', MAX($0) OVER ())))], __period__=[+(-(+(*(EXTRACT('YEAR', MAX($0) OVER ()), 12), EXTRACT('MONTH', MAX($0) OVER ())), +(*(EXTRACT('YEAR', $0), 12), EXTRACT('MONTH', $0))), 1)]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], avg(cpu_usage)=[$1]) + LogicalAggregate(group=[{1}], avg(cpu_usage)=[AVG($0)]) + LogicalProject(cpu_usage=[$7], @timestamp0=[SPAN($1, 1, 'd')]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + LogicalFilter(condition=[<=($1, TIMESTAMP('2024-07-03 18:00:00':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[UNIX_TIMESTAMP($t4)], expr#6=[CAST($t5):BIGINT NOT NULL], expr#7=['DAY'], expr#8=[EXTRACT($t7, $t4)], expr#9=[1:BIGINT], expr#10=[-($t8, $t9)], expr#11=[86400:BIGINT], expr#12=[*($t10, $t11)], expr#13=[-($t6, $t12)], expr#14=['HOUR'], expr#15=[EXTRACT($t14, $t4)], expr#16=[3600:BIGINT], expr#17=[*($t15, $t16)], expr#18=['MINUTE'], expr#19=[EXTRACT($t18, $t4)], expr#20=[60:BIGINT], expr#21=[*($t19, $t20)], expr#22=[+($t17, $t21)], expr#23=['SECOND'], expr#24=[EXTRACT($t23, $t4)], expr#25=[+($t22, $t24)], expr#26=[-($t13, $t25)], expr#27=[+($t26, $t2)], expr#28=[FROM_UNIXTIME($t27)], expr#29=[24295:BIGINT], expr#30=['YEAR'], expr#31=[EXTRACT($t30, $t4)], expr#32=[12:BIGINT], expr#33=[*($t31, $t32)], expr#34=['MONTH'], expr#35=[EXTRACT($t34, $t4)], expr#36=[+($t33, $t35)], expr#37=[-($t29, $t36)], expr#38=[-($t36, $t3)], expr#39=[+($t38, $t9)], @timestamp=[$t28], avg(cpu_usage)=[$t1], __base_offset__=[$t37], __period__=[$t39]) + EnumerableWindow(window#0=[window(aggs [MAX($0)])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], expr#8=[UNIX_TIMESTAMP($t0)], expr#9=[CAST($t8):BIGINT NOT NULL], expr#10=['DAY'], expr#11=[EXTRACT($t10, $t0)], expr#12=[1:BIGINT], expr#13=[-($t11, $t12)], expr#14=[86400:BIGINT], expr#15=[*($t13, $t14)], expr#16=[-($t9, $t15)], expr#17=['HOUR'], expr#18=[EXTRACT($t17, $t0)], expr#19=[3600:BIGINT], expr#20=[*($t18, $t19)], expr#21=['MINUTE'], expr#22=[EXTRACT($t21, $t0)], expr#23=[60:BIGINT], expr#24=[*($t22, $t23)], expr#25=[+($t20, $t24)], expr#26=['SECOND'], expr#27=[EXTRACT($t26, $t0)], expr#28=[+($t25, $t27)], expr#29=[-($t16, $t28)], expr#30=[-($t9, $t29)], expr#31=['YEAR'], expr#32=[EXTRACT($t31, $t0)], expr#33=[12:BIGINT], expr#34=[*($t32, $t33)], expr#35=['MONTH'], expr#36=[EXTRACT($t35, $t0)], expr#37=[+($t34, $t36)], @timestamp0=[$t0], avg(cpu_usage)=[$t7], $2=[$t30], $3=[$t37]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['d'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=['2024-07-03 18:00:00':EXPR_TIMESTAMP VARCHAR], expr#20=[<=($t1, $t19)], expr#21=[IS NOT NULL($t7)], expr#22=[AND($t20, $t21)], cpu_usage=[$t7], @timestamp0=[$t18], $condition=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/test/resources/game_of_thrones_complex.json b/integ-test/src/test/resources/game_of_thrones_complex.json index 240344e25eb..7be567b6de9 100644 --- a/integ-test/src/test/resources/game_of_thrones_complex.json +++ b/integ-test/src/test/resources/game_of_thrones_complex.json @@ -1,14 +1,14 @@ {"index":{"_id":"1"}} -{"name":{"firstname":"Daenerys","lastname":"Targaryen","ofHerName":1},"nickname":"Daenerys \"Stormborn\"","house":"Targaryen","gender":"F","parents":{"father":"Aerys" , "mother":"Rhaella"},"titles":["motherOfDragons","queenOfTheAndals","breakerOfChains","Khaleesi"]} +{"name": {"firstname": "Daenerys", "lastname": "Targaryen", "ofHerName": 1}, "nickname": "Daenerys \"Stormborn\"", "house": "Targaryen", "gender": "F", "parents": {"father": "Aerys", "mother": "Rhaella"}} {"index":{"_id":"2"}} -{"name":{"firstname":"Eddard","lastname":"Stark","ofHisName":1},"house":"Stark", "parents":{"father":"Rickard" , "mother":"Lyarra"} ,"gender":"M","titles":["lordOfWinterfell","wardenOfTheNorth","handOfTheKing"]} +{"name": {"firstname": "Eddard", "lastname": "Stark", "ofHisName": 1}, "house": "Stark", "parents": {"father": "Rickard", "mother": "Lyarra"}, "gender": "M"} {"index":{"_id":"3"}} -{"name":{"firstname":"Brandon","lastname":"Stark","ofHisName":4},"house":"Stark","parents":{"father":"Eddard","mother":"Catelyn"},"gender":"M","titles":["princeOfWinterfell"],"@wolf":"Summer"} +{"name": {"firstname": "Brandon", "lastname": "Stark", "ofHisName": 4}, "house": "Stark", "parents": {"father": "Eddard", "mother": "Catelyn"}, "gender": "M", "@wolf": "Summer"} {"index":{"_id":"4"}} -{"name":{"firstname":"Jaime","lastname":"Lannister","ofHisName":1},"gender":"M","house":"Lannister","parents":{"father":"Tywin","mother":"Joanna"},"titles":["kingSlayer","lordCommanderOfTheKingsguard","Ser"]} +{"name": {"firstname": "Jaime", "lastname": "Lannister", "ofHisName": 1}, "gender": "M", "house": "Lannister", "parents": {"father": "Tywin", "mother": "Joanna"}} {"index":{"_id":"5"}} -{"words":"fireAndBlood","hname":"Targaryen","sigil":"Dragon","seat":"Dragonstone"} +{"words": "fireAndBlood", "hname": "Targaryen", "sigil": "Dragon", "seat": "Dragonstone"} {"index":{"_id":"6"}} -{"words":"winterIsComing" , "hname":"Stark","sigil":"direwolf","seat":"Winterfell"} +{"words": "winterIsComing", "hname": "Stark", "sigil": "direwolf", "seat": "Winterfell"} {"index":{"_id":"7"}} -{"words":"hearMeRoar" , "hname":"Lannister","sigil":"lion","seat":"CasterlyRock"} +{"words": "hearMeRoar", "hname": "Lannister", "sigil": "lion", "seat": "CasterlyRock"} diff --git a/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json new file mode 100644 index 00000000000..02fbab79a3c --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json @@ -0,0 +1,29 @@ +{ + "ruleId": "unsupported-window-function-in-eventstats", + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "eventstats-rank", + "query": "source={{index}} | eventstats rank() as rank_value", + "frontendDiagnosticCount": 1, + "backendExpected": { + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } + } + } + }, + { + "id": "eventstats-avg-control", + "query": "source={{index}} | eventstats avg(age) as avg_age", + "frontendDiagnosticCount": 0, + "backendExpected": { + "httpStatus": 200 + } + } + ] +} diff --git a/integ-test/src/test/resources/timewrap_test.json b/integ-test/src/test/resources/timewrap_test.json new file mode 100644 index 00000000000..5ba82f8fabf --- /dev/null +++ b/integ-test/src/test/resources/timewrap_test.json @@ -0,0 +1,66 @@ +{"index":{"_id":"1"}} +{"@timestamp":"2024-07-01T00:00:00","host":"web-01","requests":100,"errors":2} +{"index":{"_id":"2"}} +{"@timestamp":"2024-07-01T06:00:00","host":"web-01","requests":150,"errors":5} +{"index":{"_id":"3"}} +{"@timestamp":"2024-07-01T12:00:00","host":"web-01","requests":200,"errors":3} +{"index":{"_id":"4"}} +{"@timestamp":"2024-07-01T18:00:00","host":"web-01","requests":120,"errors":1} +{"index":{"_id":"5"}} +{"@timestamp":"2024-07-01T00:00:00","host":"web-02","requests":80,"errors":0} +{"index":{"_id":"6"}} +{"@timestamp":"2024-07-01T06:00:00","host":"web-02","requests":90,"errors":1} +{"index":{"_id":"7"}} +{"@timestamp":"2024-07-01T12:00:00","host":"web-02","requests":110,"errors":2} +{"index":{"_id":"8"}} +{"@timestamp":"2024-07-01T18:00:00","host":"web-02","requests":70,"errors":0} +{"index":{"_id":"9"}} +{"@timestamp":"2024-07-02T00:00:00","host":"web-01","requests":110,"errors":3} +{"index":{"_id":"10"}} +{"@timestamp":"2024-07-02T06:00:00","host":"web-01","requests":160,"errors":4} +{"index":{"_id":"11"}} +{"@timestamp":"2024-07-02T12:00:00","host":"web-01","requests":210,"errors":6} +{"index":{"_id":"12"}} +{"@timestamp":"2024-07-02T18:00:00","host":"web-01","requests":130,"errors":2} +{"index":{"_id":"13"}} +{"@timestamp":"2024-07-02T00:00:00","host":"web-02","requests":95,"errors":1} +{"index":{"_id":"14"}} +{"@timestamp":"2024-07-02T06:00:00","host":"web-02","requests":100,"errors":2} +{"index":{"_id":"15"}} +{"@timestamp":"2024-07-02T12:00:00","host":"web-02","requests":120,"errors":3} +{"index":{"_id":"16"}} +{"@timestamp":"2024-07-02T18:00:00","host":"web-02","requests":85,"errors":1} +{"index":{"_id":"17"}} +{"@timestamp":"2024-07-03T00:00:00","host":"web-01","requests":90,"errors":1} +{"index":{"_id":"18"}} +{"@timestamp":"2024-07-03T06:00:00","host":"web-01","requests":140,"errors":2} +{"index":{"_id":"19"}} +{"@timestamp":"2024-07-03T12:00:00","host":"web-01","requests":180,"errors":4} +{"index":{"_id":"20"}} +{"@timestamp":"2024-07-03T18:00:00","host":"web-01","requests":100,"errors":1} +{"index":{"_id":"21"}} +{"@timestamp":"2024-07-03T00:00:00","host":"web-02","requests":75,"errors":0} +{"index":{"_id":"22"}} +{"@timestamp":"2024-07-03T06:00:00","host":"web-02","requests":85,"errors":1} +{"index":{"_id":"23"}} +{"@timestamp":"2024-07-03T12:00:00","host":"web-02","requests":105,"errors":2} +{"index":{"_id":"24"}} +{"@timestamp":"2024-07-03T18:00:00","host":"web-02","requests":65,"errors":0} +{"index":{"_id":"25"}} +{"@timestamp":"2024-07-04T00:00:00","host":"web-01","requests":50,"errors":0} +{"index":{"_id":"26"}} +{"@timestamp":"2024-07-04T06:00:00","host":"web-01","requests":60,"errors":1} +{"index":{"_id":"27"}} +{"@timestamp":"2024-07-04T00:00:00","host":"web-02","requests":30,"errors":0} +{"index":{"_id":"28"}} +{"@timestamp":"2024-07-04T06:00:00","host":"web-02","requests":40,"errors":0} +{"index":{"_id":"29"}} +{"@timestamp":"2024-01-15T12:00:00","host":"web-01","requests":300,"errors":5} +{"index":{"_id":"30"}} +{"@timestamp":"2024-04-15T12:00:00","host":"web-01","requests":350,"errors":3} +{"index":{"_id":"31"}} +{"@timestamp":"2024-06-15T12:00:00","host":"web-01","requests":200,"errors":2} +{"index":{"_id":"32"}} +{"@timestamp":"2024-08-15T12:00:00","host":"web-01","requests":250,"errors":4} +{"index":{"_id":"33"}} +{"@timestamp":"2025-01-15T12:00:00","host":"web-01","requests":400,"errors":6} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml new file mode 100644 index 00000000000..930e61fdf75 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml @@ -0,0 +1,170 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5164 +# Integer/long arithmetic (+, -, *) must not silently wrap on overflow. +# +# byte/short/int overflow is prevented by operand widening (short/byte -> INT, any int/long -> +# BIGINT) in PPLFuncImpTable, so those produce the correct wider value (HTTP 200). long (BIGINT) +# arithmetic has no wider integer type to widen into, so overflow is detected via checked +# arithmetic (Math.addExact / multiplyExact) and surfaced as a 4xx client error instead of wrapping. + +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + - do: + indices.create: + index: test_overflow_5164 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + int_field: + type: integer + long_field: + type: long + small_int: + type: integer + short_field: + type: short + - do: + bulk: + index: test_overflow_5164 + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"int_field": 2147483647, "long_field": 9223372036854775807, "small_int": 10, "short_field": 30000}' + - '{"index": {"_id": "2"}}' + - '{"int_field": 100, "long_field": 200, "small_int": 5, "short_field": 3}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + - do: + indices.delete: + index: test_overflow_5164 + ignore_unavailable: true + +--- +"Normal integer addition does not error": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where small_int = 5 | eval sum = int_field + small_int | fields int_field, small_int, sum + - match: { total: 1 } + - match: { datarows: [[100, 5, 105]] } + +--- +"Integer addition overflow widens instead of wrapping": + - skip: + features: + - headers + # int + int is widened to BIGINT before the operation; 2147483647 + 1 = 2147483648 fits exactly. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where int_field = 2147483647 | eval widened = int_field + 1 | fields widened + - match: { total: 1 } + - match: { datarows: [[2147483648]] } + +--- +"Integer multiplication overflow widens instead of wrapping": + - skip: + features: + - headers + # 2147483647 * 2 = 4294967294 exceeds INT range but is exact once widened to BIGINT. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where int_field = 2147483647 | eval widened = int_field * 2 | fields widened + - match: { total: 1 } + - match: { datarows: [[4294967294]] } + +--- +"Short multiplication overflow widens instead of wrapping": + - skip: + features: + - headers + # short * short historically wrapped into the 16-bit SMALLINT range (bug bash #8: 30000 * 30000 + # wrapped). Widening to INT keeps the exact value 900000000. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where short_field = 30000 | eval widened = short_field * short_field | fields widened + - match: { total: 1 } + - match: { datarows: [[900000000]] } + +--- +"Long addition overflow throws error": + - skip: + features: + - headers + # long + long has no wider integer type; overflow is detected and surfaced as a 4xx. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field + 1 | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Long multiplication overflow throws error": + - skip: + features: + - headers + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field * 2 | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Long subtraction overflow throws error": + - skip: + features: + - headers + # long_field is i64::MAX; subtracting -1 overflows past the top of the BIGINT range. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field - (-1) | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Normal long arithmetic does not error": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 200 | eval product = long_field * 2 | fields product + - match: { total: 1 } + - match: { datarows: [[400]] } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 68350c5a0fd..3b9c3619521 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,4 +114,105 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); + + /** + * Read-only cluster health snapshot for the {@code rest} command (backs {@code + * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's + * security thread-context; performs no privilege escalation and mutates nothing. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of health field name to value + */ + default Map clusterHealth(Map params) { + throw new UnsupportedOperationException("clusterHealth is not supported by this client"); + } + + /** + * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One + * map per index. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per index + */ + default List> catIndices(Map params) { + throw new UnsupportedOperationException("catIndices is not supported by this client"); + } + + /** + * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map + * per node with resource state. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per node + */ + default List> catNodes(Map params) { + throw new UnsupportedOperationException("catNodes is not supported by this client"); + } + + /** + * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code + * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map describing the cluster manager node + */ + default List> catClusterManager(Map params) { + throw new UnsupportedOperationException("catClusterManager is not supported by this client"); + } + + /** + * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One + * map per installed plugin per node. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per plugin + */ + default List> catPlugins(Map params) { + throw new UnsupportedOperationException("catPlugins is not supported by this client"); + } + + /** + * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map + * per shard. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per shard + */ + default List> catShards(Map params) { + throw new UnsupportedOperationException("catShards is not supported by this client"); + } + + /** + * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code + * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, + * cluster_manager_node). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of cluster-state field name to value + */ + default Map clusterState(Map params) { + throw new UnsupportedOperationException("clusterState is not supported by this client"); + } + + /** + * Read-only cluster-settings listing for the {@code rest} command (backs {@code + * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per setting + */ + default List> clusterSettings(Map params) { + throw new UnsupportedOperationException("clusterSettings is not supported by this client"); + } + + /** + * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). + * One map per resolved index, alias, or data stream (name, type). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per resolved name + */ + default List> resolveIndex(Map params) { + throw new UnsupportedOperationException("resolveIndex is not supported by this client"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index b491f38ef80..080a8627894 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,6 +18,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -25,6 +27,7 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; @@ -285,4 +288,274 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); + return flattenHealth(response); + } + + @Override + public List> catIndices(Map params) { + ClusterHealthResponse response = + client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new java.util.LinkedHashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } + + @Override + public List> catNodes(Map params) { + org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = + new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); + statsRequest.all(); + org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = + client.admin().cluster().nodesStats(statsRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { + org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); + Map row = new java.util.LinkedHashMap<>(); + row.put("name", node.getName()); + row.put("ip", node.getHostAddress()); + row.put( + "node_role", + node.getRoles().stream() + .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) + .sorted() + .collect(java.util.stream.Collectors.joining(","))); + row.put( + "heap_percent", + ns.getJvm() == null || ns.getJvm().getMem() == null + ? null + : (int) ns.getJvm().getMem().getHeapUsedPercent()); + row.put( + "ram_percent", + ns.getOs() == null || ns.getOs().getMem() == null + ? null + : (int) ns.getOs().getMem().getUsedPercent()); + row.put( + "cpu", + ns.getProcess() == null || ns.getProcess().getCpu() == null + ? null + : (int) ns.getProcess().getCpu().getPercent()); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + List> rows = new java.util.ArrayList<>(); + if (cm != null) { + Map row = new java.util.LinkedHashMap<>(); + row.put("id", cm.getId()); + row.put("host", cm.getHostName()); + row.put("ip", cm.getHostAddress()); + row.put("node", cm.getName()); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = + new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); + infoRequest.all(); + org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = + client.admin().cluster().nodesInfo(infoRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { + org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = + info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); + if (plugins == null) { + continue; + } + for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", info.getNode().getName()); + row.put("component", pi.getName()); + row.put("version", pi.getVersion()); + rows.add(row); + } + } + return rows; + } + + @Override + public List> catShards(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.cluster.routing.ShardRouting sr : + response.getState().getRoutingTable().allShards()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("index", sr.getIndexName()); + row.put("shard", sr.id()); + row.put("prirep", sr.primary() ? "p" : "r"); + row.put("state", sr.state().name()); + org.opensearch.cluster.node.DiscoveryNode n = + sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); + row.put("node", n == null ? null : n.getName()); + rows.add(row); + } + return rows; + } + + @Override + public Map clusterState(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName().value()); + row.put("state_uuid", response.getState().stateUUID()); + row.put("version", response.getState().version()); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + row.put("cluster_manager_node", cm == null ? null : cm.getName()); + return row; + } + + @Override + public List> clusterSettings(Map params) { + // The transport path has no SettingsFilter of its own; it is published from + // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings + // into memory when we cannot redact them, matching native GET /_cluster/settings. + org.opensearch.common.settings.SettingsFilter filter = + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); + if (filter == null) { + throw new IllegalStateException( + "cluster settings redaction filter is not initialized; refusing to return unredacted" + + " settings"); + } + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + org.opensearch.common.settings.Settings persistent = + filter.filter(response.getState().metadata().persistentSettings()); + org.opensearch.common.settings.Settings transientSettings = + filter.filter(response.getState().metadata().transientSettings()); + collectSettings(persistent, "persistent", rows); + collectSettings(transientSettings, "transient", rows); + return rows; + } + + private void collectSettings( + org.opensearch.common.settings.Settings settings, + String tier, + List> rows) { + if (settings == null) { + return; + } + for (String key : settings.keySet()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("setting", key); + String value = settings.get(key); + if (value == null) { + // List-valued settings return null from get(); fall back to the joined list form. + java.util.List list = settings.getAsList(key); + value = list.isEmpty() ? null : String.join(",", list); + } + row.put("value", value); + row.put("tier", tier); + rows.add(row); + } + } + + @Override + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = + expandWildcards == null + ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}) + : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}, + org.opensearch.action.support.IndicesOptions.fromParameters( + expandWildcards, + null, + null, + null, + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request + .DEFAULT_INDICES_OPTIONS)); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = + client + .execute( + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : + response.getIndices()) { + rows.add(resolveRow(idx.getName(), "index")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : + response.getAliases()) { + rows.add(resolveRow(alias.getName(), "alias")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : + response.getDataStreams()) { + rows.add(resolveRow(ds.getName(), "data_stream")); + } + return rows; + } + + private Map resolveRow(String name, String type) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", name); + row.put("type", type); + return row; + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index f369c0003b8..e98c5bf95f4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,15 +8,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -28,6 +32,7 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -272,4 +277,267 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + try { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); + return flattenHealth(response); + } catch (IOException e) { + throw new IllegalStateException("Failed to get cluster health", e); + } + } + + @Override + public List> catIndices(Map params) { + try { + ClusterHealthResponse response = + client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); + List> rows = new ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new HashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } catch (IOException e) { + throw new IllegalStateException("Failed to get cat indices", e); + } + } + + @Override + public List> catNodes(Map params) { + List> raw = + catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("ip", r.get("ip")); + row.put("node_role", r.get("node.role")); + row.put("heap_percent", asInt(r.get("heap.percent"))); + row.put("ram_percent", asInt(r.get("ram.percent"))); + row.put("cpu", asInt(r.get("cpu"))); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("id", r.get("id")); + row.put("host", r.get("host")); + row.put("ip", r.get("ip")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + List> raw = catJson("/_cat/plugins", "name,component,version"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("component", r.get("component")); + row.put("version", r.get("version")); + rows.add(row); + } + return rows; + } + + @Override + public List> catShards(Map params) { + List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("index", r.get("index")); + row.put("shard", asInt(r.get("shard"))); + row.put("prirep", r.get("prirep")); + row.put("state", r.get("state")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ + @SuppressWarnings("unchecked") + private List> catJson(String path, String columns) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + request.addParameter("format", "json"); + request.addParameter("h", columns); + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + List list = parser.list(); + List> rows = new ArrayList<>(); + for (Object o : list) { + rows.add((Map) o); + } + return rows; + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Integer asInt(Object value) { + if (value == null) { + return null; + } + try { + return (int) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public Map clusterState(Map params) { + Map state = + getJsonMap( + "/_cluster/state/master_node,version,metadata,nodes", + // nodes.*.name resolves the manager id to a name without over-fetching node IPs. + Map.of( + "filter_path", + "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); + Map row = new HashMap<>(); + row.put("cluster_name", state.get("cluster_name")); + row.put("state_uuid", state.get("state_uuid")); + row.put("version", asLong(state.get("version"))); + Object cmId = state.get("cluster_manager_node"); + String cmName = null; + Object nodes = state.get("nodes"); + if (cmId != null && nodes instanceof Map) { + Object n = ((Map) nodes).get(cmId.toString()); + if (n instanceof Map) { + Object name = ((Map) n).get("name"); + cmName = name == null ? null : name.toString(); + } + } + row.put("cluster_manager_node", cmName); + return row; + } + + @Override + @SuppressWarnings("unchecked") + public List> clusterSettings(Map params) { + Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); + List> rows = new ArrayList<>(); + for (String tier : new String[] {"persistent", "transient"}) { + Object section = body.get(tier); + if (section instanceof Map) { + for (Map.Entry e : ((Map) section).entrySet()) { + Map row = new HashMap<>(); + row.put("setting", e.getKey()); + row.put("value", e.getValue() == null ? null : e.getValue().toString()); + row.put("tier", tier); + rows.add(row); + } + } + } + return rows; + } + + /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ + @SuppressWarnings("unchecked") + private Map getJsonMap(String path, Map params) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + if (params != null) { + params.forEach(request::addParameter); + } + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + return parser.map(); + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Long asLong(Object value) { + if (value == null) { + return null; + } + try { + return (long) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + Map body = + getJsonMap( + "/_resolve/index/*", + expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); + List> rows = new ArrayList<>(); + addResolved(body.get("indices"), "index", rows); + addResolved(body.get("aliases"), "alias", rows); + addResolved(body.get("data_streams"), "data_stream", rows); + return rows; + } + + @SuppressWarnings("unchecked") + private void addResolved(Object section, String type, List> rows) { + if (section instanceof List) { + for (Object o : (List) section) { + if (o instanceof Map) { + Map row = new HashMap<>(); + row.put("name", ((Map) o).get("name")); + row.put("type", type); + rows.add(row); + } + } + } + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new HashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index d772b3e603b..b4fa498e2ac 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -209,7 +209,11 @@ private ExprValue parse( final ExprType type = fieldType.get(); if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint))) { - return parseGeoPoint(content, supportArrays); + try { + return parseGeoPoint(content, supportArrays); + } catch (Exception e) { + return ExprNullValue.of(); + } } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) || content.isArray()) { return parseArray(content, field, type, supportArrays); @@ -217,9 +221,14 @@ private ExprValue parse( || type == STRUCT) { return parseStruct(content, field, supportArrays); } else if (typeActionMap.containsKey(type)) { - return content.isArray() - ? parseArray(content, field, type, supportArrays) - : typeActionMap.get(type).apply(content, type); + if (content.isArray()) { + return parseArray(content, field, type, supportArrays); + } + try { + return typeActionMap.get(type).apply(content, type); + } catch (Exception e) { + return ExprNullValue.of(); + } } else { throw new IllegalStateException( String.format( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index 32b7891d344..e8c7cfc7c68 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -5,6 +5,7 @@ package org.opensearch.sql.opensearch.executor; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Suppliers; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -24,6 +25,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.externalize.RelJsonWriter; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.runtime.Hook; @@ -42,6 +44,7 @@ import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.TimewrapPivot; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.model.ExprTupleValue; @@ -65,12 +68,14 @@ import org.opensearch.sql.opensearch.functions.DistinctCountApproxAggFunction; import org.opensearch.sql.opensearch.functions.GeoIpFunction; import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.protocol.response.format.Format; import org.opensearch.sql.storage.TableScanOperator; import org.opensearch.transport.client.node.NodeClient; /** OpenSearch execution engine implementation. */ public class OpenSearchExecutionEngine implements ExecutionEngine { private static final Logger logger = LogManager.getLogger(OpenSearchExecutionEngine.class); + private static final ObjectMapper objectMapper = new ObjectMapper(); private final OpenSearchClient client; @@ -167,38 +172,146 @@ private Hook.Closeable getCodegenInHook(AtomicReference codegen) { }); } + /** + * Parse sourceBuilder JSON strings within the physical plan tree to objects. This finds any + * sourceBuilder fields (which are serialized as JSON strings by RelJsonWriter) and parses them to + * JSON objects for easier client consumption. + */ + @SuppressWarnings("unchecked") + private void parseSourceBuilderInPhysicalTree(Object physicalTree) { + try { + if (!(physicalTree instanceof Map)) { + return; + } + Map tree = (Map) physicalTree; + Object relsObj = tree.get("rels"); + if (!(relsObj instanceof List)) { + return; + } + + List rels = (List) relsObj; + for (Object relObj : rels) { + if (!(relObj instanceof Map)) { + continue; + } + Map rel = (Map) relObj; + + // Parse sourceBuilder if it exists as a JSON string + Object sourceBuilderObj = rel.get("sourceBuilder"); + if (sourceBuilderObj instanceof String) { + try { + String sourceBuilderJson = (String) sourceBuilderObj; + Object parsed = objectMapper.readValue(sourceBuilderJson, Object.class); + rel.put("sourceBuilder", parsed); + } catch (Exception e) { + logger.debug("Failed to parse sourceBuilder JSON: {}", e.getMessage()); + } + } + } + } catch (Exception e) { + logger.warn("Failed to parse sourceBuilder in physical tree: " + e.getMessage()); + } + } + + @Override + public void explain( + RelNode rel, + ExplainMode mode, + CalcitePlanContext context, + ResponseListener listener) { + explain(rel, mode, null, context, listener); + } + @Override public void explain( RelNode rel, ExplainMode mode, + Format format, CalcitePlanContext context, ResponseListener listener) { client.schedule( () -> { try { - if (mode == ExplainMode.SIMPLE) { - String logical = RelOptUtil.toString(rel, SqlExplainLevel.NO_ATTRIBUTES); - listener.onResponse( - new ExplainResponse(new ExplainResponseNodeV2(logical, null, null))); + if (format == Format.JSON_TREE) { + // Use RelJsonWriter for structured JSON tree output + try { + RelJsonWriter logicalWriter = new RelJsonWriter(); + rel.explain(logicalWriter); + String logicalJson = logicalWriter.asString(); + + AtomicReference physicalJson = new AtomicReference<>(); + AtomicReference physicalError = new AtomicReference<>(); + SqlExplainLevel level = + mode == ExplainMode.COST + ? SqlExplainLevel.ALL_ATTRIBUTES + : SqlExplainLevel.EXPPLAN_ATTRIBUTES; + + try (Hook.Closeable closeable = + Hook.PLAN_BEFORE_IMPLEMENTATION.addThread( + obj -> { + try { + RelRoot relRoot = (RelRoot) obj; + RelJsonWriter physicalWriter = new RelJsonWriter(); + relRoot.rel.explain(physicalWriter); + physicalJson.set(physicalWriter.asString()); + } catch (Exception e) { + physicalError.set(e); + } + })) { + // triggers the hook + OpenSearchRelRunners.run(context, rel); + } + + if (physicalError.get() != null) { + throw physicalError.get(); + } + + // Parse JSON strings to objects for structured output + Object logicalTree = objectMapper.readValue(logicalJson, Object.class); + Object physicalTree = objectMapper.readValue(physicalJson.get(), Object.class); + + // Parse sourceBuilder JSON if present in physical plan + parseSourceBuilderInPhysicalTree(physicalTree); + + ExplainResponseNodeV2 response = + new ExplainResponseNodeV2(logicalJson, physicalJson.get(), null); + response.setLogicalTree(logicalTree); + response.setPhysicalTree(physicalTree); + + listener.onResponse(new ExplainResponse(response)); + } catch (Exception e) { + // RelJsonWriter can't handle some custom types (e.g., SystemLimitType enum) + listener.onFailure( + new UnsupportedOperationException( + "Cannot serialize plan to json_tree format: " + e.getMessage(), e)); + return; + } } else { - SqlExplainLevel level = - mode == ExplainMode.COST - ? SqlExplainLevel.ALL_ATTRIBUTES - : SqlExplainLevel.EXPPLAN_ATTRIBUTES; - String logical = RelOptUtil.toString(rel, level); - AtomicReference physical = new AtomicReference<>(); - AtomicReference javaCode = new AtomicReference<>(); - try (Hook.Closeable closeable = getPhysicalPlanInHook(physical, level)) { - if (mode == ExplainMode.EXTENDED) { - getCodegenInHook(javaCode); - CalcitePlanContext.skipEncoding.set(true); + // Original string format for json/yaml + if (mode == ExplainMode.SIMPLE) { + String logical = RelOptUtil.toString(rel, SqlExplainLevel.NO_ATTRIBUTES); + listener.onResponse( + new ExplainResponse(new ExplainResponseNodeV2(logical, null, null))); + } else { + SqlExplainLevel level = + mode == ExplainMode.COST + ? SqlExplainLevel.ALL_ATTRIBUTES + : SqlExplainLevel.EXPPLAN_ATTRIBUTES; + String logical = RelOptUtil.toString(rel, level); + AtomicReference physical = new AtomicReference<>(); + AtomicReference javaCode = new AtomicReference<>(); + try (Hook.Closeable closeable = getPhysicalPlanInHook(physical, level)) { + if (mode == ExplainMode.EXTENDED) { + getCodegenInHook(javaCode); + CalcitePlanContext.skipEncoding.set(true); + } + // triggers the hook + OpenSearchRelRunners.run(context, rel); } - // triggers the hook - OpenSearchRelRunners.run(context, rel); + listener.onResponse( + new ExplainResponse( + new ExplainResponseNodeV2(logical, physical.get(), javaCode.get()))); } - listener.onResponse( - new ExplainResponse( - new ExplainResponseNodeV2(logical, physical.get(), javaCode.get()))); } } catch (Exception e) { listener.onFailure(e); @@ -320,6 +433,23 @@ private QueryResponse buildResultSet( } columns.add(new Column(columnName, null, exprType)); } + // Timewrap post-processing: pivot unpivoted rows into period columns. The pivot is shared with + // the analytics route (AnalyticsExecutionEngine) so both engines produce identical output. + if (TimewrapPivot.isTimewrap()) { + try { + TimewrapPivot.Result pivoted = + TimewrapPivot.pivot( + columns, + values, + CalcitePlanContext.timewrapUnitName.get(), + CalcitePlanContext.timewrapSeries.get()); + columns = pivoted.columns(); + values = pivoted.values(); + } finally { + CalcitePlanContext.clearTimewrapSignals(); + } + } + Schema schema = new Schema(columns); QueryResponse response = new QueryResponse(schema, values, null); return response; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java new file mode 100644 index 00000000000..9905ca564ab --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteLogicalCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteScannableCatalogScan; + +/** + * Rule to convert a {@link CalciteLogicalCatalogScan} into an enumerable scan: a {@link + * CalciteScannableCatalogScan} when the source opts into {@code Scannable}, otherwise a plain + * {@link CalciteEnumerableCatalogScan}. + */ +public class EnumerableCatalogScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalCatalogScan.class, + s -> s.getCatalogTable() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableCatalogScanRule") + .withRuleFactory(EnumerableCatalogScanRule::new); + + protected EnumerableCatalogScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalCatalogScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalCatalogScan scan = (CalciteLogicalCatalogScan) rel; + if (scan.getCatalogTable().getSource().isScannable()) { + return new CalciteScannableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } + return new CalciteEnumerableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java deleted file mode 100644 index 616d1178873..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan; -import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan; - -/** - * Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link - * CalciteEnumerableSystemIndexScan}. - */ -public class EnumerableSystemIndexScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalSystemIndexScan.class, - s -> s.getSysIndex() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableSystemIndexScanRule") - .withRuleFactory(EnumerableSystemIndexScanRule::new); - - /** Creates an EnumerableProjectRule. */ - protected EnumerableSystemIndexScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalSystemIndexScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel; - return new CalciteEnumerableSystemIndexScan( - scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 3c8508cc455..c200ffa8909 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -12,8 +12,8 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = - EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule CATALOG_SCAN_RULE = + EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -26,7 +26,7 @@ public class OpenSearchIndexRules { public static final List OPEN_SEARCH_NON_PUSHDOWN_RULES = ImmutableList.of( INDEX_SCAN_RULE, - SYSTEM_INDEX_SCAN_RULE, + CATALOG_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index f919fdc0e30..775b0278683 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -38,6 +38,7 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -618,7 +619,31 @@ yield switch (functionName) { topHitsAggregationBuilder.sort( SortBuilders.fieldSort(key.field()).order(order).missing(missing)); } - yield Pair.of(topHitsAggregationBuilder, new TopHitsParser(aggName, false, false)); + // Build a mapping from original index field name to output (renamed) field names. + // top_hits returns _source / fields keyed by the original index name, but the Calcite + // row-type uses the renamed output name. A single original field may map to multiple + // output names when both a rename and an eval column-ref resolve to the same source + // field (issue #5197), so the value is a List rather than a single String. + Map> fieldNameMapping = new HashMap<>(); + for (Pair arg : args) { + if (arg.getKey() instanceof RexInputRef) { + NamedFieldExpression namedField = helper.inferNamedField(arg.getKey()); + if (namedField == null) { + continue; + } + String originalName = namedField.getRootName(); + String outputName = arg.getValue(); + if (!originalName.equals(outputName)) { + fieldNameMapping + .computeIfAbsent(originalName, k -> new ArrayList<>()) + .add(outputName); + } + } + } + yield Pair.of( + topHitsAggregationBuilder, + new TopHitsParser( + aggName, false, false, fieldNameMapping.isEmpty() ? null : fieldNameMapping)); } default -> throw new AggregateAnalyzer.AggregateAnalyzerException( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 67a31a7f56b..476e0018fcd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -1394,7 +1394,7 @@ public QueryExpression like(LiteralExpression literal, boolean caseSensitive) { .caseInsensitive(!caseSensitive); return this; } - throw new UnsupportedOperationException("Like query is not supported for text field"); + throw new PredicateAnalyzerException("Like query is not supported for text field"); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java index 0a47dc64a5e..03e2d546418 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java @@ -18,10 +18,13 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.TestOnly; import org.opensearch.action.search.SearchResponse; import org.opensearch.core.common.text.Text; @@ -42,6 +45,8 @@ @ToString public class OpenSearchResponse implements Iterable { + private static final Logger LOG = LogManager.getLogger(); + public static final OpenSearchResponse EMPTY = empty(); /** Search query result (non-aggregation). */ @@ -155,12 +160,18 @@ public Iterator iterator() { return Arrays.stream(hits.getHits()) .map( hit -> { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - addParsedHitsToBuilder(builder, hit); - addMetaDataFieldsToBuilder(builder, hit); - addHighlightsToBuilder(builder, hit); - return (ExprValue) ExprTupleValue.fromExprValueMap(builder.build()); + try { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + addParsedHitsToBuilder(builder, hit); + addMetaDataFieldsToBuilder(builder, hit); + addHighlightsToBuilder(builder, hit); + return (ExprValue) ExprTupleValue.fromExprValueMap(builder.build()); + } catch (Exception e) { + LOG.warn("Failed to parse document {}, skipping", hit.getId(), e); + return null; + } }) + .filter(Objects::nonNull) .iterator(); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java index f9c3d5bb5d2..9fa3589f1fb 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import javax.annotation.Nullable; import lombok.EqualsAndHashCode; import lombok.Getter; import org.opensearch.common.document.DocumentField; @@ -27,10 +28,29 @@ public class TopHitsParser implements MetricParser { private final boolean returnSingleValue; private final boolean returnMergeValue; + /** + * Maps each original OpenSearch field name to one or more output (renamed) field names. Used by + * the dedup aggregation pushdown path when {@code rename} creates aliases that differ from the + * index field name: top_hits returns {@code _source} / {@code fields} entries keyed by the + * original name, while the Calcite row-type expects the renamed name. A single original field may + * map to multiple output names when both {@code rename} and an {@code eval} column reference + * resolve to the same source field (issue #5197). + */ + @Nullable private final Map> fieldNameMapping; + public TopHitsParser(String name, boolean returnSingleValue, boolean returnMergeValue) { + this(name, returnSingleValue, returnMergeValue, null); + } + + public TopHitsParser( + String name, + boolean returnSingleValue, + boolean returnMergeValue, + @Nullable Map> fieldNameMapping) { this.name = name; this.returnSingleValue = returnSingleValue; this.returnMergeValue = returnMergeValue; + this.fieldNameMapping = fieldNameMapping; } @Override @@ -129,12 +149,43 @@ public List> parse(Aggregation agg) { ? new LinkedHashMap<>() : new LinkedHashMap<>(hit.getSourceAsMap()); hit.getFields().values().forEach(f -> map.put(f.getName(), f.getValue())); + applyFieldNameMapping(map); return map; }) .toList(); } } + /** + * Applies {@link #fieldNameMapping} to a parsed hit map in-place. + * + *

    For each {@code (originalName → [outputName1, outputName2, ...])} entry: the value stored + * under {@code originalName} is copied to every output name, and {@code originalName} is removed + * unless it is itself one of the expected output names. This handles both the single-rename case + * (issue #5150) and the many-to-one collision case where two aliases resolve to the same source + * field (issue #5197). + */ + private void applyFieldNameMapping(Map map) { + if (fieldNameMapping == null || fieldNameMapping.isEmpty()) { + return; + } + for (Map.Entry> entry : fieldNameMapping.entrySet()) { + String originalName = entry.getKey(); + Object value = map.get(originalName); + if (value == null && !map.containsKey(originalName)) { + continue; + } + List outputNames = entry.getValue(); + for (String outputName : outputNames) { + map.put(outputName, value); + } + // Remove the original key only when it is not itself one of the expected output names. + if (!outputNames.contains(originalName)) { + map.remove(originalName); + } + } + } + private boolean isEmptyHits(SearchHit[] hits) { return isFieldsEmpty(hits) && isSourceEmpty(hits); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index bd8001f589d..ce2bdd4960b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -29,6 +29,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexSettings; import org.opensearch.search.aggregations.MultiBucketConsumerService; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.setting.Settings; /** Setting implementation on OpenSearch. */ @@ -70,6 +71,17 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = + Setting.boolSetting( + Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); + + public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = + Setting.listSetting( + Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), + List.of(), + Function.identity(), + Setting.Property.NodeScope); + public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( Key.PPL_QUERY_TIMEOUT.getKeyValue(), @@ -201,6 +213,14 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting MAX_EXPRESSION_DEPTH_SETTING = + Setting.intSetting( + Key.MAX_EXPRESSION_DEPTH.getKeyValue(), + AstBuildGuard.DEFAULT_MAX_DEPTH, + 0, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + // Set the default value to QUERY_SIZE_LIMIT_SETTING public static final Setting QUERY_BUCKET_SIZE_SETTING = Setting.intSetting( @@ -371,6 +391,16 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); + registerNonDynamicSettings( + settingBuilder, + clusterSettings, + Key.PPL_REST_REDACTION_ENABLED, + PPL_REST_REDACTION_ENABLED_SETTING); + registerNonDynamicSettings( + settingBuilder, + clusterSettings, + Key.PPL_REST_ALLOWED_ENDPOINTS, + PPL_REST_ALLOWED_ENDPOINTS_SETTING); register( settingBuilder, clusterSettings, @@ -479,6 +509,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.QUERY_SIZE_LIMIT, QUERY_SIZE_LIMIT_SETTING, new Updater(Key.QUERY_SIZE_LIMIT)); + register( + settingBuilder, + clusterSettings, + Key.MAX_EXPRESSION_DEPTH, + MAX_EXPRESSION_DEPTH_SETTING, + new Updater(Key.MAX_EXPRESSION_DEPTH)); register( settingBuilder, clusterSettings, @@ -625,7 +661,9 @@ private void registerNonDynamicSettings( Settings.Key key, Setting setting) { settingBuilder.put(key, setting); - latestSettings.put(key, clusterSettings.get(setting)); + if (clusterSettings.get(setting) != null) { + latestSettings.put(key, clusterSettings.get(setting)); + } } /** @@ -650,6 +688,7 @@ public static List> pluginSettings() { .add(SQL_ENABLED_SETTING) .add(SQL_SLOWLOG_SETTING) .add(SQL_CURSOR_KEEP_ALIVE_SETTING) + .add(MAX_EXPRESSION_DEPTH_SETTING) .add(PPL_ENABLED_SETTING) .add(PPL_QUERY_TIMEOUT_SETTING) .add(PPL_SYNTAX_LEGACY_PREFERRED_SETTING) @@ -695,6 +734,8 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) + .add(PPL_REST_REDACTION_ENABLED_SETTING) + .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 1b7de315fb6..7b911471242 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage; +import static org.opensearch.sql.utils.SystemIndexUtils.decodeRestSpec; +import static org.opensearch.sql.utils.SystemIndexUtils.isRestSource; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; @@ -15,9 +17,13 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; +import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** OpenSearch storage engine implementation. */ @RequiredArgsConstructor @@ -35,10 +41,29 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { - if (isSystemIndex(name)) { - return new OpenSearchSystemIndex(client, settings, name); + if (isRestSource(name)) { + return restTable(name); + } else if (isSystemIndex(name)) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); } else { return new OpenSearchIndex(client, settings, name); } } + + private Table restTable(String name) { + RestSpec spec = decodeRestSpec(name); + RestEndpointRegistry.resolve(spec.getEndpoint()); + List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); + if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { + throw new IllegalArgumentException( + allowed == null || allowed.isEmpty() + ? "the rest command is disabled on this cluster" + : "rest endpoint [" + + spec.getEndpoint() + + "] is not enabled on this cluster. Enabled endpoints: " + + allowed); + } + boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); + return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java new file mode 100644 index 00000000000..96779726a7e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Map; +import lombok.Getter; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.opensearch.storage.system.CatalogSource; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management + * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. + * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. + */ +@Getter +public class RestCatalogSource implements CatalogSource { + + private final OpenSearchClient client; + private final RestSpec spec; + private final RestEndpointRegistry.Endpoint endpoint; + private final boolean redact; + + public RestCatalogSource(OpenSearchClient client, RestSpec spec) { + this(client, spec, false); + } + + public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { + this.client = client; + this.spec = spec; + this.redact = redact; + // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. + this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry.validate(spec); + } + + @Override + public Map getFieldTypes() { + return endpoint.getSchema(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return new RestRequest(client, endpoint, spec, redact); + } + + @Override + public boolean isScannable() { + return true; + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java new file mode 100644 index 00000000000..e64a91ab54a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -0,0 +1,428 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; +import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; +import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.model.ExprValueUtils.longValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Getter; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps + * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so + * the Calcite plan can fix its row type at plan time), and the query args it accepts. + * + *

    This is the single place the read-only allow-list is enforced. Endpoints outside the registry, + * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. + * Adding an endpoint is a reviewed change here, never arbitrary pass-through. + */ +public final class RestEndpointRegistry { + + private RestEndpointRegistry() {} + + /** Produces the raw rows for an endpoint via a read-only client call. */ + @FunctionalInterface + public interface RowFetcher { + List> fetch(OpenSearchClient client, RestSpec spec); + } + + /** A single allow-listed endpoint description. */ + @Getter + public static final class Endpoint { + private final String path; + private final LinkedHashMap schema; + private final Set allowedArgs; + private final RowFetcher fetcher; + + Endpoint( + String path, + LinkedHashMap schema, + Set allowedArgs, + RowFetcher fetcher) { + this.path = path; + this.schema = schema; + this.allowedArgs = allowedArgs; + this.fetcher = fetcher; + } + + /** Dispatch the read-only call and shape the response into fixed-schema rows. */ + public List toRows(OpenSearchClient client, RestSpec spec) { + return toRows(client, spec, false); + } + + /** + * Shape the response into fixed-schema rows, masking network identifiers when redaction is + * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value + * column is zone-masked. Off by default. + */ + public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { + boolean redactCat = redact && path.startsWith("/_cat"); + boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); + List out = new ArrayList<>(); + for (Map raw : fetcher.fetch(client, spec)) { + LinkedHashMap tuple = new LinkedHashMap<>(); + for (Map.Entry col : schema.entrySet()) { + ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); + tuple.put( + col.getKey(), + maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); + } + out.add(new ExprTupleValue(tuple)); + } + return out; + } + + private static ExprValue maskCell( + String column, + ExprType type, + ExprValue value, + boolean redactCat, + boolean redactSettingsValue) { + if (type != STRING || value.isNull()) { + return value; + } + if (redactCat) { + return stringValue(RestResponseRedactor.redact(value.stringValue())); + } + if (redactSettingsValue && "value".equals(column)) { + return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); + } + return value; + } + } + + private static final Map REGISTRY = buildRegistry(); + + private static Map buildRegistry() { + Map m = new LinkedHashMap<>(); + + // /_cluster/health — single-row cluster health snapshot (read-only monitor action). + LinkedHashMap healthSchema = new LinkedHashMap<>(); + healthSchema.put("cluster_name", STRING); + healthSchema.put("status", STRING); + healthSchema.put("number_of_nodes", INTEGER); + healthSchema.put("number_of_data_nodes", INTEGER); + healthSchema.put("active_primary_shards", INTEGER); + healthSchema.put("active_shards", INTEGER); + healthSchema.put("relocating_shards", INTEGER); + healthSchema.put("initializing_shards", INTEGER); + healthSchema.put("unassigned_shards", INTEGER); + healthSchema.put("timed_out", BOOLEAN); + m.put( + "/_cluster/health", + new Endpoint( + "/_cluster/health", + healthSchema, + Set.of("local"), + (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); + + // /_cat/indices — one row per index (read-only monitor action). + LinkedHashMap catSchema = new LinkedHashMap<>(); + catSchema.put("index", STRING); + catSchema.put("health", STRING); + catSchema.put("pri", INTEGER); + catSchema.put("rep", INTEGER); + catSchema.put("active_shards", INTEGER); + m.put( + "/_cat/indices", + new Endpoint( + "/_cat/indices", + catSchema, + Set.of("health"), + (client, spec) -> client.catIndices(spec.getArgs()))); + + // /_cat/nodes — one row per node with resource state (read-only monitor action). + LinkedHashMap nodesSchema = new LinkedHashMap<>(); + nodesSchema.put("name", STRING); + nodesSchema.put("ip", STRING); + nodesSchema.put("node_role", STRING); + nodesSchema.put("heap_percent", INTEGER); + nodesSchema.put("ram_percent", INTEGER); + nodesSchema.put("cpu", INTEGER); + m.put( + "/_cat/nodes", + new Endpoint( + "/_cat/nodes", + nodesSchema, + Set.of(), + (client, spec) -> client.catNodes(spec.getArgs()))); + + // /_cat/cluster_manager — single row identifying the elected cluster manager node. + LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); + clusterManagerSchema.put("id", STRING); + clusterManagerSchema.put("host", STRING); + clusterManagerSchema.put("ip", STRING); + clusterManagerSchema.put("node", STRING); + m.put( + "/_cat/cluster_manager", + new Endpoint( + "/_cat/cluster_manager", + clusterManagerSchema, + Set.of(), + (client, spec) -> client.catClusterManager(spec.getArgs()))); + + // /_cat/plugins — one row per installed plugin per node (read-only monitor action). + LinkedHashMap pluginsSchema = new LinkedHashMap<>(); + pluginsSchema.put("name", STRING); + pluginsSchema.put("component", STRING); + pluginsSchema.put("version", STRING); + m.put( + "/_cat/plugins", + new Endpoint( + "/_cat/plugins", + pluginsSchema, + Set.of(), + (client, spec) -> client.catPlugins(spec.getArgs()))); + + // /_cat/shards — one row per shard (read-only monitor action). + LinkedHashMap shardsSchema = new LinkedHashMap<>(); + shardsSchema.put("index", STRING); + shardsSchema.put("shard", INTEGER); + shardsSchema.put("prirep", STRING); + shardsSchema.put("state", STRING); + shardsSchema.put("node", STRING); + m.put( + "/_cat/shards", + new Endpoint( + "/_cat/shards", + shardsSchema, + Set.of(), + (client, spec) -> client.catShards(spec.getArgs()))); + + // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). + LinkedHashMap stateSchema = new LinkedHashMap<>(); + stateSchema.put("cluster_name", STRING); + stateSchema.put("state_uuid", STRING); + stateSchema.put("version", LONG); + stateSchema.put("cluster_manager_node", STRING); + m.put( + "/_cluster/state", + new Endpoint( + "/_cluster/state", + stateSchema, + Set.of(), + (client, spec) -> List.of(client.clusterState(spec.getArgs())))); + + // /_cluster/settings — one row per configured setting (persistent/transient tier). + LinkedHashMap settingsSchema = new LinkedHashMap<>(); + settingsSchema.put("setting", STRING); + settingsSchema.put("value", STRING); + settingsSchema.put("tier", STRING); + m.put( + "/_cluster/settings", + new Endpoint( + "/_cluster/settings", + settingsSchema, + Set.of(), + (client, spec) -> client.clusterSettings(spec.getArgs()))); + + // /_resolve/index — one row per resolved index/alias/data_stream name. + LinkedHashMap resolveSchema = new LinkedHashMap<>(); + resolveSchema.put("name", STRING); + resolveSchema.put("type", STRING); + m.put( + "/_resolve/index", + new Endpoint( + "/_resolve/index", + resolveSchema, + Set.of("expand_wildcards"), + (client, spec) -> client.resolveIndex(spec.getArgs()))); + + return m; + } + + /** + * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, + * {@code /services/*}, plugin admin endpoints) is refused here. + */ + public static Endpoint resolve(String path) { + if (path == null || path.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint must be a non-empty path. Supported read-only endpoints: " + + REGISTRY.keySet()); + } + Endpoint endpoint = REGISTRY.get(path); + if (endpoint == null) { + throw new IllegalArgumentException( + "rest endpoint [" + + path + + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " + + REGISTRY.keySet()); + } + return endpoint; + } + + /** Validate that every supplied query arg is accepted by the endpoint. */ + public static void validate(RestSpec spec) { + Endpoint endpoint = resolve(spec.getEndpoint()); + if (spec.getCount() != null && spec.getCount() < 0) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] count must be a non-negative integer, got [" + + spec.getCount() + + "]"); + } + if (spec.getTimeout() != null) { + // The timeout token is reserved in the grammar for forward compatibility, but a single + // uniform timeout cannot map cleanly across the endpoints (wait-for-status vs + // cluster-manager vs client socket timeouts differ per action). Reject it with a clear + // client error rather than silently ignoring it. + throw new IllegalArgumentException( + "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); + } + if (spec.getArgs() != null) { + for (String arg : spec.getArgs().keySet()) { + if (!endpoint.getAllowedArgs().contains(arg)) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] does not accept arg [" + + arg + + "]. Allowed args: " + + endpoint.getAllowedArgs()); + } + validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); + } + } + } + + // Allowed value domains for the get-args that are applied server-side. Keys are validated against + // the per-endpoint allow-list above; values are validated here so a user-supplied value is never + // passed unchecked into an admin transport request. + private static final Map> ARG_VALUE_DOMAINS = + Map.of( + "local", Set.of("true", "false"), + "health", Set.of("green", "yellow", "red")); + + private static final Set EXPAND_WILDCARDS_VALUES = + Set.of("open", "closed", "hidden", "none", "all"); + + /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ + private static void validateArgValue(String endpoint, String arg, String value) { + Set domain = ARG_VALUE_DOMAINS.get(arg); + if (domain != null) { + if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [" + + arg + + "] has an unsupported value [" + + value + + "]. Allowed values: " + + domain); + } + } else if ("expand_wildcards".equals(arg)) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { + if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + } + } + } + + private static ExprValue coerce(String column, ExprType type, Object value) { + if (value == null) { + return ExprNullValue.of(); + } + try { + if (type == INTEGER) { + return integerValue(toNumber(value).intValue()); + } + if (type == LONG) { + return longValue(toNumber(value).longValue()); + } + if (type == DOUBLE) { + return doubleValue(toNumber(value).doubleValue()); + } + if (type == BOOLEAN) { + return booleanValue(toBoolean(value)); + } + } catch (IllegalArgumentException | ClassCastException e) { + // Surface a clear client error (HTTP 400) instead of a raw HTTP 500 when an endpoint + // returns an unexpected value shape. NumberFormatException extends IllegalArgumentException, + // so toNumber parse failures and toBoolean's "not a boolean" are both caught here; genuinely + // unexpected faults (NPE, etc.) are left to propagate. + throw new IllegalArgumentException( + "rest endpoint value for column [" + + column + + "] could not be coerced to " + + type + + ": [" + + value + + "]"); + } + return stringValue(String.valueOf(value)); + } + + /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ + private static Number toNumber(Object value) { + if (value instanceof Number n) { + return n; + } + String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new NumberFormatException("empty string"); + } + if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { + return Double.parseDouble(s); + } + return Long.parseLong(s); + } + + /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ + private static boolean toBoolean(Object value) { + if (value instanceof Boolean b) { + return b; + } + String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new IllegalArgumentException("empty string is not a boolean"); + } + if (s.equalsIgnoreCase("true")) { + return true; + } + if (s.equalsIgnoreCase("false")) { + return false; + } + throw new IllegalArgumentException("not a boolean: " + value); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java new file mode 100644 index 00000000000..868dabdd64e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Dispatches an allow-listed, read-only management endpoint through the transport node client under + * the caller's security thread-context and returns the response shaped to the endpoint's fixed + * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link + * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to + * the system-index scan family. + */ +public class RestRequest implements OpenSearchSystemRequest { + + private final OpenSearchClient client; + private final RestEndpointRegistry.Endpoint endpoint; + private final RestSpec spec; + private final boolean redact; + + public RestRequest( + OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { + this(client, endpoint, spec, false); + } + + public RestRequest( + OpenSearchClient client, + RestEndpointRegistry.Endpoint endpoint, + RestSpec spec, + boolean redact) { + this.client = client; + this.endpoint = endpoint; + this.spec = spec; + this.redact = redact; + } + + @Override + public List search() { + List rows = endpoint.toRows(client, spec, redact); + if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { + return rows.subList(0, spec.getCount()); + } + return rows; + } + + @Override + public String toString() { + return "RestRequest{endpoint=" + endpoint.getPath() + "}"; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java new file mode 100644 index 00000000000..fd674dccde4 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import java.util.regex.Pattern; + +/** + * Masks network identifiers in rest command cell values. Enabled per deployment via {@code + * plugins.ppl.rest.redaction.enabled}; off by default. + */ +public final class RestResponseRedactor { + + private RestResponseRedactor() {} + + private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; + private static final Pattern IPV4 = + Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); + private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); + private static final Pattern EC2_HOST = + Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); + private static final Pattern IPV6 = + Pattern.compile( + "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" + + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", + Pattern.CASE_INSENSITIVE); + private static final Pattern AZ_NAME = + Pattern.compile( + "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", + Pattern.CASE_INSENSITIVE); + + private record Mask(Pattern pattern, String replacement) {} + + private static final List MASKS = + List.of( + new Mask(IPV4, "x.x.x.x"), + new Mask(INET, "inet[/x.x.x.x:y]"), + new Mask(EC2_HOST, ""), + new Mask(IPV6, "x.x.x.x"), + new Mask(AZ_NAME, "xx-xxxxx-xx")); + + /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ + public static String redact(String text) { + if (text == null || text.isEmpty()) { + return text; + } + String out = text; + for (Mask mask : MASKS) { + out = mask.pattern().matcher(out).replaceAll(mask.replacement()); + } + return out; + } + + /** Mask availability-zone names only. */ + public static String maskAvailabilityZone(String text) { + if (text == null || text.isEmpty()) { + return text; + } + return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java new file mode 100644 index 00000000000..d425c361e6a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import org.opensearch.common.settings.SettingsFilter; + +/** + * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' + * /_cluster/settings'} fetcher. + * + *

    The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with + * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the + * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads + * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the + * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction + * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. + * + *

    Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in + * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link + * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code + * getRestHandlers} fires lets the fetcher read the same instance without going back through the + * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. + */ +public final class RestSettingsFilterHolder { + + private static volatile SettingsFilter settingsFilter; + + private RestSettingsFilterHolder() {} + + public static void set(SettingsFilter instance) { + settingsFilter = instance; + } + + public static SettingsFilter get() { + return settingsFilter; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 609a5aaa92f..f3773cfe246 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -33,6 +33,7 @@ import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.externalize.RelJsonWriter; import org.apache.calcite.rel.externalize.RelWriterImpl; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -105,13 +106,29 @@ public RelDataType deriveRowType() { @Override public RelWriter explainTerms(RelWriter pw) { + // Build explain string with context and request builder info String explainString = String.valueOf(pushDownContext); - if (pw instanceof RelWriterImpl) { - // Only add request builder to the explain plan - explainString += ", " + pushDownContext.createRequestBuilder(); + if (pw instanceof RelJsonWriter) { + // For JSON output, add structured items + super.explainTerms(pw); + if (!pushDownContext.isEmpty()) { + pw.item("PushDownContext", explainString); + try { + OpenSearchRequestBuilder requestBuilder = pushDownContext.createRequestBuilder(); + pw.item("sourceBuilder", requestBuilder.getSourceBuilder().toString()); + } catch (Exception e) { + // Ignore if request builder cannot be created + } + } + return pw; + } else { + // For text output, use original chained format + if (pw instanceof RelWriterImpl && !pushDownContext.isEmpty()) { + explainString += ", " + pushDownContext.createRequestBuilder(); + } + return super.explainTerms(pw) + .itemIf("PushDownContext", explainString, !pushDownContext.isEmpty()); } - return super.explainTerms(pw) - .itemIf("PushDownContext", explainString, !pushDownContext.isEmpty()); } protected Integer getQuerySizeLimit() { @@ -323,7 +340,7 @@ && isAnyCollationNameInAggregators(collationNames)) { // aggregators. return null; } - RelTraitSet traitsWithCollations = getTraitSet().plus(RelCollations.of(collations)); + RelTraitSet traitsWithCollations = getTraitSet().replace(RelCollations.of(collations)); PushDownContext pushDownContextWithoutSort = this.pushDownContext.cloneWithoutSort(); AbstractAction action; Object digest; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index 740801ff418..2017437e7bd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -321,7 +321,7 @@ private RelTraitSet reIndexCollations(List selectedColumns) { collation -> collation.withFieldIndex(selectedColumns.indexOf(collation.getFieldIndex()))) .collect(Collectors.toList()); - newTraitSet = getTraitSet().plus(RelCollations.of(newCollations)); + newTraitSet = getTraitSet().replace(RelCollations.of(newCollations)); } else { newTraitSet = getTraitSet(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java index d77dee3e297..8a0b4daa57a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java @@ -232,6 +232,7 @@ public Object toJson(RexNode node) { map.put("operands", list); switch (node.getKind()) { case MINUS: + case CHECKED_MINUS: case CAST: case SAFE_CAST: map.put("type", toJson(node.getType())); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java similarity index 68% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java index fa543ab266b..19d585ea684 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java @@ -16,21 +16,21 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -/** An abstract relational operator representing a scan of an OpenSearchSystemIndex type. */ +/** An abstract relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ @Getter -public abstract class AbstractCalciteSystemIndexScan extends TableScan { - public final OpenSearchSystemIndex sysIndex; +public abstract class AbstractCalciteCatalogScan extends TableScan { + public final OpenSearchCatalogTable catalogTable; protected final RelDataType schema; - protected AbstractCalciteSystemIndexScan( + protected AbstractCalciteCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super(cluster, traitSet, hints, table); - this.sysIndex = requireNonNull(sysIndex, "OpenSearch system index"); + this.catalogTable = requireNonNull(catalogTable, "OpenSearch catalog table"); this.schema = schema; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java similarity index 80% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java index b0c92dce8f9..58f86874c73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java @@ -26,17 +26,22 @@ import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; -/** The physical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteEnumerableSystemIndexScan extends AbstractCalciteSystemIndexScan +/** The physical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteEnumerableCatalogScan extends AbstractCalciteCatalogScan implements EnumerableRel { - public CalciteEnumerableSystemIndexScan( + public CalciteEnumerableCatalogScan( RelOptCluster cluster, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super( - cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, sysIndex, schema); + cluster, + cluster.traitSetOf(EnumerableConvention.INSTANCE), + hints, + table, + catalogTable, + schema); } @Override @@ -60,7 +65,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - Expression scanOperator = implementor.stash(this, CalciteEnumerableSystemIndexScan.class); + Expression scanOperator = implementor.stash(this, CalciteEnumerableCatalogScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); } @@ -68,10 +73,10 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new OpenSearchSystemIndexEnumerator( + return new OpenSearchCatalogEnumerator( getFieldPath(), - sysIndex.getSystemIndexBundle().getRight(), - sysIndex.createOpenSearchResourceMonitor()); + catalogTable.getSource().createRequest(), + catalogTable.createOpenSearchResourceMonitor()); } }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java similarity index 59% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java index 012ceec8c13..4278539c2b0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java @@ -14,35 +14,35 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableCatalogScanRule; -/** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { +/** The logical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteLogicalCatalogScan extends AbstractCalciteCatalogScan { - public CalciteLogicalSystemIndexScan( - RelOptCluster cluster, RelOptTable table, OpenSearchSystemIndex sysIndex) { + public CalciteLogicalCatalogScan( + RelOptCluster cluster, RelOptTable table, OpenSearchCatalogTable catalogTable) { this( cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, - sysIndex, + catalogTable, table.getRowType()); } - protected CalciteLogicalSystemIndexScan( + protected CalciteLogicalCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { - super(cluster, traitSet, hints, table, sysIndex, schema); + super(cluster, traitSet, hints, table, catalogTable, schema); } @Override public void register(RelOptPlanner planner) { super.register(planner); - planner.addRule(EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule()); + planner.addRule(EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java new file mode 100644 index 00000000000..1d021cda354 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.List; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.sql.calcite.plan.Scannable; + +/** + * A {@link CalciteEnumerableCatalogScan} that additionally carries the {@link Scannable} marker, + * enabling the {@code collect} short-circuit. Produced when the {@link CatalogSource} opts in via + * {@link CatalogSource#isScannable()}. + */ +public class CalciteScannableCatalogScan extends CalciteEnumerableCatalogScan implements Scannable { + public CalciteScannableCatalogScan( + RelOptCluster cluster, + List hints, + RelOptTable table, + OpenSearchCatalogTable catalogTable, + RelDataType schema) { + super(cluster, hints, table, catalogTable, schema); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java new file mode 100644 index 00000000000..85dabe06b9a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * Strategy supplying the schema and row source for an {@link OpenSearchCatalogTable} backed by a + * read-only OpenSearch admin API. Each endpoint family plugs in its own {@code CatalogSource} + * rather than defining a separate table and Calcite scan hierarchy. + */ +public interface CatalogSource { + + /** Fixed schema mapping each column name to its type. */ + Map getFieldTypes(); + + /** + * Builds the read-only request whose {@link OpenSearchSystemRequest#search()} yields the rows. + */ + OpenSearchSystemRequest createRequest(); + + /** + * Whether the enumerable scan should carry the {@link org.opensearch.sql.calcite.plan.Scannable} + * marker for the {@code collect} short-circuit. Defaults to {@code false}. + */ + default boolean isScannable() { + return false; + } + + /** The V2 physical plan path, or throws when the source is Calcite only. */ + PhysicalPlan implementV2(LogicalPlan plan); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java similarity index 90% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java index 1bb8f9d5293..dff9b47265a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java @@ -16,8 +16,11 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -/** Supports a simple iteration over a collection for OpenSearch system index */ -public class OpenSearchSystemIndexEnumerator implements Enumerator { +/** + * Resource-monitored iteration over the rows produced by a read-only catalog {@link + * OpenSearchSystemRequest}. + */ +public class OpenSearchCatalogEnumerator implements Enumerator { /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -35,7 +38,7 @@ public class OpenSearchSystemIndexEnumerator implements Enumerator { /** ResourceMonitor. */ private final ResourceMonitor monitor; - public OpenSearchSystemIndexEnumerator( + public OpenSearchCatalogEnumerator( List fields, OpenSearchSystemRequest request, ResourceMonitor monitor) { this.fields = fields; this.request = request; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java new file mode 100644 index 00000000000..9bcb3a3ff0c --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * A single generic Calcite and V2 table over a read-only OpenSearch catalog endpoint. Per-endpoint + * behavior (schema, row source, V2 support, and the {@code Scannable} marker) is supplied by a + * pluggable {@link CatalogSource}. + */ +@Getter +public class OpenSearchCatalogTable extends AbstractOpenSearchTable { + + private final CatalogSource source; + private final Settings settings; + + public OpenSearchCatalogTable(CatalogSource source, Settings settings) { + this.source = source; + this.settings = settings; + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch catalog table is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return source.getFieldTypes(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalCatalogScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return source.implementV2(plan); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java deleted file mode 100644 index 6bae4d21aba..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; - -import com.google.common.annotations.VisibleForTesting; -import java.util.Map; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.DefaultImplementor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** OpenSearch System Index Table Implementation. */ -@Getter -public class OpenSearchSystemIndex extends AbstractOpenSearchTable { - /** System Index Name. */ - private final Pair systemIndexBundle; - - @Getter private final Settings settings; - - public OpenSearchSystemIndex(OpenSearchClient client, Settings settings, String indexName) { - this.systemIndexBundle = buildIndexBundle(client, indexName); - this.settings = settings; - } - - @Override - public boolean exists() { - return true; // TODO: implement for system index later - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException( - "OpenSearch system index is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return systemIndexBundle.getLeft().getMapping(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalSystemIndexScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - return plan.accept(new OpenSearchSystemIndexDefaultImplementor(), null); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); - } - - @VisibleForTesting - @RequiredArgsConstructor - public class OpenSearchSystemIndexDefaultImplementor extends DefaultImplementor { - - @Override - public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new OpenSearchSystemIndexScan(systemIndexBundle.getRight()); - } - } - - /** - * Constructor of ElasticsearchSystemIndexName. - * - * @param indexName index name; - */ - private Pair buildIndexBundle( - OpenSearchClient client, String indexName) { - SystemIndexUtils.SystemTable systemTable = systemTable(indexName); - if (systemTable.isSystemInfoTable()) { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); - } else { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, - new OpenSearchDescribeIndexRequest( - client, systemTable.getTableName(), systemTable.getLangSpec())); - } - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java new file mode 100644 index 00000000000..3541e745e9d --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * {@link CatalogSource} for the SHOW TABLES and DESCRIBE system tables, backed by index listing and + * index field mappings. + */ +public class SystemIndexCatalogSource implements CatalogSource { + + private final Pair bundle; + + public SystemIndexCatalogSource(OpenSearchClient client, String indexName) { + this.bundle = buildBundle(client, indexName); + } + + @Override + public Map getFieldTypes() { + return bundle.getLeft().getMapping(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return bundle.getRight(); + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + return plan.accept( + new DefaultImplementor() { + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new OpenSearchSystemIndexScan(bundle.getRight()); + } + }, + null); + } + + private static Pair buildBundle( + OpenSearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); + } else { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new OpenSearchDescribeIndexRequest( + client, systemTable.getTableName(), systemTable.getLangSpec())); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java index dab778923b6..aad2623a6f7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java @@ -55,7 +55,7 @@ public static Optional> getOrderEquivalentInputInfo(RexNo case MINUS_PREFIX: return getOrderEquivalentInputInfo(((RexCall) expr).getOperands().get(0)) .map(inputInfo -> Pair.of(inputInfo.getLeft(), !inputInfo.getRight())); - case PLUS, MINUS: + case PLUS, MINUS, CHECKED_PLUS, CHECKED_MINUS: { RexNode operand0 = ((RexCall) expr).getOperands().get(0); RexNode operand1 = ((RexCall) expr).getOperands().get(1); @@ -68,12 +68,14 @@ public static Optional> getOrderEquivalentInputInfo(RexNo } RexNode variable = operand0Lit ? operand1 : operand0; - boolean flipped = (expr.getKind() == SqlKind.MINUS) && operand0Lit; + boolean isMinus = + expr.getKind() == SqlKind.MINUS || expr.getKind() == SqlKind.CHECKED_MINUS; + boolean flipped = isMinus && operand0Lit; return getOrderEquivalentInputInfo(variable) .map(inputInfo -> Pair.of(inputInfo.getLeft(), flipped != inputInfo.getRight())); } - case TIMES: + case TIMES, CHECKED_TIMES: { RexNode operand0 = ((RexCall) expr).getOperands().get(0); RexNode operand1 = ((RexCall) expr).getOperands().get(1); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java new file mode 100644 index 00000000000..33be6cc8482 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.client; + +import static java.util.stream.Collectors.toSet; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using + * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. + */ +class OpenSearchNodeClientClusterSettingsFilterTest { + + @AfterEach + void clearHolder() { + RestSettingsFilterHolder.set(null); + } + + private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { + NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); + ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); + when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) + .thenReturn(stateResp); + when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); + when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); + return new OpenSearchNodeClient(nodeClient); + } + + @Test + void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("plugins.secret.token", "supersecret") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // Publish a filter that redacts the secret key, exactly as the native endpoint would. + RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); + + List> rows = client.clusterSettings(Map.of()); + Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); + assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); + } + + @Test + void clusterSettingsRedactsByGlobPattern() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("s3.client.default.secret_key", "AKIAEXAMPLE") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); + + Set keys = + client.clusterSettings(Map.of()).stream() + .map(r -> (String) r.get("setting")) + .collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable")); + assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); + } + + @Test + void clusterSettingsFailsClosedWhenNoFilterPublished() { + Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw + // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter + // before any query, so this path is unreachable in cluster. + assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 031b9243f38..7c27c995e4c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -48,7 +48,6 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.junit.jupiter.api.Test; -import org.opensearch.OpenSearchParseException; import org.opensearch.geometry.utils.Geohash; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; @@ -230,6 +229,25 @@ public void constructIp() { constructFromObject("ipV", "2001:db7::ff00:42:8329"))); } + @Test + public void constructIpFromInvalidString_ReturnsNull() { + assertEquals(nullValue(), tupleValue("{\"ipV\":\"not-an-ip\"}").get("ipV")); + assertEquals(nullValue(), constructFromObject("ipV", "garbage")); + } + + @Test + public void constructNumericFromObjectNode_ReturnsNull() { + assertEquals(nullValue(), tupleValue("{\"intV\":{\"nested\":\"value\"}}").get("intV")); + assertEquals(nullValue(), tupleValue("{\"longV\":{\"nested\":\"value\"}}").get("longV")); + assertEquals(nullValue(), tupleValue("{\"floatV\":{\"nested\":\"value\"}}").get("floatV")); + assertEquals(nullValue(), tupleValue("{\"doubleV\":{\"nested\":\"value\"}}").get("doubleV")); + } + + @Test + public void constructBooleanFromObjectNode_ReturnsNull() { + assertEquals(nullValue(), tupleValue("{\"boolV\":{\"nested\":\"value\"}}").get("boolV")); + } + @Test public void constructBoolean() { assertAll( @@ -374,13 +392,7 @@ public void constructDatetime_fromCustomFormat() { new ExprTimestampValue("2015-01-01 12:10:30"), constructFromObject("customFormatV", "2015-01-01-12-10-30")); - IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, - () -> constructFromObject("customFormatV", "2015-01-01 12-10-30")); - assertEquals( - "Construct TIMESTAMP from \"2015-01-01 12-10-30\" failed, unsupported format.", - exception.getMessage()); + assertEquals(nullValue(), constructFromObject("customFormatV", "2015-01-01 12-10-30")); assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), @@ -388,52 +400,21 @@ public void constructDatetime_fromCustomFormat() { } @Test - public void constructDatetimeFromUnsupportedFormat_ThrowIllegalArgumentException() { - IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, - () -> constructFromObject("timestampV", "2015-01-01 12:10")); - assertEquals( - "Construct TIMESTAMP from \"2015-01-01 12:10\" failed, unsupported format.", - exception.getMessage()); - - // fail with missing seconds - exception = - assertThrows( - IllegalArgumentException.class, - () -> constructFromObject("dateOrEpochMillisV", "2015-01-01 12:10")); - assertEquals( - "Construct TIMESTAMP from \"2015-01-01 12:10\" failed, unsupported format.", - exception.getMessage()); + public void constructDatetimeFromUnsupportedFormat_ReturnsNull() { + assertEquals(nullValue(), constructFromObject("timestampV", "2015-01-01 12:10")); + assertEquals(nullValue(), constructFromObject("dateOrEpochMillisV", "2015-01-01 12:10")); } @Test - public void constructTimeFromUnsupportedFormat_ThrowIllegalArgumentException() { - IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, () -> constructFromObject("timeV", "2015-01-01")); - assertEquals( - "Construct TIME from \"2015-01-01\" failed, unsupported format.", exception.getMessage()); - - exception = - assertThrows( - IllegalArgumentException.class, () -> constructFromObject("timeStringV", "10:10")); - assertEquals( - "Construct TIME from \"10:10\" failed, unsupported format.", exception.getMessage()); + public void constructTimeFromUnsupportedFormat_ReturnsNull() { + assertEquals(nullValue(), constructFromObject("timeV", "2015-01-01")); + assertEquals(nullValue(), constructFromObject("timeStringV", "10:10")); } @Test - public void constructDateFromUnsupportedFormat_ThrowIllegalArgumentException() { - IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, () -> constructFromObject("dateV", "12:10:10")); - assertEquals( - "Construct DATE from \"12:10:10\" failed, unsupported format.", exception.getMessage()); - - exception = - assertThrows( - IllegalArgumentException.class, () -> constructFromObject("dateStringV", "abc")); - assertEquals("Construct DATE from \"abc\" failed, unsupported format.", exception.getMessage()); + public void constructDateFromUnsupportedFormat_ReturnsNull() { + assertEquals(nullValue(), constructFromObject("dateV", "12:10:10")); + assertEquals(nullValue(), constructFromObject("dateStringV", "abc")); } @Test @@ -803,24 +784,12 @@ public void constructGeoPoint() { } @Test - public void constructGeoPointFromUnsupportedFormatShouldThrowException() { - OpenSearchParseException exception = - assertThrows( - OpenSearchParseException.class, - () -> tupleValue("{\"geoV\": [42.60355556, false]}").get("geoV")); - assertEquals("lat must be a number, got false", exception.getMessage()); - - exception = - assertThrows( - OpenSearchParseException.class, - () -> tupleValue("{\"geoV\":{\"lon\":-97.25263889}}").get("geoV")); - assertEquals("field [lat] missing", exception.getMessage()); - - exception = - assertThrows( - OpenSearchParseException.class, - () -> tupleValue("{\"geoV\":{\"lat\":true,\"lon\":-97.25263889}}").get("geoV")); - assertEquals("lat must be a number", exception.getMessage()); + public void constructGeoPointFromUnsupportedFormat_ReturnsNull() { + assertEquals(nullValue(), tupleValue("{\"geoV\": [42.60355556, false]}").get("geoV")); + assertEquals(nullValue(), tupleValue("{\"geoV\":{\"lon\":-97.25263889}}").get("geoV")); + assertEquals( + nullValue(), tupleValue("{\"geoV\":{\"lat\":true,\"lon\":-97.25263889}}").get("geoV")); + assertEquals(nullValue(), tupleValue("{\"geoV\":\"not-a-geo-point\"}").get("geoV")); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 041063f62a0..01c5e6108ef 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -1649,4 +1649,48 @@ void notIsFalse_generatesOnlyMustNotTerm() throws ExpressionNotAnalyzableExcepti """, result.toString()); } + + @Test + void andWithUnpushableLike_partiallyPushesOtherPredicates() + throws ExpressionNotAnalyzableException { + // field3 (c) is text without .keyword → LIKE throws PredicateAnalyzerException + // field4 (d) is date → timestamp range should push as RangeQueryBuilder + final RelDataType rowType = + builder + .getTypeFactory() + .builder() + .kind(StructKind.FULLY_QUALIFIED) + .add("a", typeFactory.createSqlType(SqlTypeName.INTEGER)) + .add("b", typeFactory.createSqlType(SqlTypeName.VARCHAR)) + .add("c", typeFactory.createSqlType(SqlTypeName.VARCHAR)) + .add("d", typeFactory.createUDT(ExprUDT.EXPR_TIMESTAMP)) + .add("e", typeFactory.createSqlType(SqlTypeName.BOOLEAN)) + .build(); + Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); + + RexInputRef field3 = builder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 2); + RexNode likeCall = + builder.makeCall( + SqlStdOperatorTable.LIKE, field3, stringLiteral, builder.makeLiteral("\\")); + RexNode rangeCall = + builder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, field4, dateTimeLiteral); + RexNode andCall = builder.makeCall(SqlStdOperatorTable.AND, rangeCall, likeCall); + + QueryBuilder result = PredicateAnalyzer.analyze(andCall, schema, fieldTypes, rowType, cluster); + + // Should be a BoolQueryBuilder with range in must[] and LIKE as script + assertInstanceOf(BoolQueryBuilder.class, result); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) result; + assertEquals(2, boolQuery.must().size()); + + // First must clause should be the range query (pushable) + QueryBuilder firstMust = boolQuery.must().get(0); + assertInstanceOf(RangeQueryBuilder.class, firstMust); + RangeQueryBuilder rangeQuery = (RangeQueryBuilder) firstMust; + assertEquals("d", rangeQuery.fieldName()); + + // Second must clause should be script query (unpushable LIKE) + QueryBuilder secondMust = boolQuery.must().get(1); + assertInstanceOf(ScriptQueryBuilder.class, secondMust); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java index 7ba64eaa475..1147fccf063 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java @@ -570,6 +570,96 @@ void two_bucket_percentiles_should_pass() { ImmutableMap.of("percentiles", List.of(21.0, 27.0, 30.0, 35.0, 55.0, 58.0, 60.0)))); } + /** + * Dedup pushdown (LITERAL_AGG) with a renamed field: {@code rename value as val | dedup + * category}. The top_hits response returns {@code _source: {category, value}} but the output + * schema expects {@code val}. The {@code fieldNameMapping} must translate {@code value -> val}. + * + *

    Regression test for https://github.com/opensearch-project/sql/issues/5150 + */ + @Test + void top_hits_field_name_mapping_single_rename_should_pass() { + String response = + "{\n" + + " \"composite#composite_buckets\": {\n" + + " \"buckets\": [\n" + + " {\n" + + " \"key\": { \"category\": \"A\" },\n" + + " \"doc_count\": 2,\n" + + " \"top_hits#dedup\": {\n" + + " \"hits\": {\n" + + " \"total\": { \"value\": 2, \"relation\": \"eq\" },\n" + + " \"hits\": [\n" + + " {\n" + + " \"_index\": \"idx\",\n" + + " \"_id\": \"1\",\n" + + " \"fields\": { \"value\": [10.5] }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + + Map> mapping = Map.of("value", List.of("val")); + OpenSearchAggregationResponseParser parser = + new CompositeAggregationParser(new TopHitsParser("dedup", false, false, mapping)); + assertThat( + parse(parser, response), + contains(ImmutableMap.of("category", "A"), ImmutableMap.of("val", 10.5))); + } + + /** + * Dedup pushdown (LITERAL_AGG) where two output names ({@code pay} from rename, {@code pay2} from + * eval column-ref) both resolve to the same original field {@code salary}. The old {@code + * Map} approach silently dropped one mapping on collision; with {@code + * Map>} both aliases must appear in the result. + * + *

    Regression test for https://github.com/opensearch-project/sql/issues/5197 + */ + @Test + void top_hits_field_name_mapping_collision_should_duplicate_value() { + String response = + "{\n" + + " \"composite#composite_buckets\": {\n" + + " \"buckets\": [\n" + + " {\n" + + " \"key\": { \"dept_id\": \"eng\" },\n" + + " \"doc_count\": 3,\n" + + " \"top_hits#dedup\": {\n" + + " \"hits\": {\n" + + " \"total\": { \"value\": 3, \"relation\": \"eq\" },\n" + + " \"hits\": [\n" + + " {\n" + + " \"_index\": \"idx\",\n" + + " \"_id\": \"1\",\n" + + " \"fields\": { \"salary\": [50000] }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + + // salary -> [pay, pay2]: both rename and eval-column-ref resolve to the same source field + Map> mapping = Map.of("salary", List.of("pay", "pay2")); + OpenSearchAggregationResponseParser parser = + new CompositeAggregationParser(new TopHitsParser("dedup", false, false, mapping)); + + List> result = parse(parser, response); + // Bucket key row + assertThat(result.get(0), org.hamcrest.Matchers.hasEntry("dept_id", "eng")); + // Hit row must contain both aliases with the same value; original key must be absent + Map hitRow = result.get(1); + assertEquals(50000, hitRow.get("pay")); + assertEquals(50000, hitRow.get("pay2")); + assertNull(hitRow.get("salary"), "original field name must be removed after mapping"); + } + public List> parse(OpenSearchAggregationResponseParser parser, String json) { return parser.parse(fromJson(json)); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index f9897f48dd2..938fb56f47b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.opensearch.response; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -17,8 +18,10 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -340,4 +343,36 @@ void highlight_iterator() { assertTrue(expected.equals(result)); } } + + @Test + void iterator_skipsDocumentWhenConstructThrows() { + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit1, searchHit2}, + new TotalHits(2L, TotalHits.Relation.EQUAL_TO), + 1.0F)); + + when(searchHit1.getSourceAsString()).thenReturn("{\"id1\": 1}"); + when(searchHit1.getInnerHits()).thenReturn(null); + when(searchHit1.getId()).thenReturn("doc1"); + when(searchHit2.getSourceAsString()).thenReturn("{\"id1\": 2}"); + when(searchHit2.getInnerHits()).thenReturn(null); + + when(factory.construct(any(), anyBoolean())) + .thenThrow(new RuntimeException("simulated parse failure")) + .thenReturn(exprTupleValue2); + + List results = new ArrayList<>(); + assertDoesNotThrow( + () -> { + Iterator it = + OpenSearchResponse.of(searchResponse, factory, List.of("id1")).iterator(); + while (it.hasNext()) { + results.add(it.next()); + } + }); + assertEquals(1, results.size()); + assertEquals(exprTupleValue2, results.get(0)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 5024d416086..0c570098924 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -9,12 +9,15 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalMatchers.not; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -74,6 +77,15 @@ void pluginNonDynamicSettings() { assertFalse(settings.isEmpty()); } + @Test + void restSettingsAreNonDynamic() { + assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); + assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); + List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); + assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); + assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); + } + @Test void getSettings() { when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index fa04395e065..102ec4da8f7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -7,11 +7,15 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -20,8 +24,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils; @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { @@ -52,6 +57,68 @@ public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); - assertAll(() -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchSystemIndex)); + assertAll( + () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); + } + + @Test + public void getRestTableAllowedByWildcard() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("*")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + Table table = + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); + assertTrue(table instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableAllowedBySubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + assertTrue( + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) + instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableRejectedWhenEndpointNotInSubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("is not enabled on this cluster")); + } + + @Test + public void getRestTableDisabledWhenListEmpty() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)).thenReturn(List.of()); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("disabled on this cluster")); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/VectorSearchTableFunctionResolverTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/VectorSearchTableFunctionResolverTest.java index c6fece7bf32..afa4e9e725e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/VectorSearchTableFunctionResolverTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/VectorSearchTableFunctionResolverTest.java @@ -21,6 +21,7 @@ import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.FunctionBuilder; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionProperties; @@ -182,6 +183,64 @@ void resolve_rejectsDuplicateNamedArgument() { assertTrue(ex.getMessage().contains("table")); } + @Test + void resolverArityGuardRejectsExtraNamedArgument() { + // The resolver's own builder runs validateArguments() first, so applying it directly to a + // duplicate named argument plus all four required arguments (five total) throws a clean + // ExpressionEvaluationException. This covers the resolver's arity guard only; the + // repository-level cast path that originally crashed is covered by + // resolveThroughRepositoryRejectsExtraNamedArgument and BuiltinFunctionRepositoryTest. + VectorSearchTableFunctionResolver resolver = + new VectorSearchTableFunctionResolver(client, settings); + FunctionName functionName = FunctionName.of("vectorsearch"); + List expressions = + List.of( + DSL.namedArgument("table", DSL.literal("a")), + DSL.namedArgument("table", DSL.literal("b")), + DSL.namedArgument("field", DSL.literal("embedding")), + DSL.namedArgument("vector", DSL.literal("[1.0]")), + DSL.namedArgument("option", DSL.literal("k=5"))); + FunctionSignature functionSignature = + new FunctionSignature( + functionName, expressions.stream().map(Expression::type).collect(Collectors.toList())); + FunctionBuilder builder = resolver.resolve(functionSignature).getValue(); + + ExpressionEvaluationException ex = + assertThrows( + ExpressionEvaluationException.class, + () -> builder.apply(functionProperties, expressions)); + assertTrue(ex.getMessage().contains("requires 4 arguments")); + } + + @Test + void resolveThroughRepositoryRejectsExtraNamedArgument() { + // Routes the vectorSearch resolver through BuiltinFunctionRepository the way production does + // (storage-engine resolver registered as a datasource function). A duplicate named argument + // plus all four required arguments gives five arguments against the resolver's fixed four-arg + // signature, which exercises the cast-wrapper path that originally crashed with an unchecked + // IndexOutOfBoundsException (surfacing as HTTP 500). It must now fail with a clean + // ExpressionEvaluationException instead. + VectorSearchTableFunctionResolver resolver = + new VectorSearchTableFunctionResolver(client, settings); + BuiltinFunctionRepository repository = BuiltinFunctionRepository.getInstance(); + List expressions = + List.of( + DSL.namedArgument("table", DSL.literal("a")), + DSL.namedArgument("table", DSL.literal("b")), + DSL.namedArgument("field", DSL.literal("embedding")), + DSL.namedArgument("vector", DSL.literal("[1.0]")), + DSL.namedArgument("option", DSL.literal("k=5"))); + + assertThrows( + ExpressionEvaluationException.class, + () -> + repository.compile( + functionProperties, + List.of(resolver), + FunctionName.of("vectorsearch"), + expressions)); + } + @Test void resolve_rejectsUnknownArgumentName() { VectorSearchTableFunctionResolver resolver = diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java new file mode 100644 index 00000000000..8676373d2a6 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java @@ -0,0 +1,123 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, + * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) + * path. + */ +@ExtendWith(MockitoExtension.class) +class RestCatalogSourceTest { + + @Mock private OpenSearchClient client; + + private RestSpec healthSpec() { + return new RestSpec("/_cluster/health", Map.of(), null, null); + } + + @Test + void getFieldTypesReturnsFixedEndpointSchema() { + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + Map fieldTypes = source.getFieldTypes(); + assertThat(fieldTypes, hasEntry("status", STRING)); + assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); + } + + @Test + void isScannable() { + assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); + } + + @Test + void implementV2IsUnsupported() { + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); + } + + @Test + void constructorRejectsNonAllowListedEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + } + + @Test + void constructorRejectsDisallowedArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource( + client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + } + + @Test + void constructorRejectsNegativeCount() { + assertThrows( + IllegalArgumentException.class, + () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); + } + + @Test + void constructorRejectsTimeoutArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + } + + @Test + void restRequestShapesResponseRows() { + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + List rows = source.createRequest().search(); + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void countTruncatesRows() { + Map idx1 = new LinkedHashMap<>(); + idx1.put("index", "a"); + Map idx2 = new LinkedHashMap<>(); + idx2.put("index", "b"); + when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); + + RestCatalogSource source = + new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); + List rows = source.createRequest().search(); + assertEquals(1, rows.size()); + assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java new file mode 100644 index 00000000000..c5a978bb495 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -0,0 +1,290 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +@ExtendWith(MockitoExtension.class) +class RestEndpointRegistryTest { + + @Mock private OpenSearchClient client; + + @Test + void resolveAllowListedEndpoint() { + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + assertEquals("/_cluster/health", endpoint.getPath()); + assertEquals(STRING, endpoint.getSchema().get("status")); + assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); + } + + @Test + void resolveRejectsNonAllowListedEndpoint() { + // A mutating endpoint is simply absent from the registry and is refused here. + assertThrows( + IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); + assertThrows( + IllegalArgumentException.class, + () -> RestEndpointRegistry.resolve("/services/server/info")); + } + + @Test + void validateRejectsUnknownArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsAllowedArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); + RestEndpointRegistry.validate(spec); // no throw + } + + @Test + void catEndpointsRedactAddressesWhenRedactionEnabled() { + Map node = new LinkedHashMap<>(); + node.put("name", "ip-10-0-0-7"); + node.put("ip", "10.0.0.7"); + node.put("node_role", "dir"); + node.put("heap_percent", 44); + node.put("ram_percent", 95); + node.put("cpu", 2); + when(client.catNodes(any())).thenReturn(List.of(node)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); + RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("", redacted.get("name").stringValue()); + assertEquals(44, redacted.get("heap_percent").integerValue()); + + Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); + assertEquals("10.0.0.7", plain.get("ip").stringValue()); + assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); + } + + @Test + void catClusterManagerRedactsHostAndIp() { + Map row = new LinkedHashMap<>(); + row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); + row.put("host", "10.0.0.7"); + row.put("ip", "10.0.0.7"); + row.put("node", "71d03b567bb755839a73d437b2b066d4"); + when(client.catClusterManager(any())).thenReturn(List.of(row)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); + RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("host").stringValue()); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); + assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); + } + + @Test + void nonCatEndpointNotRedactedEvenWhenEnabled() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "10.0.0.7"); + health.put("status", "green"); + health.put("number_of_nodes", 3); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); + } + + @Test + void clusterSettingsMasksAvailabilityZoneInValue() { + Map setting = new LinkedHashMap<>(); + setting.put("setting", "cluster.routing.allocation.awareness.attributes"); + setting.put("value", "zone:us-east-1a"); + setting.put("tier", "persistent"); + when(client.clusterSettings(any())).thenReturn(List.of(setting)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); + RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); + assertEquals( + "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); + assertEquals("persistent", row.get("tier").stringValue()); + } + + @Test + void validateRejectsDroppedLevelArg() { + // level was dropped (no-op against the fixed cluster-level health schema); now unknown. + RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void validateRejectsDroppedFlatSettingsArg() { + // flat_settings was dropped (redundant: settings are already flattened to dotted keys). + RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsValidArgValues() { + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); + } + + @Test + void validateRejectsBadArgValue() { + IllegalArgumentException health = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); + assertTrue(health.getMessage().contains("unsupported value")); + + IllegalArgumentException local = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); + assertTrue(local.getMessage().contains("unsupported value")); + + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec( + "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); + } + + @Test + void resolveRejectsBlankEndpoint() { + IllegalArgumentException emptyEx = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); + assertTrue(emptyEx.getMessage().contains("non-empty path")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); + } + + @Test + void validateRejectsNegativeCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("non-negative")); + } + + @Test + void validateAcceptsZeroCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); + RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit + } + + @Test + void validateRejectsTimeoutArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("timeout")); + } + + @Test + void coerceParsesNumericStringValues() { + // The cat JSON API returns numeric columns as strings; coerce must parse them. + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "3"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void coerceThrowsClearErrorOnUncoercibleValue() { + // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), + // not a raw ClassCastException / NumberFormatException (HTTP 500). + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "not-a-number"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); + assertTrue(ex.getMessage().contains("number_of_nodes")); + assertTrue(ex.getMessage().contains("not-a-number")); + } + + @Test + void clusterHealthRowsAreShapedToFixedSchema() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "test-cluster"); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + // a declared column the action did not return becomes null, never absent. + assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); + } + + @Test + void catIndicesRowsAreShapedToFixedSchema() { + Map idx = new LinkedHashMap<>(); + idx.put("index", "books"); + idx.put("health", "yellow"); + idx.put("pri", 1); + idx.put("rep", 1); + when(client.catIndices(any())).thenReturn(List.of(idx)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); + List rows = + endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); + assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java new file mode 100644 index 00000000000..3b038306da5 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class RestResponseRedactorTest { + + @Test + void masksIpv4() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); + assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); + assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); + } + + @Test + void doesNotMaskInvalidOrPartialIpv4() { + assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); + assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); + assertEquals("44", RestResponseRedactor.redact("44")); + } + + @Test + void masksEc2HostName() { + assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); + assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); + assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); + assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); + } + + @Test + void masksFullIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); + } + + @Test + void masksCompressedIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); + } + + @Test + void masksInetAddress() { + assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); + } + + @Test + void masksAvailabilityZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); + // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); + assertEquals( + "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); + } + + @Test + void maskAvailabilityZoneMasksOnlyZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); + assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); + assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); + } + + @Test + void leavesNonAddressesIntact() { + assertEquals( + "e4e136ea81e27370ff73cf753ba22d39", + RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); + assertEquals( + "data,ingest,remote_cluster_client", + RestResponseRedactor.redact("data,ingest,remote_cluster_client")); + assertEquals( + "x.x.x.x 44 95 imr - e4e136ea", + RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); + } + + @Test + void handlesNullAndEmpty() { + assertEquals(null, RestResponseRedactor.redact(null)); + assertEquals("", RestResponseRedactor.redact("")); + assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); + assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java similarity index 76% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java index 0b0aa1ec521..df81225afbe 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java @@ -32,8 +32,12 @@ import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.storage.Table; +/** + * Covers the generic {@link OpenSearchCatalogTable} through a {@link SystemIndexCatalogSource}: + * schema delegation, the predefined-table contract, and the V2 physical path. + */ @ExtendWith(MockitoExtension.class) -class OpenSearchSystemIndexTest { +class OpenSearchCatalogTableTest { @Mock private OpenSearchClient client; @@ -41,36 +45,37 @@ class OpenSearchSystemIndexTest { @Mock private Settings settings; + private OpenSearchCatalogTable systemTable(String name) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); + } + @Test void testGetFieldTypesOfMetaTable() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = systemTable(TABLE_INFO).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("TABLE_CAT", STRING))); } @Test void testGetFieldTypesOfMappingTable() { - OpenSearchSystemIndex systemIndex = - new OpenSearchSystemIndex(client, settings, mappingTable("test_index")); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = + systemTable(mappingTable("test_index")).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("COLUMN_NAME", STRING))); } @Test void testIsExist() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - assertTrue(systemIndex.exists()); + assertTrue(systemTable(TABLE_INFO).exists()); } @Test void testCreateTable() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + Table systemIndex = systemTable(TABLE_INFO); assertThrows(UnsupportedOperationException.class, () -> systemIndex.create(ImmutableMap.of())); } @Test void implement() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + OpenSearchCatalogTable systemIndex = systemTable(TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index ab8b923e3ae..d9437fded5b 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -107,6 +107,7 @@ import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine; import org.opensearch.sql.plugin.config.EngineExtensionsHolder; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; +import org.opensearch.sql.plugin.rest.AnalyticsEngineFormatSupport; import org.opensearch.sql.plugin.rest.AnalyticsExecutorHolder; import org.opensearch.sql.plugin.rest.RestPPLGrammarAction; import org.opensearch.sql.plugin.rest.RestPPLQueryAction; @@ -197,6 +198,10 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); + // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can + // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); + return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), @@ -271,6 +276,13 @@ public void onFailure(Exception e) { } }); } else { + // Analytics route only emits JSON; reject unsupported formats (e.g. csv) with a 4xx. + try { + AnalyticsEngineFormatSupport.validateFormat(sqlRequest.format()); + } catch (Exception e) { + RestSqlAction.handleException(channel, e); + return true; + } unifiedQueryHandler.execute( sqlRequest.getQuery(), QueryType.SQL, diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java index d9406935ee5..057c88c9a02 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java @@ -106,8 +106,9 @@ public PPLService pplService( } @Provides - public SQLService sqlService(QueryManager queryManager, QueryPlanFactory queryPlanFactory) { - return new SQLService(new SQLSyntaxParser(), queryManager, queryPlanFactory); + public SQLService sqlService( + QueryManager queryManager, QueryPlanFactory queryPlanFactory, Settings settings) { + return new SQLService(new SQLSyntaxParser(), queryManager, queryPlanFactory, settings); } /** {@link QueryPlanFactory}. */ diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupport.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupport.java new file mode 100644 index 00000000000..b4ac9dd0678 --- /dev/null +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupport.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.rest; + +import java.util.Locale; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.QueryProcessingStage; +import org.opensearch.sql.protocol.response.format.Format; + +/** + * Guards the analytics-engine query route, which only produces JSON. Rejects the alternate output + * formats (csv/raw/viz) the route does not implement, instead of silently answering with JSON. + */ +public final class AnalyticsEngineFormatSupport { + + private AnalyticsEngineFormatSupport() {} + + /** + * Throw an {@link ErrorReport} if the requested output format is unsupported on the analytics + * engine. JSON/JDBC (the default) is supported; csv/raw/viz are not. + */ + public static void validateFormat(Format format) { + // JDBC (the default) is the JSON contract the analytics route emits. + if (format == Format.JDBC) { + return; + } + throw ErrorReport.wrap( + new IllegalArgumentException( + String.format( + Locale.ROOT, + "response in %s format is not supported on the analytics engine", + format.getFormatName()))) + .code(ErrorCode.UNSUPPORTED_OPERATION) + .stage(QueryProcessingStage.ANALYZING) + .location("while selecting the response format for the analytics engine") + .suggestion( + "The analytics engine only returns JSON. Remove the 'format' parameter, or run this" + + " query against a non-analytics index to use the requested format.") + .build(); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 5c6266beee1..b6347bdf8e1 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -22,6 +22,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestCancellableNodeClient; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.datasources.exceptions.DataSourceClientException; @@ -113,27 +114,32 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod TransportPPLQueryRequest transportPPLQueryRequest = new TransportPPLQueryRequest(PPLQueryRequestFactory.getPPLRequest(request)); - return channel -> - nodeClient.execute( - PPLQueryAction.INSTANCE, - transportPPLQueryRequest, - new ActionListener<>() { - @Override - public void onResponse(TransportPPLQueryResponse response) { - sendResponse(channel, OK, response.getContentType(), response.getResult()); + // RestCancellableNodeClient cancels the PPLQueryTask on client disconnect, which cascades to + // the analytics query + fragments. + return channel -> { + RestCancellableNodeClient cancellableClient = + new RestCancellableNodeClient(nodeClient, request.getHttpChannel()); + cancellableClient.execute( + PPLQueryAction.INSTANCE, + transportPPLQueryRequest, + new ActionListener<>() { + @Override + public void onResponse(TransportPPLQueryResponse response) { + sendResponse(channel, OK, response.getContentType(), response.getResult()); + } + + @Override + public void onFailure(Exception e) { + RestStatus status = loggedErrorCode(e); + if (transportPPLQueryRequest.isExplainRequest()) { + LOG.error("Error happened during explain (status {})", status, e); + } else { + LOG.error("Error happened during query handling (status {})", status, e); } - - @Override - public void onFailure(Exception e) { - RestStatus status = loggedErrorCode(e); - if (transportPPLQueryRequest.isExplainRequest()) { - LOG.error("Error happened during explain (status {})", status, e); - } else { - LOG.error("Error happened during query handling (status {})", status, e); - } - reportError(channel, e, status); - } - }); + reportError(channel, e, status); + } + }); + }; } private void sendResponse( diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index d81ce7b5137..5685d539541 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -10,6 +10,9 @@ import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_WORKER_THREAD_POOL_NAME; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.util.Map; import java.util.Optional; import org.apache.calcite.rel.RelNode; @@ -17,11 +20,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.ThreadContext; +import org.opensearch.analytics.QueryRequestContext; import org.opensearch.analytics.exec.QueryPlanExecutor; import org.opensearch.analytics.exec.profile.QueryProfile; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.indices.IndicesService; import org.opensearch.sql.api.UnifiedQueryContext; @@ -37,11 +44,14 @@ import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.executor.analytics.AnalyticsExecutionEngine; import org.opensearch.sql.lang.LangSpec; +import org.opensearch.sql.monitor.profile.ProfileContext; +import org.opensearch.sql.monitor.profile.QueryProfiling; import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse; import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; import org.opensearch.sql.utils.SystemIndexUtils; +import org.opensearch.tasks.Task; import org.opensearch.transport.client.node.NodeClient; /** @@ -97,13 +107,14 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline + // Analytics engine serves neither the system catalog nor the rest command's reserved + // in-cluster source; both fall back to the default (Calcite) pipeline. try (UnifiedQueryContext context = buildParsingContext(queryType)) { - boolean systemCatalog = + boolean defaultPipeline = extractIndexName(query, queryType, context) - .map(RestUnifiedQueryAction::isSystemCatalog) + .map(name -> isSystemCatalog(name) || SystemIndexUtils.isRestSource(name)) .orElse(false); - return !systemCatalog; + return !defaultPipeline; } catch (Exception e) { // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. return !isLegacySystemCatalogQuery(query); @@ -112,13 +123,35 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { try (UnifiedQueryContext context = buildParsingContext(queryType)) { return extractIndexName(query, queryType, context) .map(this::stripSchemaPrefix) - .map(this::isPluggableDataformatIndex) + .map(this::allIndicesArePluggableDataformat) .orElse(false); } catch (Exception e) { return false; } } + /** + * Checks if all indices in a (possibly comma-separated) index expression are pluggable + * dataformat. For multi-index queries (source=idx1,idx2), each index is checked independently. + * Returns true only if every index is composite — mixed or all-lucene returns false. + */ + private boolean allIndicesArePluggableDataformat(String indexExpression) { + String[] indices = indexExpression.split(","); + if (indices.length == 0) { + return false; + } + for (String idx : indices) { + String trimmed = idx.trim(); + if (trimmed.isEmpty()) { + continue; + } + if (!isPluggableDataformatIndex(trimmed)) { + return false; + } + } + return true; + } + private static boolean isSystemCatalog(String name) { return SystemIndexUtils.isSystemIndex(name) || SystemIndexUtils.DATASOURCES_TABLE_NAME.equals(name); @@ -144,11 +177,33 @@ private boolean isPluggableDataformatIndex(String indexName) { && "composite".equals(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(settings)); } - /** Execute a query through the unified query pipeline on the sql-worker thread pool. */ + /** Execute with no parent task (SQL path): the analytics query runs detached, not cancellable. */ + public void execute( + String query, + QueryType queryType, + boolean profiling, + ActionListener listener) { + doExecute(query, queryType, profiling, 0, null, listener); + } + + /** Execute linked to {@code parentTask} so a front-end cancel propagates into the engine. */ public void execute( String query, QueryType queryType, boolean profiling, + int fetchSize, + Task parentTask, + ActionListener listener) { + assert parentTask != null : "parentTask required for cancellation propagation"; + doExecute(query, queryType, profiling, fetchSize, parentTask, listener); + } + + private void doExecute( + String query, + QueryType queryType, + boolean profiling, + int fetchSize, + Task parentTask, ActionListener listener) { client .threadPool() @@ -158,56 +213,83 @@ public void execute( // Ask the engine for a per-query context — it binds the snapshot // (cluster state + schema built from it) and returns the pair, so the // schema we plan against and the state the executor uses are the same view. - org.opensearch.analytics.QueryRequestContext queryCtx = - contextProvider.getContext(); - // Disable SQL-layer phase profiling when analytics engine profiling is active. - // Our QueryProfile (stages, tasks, timing) is strictly more detailed and replaces - // it. - UnifiedQueryContext context = buildContext(queryType, false, queryCtx); + // Carry the front-end task so cancellation propagates into the engine. + QueryRequestContext queryCtx = + withParentTask(contextProvider.getContext(), parentTask); + + UnifiedQueryContext context = buildContext(queryType, profiling, queryCtx); + ProfileContext profileCtx = QueryProfiling.current(); ActionListener closingListener = wrapWithContextClose(context, listener); try { UnifiedQueryPlanner planner = new UnifiedQueryPlanner(context); RelNode plan = planner.plan(query); CalcitePlanContext planContext = context.getPlanContext(); + // PPL fetch_size caps the response to N rows (no cursor) — the V2 path attaches + // a `head N` in AstStatementBuilder; the unified path parses only the query + // string, so apply the equivalent top-level limit here before the system cap. + plan = addFetchSizeLimit(plan, planContext, fetchSize); plan = addQuerySizeLimit(plan, planContext); if (profiling) { analyticsEngine.executeWithProfile( plan, planContext, queryCtx, - createQueryListener(queryType, closingListener)); + createQueryListener(queryType, profileCtx, closingListener)); } else { analyticsEngine.execute( plan, planContext, queryCtx, - createQueryListener(queryType, closingListener)); + createQueryListener(queryType, profileCtx, closingListener)); } } catch (Exception e) { closingListener.onFailure(e); + } finally { + // visitTimewrap (run inside planner.plan) sets timewrap thread-locals on this + // worker thread. execute()/executeWithProfile() capture-and-clear them on the + // happy path, but a planning exception bypasses that — clear here so the + // signals never leak onto the next query reusing this pooled thread. + CalcitePlanContext.clearTimewrapSignals(); } }), new TimeValue(0), SQL_WORKER_THREAD_POOL_NAME); } - /** - * Explain a query through the unified query pipeline on the sql-worker thread pool. Returns - * ExplainResponse via ResponseListener so the caller can format it. - */ + /** Explain with no parent task (SQL path). */ public void explain( String query, QueryType queryType, ExplainMode mode, ResponseListener listener) { + doExplain(query, queryType, mode, null, listener); + } + + /** Explain linked to {@code parentTask} so a front-end cancel propagates into the engine. */ + public void explain( + String query, + QueryType queryType, + ExplainMode mode, + Task parentTask, + ResponseListener listener) { + assert parentTask != null : "parentTask required for cancellation propagation"; + doExplain(query, queryType, mode, parentTask, listener); + } + + private void doExplain( + String query, + QueryType queryType, + ExplainMode mode, + Task parentTask, + ResponseListener listener) { client .threadPool() .schedule( withCurrentContext( () -> { - org.opensearch.analytics.QueryRequestContext queryCtx = - contextProvider.getContext(); + QueryRequestContext queryCtx = + withParentTask(contextProvider.getContext(), parentTask); try (UnifiedQueryContext context = buildContext(queryType, false, queryCtx)) { UnifiedQueryPlanner planner = new UnifiedQueryPlanner(context); RelNode plan = planner.plan(query); @@ -216,6 +298,11 @@ public void explain( analyticsEngine.explain(plan, mode, planContext, listener); } catch (Exception e) { listener.onFailure(e); + } finally { + // explain plans a timewrap query (visitTimewrap sets thread-locals) but never + // executes, so nothing captures-and-clears them — clear here to avoid leaking + // onto the next query on this pooled thread. + CalcitePlanContext.clearTimewrapSignals(); } }), new TimeValue(0), @@ -231,9 +318,7 @@ private UnifiedQueryContext buildParsingContext(QueryType queryType) { } private UnifiedQueryContext buildContext( - QueryType queryType, - boolean profiling, - org.opensearch.analytics.QueryRequestContext queryCtx) { + QueryType queryType, boolean profiling, QueryRequestContext queryCtx) { return applyClusterOverrides( UnifiedQueryContext.builder() .language(queryType) @@ -244,6 +329,14 @@ private UnifiedQueryContext buildContext( .build(); } + /** Returns {@code ctx} carrying {@code parentTask}, or unchanged when there is none. */ + private static QueryRequestContext withParentTask(QueryRequestContext ctx, Task parentTask) { + if (parentTask == null) { + return ctx; + } + return new QueryRequestContext(ctx.clusterState(), ctx.schema(), ctx.querySource(), parentTask); + } + /** * Routes operator-configured cluster overrides into the builder via the existing {@code * setting(String, Object)} API, keeping {@link UnifiedQueryContext} decoupled from any specific @@ -257,6 +350,8 @@ private UnifiedQueryContext.Builder applyClusterOverrides(UnifiedQueryContext.Bu builder, org.opensearch.sql.common.setting.Settings.Key.PPL_REX_MAX_MATCH_LIMIT); forwardClusterSetting( builder, org.opensearch.sql.common.setting.Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED); + forwardClusterSetting( + builder, org.opensearch.sql.common.setting.Settings.Key.MAX_EXPRESSION_DEPTH); return builder; } @@ -293,20 +388,48 @@ private static RelNode addQuerySizeLimit(RelNode plan, CalcitePlanContext contex context.relBuilder.literal(context.sysLimit.querySizeLimit())); } + /** + * Cap the result to {@code fetchSize} rows when {@code fetchSize > 0}, mirroring PPL's {@code + * fetch_size} (which the V2 path lowers to a top-level {@code head N} in AstStatementBuilder). + * {@code fetchSize <= 0} means "use system default", so no limit is added. Uses the same {@code + * relBuilder.limit} primitive that {@code head} lowers to, so the analytics backend sees an + * ordinary fetch limit. + */ + private static RelNode addFetchSizeLimit( + RelNode plan, CalcitePlanContext context, int fetchSize) { + if (fetchSize <= 0) { + return plan; + } + return context.relBuilder.push(plan).limit(0, fetchSize).build(); + } + private ResponseListener createQueryListener( - QueryType queryType, ActionListener transportListener) { + QueryType queryType, + ProfileContext profileCtx, + ActionListener transportListener) { ResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); return new ResponseListener() { @Override public void onResponse(QueryResponse response) { LangSpec langSpec = queryType == QueryType.PPL ? PPL_SPEC : LangSpec.SQL_SPEC; - String result = - formatter.format( - new QueryResult( - response.getSchema(), response.getResults(), response.getCursor(), langSpec)); + + // Set the engine profile as the plan so the formatter serializes it in one pass. if (response.getProfile() != null) { - // Append profile and error (if any) to the JSON response - result = appendProfileToJson(result, response.getProfile(), response.getError()); + profileCtx.setEnginePlan(toJsonElement(response.getProfile())); + } + + String result = + QueryProfiling.withCurrentContext( + profileCtx, + () -> + formatter.format( + new QueryResult( + response.getSchema(), + response.getResults(), + response.getCursor(), + langSpec))); + if (response.getError() != null) { + result = appendError(result, response.getError()); } transportListener.onResponse(new TransportPPLQueryResponse(result)); } @@ -318,30 +441,24 @@ public void onFailure(Exception e) { }; } - private static String appendProfileToJson(String json, QueryProfile profile, Throwable error) { + private static JsonElement toJsonElement(QueryProfile profile) { try { - StringBuilder extra = new StringBuilder(); - // Append profile - org.opensearch.core.xcontent.XContentBuilder builder = - org.opensearch.common.xcontent.XContentFactory.jsonBuilder(); - profile.toXContent(builder, org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS); - extra.append(",\"profile\":").append(builder.toString()); - // Append error if query partially failed - if (error != null) { - extra - .append(",\"error\":{\"type\":\"") - .append(error.getClass().getSimpleName()) - .append("\",\"reason\":\"") - .append(error.getMessage() != null ? error.getMessage().replace("\"", "\\\"") : "") - .append("\"}"); - } - if (json.endsWith("}")) { - return json.substring(0, json.length() - 1) + extra + "}"; - } - return json; + XContentBuilder builder = XContentFactory.jsonBuilder(); + profile.toXContent(builder, ToXContent.EMPTY_PARAMS); + return JsonParser.parseString(builder.toString()); } catch (Exception e) { + return null; + } + } + + private static String appendError(String json, Throwable error) { + if (!json.endsWith("}")) { return json; } + JsonObject err = new JsonObject(); + err.addProperty("type", error.getClass().getSimpleName()); + err.addProperty("reason", error.getMessage() != null ? error.getMessage() : ""); + return json.substring(0, json.length() - 1) + ",\"error\":" + err + "}"; } private static Runnable withCurrentContext(final Runnable task) { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index fd231c29076..678ed58f37f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -10,7 +10,9 @@ import static org.opensearch.sql.lang.PPLLangSpec.PPL_SPEC; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +import java.util.LinkedHashMap; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; import org.apache.calcite.rel.RelNode; @@ -40,6 +42,7 @@ import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.plugin.config.EngineExtensionsHolder; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; +import org.opensearch.sql.plugin.rest.AnalyticsEngineFormatSupport; import org.opensearch.sql.plugin.rest.AnalyticsExecutorHolder; import org.opensearch.sql.plugin.rest.RestUnifiedQueryAction; import org.opensearch.sql.ppl.PPLService; @@ -176,17 +179,28 @@ protected void doExecute( if (unifiedQueryHandler != null && unifiedQueryHandler.isAnalyticsIndex(transformedRequest.getRequest(), QueryType.PPL)) { LOG.info("[{}] Routing PPL query to analytics engine", QueryContext.getRequestId()); + // Pass this PPL task so the analytics engine links its query task to it for cancellation. if (transformedRequest.isExplainRequest()) { unifiedQueryHandler.explain( transformedRequest.getRequest(), QueryType.PPL, transformedRequest.mode(), + task, createExplainResponseListener(transformedRequest, clearingListener)); } else { + // Analytics route only emits JSON; reject unsupported formats (e.g. csv) with a 4xx. + try { + AnalyticsEngineFormatSupport.validateFormat(format(transformedRequest)); + } catch (Exception e) { + clearingListener.onFailure(e); + return; + } unifiedQueryHandler.execute( transformedRequest.getRequest(), QueryType.PPL, transformedRequest.profile(), + transformedRequest.getFetchSize(), + task, clearingListener); } return; @@ -231,6 +245,18 @@ protected Object buildYamlObject(ExecutionEngine.ExplainResponse response) { new JsonResponseFormatter<>(PRETTY) { @Override protected Object buildJsonObject(ExecutionEngine.ExplainResponse response) { + // For json_tree format, use parsed tree objects instead of strings + if (response.getCalcite() != null + && response.getCalcite().getLogicalTree() != null) { + Map result = new LinkedHashMap<>(); + Map calcite = new LinkedHashMap<>(); + calcite.put("logical", response.getCalcite().getLogicalTree()); + if (response.getCalcite().getPhysicalTree() != null) { + calcite.put("physical", response.getCalcite().getPhysicalTree()); + } + result.put("calcite", calcite); + return result; + } return response; } }; diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupportTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupportTest.java new file mode 100644 index 00000000000..e7de2267e78 --- /dev/null +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupportTest.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.rest; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.protocol.response.format.Format; + +/** + * Verifies {@link AnalyticsEngineFormatSupport#validateFormat(Format)} accepts the JSON/JDBC + * default the analytics route emits and rejects unsupported output formats (csv/raw/viz) with a + * structured {@link ErrorReport} so the REST layer returns a clean 4xx instead of silently + * returning JSON. + */ +public class AnalyticsEngineFormatSupportTest { + + @Test + public void jdbcFormatIsAccepted() { + // No exception expected — JDBC is the JSON contract the analytics route emits. + AnalyticsEngineFormatSupport.validateFormat(Format.JDBC); + } + + @Test + public void csvFormatIsRejectedAsUnsupportedOperation() { + ErrorReport report = + assertThrows(() -> AnalyticsEngineFormatSupport.validateFormat(Format.CSV)); + assertEquals(ErrorCode.UNSUPPORTED_OPERATION, report.getCode()); + assertTrue(report.getMessage().toLowerCase().contains("csv")); + assertTrue(report.getMessage().contains("analytics engine")); + assertNotNull(report.getSuggestion()); + } + + @Test + public void rawFormatIsRejected() { + ErrorReport report = + assertThrows(() -> AnalyticsEngineFormatSupport.validateFormat(Format.RAW)); + assertEquals(ErrorCode.UNSUPPORTED_OPERATION, report.getCode()); + assertTrue(report.getMessage().toLowerCase().contains("raw")); + } + + @Test + public void vizFormatIsRejected() { + ErrorReport report = + assertThrows(() -> AnalyticsEngineFormatSupport.validateFormat(Format.VIZ)); + assertEquals(ErrorCode.UNSUPPORTED_OPERATION, report.getCode()); + } + + private static ErrorReport assertThrows(Runnable runnable) { + try { + runnable.run(); + } catch (ErrorReport e) { + return e; + } + throw new AssertionError("Expected ErrorReport to be thrown, but nothing was thrown"); + } +} diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 111597bb587..0cf87f0604e 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -178,6 +178,12 @@ public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } + @Test + public void restCommandNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("| rest '/_cluster/health'", QueryType.PPL)); + } + @Test public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { enableClusterComposite(); diff --git a/ppl/build.gradle b/ppl/build.gradle index caf5223103c..e883c891fd3 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -63,7 +63,7 @@ dependencies { testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" - testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0' + testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: "${calcite_version}" testImplementation(testFixtures(project(":core"))) } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 4bc69a8f295..fab30b66c3f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,6 +13,8 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +REST: 'REST'; +TIMEOUT: 'TIMEOUT'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; @@ -26,6 +28,7 @@ STREAMSTATS: 'STREAMSTATS'; DEDUP: 'DEDUP'; SORT: 'SORT'; EVAL: 'EVAL'; +FOREACH: 'FOREACH'; FIELDFORMAT: 'FIELDFORMAT'; HEAD: 'HEAD'; BIN: 'BIN'; @@ -55,6 +58,12 @@ APPENDCOL: 'APPENDCOL'; ADDTOTALS: 'ADDTOTALS'; ADDCOLTOTALS: 'ADDCOLTOTALS'; GRAPHLOOKUP: 'GRAPHLOOKUP'; +TIMEWRAP: 'TIMEWRAP'; +ALIGN: 'ALIGN'; +SERIES: 'SERIES'; +RELATIVE: 'RELATIVE'; +SHORT: 'SHORT'; +EXACT: 'EXACT'; EDGE: 'EDGE'; MAX_DEPTH: 'MAXDEPTH'; DEPTH_FIELD: 'DEPTHFIELD'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 98f85b08282..fda8d66d135 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -45,6 +45,7 @@ subSearch // commands pplCommands : describeCommand + | restCommand | showDataSourcesCommand | searchCommand | multisearchCommand @@ -64,6 +65,7 @@ commands | dedupCommand | sortCommand | evalCommand + | foreachCommand | headCommand | binCommand | rareTopCommand @@ -98,11 +100,13 @@ commands | nomvCommand | graphLookupCommand | unionCommand + | timewrapCommand ; commandName : SEARCH | DESCRIBE + | REST | SHOW | WHERE | FIELDS @@ -115,6 +119,7 @@ commandName | DEDUP | SORT | EVAL + | FOREACH | FIELDFORMAT | HEAD | BIN @@ -149,6 +154,7 @@ commandName | NOMV | TRANSPOSE | GRAPHLOOKUP + | TIMEWRAP ; searchCommand @@ -206,6 +212,16 @@ describeCommand : DESCRIBE tableSourceClause ; + +restCommand + : REST stringLiteral (restArgument)* + ; + +restArgument + : COUNT EQUAL integerLiteral + | TIMEOUT EQUAL stringLiteral + | ident EQUAL literalValue + ; showDataSourcesCommand : SHOW DATASOURCES ; @@ -355,6 +371,27 @@ transposeParameter | (COLUMN_NAME EQUAL stringLiteral) ; +timewrapCommand + : TIMEWRAP spanLiteral timewrapParameter* + ; + +timewrapParameter + : ALIGN EQUAL timewrapAlign + | SERIES EQUAL timewrapSeries + | TIME_FORMAT EQUAL stringLiteral + ; + +timewrapAlign + : NOW + | END + ; + +timewrapSeries + : RELATIVE + | SHORT + | EXACT + ; + timechartParameter : LIMIT EQUAL integerLiteral @@ -375,6 +412,38 @@ evalCommand : EVAL evalClause (COMMA evalClause)* ; +foreachCommand + : FOREACH foreachArgument* LT_SQR_PRTHS foreachEvalCommand RT_SQR_PRTHS + ; + +foreachArgument + : foreachOption + | foreachTarget + ; + +foreachTarget + : functionCall + | stringLiteral + | wcFieldExpression + | STAR + ; + +foreachOption + : ident EQUAL (ident | foreachPlaceholder) + ; + +foreachEvalCommand + : EVAL foreachEvalClause (COMMA foreachEvalClause)* + ; + +foreachEvalClause + : target = foreachEvalTarget EQUAL logicalExpression + ; + +foreachEvalTarget + : (foreachPlaceholder | ident | DOT | MODULE)+ + ; + fieldformatCommand : FIELDFORMAT fieldFormatEvalClause (COMMA fieldFormatEvalClause)* ; @@ -942,6 +1011,7 @@ valueExpression | literalValue # literalValueExpr | functionCall # functionCallExpr | lambda # lambdaExpr + | foreachPlaceholder # foreachPlaceholderExpr | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr | valueExpression NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr | LT_PRTHS valueExpression (COMMA valueExpression)* RT_PRTHS NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr @@ -1044,6 +1114,16 @@ wcFieldExpression : wcQualifiedName ; +foreachPlaceholder + : LESS LESS foreachPlaceholderName GREATER GREATER + ; + +foreachPlaceholderName + : FIELD + | ID + | NUMERIC_ID + ; + selectFieldExpression : wcQualifiedName | STAR @@ -1773,4 +1853,6 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + // rest command token, also usable as a free-text search term / identifier + | TIMEOUT ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index ea353066cb0..6ad9032432c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -99,7 +99,12 @@ private AbstractPlan plan( .isExplain(request.isExplainRequest()) .fetchSize(request.getFetchSize()) .highlightConfig(request.getHighlightConfig()) - .format(request.getFormat()) + .format( + request.getFormat() != null && !request.getFormat().isEmpty() + ? org.opensearch.sql.protocol.response.format.Format.ofExplain( + request.getFormat()) + .orElse(null) + : null) .explainMode(request.getExplainMode()) .build())); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 3741137f5a9..e7d42793fe8 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -54,6 +54,7 @@ import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.Let; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Map; @@ -64,6 +65,7 @@ import org.opensearch.sql.ast.expression.SearchAnd; import org.opensearch.sql.ast.expression.SearchExpression; import org.opensearch.sql.ast.expression.SearchGroup; +import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedArgument; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.expression.WindowFrame; @@ -86,6 +88,8 @@ import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; +import org.opensearch.sql.ast.tree.Foreach; +import org.opensearch.sql.ast.tree.Foreach.ForeachEvalClause; import org.opensearch.sql.ast.tree.GraphLookup; import org.opensearch.sql.ast.tree.GraphLookup.Direction; import org.opensearch.sql.ast.tree.Head; @@ -109,6 +113,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.ReplacePair; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -118,12 +123,14 @@ import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Timewrap; import org.opensearch.sql.ast.tree.Transpose; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Window; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; @@ -140,6 +147,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; +import org.opensearch.sql.utils.SystemIndexUtils; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -159,7 +167,7 @@ public AstBuilder(String query) { } public AstBuilder(String query, Settings settings) { - this.expressionBuilder = new AstExpressionBuilder(this); + this.expressionBuilder = new AstExpressionBuilder(this, AstBuildGuard.fromSettings(settings)); this.query = query; this.settings = settings; } @@ -251,6 +259,37 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } + /** + * Rest command.
    + * Leading command that reads an allow-listed, read-only in-cluster management endpoint + * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved + * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves + * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE + * resolves to a system index. Allow-list/authorization enforcement happens at source-table + * construction in the storage engine (it owns the transport actions and per-endpoint schemas). + */ + @Override + public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { + String endpoint = StringUtils.unquoteText(ctx.stringLiteral().getText()); + LinkedHashMap args = new LinkedHashMap<>(); + Integer count = null; + String timeout = null; + for (OpenSearchPPLParser.RestArgumentContext arg : ctx.restArgument()) { + if (arg.COUNT() != null) { + count = Integer.parseInt(arg.integerLiteral().getText()); + } else if (arg.TIMEOUT() != null) { + timeout = StringUtils.unquoteText(arg.stringLiteral().getText()); + } else { + args.put( + StringUtils.unquoteIdentifier(arg.ident().getText()), + StringUtils.unquoteText(arg.literalValue().getText())); + } + } + String token = + SystemIndexUtils.restTable(new SystemIndexUtils.RestSpec(endpoint, args, count, timeout)); + return new RestRelation(new QualifiedName(token)); + } + /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { @@ -822,6 +861,39 @@ public UnresolvedPlan visitTimechartCommand(OpenSearchPPLParser.TimechartCommand .build(); } + /** Timewrap command. */ + @Override + public UnresolvedPlan visitTimewrapCommand(OpenSearchPPLParser.TimewrapCommandContext ctx) { + Literal spanLiteral = (Literal) expressionBuilder.visit(ctx.spanLiteral()); + String spanText = spanLiteral.getValue().toString(); + String valueStr = spanText.replaceAll("[^0-9]", ""); + String unitStr = spanText.replaceAll("[0-9]", ""); + int value = valueStr.isEmpty() ? 1 : Integer.parseInt(valueStr); + SpanUnit unit = SpanUnit.of(unitStr); + if (unit == SpanUnit.UNKNOWN || unit == SpanUnit.NONE) { + throw new SemanticCheckException("Invalid timewrap span unit: " + unitStr); + } + String align = "end"; + String series = "relative"; + String timeFormat = null; + for (var param : ctx.timewrapParameter()) { + if (param.timewrapAlign() != null) { + align = param.timewrapAlign().getText().toLowerCase(); + } else if (param.timewrapSeries() != null) { + series = param.timewrapSeries().getText().toLowerCase(); + } else if (param.TIME_FORMAT() != null) { + timeFormat = param.stringLiteral().getText(); + // Strip surrounding quotes + if (timeFormat.length() >= 2 + && ((timeFormat.startsWith("\"") && timeFormat.endsWith("\"")) + || (timeFormat.startsWith("'") && timeFormat.endsWith("'")))) { + timeFormat = timeFormat.substring(1, timeFormat.length() - 1); + } + } + } + return new Timewrap(unit, value, align, series, timeFormat, spanLiteral); + } + /** Eval command. */ @Override public UnresolvedPlan visitEvalCommand(EvalCommandContext ctx) { @@ -831,6 +903,66 @@ public UnresolvedPlan visitEvalCommand(EvalCommandContext ctx) { .collect(Collectors.toList())); } + @Override + public UnresolvedPlan visitForeachCommand(OpenSearchPPLParser.ForeachCommandContext ctx) { + java.util.Map options = new LinkedHashMap<>(); + List targets = new ArrayList<>(); + List patterns = new ArrayList<>(); + for (OpenSearchPPLParser.ForeachArgumentContext argument : ctx.foreachArgument()) { + OpenSearchPPLParser.ForeachOptionContext option = argument.foreachOption(); + if (option != null) { + options.put( + option.ident(0).getText().toLowerCase(Locale.ROOT), + stripForeachPlaceholderMarkers(option.getChild(2).getText())); + } else { + OpenSearchPPLParser.ForeachTargetContext target = argument.foreachTarget(); + patterns.add(getTextInQuery(target)); + if (target.functionCall() != null) { + targets.add(expressionBuilder.visit(target.functionCall())); + } else if (target.stringLiteral() != null) { + targets.add(expressionBuilder.visit(target.stringLiteral())); + } else { + targets.add(new Field(new QualifiedName(getTextInQuery(target)))); + } + } + } + Foreach.Mode mode = + options.containsKey("mode") + ? Foreach.Mode.of(options.get("mode")) + : inferForeachMode(targets); + UnresolvedExpression collectionExpression = null; + if (mode != Foreach.Mode.MULTIFIELD) { + if (targets.size() > 1 || (targets.isEmpty() && mode != Foreach.Mode.AUTO_COLLECTIONS)) { + throw new IllegalArgumentException("foreach collection modes accept exactly one field"); + } + collectionExpression = targets.isEmpty() ? null : targets.get(0); + } + List evalClauses = + ctx.foreachEvalCommand().foreachEvalClause().stream() + .map( + clause -> + new ForeachEvalClause( + getTextInQuery(clause.target), + expressionBuilder.visit(clause.logicalExpression()))) + .collect(Collectors.toList()); + return new Foreach(mode, options, patterns, collectionExpression, evalClauses); + } + + private String stripForeachPlaceholderMarkers(String value) { + return value.startsWith("<<") && value.endsWith(">>") + ? value.substring(2, value.length() - 2) + : value; + } + + /** A bare json_array(...) target implies json_array mode; anything else is multifield. */ + private Foreach.Mode inferForeachMode(List targets) { + return targets.size() == 1 + && targets.get(0) instanceof Function function + && "json_array".equalsIgnoreCase(function.getFuncName()) + ? Foreach.Mode.JSON_ARRAY + : Foreach.Mode.MULTIFIELD; + } + @Override public UnresolvedPlan visitFieldformatCommand(FieldformatCommandContext ctx) { // Use the new fieldFormatEvalClause instead of evalClause diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 77d5c77a635..b7031a1bd68 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -22,6 +22,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.RuleNode; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.Node; import org.opensearch.sql.ast.dsl.AstDSL; @@ -31,6 +32,7 @@ import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; @@ -95,8 +97,25 @@ public class AstExpressionBuilder extends OpenSearchPPLParserBaseVisitor super.visit(tree)); + } + + @Override + public UnresolvedExpression visitChildren(RuleNode node) { + return guard.enforce(() -> super.visitChildren(node)); } /** Eval clause. */ @@ -105,6 +124,18 @@ public UnresolvedExpression visitEvalClause(EvalClauseContext ctx) { return new Let((Field) visit(ctx.fieldExpression()), visit(ctx.logicalExpression())); } + @Override + public UnresolvedExpression visitForeachPlaceholderExpr( + OpenSearchPPLParser.ForeachPlaceholderExprContext ctx) { + return visit(ctx.foreachPlaceholder()); + } + + @Override + public UnresolvedExpression visitForeachPlaceholder( + OpenSearchPPLParser.ForeachPlaceholderContext ctx) { + return new ForeachPlaceholder(ctx.foreachPlaceholderName().getText()); + } + /** Field format eval clause - similar to evalClause but for fieldformat command. */ @Override public UnresolvedExpression visitFieldFormatEvalClause( @@ -903,11 +934,21 @@ public UnresolvedExpression visitMaxOption(OpenSearchPPLParser.MaxOptionContext } public QualifiedName visitIdentifiers(List ctx) { - return new QualifiedName( + List parts = ctx.stream() .map(RuleContext::getText) .map(StringUtils::unquoteIdentifier) - .collect(Collectors.toList())); + .collect(Collectors.toList()); + + // Capture source position from the first identifier for error reporting + if (!ctx.isEmpty()) { + ParserRuleContext first = ctx.get(0); + int line = first.getStart().getLine(); + int column = first.getStart().getCharPositionInLine(); + return new QualifiedName(parts, line, column); + } + + return new QualifiedName(parts); } private List singleFieldRelevanceArguments( diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java index d2c1f610238..62503923eee 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java @@ -21,6 +21,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; +import org.opensearch.sql.protocol.response.format.Format; /** Build {@link Statement} from PPL Query. */ @RequiredArgsConstructor @@ -45,12 +46,15 @@ public Statement visitPplStatement(OpenSearchPPLParser.PplStatementContext ctx) } if (ctx.explainStatement() != null) { if (ctx.explainStatement().explainMode() == null) { - return new Explain(query, PPL); + return new Explain(query, PPL, null, context.format); } else { - return new Explain(query, PPL, ctx.explainStatement().explainMode().getText()); + return new Explain( + query, PPL, ctx.explainStatement().explainMode().getText(), context.format); } } else { - return context.isExplain ? new Explain(query, PPL, context.explainMode) : query; + return context.isExplain + ? new Explain(query, PPL, context.explainMode, context.format) + : query; } } @@ -74,7 +78,7 @@ public static class StatementBuilderContext { /** Highlight config from the API request. */ private final HighlightConfig highlightConfig; - private final String format; + private final Format format; private final String explainMode; } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 4b75d444467..540869d1642 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -78,6 +78,7 @@ import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; +import org.opensearch.sql.ast.tree.Foreach; import org.opensearch.sql.ast.tree.GraphLookup; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Join; @@ -96,6 +97,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -105,6 +107,7 @@ import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Timewrap; import org.opensearch.sql.ast.tree.Transpose; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Union; @@ -122,6 +125,7 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.utils.SystemIndexUtils; /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { @@ -165,6 +169,23 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { + if (node instanceof RestRelation) { + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(node.getTableQualifiedName().toString()); + StringBuilder sb = new StringBuilder("rest ").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append(" count=").append(MASK_LITERAL); + } + if (spec.getTimeout() != null) { + sb.append(" timeout=").append(MASK_LITERAL); + } + if (spec.getArgs() != null) { + for (String key : spec.getArgs().keySet()) { + sb.append(' ').append(key).append('=').append(MASK_LITERAL); + } + } + return sb.toString(); + } if (node instanceof DescribeRelation) { return StringUtils.format("describe %s", MASK_TABLE); } @@ -584,6 +605,31 @@ public String visitMvExpand(MvExpand node, String context) { return StringUtils.format("%s | mvexpand %s", child, field); } + @Override + public String visitForeach(Foreach node, String context) { + String child = node.getChild().get(0).accept(this, context); + StringBuilder command = new StringBuilder(child).append(" | foreach"); + if (node.getMode() != Foreach.Mode.MULTIFIELD) { + command.append(" mode=").append(node.getMode()); + } + // Placeholder rename options (itemstr=X etc.) carry user-chosen identifiers, not data; the + // mode key is already rendered above. + node.getOptions().keySet().stream() + .filter(key -> !"mode".equals(key)) + .forEach(key -> command.append(' ').append(key).append('=').append(MASK_COLUMN)); + // Targets are field names or patterns in multifield mode; collection targets may embed + // literals (e.g. a JSON array string), so mask them all. + node.getFieldPatterns().forEach(pattern -> command.append(' ').append(MASK_COLUMN)); + String evalClauses = + node.getEvalClauses().stream() + .map( + clause -> + StringUtils.format( + "%s = %s", MASK_COLUMN, visitExpression(clause.getExpression()))) + .collect(Collectors.joining(", ")); + return command.append(" [ eval ").append(evalClauses).append(" ]").toString(); + } + /** Build {@link LogicalSort}. */ @Override public String visitSort(Sort node, String context) { @@ -626,6 +672,25 @@ public String visitReverse(Reverse node, String context) { return StringUtils.format("%s | reverse", child); } + @Override + public String visitTimewrap(Timewrap node, String context) { + String child = node.getChild().get(0).accept(this, context); + StringBuilder command = new StringBuilder(); + // span magnitude is masked like other span literals (see visitChart); align/series are + // constrained keywords, not user data, so they are rendered verbatim. + command.append(" | timewrap ").append(MASK_LITERAL); + if (node.getAlign() != null) { + command.append(" align=").append(node.getAlign()); + } + if (node.getSeries() != null) { + command.append(" series=").append(node.getSeries()); + } + if (node.getTimeFormat() != null) { + command.append(" time_format=").append(MASK_LITERAL); + } + return StringUtils.format("%s%s", child, command); + } + @Override public String visitChart(Chart node, String context) { String child = node.getChild().get(0).accept(this, context); @@ -1207,5 +1272,12 @@ public String visitQualifiedName( org.opensearch.sql.ast.expression.QualifiedName node, String context) { return MASK_COLUMN; } + + @Override + public String visitForeachPlaceholder( + org.opensearch.sql.ast.expression.ForeachPlaceholder node, String context) { + // Placeholder names are user-chosen identifiers; mask like any other column reference. + return MASK_COLUMN; + } } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 0825f6d1def..37398bbf7e8 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -22,7 +22,6 @@ import org.opensearch.sql.executor.DefaultQueryManager; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; -import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.executor.execution.QueryPlanFactory; @@ -134,15 +133,6 @@ public void testExecuteCsvFormatShouldPass() { @Test public void testExplainShouldPass() { - doAnswer( - invocation -> { - ResponseListener listener = invocation.getArgument(3); - listener.onResponse(new ExplainResponse(new ExplainResponseNode("test"))); - return null; - }) - .when(queryService) - .explain(any(), any(), any(), any(), any()); - pplService.explain( new PPLQueryRequest("search source=t a=1", null, EXPLAIN), new ResponseListener() { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 1238f1e7e6c..0c5ad9dd4fe 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -89,6 +89,57 @@ public void testSearchFieldsCommandShouldPass() { assertNotEquals(null, tree); } + @Test + public void testForeachCommandShouldPass() { + ParseTree tree = + new PPLSyntaxParser() + .parse("source=t | foreach a* b [ eval <>_double = <> * 2 ]"); + assertNotEquals(null, tree); + } + + @Test + public void testForeachCommandWithModeShouldPass() { + ParseTree tree = + new PPLSyntaxParser() + .parse( + "source=t | foreach mode=multifield value_* [ eval" + + " total_<> = <> + 1 ]"); + assertNotEquals(null, tree); + } + + @Test + public void testForeachCollectionModesShouldPass() { + assertNotEquals( + null, + new PPLSyntaxParser() + .parse( + "source=t | foreach mode=multivalue itemstr=<> nums [ eval total = total +" + + " <> ]")); + assertNotEquals( + null, + new PPLSyntaxParser() + .parse( + "source=t | foreach mode=json_array '[1,2,3]' [ eval total = total + <> ]")); + assertNotEquals( + null, + new PPLSyntaxParser() + .parse("source=t | foreach mode=auto_collections nums [ eval total = total + ITEM ]")); + assertNotEquals( + null, + new PPLSyntaxParser() + .parse("source=t | foreach mode=auto_collections [ eval total = total + <> ]")); + assertNotEquals( + null, + new PPLSyntaxParser() + .parse("source=t | foreach json_array(1, 2, 3) [ eval total = total + <> ]")); + assertNotEquals( + null, + new PPLSyntaxParser() + .parse( + "source=t | foreach mode=multivalue iterstr=IDX nums [ eval total = total + IDX" + + " ]")); + } + @Test public void testSearchFieldsCommandCrossClusterShouldPass() { ParseTree tree = new PPLSyntaxParser().parse("search source=c:t a=1 b=2 | fields a,b"); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java index 56ed409b4d7..dc476b77cff 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java @@ -44,18 +44,18 @@ public void testAppendPipeWithMergedColumns() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:INTEGER])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:BIGINT])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+($7, 10)])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+(CAST($7):BIGINT, 10)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); verifyResultCount(root, 28); String expectedSparkSql = - "SELECT `DEPTNO`, CAST(NULL AS INTEGER) `DEPTNO_PLUS`\n" + "SELECT `DEPTNO`, CAST(NULL AS BIGINT) `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" - + "SELECT `DEPTNO`, `DEPTNO` + 10 `DEPTNO_PLUS`\n" + + "SELECT `DEPTNO`, CAST(`DEPTNO` AS BIGINT) + 10 `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java index a163af186d5..027062485c6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java @@ -211,18 +211,18 @@ public void testAppendWithMergedColumns() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:INTEGER])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:BIGINT])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+($7, 10)])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+(CAST($7):BIGINT, 10)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); verifyResultCount(root, 28); String expectedSparkSql = - "SELECT `DEPTNO`, CAST(NULL AS INTEGER) `DEPTNO_PLUS`\n" + "SELECT `DEPTNO`, CAST(NULL AS BIGINT) `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" - + "SELECT `DEPTNO`, `DEPTNO` + 10 `DEPTNO_PLUS`\n" + + "SELECT `DEPTNO`, CAST(`DEPTNO` AS BIGINT) + 10 `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java index 1d6792b0990..fbbee80ff79 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java @@ -656,8 +656,8 @@ public void testMvmapWithNestedFunction() { verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT TRANSFORM(ARRAY_SLICE(ARRAY(1, 2, 3, 4, 5), 1, 3 - 1 + 1), `arr` -> `arr` * 10)" - + " `result`\n" + "SELECT TRANSFORM(ARRAY_SLICE(ARRAY(1, 2, 3, 4, 5), 1, 3 - 1 + 1), `arr` -> `arr`" + + " * 10) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java index 5cc257b0b0e..14d3476802d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java @@ -5,9 +5,13 @@ package org.opensearch.sql.ppl.calcite; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; +import org.opensearch.sql.exception.ExpressionEvaluationException; public class CalcitePPLCaseFunctionTest extends CalcitePPLAbstractTest { @@ -98,7 +102,19 @@ public void testCaseWhenInSubquery() { + "FROM `scott`.`EMP`\n" + "WHERE `DEPTNO` IN (SELECT CASE WHEN `DEPTNO` IN (20, 21) THEN 20 WHEN `DEPTNO` IN" + " (30, 31) THEN 30 ELSE 100 END `new_deptno`\n" - + "FROM `scott`.`EMP`)"; + + "FROM `scott`.`EMP` `EMP0`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + /** Case branches with no common supertype must be rejected cleanly, not NPE. */ + @Test + public void testCaseWithIncompatibleBranchTypesRejectsCleanly() { + String ppl = + "source=EMP | eval x = case(DEPTNO > 20, 'big'," + " DEPTNO > 10, 1 else 0.0) | fields x"; + ExpressionEvaluationException e = + assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl)); + assertTrue( + "expected message to list incompatible types, got: " + e.getMessage(), + e.getMessage().contains("case branches must have a common type")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java index 5f32c1b85bb..3c13297f8f0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java @@ -198,7 +198,7 @@ public void testDedupExpr() { + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)])\n" + " LogicalFilter(condition=[IS NOT NULL($4)])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], DEPTNO=[$7]," - + " NEW_DEPTNO=[+($7, 1)])\n" + + " NEW_DEPTNO=[+(CAST($7):BIGINT, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -216,7 +216,8 @@ public void testDedupExpr() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $3)])\n" + " LogicalFilter(condition=[IS NOT NULL($3)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -229,10 +230,10 @@ public void testDedupExpr() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); } @@ -277,10 +278,10 @@ public void testSortThenDedupWithEval() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); // After fix, the sort order (NEW_DEPTNO ASC) must be preserved through dedup. @@ -304,7 +305,8 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -317,7 +319,8 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $3)])\n" + " LogicalFilter(condition=[IS NOT NULL($3)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -330,10 +333,10 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java index 9b37ab5b407..3cd9e417a58 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java @@ -104,7 +104,7 @@ public void testEvalSum() { String ppl = "source=EMP | eval total = sum(1, 2, 3) | fields EMPNO, total"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], total=[+(1, +(2, 3))])\n" + "LogicalProject(EMPNO=[$0], total=[+(1, +(2:BIGINT, 3:BIGINT))])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -131,7 +131,8 @@ public void testEvalAvg() { String ppl = "source=EMP | eval average = avg(10, 20, 30) | fields EMPNO, average"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], average=[DIVIDE(+(10, +(20, 30)), 3.0E0:DOUBLE)])\n" + "LogicalProject(EMPNO=[$0], average=[DIVIDE(+(10, +(20:BIGINT, 30:BIGINT))," + + " 3.0E0:DOUBLE)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -210,7 +211,7 @@ public void testEvalUsingExistingFields() { "LogicalProject(EMPNO=[$0], EMPNO_PLUS=[$8])\n" + " LogicalSort(sort0=[$8], dir0=[DESC-nulls-last], fetch=[3])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO_PLUS=[+($0, 1)])\n" + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO_PLUS=[+(CAST($0):BIGINT NOT NULL, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -223,7 +224,7 @@ public void testEvalUsingExistingFields() { String expectedSparkSql = "SELECT `EMPNO`, `EMPNO_PLUS`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " `EMPNO` + 1 `EMPNO_PLUS`\n" + + " CAST(`EMPNO` AS BIGINT) + 1 `EMPNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "ORDER BY 9 DESC\n" + "LIMIT 3) `t0`"; @@ -239,7 +240,7 @@ public void testEvalOverridingExistingFields() { "LogicalProject(EMPNO=[$0], SAL=[$7])\n" + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[3])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " COMM=[$6], DEPTNO=[$7], SAL=[+($7, 10000)])\n" + + " COMM=[$6], DEPTNO=[$7], SAL=[+(CAST($7):BIGINT, 10000)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -248,7 +249,7 @@ public void testEvalOverridingExistingFields() { String expectedSparkSql = "" - + "SELECT `EMPNO`, `DEPTNO` + 10000 `SAL`\n" + + "SELECT `EMPNO`, CAST(`DEPTNO` AS BIGINT) + 10000 `SAL`\n" + "FROM `scott`.`EMP`\n" + "ORDER BY `EMPNO` DESC\n" + "LIMIT 3"; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java index 76c280db92f..2a436d30bb0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java @@ -326,7 +326,7 @@ public void testNestedCorrelatedExistsSubquery() { + "FROM (SELECT *\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE EXISTS (SELECT *\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + "WHERE `SAL` = `SALGRADE`.`HISAL`)) `t0`\n" + "WHERE `EMP`.`SAL` = `HISAL`)\n" + "ORDER BY `EMPNO` DESC"; @@ -372,7 +372,7 @@ public void testNestedUncorrelatedExistsSubquery() { + "FROM (SELECT *\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE EXISTS (SELECT *\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + "WHERE `SAL` > 1000.0)) `t0`\n" + "WHERE `HISAL` > 1000.0)\n" + "ORDER BY `EMPNO` DESC"; @@ -435,12 +435,12 @@ public void testNestedMixedExistsSubquery() { + "FROM `scott`.`SALGRADE`\n" + "WHERE EXISTS (SELECT *\n" + "FROM (SELECT *\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + "WHERE EXISTS (SELECT *\n" + "FROM (SELECT *\n" - + "FROM `scott`.`SALGRADE`\n" + + "FROM `scott`.`SALGRADE` `SALGRADE0`\n" + "WHERE EXISTS (SELECT *\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP1`\n" + "WHERE `SAL` = `SALGRADE0`.`HISAL`)) `t0`\n" + "WHERE `EMP0`.`SAL` > 1000.0)) `t2`\n" + "WHERE `SAL` = `SALGRADE`.`HISAL`)) `t4`\n" @@ -490,14 +490,14 @@ public void testCorrelatedExistsSubqueryWithOverridingFields() { + " LogicalTableScan(table=[[scott, DEPT]])\n" + "})], variablesSet=[[$cor0]])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[+($7, 1)])\n" + + " SAL=[$5], COMM=[$6], DEPTNO=[+(CAST($7):BIGINT, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = "SELECT *\n" - + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO` + 1" - + " `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " CAST(`DEPTNO` AS BIGINT) + 1 `DEPTNO`\n" + "FROM `scott`.`EMP`) `t`\n" + "WHERE EXISTS (SELECT *\n" + "FROM `scott`.`DEPT`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java index 5bef9c397eb..85dfb3eb650 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java @@ -177,7 +177,7 @@ public void testFieldFormatSum() { "source=EMP |sort EMPNO | head 3| fieldformat total = sum(1, 2, 3) | fields EMPNO, total"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], total=[+(1, +(2, 3))])\n" + "LogicalProject(EMPNO=[$0], total=[+(1, +(2:BIGINT, 3:BIGINT))])\n" + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java index e664b4f21db..ee7a1d5c2f1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java @@ -114,7 +114,8 @@ public void testFillnullAll() { String ppl = "source=EMP | fields EMPNO, MGR, SAL, COMM, DEPTNO | fillnull with 0"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], MGR=[COALESCE($3, 0)], SAL=[COALESCE($5, 0)]," + "LogicalProject(EMPNO=[CAST($0):INTEGER NOT NULL], MGR=[COALESCE($3, 0)]," + + " SAL=[COALESCE($5, 0)]," + " COMM=[COALESCE($6, 0)], DEPTNO=[COALESCE($7, 0)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -136,7 +137,8 @@ public void testFillnullAll() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0) `SAL`, COALESCE(`COMM`, 0)" + "SELECT CAST(`EMPNO` AS INTEGER) `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0)" + + " `SAL`, COALESCE(`COMM`, 0)" + " `COMM`, COALESCE(`DEPTNO`, 0) `DEPTNO`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -194,7 +196,8 @@ public void testFillnullValueSyntaxAllFields() { String ppl = "source=EMP | fields EMPNO, MGR, SAL, COMM, DEPTNO | fillnull value=0"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], MGR=[COALESCE($3, 0)], SAL=[COALESCE($5, 0)]," + "LogicalProject(EMPNO=[CAST($0):INTEGER NOT NULL], MGR=[COALESCE($3, 0)]," + + " SAL=[COALESCE($5, 0)]," + " COMM=[COALESCE($6, 0)], DEPTNO=[COALESCE($7, 0)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -216,7 +219,8 @@ public void testFillnullValueSyntaxAllFields() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0) `SAL`, COALESCE(`COMM`, 0)" + "SELECT CAST(`EMPNO` AS INTEGER) `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0)" + + " `SAL`, COALESCE(`COMM`, 0)" + " `COMM`, COALESCE(`DEPTNO`, 0) `DEPTNO`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLForeachTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLForeachTest.java new file mode 100644 index 00000000000..436af2c6259 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLForeachTest.java @@ -0,0 +1,328 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; +import org.opensearch.sql.exception.SemanticCheckException; + +public class CalcitePPLForeachTest extends CalcitePPLAbstractTest { + + public CalcitePPLForeachTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testForeachExpandsExplicitFields() { + String ppl = + "source=EMP | foreach SAL COMM [ eval <>_double = <> * 2 ] | fields" + + " EMPNO, SAL_double, COMM_double"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(EMPNO=[$0], SAL_double=[*($5, 2)], COMM_double=[*($6, 2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `SAL` * 2 `SAL_double`, `COMM` * 2 `COMM_double`\n" + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testForeachWildcardAndMatchstr() { + String ppl = + "source=EMP | foreach *NO [ eval copy_<> = <> ] | fields copy_EMP," + + " copy_DEPT"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(copy_EMP=[$0], copy_DEPT=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachCustomMultifieldPlaceholders() { + String ppl = + "source=EMP | foreach fieldstr=F matchstr=M *NO [ eval copy_<> = F ] | fields" + + " copy_EMP, copy_DEPT"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(copy_EMP=[$0], copy_DEPT=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachMultivaluePlansReduce() { + String ppl = + "source=EMP | eval nums = array(1, 2, 3), total = 0 | foreach mode=multivalue" + + " itemstr=NUMBER nums [ eval total = total + NUMBER ] | fields total"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(total=[foreach_pair_item(reduce(foreach_pair_collection(array(1, 2, 3))," + + " foreach_state(0), (__foreach_state, __foreach_pair) ->" + + " foreach_state(+(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0)))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachIterPlaceholderPlansReduce() { + String ppl = + "source=EMP | eval nums = array(10, 20, 30), total = 0 | foreach mode=multivalue" + + " iterstr=IDX nums [ eval total = total + IDX ] | fields total"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(total=[foreach_pair_item(reduce(foreach_pair_collection(array(10, 20, 30))," + + " foreach_state(0), (__foreach_state, __foreach_pair) ->" + + " foreach_state(+(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 1)))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachJsonArrayPlansReduce() { + String ppl = + "source=EMP | eval total = 0 | foreach mode=json_array '[1,2,3]' [ eval total = total +" + + " <> ] | fields total"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(total=[foreach_pair_item(reduce(foreach_pair_collection(foreach_json_array('[1,2,3]':VARCHAR," + + " 'DOUBLE':VARCHAR)), foreach_state(0.0E0:DOUBLE), (__foreach_state, __foreach_pair)" + + " -> foreach_state(+(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0)))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachJsonArrayFunctionWithoutModePlansReduce() { + String ppl = + "source=EMP | eval total = 0 | foreach json_array(1, 2, 3) [ eval total = total +" + + " <> ] | fields total"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(total=[foreach_pair_item(reduce(foreach_pair_collection(foreach_json_array(JSON_ARRAY(FLAG(NULL_ON_NULL)," + + " 1, 2, 3), 'DOUBLE':VARCHAR)), foreach_state(0.0E0:DOUBLE), (__foreach_state," + + " __foreach_pair) -> foreach_state(+(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0)))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachJsonArrayMixedStringAndNumberFails() { + String ppl = + "source=EMP | eval total = 0 | foreach json_array(1, 2, 'hello') [ eval total = total +" + + " <> ] | fields total"; + + assertThrows(SemanticCheckException.class, () -> getRelNode(ppl)); + } + + @Test + public void testForeachAutoCollectionsPlansReduce() { + String ppl = + "source=EMP | eval nums = array(1, 2, 3), total = 0 | foreach mode=auto_collections nums [" + + " eval total = total + <> ] | fields total"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(total=[foreach_pair_item(reduce(foreach_pair_collection(array(1, 2, 3))," + + " foreach_state(0), (__foreach_state, __foreach_pair) ->" + + " foreach_state(+(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0)))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachAutoCollectionsWithoutTargetPlansReduce() { + String ppl = + "source=EMP | eval nums = array(1, 2, 3), total = 0 | foreach mode=auto_collections [" + + " eval total = total + <> ] | fields total"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(total=[foreach_pair_item(reduce(foreach_pair_collection(array(1, 2, 3))," + + " foreach_state(0), (__foreach_state, __foreach_pair) ->" + + " foreach_state(+(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0)))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachStringItemWithNumericIterPlansReduce() { + String ppl = + "source=EMP | eval word = split('ABCDE', ''), nums = array('1', '2', '3', '4', '5')," + + " word_and_num = array() | foreach word mode=multivalue [ eval word_and_num =" + + " mvappend(word_and_num, concat(<>, mvindex(nums, <>))) ] | fields" + + " word_and_num"; + RelNode root = getRelNode(ppl); + + // The captured field `nums` rides in pair slot 2; ITEM is slot 0 and ITER slot 1. + String expectedLogical = + "LogicalProject(word_and_num=[foreach_pair_item(reduce(foreach_pair_collection(CASE(=('':VARCHAR," + + " ''), REGEXP_EXTRACT_ALL('ABCDE':VARCHAR, '.'), SPLIT('ABCDE':VARCHAR, '':VARCHAR))," + + " array('1', '2', '3', '4', '5')), foreach_state(array()), (__foreach_state," + + " __foreach_pair) -> foreach_state(mvappend(foreach_pair_item(__foreach_state, 0)," + + " CONCAT(foreach_pair_item(__foreach_pair, 0), ITEM(foreach_pair_item(__foreach_pair," + + " 2), CASE(<(foreach_pair_item(__foreach_pair, 1), 0)," + + " +(+(ARRAY_LENGTH(foreach_pair_item(__foreach_pair, 2))," + + " foreach_pair_item(__foreach_pair, 1)), 1), +(foreach_pair_item(__foreach_pair, 1)," + + " 1))))))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachJsonArrayFieldNumericUsage() { + // Field content is opaque at plan time; arithmetic on <> selects DOUBLE elements. + String ppl = + "source=EMP | eval total = 0 | foreach mode=json_array ENAME [ eval total = total +" + + " <> ] | fields total"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(total=[foreach_pair_item(reduce(foreach_pair_collection(foreach_json_array($1," + + " 'DOUBLE':VARCHAR)), foreach_state(0.0E0:DOUBLE), (__foreach_state, __foreach_pair)" + + " -> foreach_state(+(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0)))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachJsonArrayFieldStringUsage() { + // No arithmetic on <> keeps field-backed JSON array elements as VARCHAR. + String ppl = + "source=EMP | eval r = '' | foreach mode=json_array ENAME [ eval r = concat(r, <>)" + + " ] | fields r"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(r=[foreach_pair_item(reduce(foreach_pair_collection(foreach_json_array($1," + + " 'VARCHAR':VARCHAR)), foreach_state('':VARCHAR), (__foreach_state, __foreach_pair)" + + " -> foreach_state(CONCAT(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0)))), 0)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testForeachSubstitutesTemplateInsideStringLiteral() { + RelNode root = + getRelNode("source=EMP | foreach ENAME [ eval copy = '<>' ] | fields copy"); + + verifyLogical( + root, "LogicalProject(copy=['ENAME'])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"); + } + + @Test + public void testDefaultItemNameDoesNotShadowRowField() { + RelNode root = + getRelNode( + "source=EMP | eval ITEM=100, nums=array(1,2), total=0 | foreach mode=multivalue" + + " nums [ eval total=total+ITEM ] | fields total"); + + String logical = org.apache.calcite.plan.RelOptUtil.toString(root); + assertTrue(logical.contains("foreach_pair_collection(array(1, 2), 100)")); + assertTrue(logical.contains("foreach_pair_item(__foreach_pair, 2)")); + } + + @Test + public void testCustomItemNameShadowsRowField() { + RelNode root = + getRelNode( + "source=EMP | eval ITEM=100, nums=array(1,2), total=0 | foreach mode=multivalue" + + " itemstr=ITEM nums [ eval total=total+ITEM ] | fields total"); + + String logical = org.apache.calcite.plan.RelOptUtil.toString(root); + assertTrue(logical.contains("foreach_pair_collection(array(1, 2))")); + assertTrue(logical.contains("foreach_pair_item(__foreach_pair, 0)")); + } + + @Test + public void testJsonFieldNumericFunctionInfersDoubleItem() { + RelNode root = + getRelNode( + "source=EMP | eval result=0 | foreach mode=json_array ENAME [ eval" + + " result=abs(<>) ] | fields result"); + + String logical = org.apache.calcite.plan.RelOptUtil.toString(root); + assertTrue(logical, logical.contains("foreach_json_array($1, 'DOUBLE':VARCHAR)")); + } + + @Test + public void testJsonFieldNumericComparisonInfersDoubleItem() { + RelNode root = + getRelNode( + "source=EMP | eval count=0 | foreach mode=json_array ENAME [ eval" + + " count=count+if(<> > 9, 1, 0) ] | fields count"); + + assertTrue( + org.apache.calcite.plan.RelOptUtil.toString(root) + .contains("foreach_json_array($1, 'DOUBLE':VARCHAR)")); + } + + @Test + public void testCollectionModeMismatchIsNoOp() { + RelNode multivalue = + getRelNode( + "source=EMP | eval total=0 | foreach mode=multivalue ENAME [ eval" + + " total=total+<> ] | fields total"); + RelNode jsonArray = + getRelNode( + "source=EMP | eval nums=array(1,2), total=0 | foreach mode=json_array nums [ eval" + + " total=total+<> ] | fields total"); + + String expected = "LogicalProject(total=[0])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(multivalue, expected); + verifyLogical(jsonArray, expected); + } + + @Test + public void testExactPatternTakesPrecedenceForMatchstr() { + RelNode root = + getRelNode( + "source=EMP | foreach *NO EMPNO [ eval copy_<> = '<>' ] |" + + " fields copy_"); + + verifyLogical( + root, "LogicalProject(copy_=[''])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"); + } + + @Test + public void testCollectionAssignmentsSharePerIterationState() { + RelNode root = + getRelNode( + "source=EMP | eval nums=array(1,2), sum=0, seen=0 | foreach mode=multivalue nums" + + " [ eval sum=sum+<>, seen=seen+sum ] | fields sum, seen"); + + String logical = org.apache.calcite.plan.RelOptUtil.toString(root); + assertTrue(logical.contains("foreach_state(0, 0)")); + assertTrue( + logical.contains( + "foreach_state(+(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0))," + + " +(foreach_pair_item(__foreach_state, 1)," + + " +(foreach_pair_item(__foreach_state, 0)," + + " foreach_pair_item(__foreach_pair, 0))))")); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java index 5c26d70335b..e30fa9fd045 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java @@ -77,7 +77,7 @@ public void testSelfInSubquery() { + "SELECT `MGR`\n" + "FROM `scott`.`EMP`\n" + "WHERE `MGR` IN (SELECT `MGR`\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + "WHERE `DEPTNO` = 10)"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java index c272453b829..161a923738c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java @@ -68,7 +68,7 @@ public void testPatternsLabelMode_ShowNumberedToken_ForSimplePatternMethod() { + " '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END, `ENAME`)['pattern'] AS" + " STRING) `patterns_field`, TRY_CAST(PATTERN_PARSER(CASE WHEN `ENAME` IS NULL OR" + " `ENAME` = '' THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END," - + " `ENAME`)['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " `ENAME`)['tokens'] AS MAP< STRING, ARRAY< STRING > >) `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -95,7 +95,7 @@ public void testPatternsLabelModeWithCustomPattern_ShowNumberedToken_ForSimplePa + " '' ELSE REGEXP_REPLACE(`ENAME`, '[A-H]', '<*>') END, `ENAME`)['pattern'] AS STRING)" + " `patterns_field`, TRY_CAST(PATTERN_PARSER(CASE WHEN `ENAME` IS NULL OR `ENAME` =" + " '' THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[A-H]', '<*>') END, `ENAME`)['tokens'] AS" - + " MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " MAP< STRING, ARRAY< STRING > >) `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -142,7 +142,7 @@ public void testPatternsLabelModeWithPartitionBy_ShowNumberedToken_SimplePattern + " = '' THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END," + " `ENAME`)['pattern'] AS STRING) `patterns_field`, TRY_CAST(PATTERN_PARSER(CASE" + " WHEN `ENAME` IS NULL OR `ENAME` = '' THEN '' ELSE REGEXP_REPLACE(`ENAME`," - + " '[a-zA-Z0-9]+', '<*>') END, `ENAME`)['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >)" + + " '[a-zA-Z0-9]+', '<*>') END, `ENAME`)['tokens'] AS MAP< STRING, ARRAY< STRING > >)" + " `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -187,7 +187,7 @@ public void testPatternsLabelMode_ShowNumberedToken_ForBrainMethod() { + " OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING), TRUE)['pattern']" + " AS STRING) `patterns_field`, TRY_CAST(PATTERN_PARSER(`ENAME`, `pattern`(`ENAME`," + " 10, 100000, TRUE) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)," - + " TRUE)['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " TRUE)['tokens'] AS MAP< STRING, ARRAY< STRING > >) `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -234,7 +234,7 @@ public void testPatternsLabelModeWithPartitionBy_ShowNumberedToken_ForBrainMetho + " UNBOUNDED FOLLOWING), TRUE)['pattern'] AS STRING) `patterns_field`," + " TRY_CAST(PATTERN_PARSER(`ENAME`, `pattern`(`ENAME`, 10, 100000, TRUE) OVER" + " (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)," - + " TRUE)['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " TRUE)['tokens'] AS MAP< STRING, ARRAY< STRING > >) `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -287,7 +287,7 @@ public void testPatternsAggregationMode_ShowNumberedToken_ForSimplePatternMethod + " '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END) `pattern_count`," + " TRY_CAST(PATTERN_PARSER(CASE WHEN `ENAME` IS NULL OR `ENAME` = '' THEN '' ELSE" + " REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END, `TAKE`(`ENAME`, 10))['tokens']" - + " AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`, `TAKE`(`ENAME`, 10) `sample_logs`\n" + + " AS MAP< STRING, ARRAY< STRING > >) `tokens`, `TAKE`(`ENAME`, 10) `sample_logs`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY CASE WHEN `ENAME` IS NULL OR `ENAME` = '' THEN '' ELSE" + " REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END"; @@ -318,7 +318,7 @@ public void testPatternsAggregationModeWithGroupBy_ShowNumberedToken_ForSimplePa + " `ENAME` = '' THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END)" + " `pattern_count`, TRY_CAST(PATTERN_PARSER(CASE WHEN `ENAME` IS NULL OR `ENAME` = ''" + " THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END, `TAKE`(`ENAME`," - + " 10))['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`, `TAKE`(`ENAME`, 10)" + + " 10))['tokens'] AS MAP< STRING, ARRAY< STRING > >) `tokens`, `TAKE`(`ENAME`, 10)" + " `sample_logs`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`, CASE WHEN `ENAME` IS NULL OR `ENAME` = '' THEN '' ELSE" @@ -410,7 +410,7 @@ public void testPatternsAggregationMode_ShowNumberedToken_ForBrainMethod() { String expectedSparkSql = "SELECT TRY_CAST(`t20`.`patterns_field`['pattern'] AS STRING) `patterns_field`," + " TRY_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT) `pattern_count`," - + " TRY_CAST(`t20`.`patterns_field`['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >)" + + " TRY_CAST(`t20`.`patterns_field`['tokens'] AS MAP< STRING, ARRAY< STRING > >)" + " `tokens`, TRY_CAST(`t20`.`patterns_field`['sample_logs'] AS ARRAY< STRING >)" + " `sample_logs`\n" + "FROM (SELECT `pattern`(`ENAME`, 10, 100000, TRUE) `patterns_field`\n" @@ -475,8 +475,8 @@ public void testPatternsAggregationModeWithGroupBy_ShowNumberedToken_ForBrainMet String expectedSparkSql = "SELECT `$cor0`.`DEPTNO`, TRY_CAST(`t20`.`patterns_field`['pattern'] AS STRING)" + " `patterns_field`, TRY_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT)" - + " `pattern_count`, TRY_CAST(`t20`.`patterns_field`['tokens'] AS MAP< VARCHAR," - + " VARCHAR ARRAY >) `tokens`, TRY_CAST(`t20`.`patterns_field`['sample_logs'] AS" + + " `pattern_count`, TRY_CAST(`t20`.`patterns_field`['tokens'] AS MAP< STRING," + + " ARRAY< STRING > >) `tokens`, TRY_CAST(`t20`.`patterns_field`['sample_logs'] AS" + " ARRAY< STRING >) `sample_logs`\n" + "FROM (SELECT `DEPTNO`, `pattern`(`ENAME`, 10, 100000, TRUE) `patterns_field`\n" + "FROM `scott`.`EMP`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java new file mode 100644 index 00000000000..da9424c9c8f --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import org.junit.Test; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.RestRelation; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. + * + *

    The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> + * {@code RestCatalogSource} -> {@code CalciteLogicalCatalogScan}, which lives in the {@code + * opensearch} module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the + * OpenSearch storage engine, so the optimized {@code CalciteScannableCatalogScan} logical-plan + * assertion is exercised in {@code RestCatalogSourceTest} (source schema + request, unit) and + * {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the + * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a + * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides + * {@code visitRelation} exactly like {@code DESCRIBE}. + */ +public class CalcitePPLRestTest { + + private final PPLSyntaxParser parser = new PPLSyntaxParser(); + private final Settings settings = mock(Settings.class); + + private Node parse(String ppl) { + return new AstBuilder(ppl, settings).visit(parser.parse(ppl)); + } + + @Test + public void restHealthProjectsDeclaredColumns() { + Project project = + (Project) parse("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + RestRelation rest = (RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + // downstream fields compose on top of the rest row source. + assertEquals(2, project.getProjectList().size()); + } + + @Test + public void restReservedNameRoundTrips() { + RestRelation rest = + (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); + String reserved = rest.getTableQualifiedName().toString(); + assertTrue(SystemIndexUtils.isRestSource(reserved)); + SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); + assertEquals("/_cat/indices", spec.getEndpoint()); + assertEquals(Integer.valueOf(10), spec.getCount()); + assertEquals("5s", spec.getTimeout()); + assertEquals("green", spec.getArgs().get("health")); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java index 619cb26b64a..2275f9352de 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java @@ -306,4 +306,14 @@ public void testRexWithMaxMatchAndOffsetField() { + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + /** Verifies that rex plans correctly when it appears as the first command of a subsearch. */ + @Test + public void testRexInsideSubsearch() { + String ppl = + "source=EMP | stats count() as base_c by JOB" + + " | appendcol [ rex field=ENAME '^(?[A-Z])' | fields ENAME, first ]"; + RelNode root = getRelNode(ppl); + org.junit.Assert.assertNotNull(root); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java index e19d896283d..8844105f447 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java @@ -39,8 +39,8 @@ public void testUncorrelatedScalarSubqueryInWhere() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` > (((SELECT AVG(`SAL`) `AVG(SAL)`\n" - + "FROM `scott`.`EMP`)))"; + + "WHERE `SAL` > (SELECT AVG(`SAL`) `AVG(SAL)`\n" + + "FROM `scott`.`EMP` `EMP0`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -66,9 +66,9 @@ public void testUncorrelatedScalarSubqueryInSelect() { String expectedSparkSql = "" - + "SELECT (((SELECT MIN(`EMPNO`) `min(EMPNO)`\n" - + "FROM `scott`.`EMP`))) `min_empno`, `SAL`\n" - + "FROM `scott`.`EMP`"; + + "SELECT (SELECT MIN(`EMPNO`) `min(EMPNO)`\n" + + "FROM `scott`.`EMP` `EMP0`) `min_empno`, `SAL`\n" + + "FROM `scott`.`EMP` `EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -107,11 +107,11 @@ public void testUncorrelatedScalarSubqueryInWhereAndSelect() { String expectedSparkSql = "SELECT `min_empno`, `SAL`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " (((SELECT MIN(`EMPNO`) `min(EMPNO)`\n" - + "FROM `scott`.`EMP`))) `min_empno`\n" - + "FROM `scott`.`EMP`) `t1`\n" - + "WHERE `SAL` > (((SELECT AVG(`SAL`) `AVG(SAL)`\n" - + "FROM `scott`.`EMP`)))"; + + " (SELECT MIN(`EMPNO`) `min(EMPNO)`\n" + + "FROM `scott`.`EMP` `EMP0`) `min_empno`\n" + + "FROM `scott`.`EMP` `EMP`) `t1`\n" + + "WHERE `SAL` > (SELECT AVG(`SAL`) `AVG(SAL)`\n" + + "FROM `scott`.`EMP` `EMP1`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -140,9 +140,9 @@ public void testCorrelatedScalarSubqueryInWhere() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` > (((SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + + "WHERE `SAL` > (SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `EMP`.`SAL` = `HISAL`)))"; + + "WHERE `EMP`.`SAL` = `HISAL`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -170,10 +170,10 @@ public void testCorrelatedScalarSubqueryInSelect() { String expectedSparkSql = "" - + "SELECT (((SELECT MIN(`EMP`.`EMPNO`) `min(EMPNO)`\n" + + "SELECT (SELECT MIN(`EMP`.`EMPNO`) `min(EMPNO)`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `EMP`.`SAL` = `HISAL`))) `min_empno`, `SAL`\n" - + "FROM `scott`.`EMP`"; + + "WHERE `EMP`.`SAL` = `HISAL`) `min_empno`, `SAL`\n" + + "FROM `scott`.`EMP` `EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -201,9 +201,9 @@ public void testDisjunctiveCorrelatedScalarSubqueryInWhere() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE (((SELECT COUNT(*) `COUNT()`\n" + + "WHERE (SELECT COUNT(*) `COUNT()`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `EMP`.`SAL` = `HISAL` OR `HISAL` > 1000.0))) > 0"; + + "WHERE `EMP`.`SAL` = `HISAL` OR `HISAL` > 1000.0) > 0"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -230,10 +230,10 @@ public void testDisjunctiveCorrelatedScalarSubqueryInWhere2() { String expectedSparkSql = "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE (((SELECT COUNT(*) `COUNT()`\n" + + "WHERE (SELECT COUNT(*) `COUNT()`\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE `EMP`.`SAL` = `HISAL` AND `HISAL` > 1000.0 OR `EMP`.`SAL` = `HISAL` AND" - + " `LOSAL` > 1000.0))) > 0"; + + " `LOSAL` > 1000.0) > 0"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -268,14 +268,14 @@ public void testTwoScalarSubqueriesInOr() { String expectedSparkSql = "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` = (((SELECT MAX(`HISAL`) `max(HISAL)`\n" + + "WHERE `SAL` = (SELECT MAX(`HISAL`) `max(HISAL)`\n" + "FROM (SELECT `HISAL`\n" + "FROM `scott`.`SALGRADE`\n" - + "ORDER BY `LOSAL`) `t0`))) OR `SAL` = (((SELECT MIN(`HISAL`) `min(HISAL)`\n" + + "ORDER BY `LOSAL`) `t0`) OR `SAL` = (SELECT MIN(`HISAL`) `min(HISAL)`\n" + "FROM (SELECT `HISAL`\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE `LOSAL` > 1000.0\n" - + "ORDER BY `HISAL` DESC) `t4`)))"; + + "ORDER BY `HISAL` DESC) `t4`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -331,13 +331,13 @@ public void testNestedScalarSubquery() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` = (((SELECT MAX(`HISAL`) `max_hisal`\n" + + "WHERE `SAL` = (SELECT MAX(`HISAL`) `max_hisal`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `HISAL` = (((SELECT MAX(`SAL`) `max_sal`\n" - + "FROM `scott`.`EMP`\n" - + "GROUP BY `JOB`)))\n" + + "WHERE `HISAL` = (SELECT MAX(`SAL`) `max_sal`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + + "GROUP BY `JOB`)\n" + "GROUP BY `GRADE`\n" - + "LIMIT 1)))"; + + "LIMIT 1)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -366,9 +366,9 @@ public void testCorrelatedScalarSubqueryInWhereMaxOut() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` > (((SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + + "WHERE `SAL` > (SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `EMP`.`SAL` = `HISAL`)))"; + + "WHERE `EMP`.`SAL` = `HISAL`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index 43912b90572..4b19990ea1a 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -286,7 +286,7 @@ public void testLike() { String expectedLogical = "" + "LogicalAggregate(group=[{}], cnt=[COUNT()])\n" - + " LogicalFilter(condition=[LIKE($2, 'SALE%', '\\')])\n" + + " LogicalFilter(condition=[LIKE($2, 'SALE%':VARCHAR, '\\')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "cnt=4\n"; @@ -307,7 +307,7 @@ public void testILike() { String expectedLogical = "" + "LogicalAggregate(group=[{}], cnt=[COUNT()])\n" - + " LogicalFilter(condition=[ILIKE($2, 'SALE%', '\\')])\n" + + " LogicalFilter(condition=[ILIKE($2, 'SALE%':VARCHAR, '\\')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "cnt=4\n"; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java new file mode 100644 index 00000000000..66027839f8e --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java @@ -0,0 +1,174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.apache.calcite.DataContext; +import org.apache.calcite.config.CalciteConnectionConfig; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.plan.RelTraitDef; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelProtoDataType; +import org.apache.calcite.schema.ScannableTable; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Statistic; +import org.apache.calcite.schema.Statistics; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Programs; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.Test; + +/** + * Unit tests for the {@code timewrap} command's Calcite plan construction. Timewrap reshapes {@code + * timechart} output, so every query pipes through {@code timechart span=...} first. The pivot + * itself is post-processing in the execution engine (see {@code TimewrapPivot}); these tests cover + * the {@link org.opensearch.sql.calcite.CalciteRelNodeVisitor#visitTimewrap} lowering — the + * unpivoted RelNode and its generated Spark SQL. + */ +public class CalcitePPLTimewrapTest extends CalcitePPLAbstractTest { + + public CalcitePPLTimewrapTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Override + protected Frameworks.ConfigBuilder config(CalciteAssert.SchemaSpec... schemaSpecs) { + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus schema = CalciteAssert.addSchema(rootSchema, schemaSpecs); + ImmutableList rows = + ImmutableList.of( + new Object[] {java.sql.Timestamp.valueOf("2024-07-01 00:00:00"), 180}, + new Object[] {java.sql.Timestamp.valueOf("2024-07-01 06:00:00"), 240}, + new Object[] {java.sql.Timestamp.valueOf("2024-07-02 00:00:00"), 205}, + new Object[] {java.sql.Timestamp.valueOf("2024-07-03 00:00:00"), 165}); + schema.add("events", new EventsTable(rows)); + return Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(schema) + .traitDefs((List) null) + .programs(Programs.heuristicJoinOrder(Programs.RULE_SET, true, 2)); + } + + // align=end query with a deterministic WHERE upper bound (2024-07-03 18:00:00 = epoch + // 1720029600), so the base_offset reference is stable across runs rather than the query clock. + private static final String TIMEWRAP_DAY = + "source=events | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-03 18:00:00' | timechart span=6h sum(value) | timewrap 1day align=end"; + + @Test + public void testTimewrapDayProducesUnpivotedPlan() { + RelNode root = getRelNode(TIMEWRAP_DAY); + // The pivot is post-processing in the execution engine; the RelNode is intentionally unpivoted: + // [display_ts, value, __base_offset__, __period__], sorted by (display_ts, period). base_offset + // uses FLOOR(.../span) (not truncating integer divide) so future-dated align=now stays correct. + String expectedLogical = + "LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC])\n" + + " LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT" + + " NOT NULL) OVER (), MOD(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER ()," + + " 86400:BIGINT)), MOD(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, 86400:BIGINT)))]," + + " sum(value)=[$1], __base_offset__=[CAST(FLOOR(/(CAST(-(1720029600," + + " MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER ())):DOUBLE NOT NULL," + + " 86400:BIGINT))):BIGINT NOT NULL], __period__=[+(/(-(MAX(CAST(UNIX_TIMESTAMP($0))" + + ":BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(@timestamp=[$0], sum(value)=[$1])\n" + + " LogicalAggregate(group=[{1}], sum(value)=[SUM($0)])\n" + + " LogicalProject(value=[$1], @timestamp0=[SPAN($0, 6, 'h')])\n" + + " LogicalFilter(condition=[AND(>=($0, TIMESTAMP('2024-07-01" + + " 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2024-07-03 18:00:00':VARCHAR)), IS NOT" + + " NULL($1))])\n" + + " LogicalTableScan(table=[[scott, events]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testTimewrapDaySparkSql() { + RelNode root = getRelNode(TIMEWRAP_DAY); + String expectedSparkSql = + "SELECT FROM_UNIXTIME((MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE" + + " BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + + " MOD(MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING), 86400) + MOD(CAST(UNIX_TIMESTAMP(`@timestamp`)" + + " AS BIGINT), 86400)) `@timestamp`, `sum(value)`, CAST(FLOOR(CAST(1720029600 -" + + " (MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING)) AS DOUBLE) / 86400) AS BIGINT)" + + " `__base_offset__`, ((MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE" + + " BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + + " CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) / 86400 + 1 `__period__`\n" + + "FROM (SELECT SPAN(`@timestamp`, 6, 'h') `@timestamp`, SUM(`value`) `sum(value)`\n" + + "FROM `scott`.`events`\n" + + "WHERE `@timestamp` >= TIMESTAMP('2024-07-01 00:00:00') AND `@timestamp` <=" + + " TIMESTAMP('2024-07-03 18:00:00') AND `value` IS NOT NULL\n" + + "GROUP BY SPAN(`@timestamp`, 6, 'h')\n" + + "ORDER BY 1 NULLS LAST) `t3`\n" + + "ORDER BY 1 NULLS LAST, 4 NULLS LAST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + /** Minimal time-series table: a nullable timestamp and an integer measure. */ + public static class EventsTable implements ScannableTable { + private final ImmutableList rows; + + public EventsTable(ImmutableList rows) { + this.rows = rows; + } + + protected final RelProtoDataType protoRowType = + factory -> + factory + .builder() + .add("@timestamp", SqlTypeName.TIMESTAMP) + .nullable(true) + .add("value", SqlTypeName.INTEGER) + .nullable(true) + .build(); + + @Override + public Enumerable<@Nullable Object[]> scan(DataContext root) { + return Linq4j.asEnumerable(rows); + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return protoRowType.apply(typeFactory); + } + + @Override + public Statistic getStatistic() { + return Statistics.of(0d, ImmutableList.of(), RelCollations.createSingleton(0)); + } + + @Override + public Schema.TableType getJdbcTableType() { + return Schema.TableType.TABLE; + } + + @Override + public boolean isRolledUp(String column) { + return false; + } + + @Override + public boolean rolledUpColumnValidInsideAgg( + String column, + SqlCall call, + @Nullable SqlNode parent, + @Nullable CalciteConnectionConfig config) { + return false; + } + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 9d70487c741..d57ca8a69bb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1939,4 +1939,32 @@ public void testJoinNoPrefixComparisonStaysCondition() { public void testJoinPrefixWithoutCriteriaKeywordIsSyntaxError() { assertThrows(SyntaxCheckException.class, () -> plan("source=t1 | inner join a t2")); } + + // rest command tests + + @Test + public void testRestCommand() { + org.opensearch.sql.ast.tree.Project project = + (org.opensearch.sql.ast.tree.Project) + plan("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertTrue(spec.getArgs().isEmpty()); + } + + @Test + public void testRestCommandWithArgs() { + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) + plan("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\""); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertEquals(Integer.valueOf(5), spec.getCount()); + assertEquals("30s", spec.getTimeout()); + assertEquals("indices", spec.getArgs().get("level")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index ce7a120ff56..bf90b8bbbc9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -55,7 +55,9 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import org.antlr.v4.runtime.CommonTokenStream; import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.ast.Node; @@ -65,10 +67,50 @@ import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.AstBuildGuard; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; +import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; public class AstExpressionBuilderTest extends AstBuilderTest { + + @Test + public void deeplyNestedExpressionShouldBeRejected() { + for (String expr : + List.of( + nest(30, "a = 0", e -> "(" + e + " or a = 1)"), + nest(30, "a = 0", e -> "(" + e + " and a = 1)"), + nest(30, "a", e -> "abs(" + e + ")") + " = 1")) { + assertThrows(IllegalArgumentException.class, () -> parseWithGuard(expr, 20)); + } + } + + @Test + public void shallowExpressionWithinLimitIsAccepted() { + parseWithGuard("a = 0 or a = 1 or a = 2", 20); + } + + private void parseWithGuard(String expr, int maxDepth) { + OpenSearchPPLParser parser = + new OpenSearchPPLParser( + new CommonTokenStream(new OpenSearchPPLLexer(new CaseInsensitiveCharStream(expr)))); + parser.addErrorListener(new SyntaxAnalysisErrorListener()); + parser + .logicalExpression() + .accept(new AstExpressionBuilder(new AstBuilder(expr), new AstBuildGuard(maxDepth))); + } + + private static String nest(int depth, String base, UnaryOperator wrap) { + String expr = base; + for (int i = 0; i < depth; i++) { + expr = wrap.apply(expr); + } + return expr; + } + @Test public void testLogicalNotExpr() { assertEqual( diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 6756cdc198a..1822b690feb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -48,6 +48,18 @@ public void testPrometheusPPLCommand() { assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } + @Test + public void testRestCommand() { + assertEquals("rest /_cluster/health", anonymize("| rest \"/_cluster/health\"")); + } + + @Test + public void testRestCommandMasksArgValues() { + assertEquals( + "rest /_cluster/health count=*** timeout=*** level=***", + anonymize("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\"")); + } + @Test public void testWhereCommand() { assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1")); @@ -283,6 +295,17 @@ public void testTimechartCommand() { anonymize("source=t | timechart timefield=month max(revenue)")); } + @Test + public void testTimewrapCommand() { + assertEquals( + "source=table | timechart count() | timewrap *** align=end series=relative", + anonymize("source=t | timechart count() | timewrap 1day")); + + assertEquals( + "source=table | timechart count() | timewrap *** align=now series=short", + anonymize("source=t | timechart count() | timewrap 1week align=now series=short")); + } + @Test public void testChartCommand() { assertEquals( @@ -1203,6 +1226,32 @@ public void testMvexpandCommandWithLimit() { anonymize("source=t | mvexpand skills limit=5")); } + @Test + public void testForeachMultifieldCommand() { + assertEquals( + "source=table | foreach identifier identifier [ eval identifier = *(identifier,***) ]", + anonymize("source=t | foreach a b [ eval <>_double = <> * 2 ]")); + } + + @Test + public void testForeachMultivalueCommandWithOptions() { + assertEquals( + "source=table | foreach mode=multivalue itemstr=identifier identifier" + + " [ eval identifier = +(identifier,identifier) ]", + anonymize( + "source=t | foreach mode=multivalue itemstr=NUMBER nums" + + " [ eval total = total + NUMBER ]")); + } + + @Test + public void testForeachJsonArrayCommandMasksLiteralTarget() { + assertEquals( + "source=table | foreach mode=json_array identifier [ eval identifier =" + + " +(identifier,identifier) ]", + anonymize( + "source=t | foreach mode=json_array '[1,2,3]' [ eval total = total + <> ]")); + } + @Test public void testUnion() { assertEquals( diff --git a/release-notes/opensearch-sql.release-notes-2.10.0.0.md b/release-notes/opensearch-sql.release-notes-2.10.0.0.md index f1a730ce46b..67ce2c09275 100644 --- a/release-notes/opensearch-sql.release-notes-2.10.0.0.md +++ b/release-notes/opensearch-sql.release-notes-2.10.0.0.md @@ -40,4 +40,4 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.10.0 * [Backport 2.x] [spotless] Removes Checkstyle in favor of spotless by @MitchellGale in https://github.com/opensearch-project/sql/pull/2018 * [Backport 2.x] [Spotless] Entire project running spotless by @MitchellGale in https://github.com/opensearch-project/sql/pull/2016 --- -**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...v.2.10.0.0 \ No newline at end of file +**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...2.10.0.0 diff --git a/release-notes/opensearch-sql.release-notes-2.11.0.0.md b/release-notes/opensearch-sql.release-notes-2.11.0.0.md index a560d5c8dd7..a517b86bc48 100644 --- a/release-notes/opensearch-sql.release-notes-2.11.0.0.md +++ b/release-notes/opensearch-sql.release-notes-2.11.0.0.md @@ -52,4 +52,4 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.11.0 * bump okio to 3.4.0 by @joshuali925 in https://github.com/opensearch-project/sql/pull/2047 --- -**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...v.2.11.0.0 \ No newline at end of file +**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...2.11.0.0 diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh new file mode 100755 index 00000000000..1552353d052 --- /dev/null +++ b/scripts/ppl-lint-rule-validation.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Local developer entry point for the PPL lint rule validation contract. +# +# Runs both halves of the cross-repository check from a SQL checkout: +# 1. Frontend: loads the compiled OpenSearch-Dashboards (OSD) PPL analyzer and +# asserts the rule's diagnostic counts against the shared contract. +# 2. Backend: runs the Gradle integration test against a live /_plugins/_ppl +# endpoint on the SQL plugin built from this checkout. +# +# Usage: +# # OSD main frontend check plus SQL backend IT (fetches OSD into .ci/) +# ./scripts/ppl-lint-rule-validation.sh +# +# # Reuse an existing OSD checkout (skips clone + bootstrap if node_modules present) +# OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh +# +# # Reproduce a CI run against a specific OSD revision +# OSD_REF= ./scripts/ppl-lint-rule-validation.sh +# +# # Skip one half +# SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh +# SKIP_FRONTEND=1 ./scripts/ppl-lint-rule-validation.sh + +set -euo pipefail + +SQL_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$SQL_ROOT" + +OSD_REPO_URL="${OSD_REPO_URL:-https://github.com/opensearch-project/OpenSearch-Dashboards.git}" +OSD_REF="${OSD_REF:-main}" +DEFAULT_OSD_CHECKOUT="$SQL_ROOT/.ci/OpenSearch-Dashboards" +CONTRACT_FILE="$SQL_ROOT/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json" +FRONTEND_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" +IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + +log() { echo "[ppl-lint-rule-validation] $*"; } + +resolve_opensearch_version() { + local raw + raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | + sed -E 's/.*"opensearch.version", "([^"]+)"/\1/') + echo "${raw%%-*}" +} + +run_frontend() { + local osd_checkout="$1" + + if [[ ! -d "$osd_checkout/node_modules" ]]; then + log "Bootstrapping OSD at $osd_checkout (this can take a while)..." + (cd "$osd_checkout" && yarn osd bootstrap) + else + log "Reusing bootstrapped OSD at $osd_checkout (node_modules present)." + fi + + local os_version + os_version="$(resolve_opensearch_version)" + log "Running frontend contract against OSD analyzer (PPL_SQL_VERSION=$os_version)..." + ( + cd "$osd_checkout" + PPL_LINT_CONTRACT_FILE="$CONTRACT_FILE" \ + PPL_SQL_VERSION="$os_version" \ + node -r ./src/setup_node_env "$FRONTEND_SCRIPT" + ) +} + +if [[ "${SKIP_FRONTEND:-0}" != "1" ]]; then + if [[ -n "${OSD_SOURCE_PATH:-}" ]]; then + OSD_CHECKOUT="$(cd "$OSD_SOURCE_PATH" && pwd)" + log "Using existing OSD checkout: $OSD_CHECKOUT" + else + OSD_CHECKOUT="$DEFAULT_OSD_CHECKOUT" + if [[ ! -d "$OSD_CHECKOUT/.git" ]]; then + log "Cloning OSD ($OSD_REF) into $OSD_CHECKOUT ..." + mkdir -p "$(dirname "$OSD_CHECKOUT")" + git clone --depth 1 --branch "$OSD_REF" "$OSD_REPO_URL" "$OSD_CHECKOUT" 2>/dev/null || + git clone "$OSD_REPO_URL" "$OSD_CHECKOUT" + fi + log "Checking out OSD ref: $OSD_REF" + git -C "$OSD_CHECKOUT" fetch --depth 1 origin "$OSD_REF" 2>/dev/null || true + git -C "$OSD_CHECKOUT" checkout "$OSD_REF" 2>/dev/null || + git -C "$OSD_CHECKOUT" checkout FETCH_HEAD + fi + + OSD_SHA="$(git -C "$OSD_CHECKOUT" rev-parse HEAD)" + log "OSD revision under test: $OSD_SHA" + + run_frontend "$OSD_CHECKOUT" + log "Frontend contract passed." +else + log "SKIP_FRONTEND=1 — skipping the OSD frontend contract." +fi + +if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then + log "Running backend integration test: $IT_CLASS" + ./gradlew :integ-test:integTest --tests "$IT_CLASS" + log "Backend integration test passed." +else + log "SKIP_BACKEND=1 — skipping the SQL backend integration test." +fi + +log "Done." diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs new file mode 100644 index 00000000000..603e3feab6c --- /dev/null +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -0,0 +1,164 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * SQL-owned frontend contract adapter for the PPL lint rule validation CI. + * + * This script is executed from inside an OpenSearch-Dashboards (OSD) checkout, + * for example: + * + * cd .ci/OpenSearch-Dashboards + * PPL_LINT_CONTRACT_FILE= \ + * PPL_SQL_VERSION= \ + * node -r ./src/setup_node_env \ + * "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + * + * `node -r ./src/setup_node_env` installs OSD's process-wide auto-transpilation + * hook (`@osd/optimizer`'s `registerNodeAutoTranspilation`), which transpiles + * `packages/osd-monaco/src/**` TypeScript on `require()` regardless of where the + * entry script lives. That is what lets this SQL-owned `.mjs` load the compiled + * OSD analyzer without OSD's own Jest. + * + * The analyzer, catalog and detector registry are NOT re-exported from the + * `@osd/monaco` package barrel, so they are loaded via their deep module paths. + * Because this is an ES module, `require` is obtained with `createRequire`, and + * the modules are resolved against the OSD checkout root (process.cwd()) rather + * than the location of this script (which lives in the SQL repo, not OSD). + */ + +import assert from 'assert'; +import fs from 'fs'; +import path from 'path'; +import { createRequire } from 'module'; + +const RULE_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; +const CATALOG_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; +const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/src/ppl/lint/detector_registry'; + +function fail(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-frontend-contract] FAIL: ${message}`); + process.exit(1); +} + +function loadContract() { + const contractFile = process.env.PPL_LINT_CONTRACT_FILE; + if (!contractFile) { + fail('PPL_LINT_CONTRACT_FILE is not set.'); + } + if (!fs.existsSync(contractFile)) { + fail(`Contract file not found: ${contractFile}`); + } + try { + return JSON.parse(fs.readFileSync(contractFile, 'utf8')); + } catch (error) { + fail(`Could not parse contract file ${contractFile}: ${error.message}`); + return undefined; // unreachable + } +} + +function loadOsdAnalyzer() { + // Resolve the OSD compiled analyzer from the checkout root. In CI this script + // runs after `cd .ci/OpenSearch-Dashboards`, so process.cwd() is that root. + const osdRoot = process.cwd(); + const require = createRequire(path.join(osdRoot, 'noop.js')); + + const resolveOsd = (relativeModule) => { + const absolute = path.join(osdRoot, relativeModule); + if (!fs.existsSync(`${absolute}.ts`) && !fs.existsSync(`${absolute}.js`)) { + fail( + `Expected OSD module not found under the checkout root: ${relativeModule}\n` + + `Resolved OSD root: ${osdRoot}\n` + + `Run this script from the OSD checkout (e.g. cd .ci/OpenSearch-Dashboards) after bootstrap.` + ); + } + return require(absolute); + }; + + const { PPLLanguageAnalyzer } = resolveOsd(RULE_MODULE); + const { getBundledCatalog } = resolveOsd(CATALOG_MODULE); + const { getDetector } = resolveOsd(DETECTOR_REGISTRY_MODULE); + + if (typeof PPLLanguageAnalyzer !== 'function') { + fail(`PPLLanguageAnalyzer was not a constructor when loaded from ${RULE_MODULE}.`); + } + return { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot }; +} + +function assertRuleIsWiredUp(ruleId, getBundledCatalog, getDetector) { + const entry = getBundledCatalog().find((candidate) => candidate.id === ruleId); + if (!entry) { + fail(`Rule "${ruleId}" is not present in the OSD bundled catalog.`); + } + if (!entry.enabled) { + fail(`Rule "${ruleId}" is present but disabled in the OSD bundled catalog.`); + } + if (entry.severity !== 'error') { + fail(`Rule "${ruleId}" severity is "${entry.severity}", expected "error".`); + } + if (typeof getDetector(entry.detector) !== 'function') { + fail(`Rule "${ruleId}" has no registered detector "${entry.detector}".`); + } + return entry; +} + +function main() { + const contract = loadContract(); + const ruleId = contract.ruleId; + const index = contract.index; + const sqlVersion = process.env.PPL_SQL_VERSION; + + const { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot } = loadOsdAnalyzer(); + + const entry = assertRuleIsWiredUp(ruleId, getBundledCatalog, getDetector); + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] OSD root: ${osdRoot}\n` + + `[ppl-lint-frontend-contract] rule "${ruleId}" enabled=${entry.enabled} severity=${entry.severity} detector="${entry.detector}"\n` + + `[ppl-lint-frontend-contract] PPL_SQL_VERSION=${sqlVersion || '(unset)'}\n` + + `[ppl-lint-frontend-contract] running ${contract.cases.length} case(s) against index "${index}"` + ); + + const analyzer = new PPLLanguageAnalyzer(); + const failures = []; + + for (const testCase of contract.cases) { + const query = testCase.query.split('{{index}}').join(index); + const result = analyzer.lint(query, { + dataSourceVersion: sqlVersion, + isCalcite: true, + }); + const matches = result.diagnostics.filter((diagnostic) => diagnostic.ruleId === ruleId); + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] ${testCase.id}: expected ${testCase.frontendDiagnosticCount}, ` + + `got ${matches.length} — ${query}` + ); + + try { + assert.strictEqual( + matches.length, + testCase.frontendDiagnosticCount, + `case "${testCase.id}": expected ${testCase.frontendDiagnosticCount} "${ruleId}" ` + + `diagnostic(s) but received ${matches.length} for query: ${query}` + ); + } catch (error) { + failures.push(error.message); + } + } + + if (failures.length > 0) { + fail(`${failures.length} case(s) failed:\n- ${failures.join('\n- ')}`); + } + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] PASS: all ${contract.cases.length} case(s) matched for "${ruleId}".` + ); +} + +main(); diff --git a/sql/src/main/java/org/opensearch/sql/sql/SQLService.java b/sql/src/main/java/org/opensearch/sql/sql/SQLService.java index d51fa1c898d..9b4cf8c1a37 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/SQLService.java +++ b/sql/src/main/java/org/opensearch/sql/sql/SQLService.java @@ -11,6 +11,7 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import org.opensearch.sql.executor.QueryManager; @@ -32,8 +33,15 @@ public class SQLService { private final QueryPlanFactory queryExecutionFactory; + private final Settings settings; + private final QueryType SQL_QUERY = QueryType.SQL; + public SQLService( + SQLSyntaxParser parser, QueryManager queryManager, QueryPlanFactory queryExecutionFactory) { + this(parser, queryManager, queryExecutionFactory, null); + } + /** * Given {@link SQLQueryRequest}, execute it. Using listener to listen result. * @@ -95,7 +103,7 @@ private AbstractPlan plan( Statement statement = cst.accept( new AstStatementBuilder( - new AstBuilder(request.getQuery()), + new AstBuilder(request.getQuery(), settings), AstStatementBuilder.StatementBuilderContext.builder() .isExplain(isExplainRequest) .fetchSize(request.getFetchSize()) diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index aaed2ba5ec2..f1f140fce8e 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -39,7 +39,9 @@ import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -54,16 +56,23 @@ public class AstBuilder extends OpenSearchSQLParserBaseVisitor { private final AstExpressionBuilder expressionBuilder; /** Parsing context stack that contains context for current query parsing. */ - private final ParsingContext context = new ParsingContext(); + protected final ParsingContext context = new ParsingContext(); /** * SQL query to get original token text. This is necessary because token.getText() returns text * without whitespaces or other characters discarded by lexer. */ - private final String query; + protected final String query; + + protected final AstBuildGuard guard; public AstBuilder(String query) { + this(query, null); + } + + public AstBuilder(String query, Settings settings) { this.query = query; + this.guard = AstBuildGuard.fromSettings(settings); this.expressionBuilder = createExpressionBuilder(); } @@ -290,7 +299,7 @@ protected UnresolvedExpression visitAstExpression(ParseTree tree) { /** Override to provide a custom expression builder (e.g., with subquery support). */ protected AstExpressionBuilder createExpressionBuilder() { - return new AstExpressionBuilder(); + return new AstExpressionBuilder(guard); } private UnresolvedExpression visitSelectItem(SelectElementContext ctx) { diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 33cfb1a56ca..e7510f31b7a 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -79,11 +79,14 @@ import java.util.Optional; import java.util.stream.Collectors; import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.RuleNode; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.*; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -101,6 +104,26 @@ /** Expression builder to parse text to expression in AST. */ public class AstExpressionBuilder extends OpenSearchSQLParserBaseVisitor { + private final AstBuildGuard guard; + + public AstExpressionBuilder() { + this(new AstBuildGuard()); + } + + public AstExpressionBuilder(AstBuildGuard guard) { + this.guard = guard; + } + + @Override + public UnresolvedExpression visit(ParseTree tree) { + return guard.enforce(() -> super.visit(tree)); + } + + @Override + public UnresolvedExpression visitChildren(RuleNode node) { + return guard.enforce(() -> super.visitChildren(node)); + } + @Override public UnresolvedExpression visitTableName(TableNameContext ctx) { return visit(ctx.qualifiedName()); diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstSortBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstSortBuilder.java index 2594709f4f4..1647bc9ee9e 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstSortBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstSortBuilder.java @@ -33,7 +33,7 @@ @RequiredArgsConstructor public class AstSortBuilder extends OpenSearchSQLParserBaseVisitor { - private final QuerySpecification querySpec; + protected final QuerySpecification querySpec; @Override public UnresolvedPlan visitOrderByClause(OrderByClauseContext ctx) { @@ -57,7 +57,7 @@ private List createSortFields() { * Argument "asc" is required. Argument "nullFirst" is optional and determined by Analyzer later * if absent. */ - private List createSortArguments(SortOption option) { + protected List createSortArguments(SortOption option) { SortOrder sortOrder = option.getSortOrder(); NullOrder nullOrder = option.getNullOrder(); ImmutableList.Builder args = ImmutableList.builder(); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index 9d504a32bb0..893731e4a10 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -37,6 +37,7 @@ import com.google.common.collect.ImmutableList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.Test; @@ -51,6 +52,7 @@ import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; @@ -777,4 +779,28 @@ public void exists_subquery_throws_syntax_check_exception() { SyntaxCheckException.class, () -> buildAST("SELECT * FROM t WHERE EXISTS (SELECT 1 FROM t2)")); } + + @Test + public void configured_max_expression_depth_from_settings_is_applied() { + Settings settings = + new Settings() { + @Override + public T getSettingValue(Key key) { + return (T) Integer.valueOf(5); + } + + @Override + public List getSettings() { + return List.of(); + } + }; + StringBuilder where = new StringBuilder("a = 1"); + for (int i = 2; i <= 20; i++) { + where.append(" OR a = ").append(i); + } + String query = "SELECT * FROM t WHERE " + where; + AstBuilder builder = new AstBuilder(query, settings); + assertThrows( + IllegalArgumentException.class, () -> new SQLSyntaxParser().parse(query).accept(builder)); + } } diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index cb981f6f45f..aba8023b07e 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql.parser; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.and; import static org.opensearch.sql.ast.dsl.AstDSL.between; @@ -35,6 +36,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.HashMap; +import java.util.List; +import java.util.function.UnaryOperator; import java.util.stream.Stream; import org.antlr.v4.runtime.CommonTokenStream; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -47,6 +50,7 @@ import org.opensearch.sql.ast.expression.WindowFrame; import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLLexer; @@ -827,10 +831,43 @@ public void canBuildInClause() { buildExprAst("age in (abs(20), abs(30))")); } + @Test + public void deeplyNestedExpressionShouldBeRejected() { + AstExpressionBuilder guarded = new AstExpressionBuilder(new AstBuildGuard(20)); + for (String expr : + List.of( + nest(30, "a = 0", e -> "(" + e + " or a = 1)"), + nest(30, "a = 0", e -> "(" + e + " and a = 1)"), + nest(30, "a", e -> "abs(" + e + ")"), + nest(30, "a", e -> "case when " + e + " > 0 then 1 else 0 end"))) { + assertThrows(IllegalArgumentException.class, () -> buildExprAst(expr, guarded)); + } + } + + @Test + public void shallowExpressionWithinLimitIsAccepted() { + AstExpressionBuilder guarded = new AstExpressionBuilder(new AstBuildGuard(20)); + assertEquals( + buildExprAst("a = 0 or a = 1 or a = 2", guarded), + buildExprAst("a = 0 or a = 1 or a = 2", guarded)); + } + + private static String nest(int depth, String base, UnaryOperator wrap) { + String expr = base; + for (int i = 0; i < depth; i++) { + expr = wrap.apply(expr); + } + return expr; + } + private Node buildExprAst(String expr) { + return buildExprAst(expr, astExprBuilder); + } + + private Node buildExprAst(String expr, AstExpressionBuilder builder) { OpenSearchSQLLexer lexer = new OpenSearchSQLLexer(new CaseInsensitiveCharStream(expr)); OpenSearchSQLParser parser = new OpenSearchSQLParser(new CommonTokenStream(lexer)); parser.addErrorListener(new SyntaxAnalysisErrorListener()); - return parser.expression().accept(astExprBuilder); + return parser.expression().accept(builder); } }