diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml new file mode 100644 index 00000000000..4ced31d611a --- /dev/null +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -0,0 +1,491 @@ +name: "[Linter] PPL multi-surface compatibility" + +# Validate the 12 active PPL lint rules on the checked-in OSD fallback grammar +# and on runtime grammar bundles exported by the latest eligible GA engine and +# this SQL pull request. Compatibility differences are data until the aggregate +# job has published the complete 12 x 3 matrix. + +on: + pull_request: + paths: + - 'build.gradle' + - 'integ-test/build.gradle' + - 'integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java' + - 'integ-test/src/test/resources/ppl-lint/**' + - 'scripts/ppl-lint/**' + - '.github/workflows/ppl-lint-multiversion-validation.yml' + workflow_dispatch: + inputs: + osd_repo: + description: OSD repository containing the detector implementation. + required: false + type: string + osd_ref: + description: OSD branch or commit containing the detector implementation. + required: false + type: string + +permissions: + contents: read + +concurrency: + group: ppl-lint-multi-surface-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + plan: + name: Plan compatibility matrix + runs-on: ubuntu-latest + outputs: + released_targets: ${{ steps.outputs.outputs.released_targets }} + osd_repo: ${{ steps.outputs.outputs.osd_repo }} + osd_ref: ${{ steps.outputs.outputs.osd_ref }} + pr_target: ${{ steps.outputs.outputs.pr_target }} + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Read official OpenSearch GA tags + run: | + set -euo pipefail + git ls-remote --tags --refs https://github.com/opensearch-project/OpenSearch.git \ + > "$RUNNER_TEMP/opensearch-release-tags.txt" + + - name: Resolve target versions and configurations + env: + REQUESTED_OSD_REPO: ${{ inputs.osd_repo }} + REQUESTED_OSD_REF: ${{ inputs.osd_ref }} + VARIABLE_OSD_REPO: ${{ vars.OSD_REPO }} + VARIABLE_OSD_REF: ${{ vars.OSD_REF }} + run: | + set -euo pipefail + osd_repo="${REQUESTED_OSD_REPO:-${VARIABLE_OSD_REPO:-opensearch-project/OpenSearch-Dashboards}}" + osd_ref="${REQUESTED_OSD_REF:-${VARIABLE_OSD_REF:-main}}" + node scripts/ppl-lint/plan-compatibility.mjs \ + --build-file build.gradle \ + --release-tags "$RUNNER_TEMP/opensearch-release-tags.txt" \ + --compiled-version 2.19.6 \ + --sql-sha "$GITHUB_SHA" \ + --osd-repository "$osd_repo" \ + --osd-ref "$osd_ref" \ + --out compatibility-plan.json + + - name: Publish plan outputs + id: outputs + run: | + set -euo pipefail + { + echo "released_targets=$(jq -c '.releasedTargets' compatibility-plan.json)" + echo "osd_repo=$(jq -r '.osd.repository' compatibility-plan.json)" + echo "osd_ref=$(jq -r '.osd.ref' compatibility-plan.json)" + echo "pr_target=$(jq -r '.prTargetVersion' compatibility-plan.json)" + } >> "$GITHUB_OUTPUT" + { + echo '## PPL lint compatibility plan' + echo + echo "- PR target: \`$(jq -r '.prTargetVersion' compatibility-plan.json)\`" + echo "- Latest eligible GA: \`$(jq -r '.latestEligibleGa' compatibility-plan.json)\`" + echo "- OSD: \`$(jq -r '.osd.repository + " @ " + .osd.ref' compatibility-plan.json)\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload compatibility plan + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-compatibility-plan + path: compatibility-plan.json + if-no-files-found: error + + observe-released: + name: Observe engine ${{ matrix.version }} (${{ matrix.label }}) + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.released_targets) }} + services: + opensearch: + image: opensearchproject/opensearch:${{ matrix.version }} + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Confirm released engine identity + run: | + set -euo pipefail + for attempt in $(seq 1 40); do + curl -sf http://localhost:9200 > "$RUNNER_TEMP/engine-root.json" && break + echo "waiting for engine (${attempt}/40)..." + sleep 5 + done + reported=$(jq -r '.version.number' "$RUNNER_TEMP/engine-root.json") + case "$reported" in + ${{ matrix.version }}*) ;; + *) echo "::error::engine reported $reported; expected ${{ matrix.version }}"; exit 1 ;; + esac + curl -sf http://localhost:9200/_cat/plugins | grep -i sql + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: temurin + java-version: 21 + + - name: Observe backend contracts + env: + EXPORT_RUNTIME_BUNDLE: ${{ matrix.export_runtime_bundle }} + run: | + set -euo pipefail + mkdir -p leg + bundle_args=() + if [ "$EXPORT_RUNTIME_BUNDLE" = 'true' ]; then + bundle_args+=("-Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json") + fi + { + printf './gradlew :integ-test:integTestRemote ' + printf '%q ' \ + '--tests' 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' \ + '-Dtests.rest.cluster=localhost:9200' \ + '-Dtests.cluster=localhost:9200' \ + '-Dtests.clustername=docker-cluster' \ + '-Dppl.lint.schedule=nightly' \ + '-Dppl.lint.observe.only=true' \ + '-Dppl.lint.execution_backend=standard' \ + "-Dppl.lint.sql_sha=$GITHUB_SHA" \ + "-Dppl.lint.report=$(pwd)/leg/backend-report.json" \ + "-Dppl.lint.target=$(pwd)/leg/target.json" \ + "${bundle_args[@]}" + echo + } > leg/backend-command.txt + ./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha="$GITHUB_SHA" \ + -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ + -Dppl.lint.target="$(pwd)/leg/target.json" \ + "${bundle_args[@]}" + + - name: Upload released observation + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ${{ matrix.artifact_name }} + path: leg + if-no-files-found: warn + + - name: Upload released observation logs + if: ${{ failure() }} + continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ${{ matrix.artifact_name }}-logs + path: | + integ-test/build/reports/** + integ-test/build/test-results/** + if-no-files-found: warn + + observe-pr-build: + name: Observe engine pr-build (runtime) + needs: + - Get-CI-Image-Tag + - plan + runs-on: ubuntu-latest + timeout-minutes: 35 + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: temurin + java-version: 21 + + - name: Observe backend contracts + run: | + set -euo pipefail + mkdir -p leg + chown -R 1000:1000 "$(pwd)" + command="./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true -Dppl.lint.execution_backend=standard -Dppl.lint.sql_sha=$GITHUB_SHA -Dppl.lint.report=$(pwd)/leg/backend-report.json -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json -Dppl.lint.target=$(pwd)/leg/target.json" + printf '%s\n' "$command" > leg/backend-command.txt + su "$(id -un 1000)" -c "$command" + + - name: Upload PR-build observation + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-observation-pr-build-runtime + path: leg + if-no-files-found: warn + + - name: Upload PR-build observation logs + if: ${{ failure() }} + continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-observation-pr-build-runtime-logs + path: | + integ-test/build/reports/** + integ-test/build/test-results/** + integ-test/build/testclusters/*/logs/* + if-no-files-found: warn + + aggregate: + name: Aggregate rule compatibility + if: ${{ always() && needs.plan.result == 'success' }} + needs: + - plan + - observe-released + - observe-pr-build + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download compatibility plan + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ppl-lint-compatibility-plan + path: plan + + - name: Download available observations + continue-on-error: true + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: ppl-lint-observation-* + path: legs + + - name: Checkout OpenSearch-Dashboards + id: osd-checkout + continue-on-error: true + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ needs.plan.outputs.osd_repo }} + ref: ${{ needs.plan.outputs.osd_ref }} + path: .ci/OpenSearch-Dashboards + + - name: Record OSD revision + id: osd-revision + if: ${{ always() }} + run: | + if sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD 2>/dev/null); then + echo "available=true" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "sha=" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Node from OSD .nvmrc + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards once + id: osd-bootstrap + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true + working-directory: .ci/OpenSearch-Dashboards + run: | + for attempt in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $attempt failed; retrying in 10 seconds." + sleep 10 + done + exit 1 + + - name: Run applicable detector passes + if: ${{ steps.osd-bootstrap.outcome == 'success' }} + continue-on-error: true + run: | + set -uo pipefail + while IFS= read -r configuration; do + id=$(jq -r '.id' <<< "$configuration") + artifact=$(jq -r '.artifactName' <<< "$configuration") + surface=$(jq -r '.surface' <<< "$configuration") + engine_mode=$(jq -r '.engineMode' <<< "$configuration") + leg="$GITHUB_WORKSPACE/legs/$artifact" + mkdir -p "$leg" + if [ ! -s "$leg/target.json" ] || [ ! -s "$leg/backend-report.json" ]; then + echo "Skipping detector pass for $id: backend evidence is incomplete." + continue + fi + + if [ "$surface" = 'compiled-simplified' ]; then + grammar_root=".ci/OpenSearch-Dashboards/packages/osd-antlr-grammar/src/opensearch_ppl_simplified" + grammar_hash=$( + find "$grammar_root" -type f -print0 | + sort -z | + xargs -0 sha256sum | + sha256sum | + awk '{print "sha256:" $1}' + ) + jq --arg hash "$grammar_hash" \ + '.grammarHash = $hash | .grammarBundle = ""' \ + "$leg/target.json" > "$leg/detector-target.json" + bundle='' + else + cp "$leg/target.json" "$leg/detector-target.json" + bundle="$leg/ppl-grammar-bundle.json" + if [ ! -s "$bundle" ]; then + echo "Skipping detector pass for $id: runtime grammar bundle is missing." + continue + fi + fi + + { + printf 'cd %q && env ' "$GITHUB_WORKSPACE/.ci/OpenSearch-Dashboards" + printf 'PPL_LINT_SURFACE=%q ' "$surface" + printf 'PPL_LINT_ENGINE_MODE=%q ' "$engine_mode" + printf 'PPL_LINT_APPLICABLE_ONLY=1 ' + printf 'PPL_LINT_GRAMMAR_BUNDLE=%q ' "$bundle" + printf 'PPL_LINT_TARGET_MANIFEST=%q ' "$leg/detector-target.json" + printf 'PPL_LINT_BACKEND_REPORT=%q ' "$leg/backend-report.json" + printf 'PPL_LINT_CONTRACT_DIR=%q ' "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" + printf 'PPL_LINT_SCHEDULE=nightly PPL_LINT_OBSERVE_ONLY=1 ' + printf 'PPL_LINT_REPORT=%q ' "$leg/detector-report.json" + printf 'node -r ./src/setup_node_env %q\n' \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + } > "$leg/detector-command.txt" + + ( + cd .ci/OpenSearch-Dashboards + env \ + PPL_LINT_SURFACE="$surface" \ + PPL_LINT_ENGINE_MODE="$engine_mode" \ + PPL_LINT_APPLICABLE_ONLY=1 \ + PPL_LINT_GRAMMAR_BUNDLE="$bundle" \ + PPL_LINT_TARGET_MANIFEST="$leg/detector-target.json" \ + PPL_LINT_BACKEND_REPORT="$leg/backend-report.json" \ + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_OBSERVE_ONLY=1 \ + PPL_LINT_REPORT="$leg/detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + ) > "$leg/detector.log" 2>&1 || true + tail -10 "$leg/detector.log" || true + done < <(jq -c '.configurations[]' plan/compatibility-plan.json) + + - name: Aggregate every planned rule and configuration + id: compatibility + if: ${{ always() }} + run: | + set +e + node scripts/ppl-lint/aggregate-compatibility.mjs \ + --plan plan/compatibility-plan.json \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --artifacts legs \ + --osd-sha "${{ steps.osd-revision.outputs.sha }}" \ + --out drift-report.json \ + --summary "$GITHUB_STEP_SUMMARY" + result=$? + echo "exit_code=$result" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Upload drift report + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-multiversion-drift + path: drift-report.json + if-no-files-found: error + + - name: Upload compatibility evidence + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-multiversion-evidence + path: | + plan/compatibility-plan.json + legs/**/backend-report.json + legs/**/detector-report.json + legs/**/target.json + legs/**/detector-target.json + legs/**/*-command.txt + legs/**/detector.log + if-no-files-found: warn + + - name: Fail after publishing compatibility results + if: ${{ always() }} + env: + AGGREGATE_EXIT: ${{ steps.compatibility.outputs.exit_code }} + run: | + set -euo pipefail + if [ ! -s drift-report.json ]; then + echo "::error::drift-report.json was not published" + exit 1 + fi + jq -e ' + .schemaVersion == 3 and + .inventory.ruleCount == 12 and + (.inventory.ruleIds | length) == 12 and + (.configurations | length) == 3 and + (.matrix | length) == 36 and + .result.cellCount == 36 and + ( + .result.compatible + .result.notApplicable + + .result.drift + .result.inconclusive + ) == 36 + ' drift-report.json > /dev/null + if [ -z "$AGGREGATE_EXIT" ]; then + echo "::error::compatibility aggregation did not complete" + exit 1 + fi + if [ "$AGGREGATE_EXIT" -ne 0 ]; then + drift=$(jq -r '.result.drift' drift-report.json) + inconclusive=$(jq -r '.result.inconclusive' drift-report.json) + echo "::error::rule compatibility validation failed after artifact publication: $drift drift, $inconclusive inconclusive" + exit "$AGGREGATE_EXIT" + fi diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml new file mode 100644 index 00000000000..d68aaf4158d --- /dev/null +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -0,0 +1,385 @@ +name: "[Linter] PPL rule validation" + +permissions: + contents: read + +concurrency: + group: ppl-lint-rule-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint detectors and +# the SQL backend must agree on the SAME candidate runtime grammar. A shared, +# reviewed corpus of contract files pins each rule's OSD detector diagnostic +# count to the live SQL engine's behavior, so neither side can drift unilaterally +# without a red build. +# +# The workflow is a linear three-job pipeline (design §3.1): +# +# backend-validation ──(artifacts)──▶ detector-validation ──▶ validation-result +# +# 1. backend-validation (Amazon Linux CI container): builds the SQL PR, starts the +# Gradle test cluster, runs the contract trigger/control queries against the +# live /_plugins/_ppl endpoint, and — while the cluster is alive — exports the +# candidate runtime grammar bundle (GET /_plugins/_ppl/_grammar) plus a target +# manifest and the observed backend report. Those three files are the ONLY +# bridge to the next job; the test cluster is never passed between jobs. +# 2. detector-validation (ubuntu-latest): checks out and bootstraps OSD as a Node +# code dependency (no OSD server, no Monaco, no browser), deserializes the +# candidate bundle through OSD's production headless lint API, runs the real +# detectors against the same queries, and asserts the detector-vs-backend +# differential. OSD needs a newer Node/glibc than the CI container provides, +# hence a separate Ubuntu job. +# 3. validation-result: the single stable required check. Fails unless BOTH +# validation jobs succeeded (an always() result job so a skipped detector +# cannot mask a backend failure), writes the compact per-rule PR summary, and +# uploads the run manifest recording the exact SQL SHA, OSD SHA, mode, backend +# version, and grammar hash. +# +# Modes (design §3.4, §4.1.1): +# - pull_request: SQL PR validation against the resolved OSD target. The ONLY +# enforcing mode; this is what branch protection pins to. Runs all 12 active +# detector contracts. The committed default is `main` on the canonical repo; +# it can be overridden by the OSD_REPO/OSD_REF repo variables — see the +# "Resolve OSD ref" step. TEMPORARY: those repo variables are currently set to +# the unmerged paired OSD branch that ships the headless lint API this job +# needs; deleting them reverts to opensearch-project/...@main once that OSD PR +# merges. +# - workflow_dispatch (osd_repo + osd_ref): pre-merge evidence for an unmerged +# OSD branch, optionally on a fork (osd_repo). Records the resolved immutable +# OSD commit SHA but CANNOT satisfy branch protection — only the pull_request +# run does. +# - schedule (nightly): the full corpus + a coverage assertion. +# +# Workflow shape deliberately mirrors the sibling SQL Java workflows so a +# maintainer sees one pattern, not a bespoke one: +# - sql-test-and-build-workflow.yml : the Get-CI-Image-Tag reusable workflow, +# the OpenSearch CI container + ci-image-start-command, and the +# `chown 1000:1000` + `su` non-root Gradle invocation (backend-validation). +# - integ-tests-with-security.yml : the report-upload-on-always() shape. +# Action SHAs are pinned to the same versions those siblings use, so dependabot +# bumps one set, not two drifting ones. +# +# CI cost (measured 2026-07-22, ~13 min wall clock): backend-validation ~5 min +# (container init ~2 min + backend IT/export ~2m50s); detector-validation ~3 min, +# of which OSD `yarn osd bootstrap` is ~2m13s and the actual lint is ~2s. The +# bootstrap dominates and is CPU-bound (it was ~2m13s even with a warm yarn +# cache), so it is NOT sharded into a per-contract matrix (that would multiply +# the 2m13s, not the 2s). Overlapping bootstrap with the backend job is a tracked +# follow-up, not done here: it would require transferring the bootstrapped OSD +# tree (multi-GB, 30+ workspace symlinks, plus built target/) between runners, +# which OSD's own CI deliberately avoids. See ~/ppl-lint-ci-fixes-impl-plan.md. + +on: + pull_request: + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + inputs: + osd_repo: + description: OSD repository to check out (a fork, for pre-merge evidence). Defaults to opensearch-project/OpenSearch-Dashboards. + required: false + type: string + osd_ref: + description: OSD commit or branch to validate instead of main (pre-merge evidence only) + required: false + type: string + schedule: + description: Contract schedule to run (pr or nightly) + required: false + default: pr + type: string + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + backend-validation: + name: Backend validation (live /_plugins/_ppl + grammar export) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve contract schedule + id: schedule + env: + REQUESTED_SCHEDULE: ${{ inputs.schedule }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ -n "$REQUESTED_SCHEDULE" ]; then + value="$REQUESTED_SCHEDULE" + elif [ "$EVENT_NAME" = "schedule" ]; then + value="nightly" + else + value="pr" + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # OpenSearch refuses to start as root, so run Gradle as a non-root user. The + # IT exports the candidate grammar bundle + target manifest while the cluster + # is alive; those become the artifacts the detector job lints against. + - name: Run backend integration test and export candidate grammar + run: | + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dppl.lint.schedule=${{ steps.schedule.outputs.value }} \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ + -Dppl.lint.report=$(pwd)/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/target.json" + + - name: Upload backend artifacts (bundle + target + report) + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-backend + path: | + backend-report.json + ppl-grammar-bundle.json + target.json + + - name: Upload backend failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-backend-logs + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + + detector-validation: + name: Detector validation (OSD headless lint on candidate bundle) + needs: backend-validation + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + osd_repo: ${{ steps.osd-ref.outputs.repo }} + osd_ref: ${{ steps.osd-ref.outputs.ref }} + osd_sha: ${{ steps.osd-rev.outputs.sha }} + schedule: ${{ steps.schedule.outputs.value }} + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + # Resolve which OSD checkout the detectors run against, in precedence order: + # 1. workflow_dispatch input (osd_repo / osd_ref) — explicit manual run + # 2. repo variable (vars.OSD_REPO / vars.OSD_REF) — the override + # point; set/cleared in repo settings with no workflow edit + # 3. canonical default opensearch-project/OpenSearch-Dashboards@main + # + # The committed default is intentionally the canonical repo + `main`, so the + # file always declares that the required check validates against upstream. + # TEMPORARY OVERRIDE: the headless lint API this job imports + # (src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint) is not yet + # on OSD `main`; it lives on the paired branch + # Hanyu-W/OpenSearch-Dashboards@ppl-lint-headless-api. Until that OSD PR + # merges, the OSD_REPO/OSD_REF repo variables are set to that branch so the + # required check validates against the OSD ref that actually ships the API. + # Deleting those two repo variables (no code change) reverts to `main`. + - name: Resolve OSD ref + id: osd-ref + env: + REQUESTED_REF: ${{ inputs.osd_ref }} + REQUESTED_REPO: ${{ inputs.osd_repo }} + VAR_REF: ${{ vars.OSD_REF }} + VAR_REPO: ${{ vars.OSD_REPO }} + run: | + ref="${REQUESTED_REF:-${VAR_REF:-main}}" + repo="${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" + echo "ref=$ref" >> "$GITHUB_OUTPUT" + echo "repo=$repo" >> "$GITHUB_OUTPUT" + + - name: Resolve contract schedule + id: schedule + env: + REQUESTED_SCHEDULE: ${{ inputs.schedule }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ -n "$REQUESTED_SCHEDULE" ]; then + value="$REQUESTED_SCHEDULE" + elif [ "$EVENT_NAME" = "schedule" ]; then + value="nightly" + else + value="pr" + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ steps.osd-ref.outputs.repo }} + ref: ${{ steps.osd-ref.outputs.ref }} + path: .ci/OpenSearch-Dashboards + + # Resolve the (possibly mutable) ref to the immutable commit SHA actually + # tested, so the run manifest pins exactly what ran (design §4.1.1, T11). + - name: Record OSD revision + id: osd-rev + run: | + sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "OSD revision: \`$sha\` (repo: ${{ steps.osd-ref.outputs.repo }}, ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" + + # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding + # it, so an OSD toolchain bump does not silently drift this job. + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + # Take the lower bound of the engines.yarn range (e.g. "^1.22.10" -> "1.22.10"). + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + # Retry-with-backoff mirrors the OSD build workflow's bootstrap step; + # `yarn osd bootstrap` occasionally fails on a transient registry hiccup. + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + # Downloaded after bootstrap (not before): the ~2m13s bootstrap does not + # need the backend artifact — only the lint step below does — so a flaky + # artifact download cannot waste a completed bootstrap. + - name: Download backend artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ppl-lint-backend + path: artifacts + + - name: Run detector validation against the candidate bundle + working-directory: .ci/OpenSearch-Dashboards + env: + PPL_LINT_CONTRACT_DIR: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/contracts + PPL_LINT_SCHEDULE: ${{ steps.schedule.outputs.value }} + PPL_LINT_GRAMMAR_BUNDLE: ${{ github.workspace }}/artifacts/ppl-grammar-bundle.json + PPL_LINT_TARGET_MANIFEST: ${{ github.workspace }}/artifacts/target.json + PPL_LINT_BACKEND_REPORT: ${{ github.workspace }}/artifacts/backend-report.json + PPL_LINT_REPORT: ${{ github.workspace }}/detector-report.json + PPL_LINT_INCLUDE_DORMANT: ${{ steps.schedule.outputs.value == 'nightly' && '1' || '0' }} + run: | + # pipefail so the runner's non-zero exit propagates through `tee` — + # otherwise the pipeline takes tee's (success) status and a real + # detector failure would go green (a vacuous pass). + set -o pipefail + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + | tee "$GITHUB_WORKSPACE/detector-contract.log" + + - name: Upload detector report and corpus + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-detector + path: | + detector-contract.log + detector-report.json + integ-test/src/test/resources/ppl-lint/contracts + + validation-result: + name: validation-result + if: ${{ always() }} + needs: + - backend-validation + - detector-validation + runs-on: ubuntu-latest + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download backend artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + continue-on-error: true + with: + name: ppl-lint-backend + path: artifacts + + - name: Download detector artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + continue-on-error: true + with: + name: ppl-lint-detector + path: artifacts + + # Assemble the run manifest and the compact per-rule PR summary from the + # reports both jobs uploaded. The manifest records the immutable SQL + OSD + # SHAs so any run is exactly reproducible (design §3.3, §4.4). + - name: Assemble run manifest and summary + env: + SQL_SHA: ${{ github.sha }} + OSD_REPO: ${{ needs.detector-validation.outputs.osd_repo }} + OSD_REF: ${{ needs.detector-validation.outputs.osd_ref }} + OSD_SHA: ${{ needs.detector-validation.outputs.osd_sha }} + EVENT_NAME: ${{ github.event_name }} + SCHEDULE: ${{ needs.detector-validation.outputs.schedule }} + BACKEND_RESULT: ${{ needs.backend-validation.result }} + DETECTOR_RESULT: ${{ needs.detector-validation.result }} + run: node "$GITHUB_WORKSPACE/scripts/ppl-lint/assemble-run-manifest.mjs" + + - name: Upload run manifest + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-run-manifest + path: | + run-manifest.json + + # The sole branch-protection check: red unless BOTH validation jobs + # succeeded. Because this job runs with always(), a skipped detector job + # (e.g. backend failed first) still reds the result instead of appearing + # green (design §4.4). A workflow_dispatch run is pre-merge evidence and is + # intentionally not what repo admins pin to branch protection. + - name: Require both validation jobs to have succeeded + if: ${{ always() }} + env: + BACKEND_RESULT: ${{ needs.backend-validation.result }} + DETECTOR_RESULT: ${{ needs.detector-validation.result }} + run: | + echo "backend-validation: $BACKEND_RESULT" + echo "detector-validation: $DETECTOR_RESULT" + if [ "$BACKEND_RESULT" != "success" ] || [ "$DETECTOR_RESULT" != "success" ]; then + echo "::error::PPL lint rule validation failed (backend=$BACKEND_RESULT detector=$DETECTOR_RESULT)." + exit 1 + fi + echo "PPL lint rule validation passed: backend and detector agree on the candidate grammar." diff --git a/.gitignore b/.gitignore index bf9002f999d..05b8f803ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ src/site-server/node_modules build/ gen/ *.tokens +.attach_pid* # various IDE files .vscode @@ -59,4 +60,13 @@ http-client.env.json !.claude/harness/ .claude/settings.local.json .clinerules -memory-bank \ No newline at end of file +memory-bank + +# PPL lint rule validation contract run artifacts (uploaded in CI, not committed) +backend-report.json +backend-report-nightly.json +detector-report.json +detector-contract.log +ppl-grammar-bundle.json +target.json +run-manifest.json diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index 3320391c0d2..ca0c524b4a1 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -286,7 +286,7 @@ SELECT department, SUM(age) AS total FROM catalog.employees GROUP BY department .assertPlan( """ LogicalProject(department=[$0], total=[$1]) - LogicalAggregate(group=[{0}], SUM(age)=[SUM($1)]) + LogicalAggregate(group=[{0}], SUM(age)=[CHECKED_LONG_SUM($1)]) LogicalProject(department=[$3], age=[$2]) LogicalTableScan(table=[[catalog, employees]]) """); @@ -366,7 +366,7 @@ SELECT department, SUM(age) FILTER(WHERE age > 30) FROM catalog.employees """) .assertPlan( """ - LogicalAggregate(group=[{0}], SUM(age) FILTER(WHERE age > 30)=[SUM($1) FILTER $2]) + LogicalAggregate(group=[{0}], SUM(age) FILTER(WHERE age > 30)=[CHECKED_LONG_SUM($1) FILTER $2]) LogicalProject(department=[$3], age=[$2], $f3=[>($2, 30)]) LogicalTableScan(table=[[catalog, employees]]) """); @@ -487,7 +487,7 @@ SELECT name, SUM(age) OVER(PARTITION BY department ORDER BY age) FROM catalog.em """) .assertPlan( """ - LogicalProject(name=[$1], SUM(age) OVER(PARTITION BY department ORDER BY age)=[SUM($2) OVER (PARTITION BY $3 ORDER BY $2 NULLS FIRST)]) + LogicalProject(name=[$1], SUM(age) OVER(PARTITION BY department ORDER BY age)=[CHECKED_LONG_SUM($2) OVER (PARTITION BY $3 ORDER BY $2 NULLS FIRST)]) LogicalTableScan(table=[[catalog, employees]]) """); } diff --git a/build.gradle b/build.gradle index 9047a9c3feb..7f32a194739 100644 --- a/build.gradle +++ b/build.gradle @@ -68,6 +68,7 @@ buildscript { repositories { mavenLocal() maven { url "https://ci.opensearch.org/maven2/" } + maven { url "https://ci.opensearch.org/m2/" } mavenCentral() maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } } @@ -93,13 +94,14 @@ apply plugin: 'opensearch.java-agent' // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{} repositories { mavenLocal() - maven { url "https://ci.opensearch.org/maven2/" } - mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available maven { url 'https://jitpack.io' content { includeGroup "com.github.babbel" } } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } + maven { url "https://ci.opensearch.org/maven2/" } + maven { url "https://ci.opensearch.org/m2/" } + mavenCentral() } spotless { @@ -175,14 +177,15 @@ allprojects { subprojects { repositories { mavenLocal() - maven { url "https://ci.opensearch.org/maven2/" } - mavenCentral() maven { url 'https://jitpack.io' content { includeGroup "com.github.babbel" } } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" } + maven { url "https://ci.opensearch.org/maven2/" } + maven { url "https://ci.opensearch.org/m2/" } + mavenCentral() } // Publish internal modules as Maven artifacts for external use, such as by opensearch-spark and opensearch-cli. diff --git a/common/src/main/java/org/opensearch/sql/common/error/ResourceLimitExceededException.java b/common/src/main/java/org/opensearch/sql/common/error/ResourceLimitExceededException.java new file mode 100644 index 00000000000..b201fb0c8ed --- /dev/null +++ b/common/src/main/java/org/opensearch/sql/common/error/ResourceLimitExceededException.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.common.error; + +/** + * Raised when a query cannot proceed because it would exceed a node or cluster resource budget -- + * e.g. the per-node Point-In-Time (PIT) context limit ({@code search.max_open_pit_context}). Pairs + * with {@link ErrorCode#RESOURCE_LIMIT_EXCEEDED}: the code is the machine-readable classifier while + * this type gives clients a stable, semantic name to match on. The message is the customer-facing + * {@code reason}; put the explanation and remedy in the {@link ErrorReport} details. + */ +public class ResourceLimitExceededException extends RuntimeException { + + public ResourceLimitExceededException(String message) { + super(message); + } + + public ResourceLimitExceededException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 5473aa8812e..5ab82f6b6c9 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,7 +36,6 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), - PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ @@ -81,7 +80,10 @@ public enum Key { ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL( "plugins.query.executionengine.async_query.external_scheduler.interval"), STREAMING_JOB_HOUSEKEEPER_INTERVAL( - "plugins.query.executionengine.spark.streamingjobs.housekeeper.interval"); + "plugins.query.executionengine.spark.streamingjobs.housekeeper.interval"), + + /** Thread Pool Settings. */ + SQL_COMPLEX_WORKER_POOL_ENABLED("plugins.sql.complex_worker_pool.enabled"); @Getter private final String keyValue; diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 8c1dbee006f..916cc00bc4a 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -82,6 +82,7 @@ import org.opensearch.sql.ast.tree.Limit; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.MvCombine; import org.opensearch.sql.ast.tree.MvExpand; @@ -111,6 +112,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -562,6 +564,11 @@ public LogicalPlan visitNoMv(NoMv node, AnalysisContext context) { throw getOnlyForCalciteException("nomv"); } + @Override + public LogicalPlan visitMakeResults(MakeResults node, AnalysisContext context) { + throw getOnlyForCalciteException("makeresults"); + } + @Override public LogicalPlan visitMvExpand(MvExpand node, AnalysisContext context) { throw getOnlyForCalciteException("mvexpand"); @@ -836,6 +843,11 @@ public LogicalPlan visitChart(Chart node, AnalysisContext context) { throw getOnlyForCalciteException("Chart"); } + @Override + public LogicalPlan visitXyseries(Xyseries node, AnalysisContext context) { + throw getOnlyForCalciteException("Xyseries"); + } + @Override public LogicalPlan visitWindow(Window node, AnalysisContext context) { throw getOnlyForCalciteException("Window"); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index a32354883bf..266f8f46f7d 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -71,6 +71,7 @@ import org.opensearch.sql.ast.tree.Limit; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.MvCombine; import org.opensearch.sql.ast.tree.MvExpand; @@ -99,6 +100,7 @@ import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; /** AST nodes visitor Defines the traverse path. */ public abstract class AbstractNodeVisitor { @@ -344,6 +346,10 @@ public T visitValues(Values node, C context) { return visitChildren(node, context); } + public T visitMakeResults(MakeResults node, C context) { + return visitChildren(node, context); + } + public T visitAlias(Alias node, C context) { return visitChildren(node, context); } @@ -515,4 +521,8 @@ public T visitMvExpand(MvExpand node, C context) { public T visitGraphLookup(GraphLookup node, C context) { return visitChildren(node, context); } + + public T visitXyseries(Xyseries node, C context) { + return visitChildren(node, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/MakeResults.java b/core/src/main/java/org/opensearch/sql/ast/tree/MakeResults.java new file mode 100644 index 00000000000..a8b9a388f92 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/MakeResults.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.Node; + +/** + * AST node for the {@code makeresults} leading command (count path). Generates {@code count} + * in-memory rows, each carrying a single {@code @timestamp} column set to query time. + * + *

The {@code format=csv|json data="..."} form is parsed into a shared {@link Values} node + * instead (see {@code MakeResultsDataParser}), so inline literal rows flow through the common + * {@code visitValues} builder. + */ +@ToString +@Getter +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class MakeResults extends UnresolvedPlan { + + private final int count; + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + throw new UnsupportedOperationException("MakeResults node is supposed to have no child node"); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitMakeResults(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java index 6c543ddc8c3..8055c4f92d8 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java @@ -56,6 +56,8 @@ public enum CommandType { public enum Option { countField, showCount, + percentField, + showPerc, useNull, } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Values.java b/core/src/main/java/org/opensearch/sql/ast/tree/Values.java index 65d7e8d7cb2..55bea273838 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Values.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Values.java @@ -9,21 +9,54 @@ import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.ToString; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.Node; import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.data.type.ExprCoreType; /** AST node class for a sequence of literal values. */ @ToString @Getter @EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor public class Values extends UnresolvedPlan { private final List> values; + private final List columnNames; + + /** + * Optional explicit column types, authoritative for the schema. Required to type a zero-row + * relation (header-only CSV / empty JSON array) where there are no literals to infer from. + */ + private final List columnTypes; + + /** + * When {@code true}, prepend an implicit {@code @timestamp = NOW()} column (from {@code + * makeresults format=json data=}). CSV data= and subsearch callers leave it {@code false}. + */ + private final boolean withImplicitTimestamp; + + public Values(List> values) { + this(values, null, null); + } + + public Values( + List> values, List columnNames, List columnTypes) { + this(values, columnNames, columnTypes, false); + } + + public Values( + List> values, + List columnNames, + List columnTypes, + boolean withImplicitTimestamp) { + this.values = values; + this.columnNames = columnNames; + this.columnTypes = columnTypes; + this.withImplicitTimestamp = withImplicitTimestamp; + } + @Override public UnresolvedPlan attach(UnresolvedPlan child) { throw new UnsupportedOperationException("Values node is supposed to have no child node"); diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java b/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java new file mode 100644 index 00000000000..84fb3019e0f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * AST node representing the xyseries command. Converts row-oriented grouped results into a wide + * table where one field is the X axis (row key), one field provides pivot values for column naming, + * and one or more data fields fill the pivoted cells. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Xyseries extends UnresolvedPlan { + + /** The x-axis field (row key in output). */ + private final UnresolvedExpression xField; + + /** The y-name field whose values become part of the output column names. */ + private final UnresolvedExpression yNameField; + + /** Explicit pivot values from the IN (...) clause. */ + private final List pivotValues; + + /** One or more y-data fields whose values fill the pivoted cells. */ + private final List yDataFields; + + /** Separator between y-data-field name and pivot value in column names. Default ":". */ + private final String separator; + + /** Optional format template for output column names using $AGG$ and $VAL$ placeholders. */ + private final String format; + + @Setter private UnresolvedPlan child; + + @Override + public Xyseries attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitXyseries(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 162a4895805..c7f3bc373ac 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -58,6 +58,11 @@ public class CalcitePlanContext { /** Timewrap series mode: "relative", "short", or "exact". */ public static final ThreadLocal timewrapSeries = new ThreadLocal<>(); + /** + * Thread-local tracking which pool executed this query ("sql-worker" or "sql-complex-worker"). + */ + public static final ThreadLocal executionPool = new ThreadLocal<>(); + /** Thread-local switch that tells whether the current query prefers legacy behavior. */ private static final ThreadLocal legacyPreferredFlag = ThreadLocal.withInitial(() -> true); @@ -119,6 +124,22 @@ public class CalcitePlanContext { /** Whether we're currently inside a lambda context. */ @Getter @Setter private boolean inLambdaContext = false; + /** + * When enabled, tracks which RelNode ids were produced by each AST command. Each entry maps an + * AST node class name to the list of RelNode ids it produced (excluding children). + */ + @Getter @Setter private boolean trackingEnabled = false; + + @Getter private final List nodeIdMappings = new ArrayList<>(); + + /** Records a mapping from an AST command to the RelNode ids it produced. */ + public void recordMapping(String astNodeType, List relNodeIds) { + nodeIdMappings.add(new NodeIdMapping(astNodeType, relNodeIds)); + } + + /** A mapping from one AST command to the RelNode ids it produced. */ + public record NodeIdMapping(String astNodeType, List relNodeIds) {} + private CalcitePlanContext(FrameworkConfig config, SysLimit sysLimit, QueryType queryType) { this.config = config; this.sysLimit = sysLimit; @@ -229,6 +250,51 @@ public static void clearTimewrapSignals() { stripNullColumns.set(false); timewrapUnitName.set(null); timewrapSeries.set(null); + executionPool.set(null); + } + + /** + * Snapshot of all thread-local state in CalcitePlanContext. Used when dispatching queries to the + * complex worker pool — capture state on the caller thread, restore on the worker thread. + */ + public static class ThreadLocalSnapshot { + final boolean skipEncoding; + final boolean stripNullColumns; + final String timewrapUnitName; + final String timewrapSeries; + final String executionPool; + + private ThreadLocalSnapshot( + boolean skipEncoding, + boolean stripNullColumns, + String timewrapUnitName, + String timewrapSeries, + String executionPool) { + this.skipEncoding = skipEncoding; + this.stripNullColumns = stripNullColumns; + this.timewrapUnitName = timewrapUnitName; + this.timewrapSeries = timewrapSeries; + this.executionPool = executionPool; + } + } + + /** Capture current thread-local state for cross-thread propagation. */ + public static ThreadLocalSnapshot snapshotThreadLocals() { + return new ThreadLocalSnapshot( + skipEncoding.get(), + stripNullColumns.get(), + timewrapUnitName.get(), + timewrapSeries.get(), + executionPool.get()); + } + + /** Restore thread-local state from a snapshot. */ + public static void restoreThreadLocals(ThreadLocalSnapshot snapshot) { + skipEncoding.set(snapshot.skipEncoding); + stripNullColumns.set(snapshot.stripNullColumns); + timewrapUnitName.set(snapshot.timewrapUnitName); + timewrapSeries.set(snapshot.timewrapSeries); + executionPool.set(snapshot.executionPool); } public void pushForeachBindings( diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 0df3c571c6b..e1f2e666c86 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -38,6 +38,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -97,6 +98,7 @@ import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.Argument.ArgumentMap; +import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.Let; @@ -140,6 +142,7 @@ import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.Lookup.OutputStrategy; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.MvCombine; import org.opensearch.sql.ast.tree.MvExpand; @@ -169,6 +172,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.AliasFieldsWrappable; import org.opensearch.sql.calcite.plan.HighlightPushDown; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -177,6 +181,7 @@ import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.utils.BinUtils; import org.opensearch.sql.calcite.utils.JoinAndLookupUtils; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLHintUtils; import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.TimewrapUtils; @@ -186,6 +191,7 @@ import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.common.patterns.PatternUtils; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; @@ -217,6 +223,8 @@ public class CalciteRelNodeVisitor extends AbstractNodeVisitor 0 ? context.relBuilder.peek().getId() : -1; + RelNode result = unresolved.accept(this, context); + int idAfter = context.relBuilder.peek().getId(); + List producedIds = new ArrayList<>(); + for (int id = idBefore + 1; id <= idAfter; id++) { + producedIds.add(id); + } + context.recordMapping(unresolved.getClass().getSimpleName(), producedIds); + return result; + } return unresolved.accept(this, context); } @Override public RelNode visitChildren(Node node, CalcitePlanContext context) { + if (context.isTrackingEnabled() && node instanceof UnresolvedPlan) { + // Track each child's total contribution (the subtree it produces) + RelNode result = null; + for (Node child : node.getChild()) { + int idBefore = context.relBuilder.size() > 0 ? context.relBuilder.peek().getId() : -1; + RelNode childResult = child.accept(this, context); + result = childResult; + // After child.accept returns, the child's visit* method has fully completed, + // so all RelNodes produced by that child (including ITS children) are on the stack. + int idAfter = context.relBuilder.peek().getId(); + if (child instanceof UnresolvedPlan) { + List producedIds = new ArrayList<>(); + for (int id = idBefore + 1; id <= idAfter; id++) { + producedIds.add(id); + } + context.recordMapping(child.getClass().getSimpleName(), producedIds); + } + } + if (node instanceof UnresolvedPlan plan) { + mapPathMaterializer.materializePaths(plan, context); + } + return result; + } RelNode result = super.visitChildren(node, context); if (node instanceof UnresolvedPlan plan) { - // Materialize MAP dotted paths as flat columns after children are analyzed - // (so MAP/struct types are known) but before the command's own visit logic runs. mapPathMaterializer.materializePaths(plan, context); } return result; @@ -3226,6 +3266,39 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { orderKeys.add(countField); orderKeys.addAll(tieBreakKeys); + // 3. compute percentage if showperc=true + Boolean showPerc = (Boolean) argumentMap.get(RareTopN.Option.showPerc.name()).getValue(); + if (showPerc) { + String percentFieldName = + (String) argumentMap.get(RareTopN.Option.percentField.name()).getValue(); + + RexNode totalWindowOver = + PlanUtils.makeOver( + context, + BuiltinFunctionName.SUM, + context.relBuilder.field(countFieldName), + List.of(), + partitionKeys, + List.of(), + WindowFrame.rowsUnbounded()); + + RexNode hundred = context.relBuilder.literal(new BigDecimal("100.0")); + RexNode countCast = + context.relBuilder.cast(context.relBuilder.field(countFieldName), SqlTypeName.DOUBLE); + RexNode totalCast = context.relBuilder.cast(totalWindowOver, SqlTypeName.DOUBLE); + RexNode numerator = context.relBuilder.call(SqlStdOperatorTable.MULTIPLY, hundred, countCast); + RexNode percValue = context.relBuilder.call(SqlStdOperatorTable.DIVIDE, numerator, totalCast); + + // Round the percent value 6 decimal places + RexNode roundedPerc = + context.relBuilder.call( + SqlStdOperatorTable.ROUND, + percValue, + context.relBuilder.literal(PERCENT_DECIMAL_PLACES)); + + context.relBuilder.projectPlus(context.relBuilder.alias(roundedPerc, percentFieldName)); + } + RexNode rowNumberWindowOver = PlanUtils.makeOver( context, @@ -3238,14 +3311,14 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { context.relBuilder.projectPlus( context.relBuilder.alias(rowNumberWindowOver, ROW_NUMBER_COLUMN_FOR_RARE_TOP)); - // 3. filter row_number() <= k in each partition + // 4. filter row_number() <= k in each partition int k = node.getNoOfResults(); context.relBuilder.filter( context.relBuilder.lessThanOrEqual( context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP), context.relBuilder.literal(k))); - // 4. project final output. the default output is group by list + field list + // 5. project final output: group by list + field list, optionally count and percent Boolean showCount = (Boolean) argumentMap.get(RareTopN.Option.showCount.name()).getValue(); if (showCount) { context.relBuilder.projectExcept(context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP)); @@ -4067,6 +4140,131 @@ static ChartConfig fromArguments(ArgumentMap argMap) { } } + @Override + public RelNode visitXyseries(Xyseries node, CalcitePlanContext context) { + visitChildren(node, context); + + RelBuilder b = context.relBuilder; + RexBuilder rx = context.rexBuilder; + + // Resolve x-field and y-name-field names + String xFieldName = resolveFieldName(node.getXField()); + String yNameFieldName = resolveFieldName(node.getYNameField()); + + // Resolve y-data field names + List yDataFieldNames = + node.getYDataFields().stream().map(this::resolveFieldName).collect(Collectors.toList()); + + List pivotValues = node.getPivotValues() != null ? node.getPivotValues() : List.of(); + String separator = node.getSeparator(); + String format = node.getFormat(); + + // Build the pivot axis - cast to VARCHAR if needed for string comparison + RexNode yNameRef = b.field(yNameFieldName); + RelDataType yNameType = yNameRef.getType(); + RexNode axis; + if (!SqlTypeUtil.isCharacter(yNameRef.getType())) { + if (!SqlTypeUtil.isAtomic(yNameType)) { + throw new IllegalArgumentException( + "xyseries y-name-field must be a scalar type, got: " + yNameType.getSqlTypeName()); + } + RelDataType varchar = + rx.getTypeFactory() + .createTypeWithNullability( + rx.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), true); + axis = rx.makeCast(varchar, yNameRef, true); + } else { + axis = yNameRef; + } + + // Build aggregate calls - MAX for each y-data field + List aggCalls = + yDataFieldNames.stream() + .map(name -> b.max(b.field(name)).as(name)) + .collect(Collectors.toList()); + + // Build pivot value entries: alias -> [literal(value)] + // LinkedHashMap preserves insertion order for deterministic column ordering + LinkedHashMap> pivotValueMap = new LinkedHashMap<>(); + for (String val : pivotValues) { + pivotValueMap.put(val, ImmutableList.of(b.literal(val))); + } + + // Execute pivot: decomposes into GROUP BY x-field with FILTER-based aggregation + // Produces columns: x-field, {val1}_{agg1}, {val1}_{agg2}, {val2}_{agg1}, ... + b.pivot( + b.groupKey(b.field(xFieldName)), + aggCalls, + ImmutableList.of(axis), + pivotValueMap.entrySet()); + + // Pivot produces value-first column ordering: val1_agg1, val1_agg2, val2_agg1, ... + // Reorder to agg-first and apply custom column naming: agg1: val1, agg1: val2, ... + List reorderProjections = new ArrayList<>(); + List reorderNames = new ArrayList<>(); + + reorderProjections.add(b.field(xFieldName)); + reorderNames.add(xFieldName); + + for (String aggName : yDataFieldNames) { + for (String pivotVal : pivotValues) { + // Reference pivot output column by its generated name: {value}_{agg} + String pivotColName = pivotVal + "_" + aggName; + try { + reorderProjections.add(b.field(pivotColName)); + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + "xyseries: expected pivot output column '" + pivotColName + "' not found", e); + } + boolean singleDataField = yDataFieldNames.size() == 1; + reorderNames.add(generateColumnName(aggName, pivotVal, separator, format, singleDataField)); + } + } + // Fail fast with a clear message if the naming scheme produced collisions + // (e.g. a format template that omits $VAL$ or $AGG$ with multiple series). + Set seenNames = new HashSet<>(); + for (String name : reorderNames) { + if (!seenNames.add(name)) { + throw new IllegalArgumentException( + "xyseries produced duplicate output column name '" + + name + + "'. Use a format template containing both $AGG$ and $VAL$ so column names" + + " are unique."); + } + } + b.project(reorderProjections, reorderNames, true); + + // Order by x-field + b.sort(b.field(0)); + + return b.peek(); + } + + private String resolveFieldName(UnresolvedExpression expr) { + if (expr instanceof Field) { + return ((Field) expr).getField().toString(); + } + if (expr instanceof Alias) { + return ((Alias) expr).getName(); + } + return expr.toString(); + } + + private String generateColumnName( + String yDataFieldName, + String pivotValue, + String separator, + String format, + boolean singleDataField) { + if (format != null) { + return format.replace("$AGG$", yDataFieldName).replace("$VAL$", pivotValue); + } + if (singleDataField) { + return pivotValue; + } + return yDataFieldName + separator + pivotValue; + } + @Override public RelNode visitTrendline(Trendline node, CalcitePlanContext context) { visitChildren(node, context); @@ -4468,17 +4666,150 @@ public RelNode visitMvExpand(MvExpand mvExpand, CalcitePlanContext context) { @Override public RelNode visitValues(Values values, CalcitePlanContext context) { List> rows = values.getValues(); - if (rows == null || rows.isEmpty()) { + RelBuilder relBuilder = context.relBuilder; + boolean hasExplicitSchema = values.getColumnNames() != null || values.getColumnTypes() != null; + if (!hasExplicitSchema && (rows == null || rows.isEmpty())) { // PPL empty subsearch (e.g., `... | append [ ]`): zero rows, no columns. - context.relBuilder.values(context.relBuilder.getTypeFactory().builder().build()); - return context.relBuilder.peek(); + relBuilder.values(relBuilder.getTypeFactory().builder().build()); + return relBuilder.peek(); } - if (rows.size() == 1 && rows.get(0).isEmpty()) { + if (rows != null && rows.size() == 1 && rows.get(0).isEmpty()) { // SQL FROM-less SELECT (dual table) encoded as Values([[]]): one-row relation for Project. - context.relBuilder.push(LogicalValues.createOneRow(context.relBuilder.getCluster())); - return context.relBuilder.peek(); + relBuilder.push(LogicalValues.createOneRow(relBuilder.getCluster())); + return relBuilder.peek(); + } + // Inline literal rows, e.g. `makeresults format=csv|json data=...`. + return buildLiteralValues( + relBuilder, + values.getColumnNames(), + values.getColumnTypes(), + rows, + values.isWithImplicitTimestamp()); + } + + /** + * Build a typed {@link LogicalValues} (+ a cast {@code Project}) from inline literal rows. Column + * names/types are taken from the explicit lists when provided (authoritative, and required to + * type a zero-row relation); otherwise names are positional and types are inferred from the + * literals. + */ + private RelNode buildLiteralValues( + RelBuilder relBuilder, + List explicitNames, + List explicitTypes, + List> rows, + boolean withImplicitTimestamp) { + int nc; + if (explicitTypes != null) { + nc = explicitTypes.size(); + } else if (explicitNames != null) { + nc = explicitNames.size(); + } else if (!rows.isEmpty() && !rows.get(0).isEmpty()) { + nc = rows.get(0).size(); + } else { + nc = 0; + } + + List names = new java.util.ArrayList<>(); + for (int i = 0; i < nc; i++) { + names.add(explicitNames != null ? explicitNames.get(i) : "column_" + i); + } + + List types = new java.util.ArrayList<>(); + for (int c = 0; c < nc; c++) { + if (explicitTypes != null) { + types.add(explicitTypes.get(c)); + } else { + // infer from the first non-null literal in this column, defaulting to STRING. + ExprCoreType t = ExprCoreType.STRING; + for (List row : rows) { + DataType dt = row.get(c).getType(); + if (dt != DataType.NULL) { + t = dt.getCoreType(); + break; + } + } + types.add(t); + } } - throw new CalciteUnsupportedException("Inline VALUES with literal rows is unsupported"); + + boolean prependTimestamp = + withImplicitTimestamp && !names.contains(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP); + RelDataType tsType = + OpenSearchTypeFactory.convertExprTypeToRelDataType(ExprCoreType.TIMESTAMP, false); + + var typeBuilder = relBuilder.getTypeFactory().builder(); + if (prependTimestamp) { + typeBuilder.add(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP, tsType); + } + for (int i = 0; i < nc; i++) { + typeBuilder.add( + names.get(i), OpenSearchTypeFactory.convertExprTypeToRelDataType(types.get(i), true)); + } + RelDataType rowType = typeBuilder.build(); + + if (rows.isEmpty()) { + // header-only CSV / empty JSON array: a zero-row relation with the resolved schema. + relBuilder.values(ImmutableList.>of(), rowType); + return relBuilder.peek(); + } + + Object[] flat = new Object[rows.size() * nc]; + int k = 0; + for (List row : rows) { + for (Literal cell : row) { + flat[k++] = cell.getValue(); + } + } + relBuilder.values(names.toArray(new String[0]), flat); + List projects = new java.util.ArrayList<>(); + if (prependTimestamp) { + projects.add( + relBuilder.alias( + relBuilder.call(PPLBuiltinOperators.NOW), + OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP)); + } + for (int i = 0; i < nc; i++) { + projects.add( + relBuilder.alias( + relBuilder.cast( + relBuilder.field(i), + rowType.getField(names.get(i), true, false).getType().getSqlTypeName()), + names.get(i))); + } + relBuilder.project(projects); + return relBuilder.peek(); + } + + @Override + public RelNode visitMakeResults(MakeResults node, CalcitePlanContext context) { + // Count path only: the `format=csv|json data=...` form is parsed into a shared Values node + // (see MakeResultsDataParser) and handled by visitValues. + RelBuilder relBuilder = context.relBuilder; + int count = node.getCount(); + RelDataType tsType = + OpenSearchTypeFactory.convertExprTypeToRelDataType(ExprCoreType.TIMESTAMP, false); + if (count == 0) { + RelDataType rowType = + relBuilder + .getTypeFactory() + .builder() + .add(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP, tsType) + .build(); + relBuilder.values(ImmutableList.>of(), rowType); + return relBuilder.peek(); + } + // The dummy column only carries row multiplicity; project it to @timestamp=NOW(), OpenSearch's + // implicit time field recognized by the time-aware commands. + Object[] dummy = new Object[count]; + for (int i = 0; i < count; i++) { + dummy[i] = i; + } + relBuilder.values(new String[] {"__makeresults_dummy__"}, dummy); + RexNode now = relBuilder.call(PPLBuiltinOperators.NOW); + relBuilder.project( + List.of(relBuilder.alias(now, OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP))); + return relBuilder.peek(); } @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java b/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java index d9ef71b2220..25a8ddef642 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java +++ b/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java @@ -432,9 +432,9 @@ private UnresolvedExpression asArrayExpression( * VARCHAR. * *

For {@code json_array()} calls and string literals the content is visible at plan time; - * mixed content is rejected. For opaque expressions — typically a field holding JSON text, the - * primary Splunk use of json_array mode — content is unknowable, so infer from usage: an item - * placeholder consumed by arithmetic means numeric elements, anything else means strings. + * mixed content is rejected. For opaque expressions, typically a field holding JSON text, content + * is unknowable, so infer from usage: an item placeholder consumed by arithmetic means numeric + * elements, anything else means strings. */ private SqlTypeName jsonElementType( UnresolvedExpression collection, CalcitePlanContext context, Foreach node) { diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java new file mode 100644 index 00000000000..4b33b9a0ffa --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +/** BIGINT average aggregate that accumulates in double to avoid an intermediate long overflow. */ +public class BigintAvgAggFunction { + + public static Accumulator init() { + return new Accumulator(); + } + + public static Accumulator add(Accumulator accumulator, Long value) { + if (value != null) { + accumulator.sum += value; + accumulator.count++; + } + return accumulator; + } + + public static Double result(Accumulator accumulator) { + return accumulator.count == 0 ? null : accumulator.sum / accumulator.count; + } + + public static class Accumulator { + private double sum; + private long count; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java new file mode 100644 index 00000000000..7e146eacedb --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +/** BIGINT sum aggregate that throws when its running long accumulator overflows. */ +public class CheckedLongSumAggFunction { + + public static long init() { + return 0L; + } + + public static long add(long accumulator, long value) { + return Math.addExact(accumulator, value); + } + + public static long result(long accumulator) { + return accumulator; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java index 54b9d4ffbaf..682a3ea17c1 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java @@ -516,8 +516,6 @@ private static void enrichErrorsForSpecialCases(ErrorReport.Builder report, SQLE public static PreparedStatement run(CalcitePlanContext context, RelNode rel) { ProfileMetric optimizeTime = QueryProfiling.current().getOrCreateMetric(OPTIMIZE); long startTime = System.nanoTime(); - // Optimize the plan by Calcite's HepPlanner before using VolcanoPlanner in prepareStatement. - rel = CalciteToolsHelper.optimize(rel, context); final RelShuttle shuttle = new RelHomogeneousShuttle() { @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index fcd361ba229..fd233c83dd5 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -5,10 +5,17 @@ package org.opensearch.sql.calcite.utils; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; + +import java.util.ArrayList; +import java.util.List; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.FamilyOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; import org.opensearch.sql.expression.function.UDFOperandMetadata; /** @@ -20,41 +27,51 @@ public class PPLOperandTypes { // This class is not meant to be instantiated. private PPLOperandTypes() {} + // Convenience RelDataType constants used to express UDF signatures via wrapUDT(...). + // UDT-backed scalar types: + public static final RelDataType DATE_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_DATE); + public static final RelDataType TIME_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIME); + public static final RelDataType TIMESTAMP_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIMESTAMP); + public static final RelDataType IP_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_IP); + public static final RelDataType BINARY_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_BINARY); + // Plain SQL scalar types: + public static final RelDataType BYTE_T = TYPE_FACTORY.createSqlType(SqlTypeName.TINYINT); + public static final RelDataType SHORT_T = TYPE_FACTORY.createSqlType(SqlTypeName.SMALLINT); + public static final RelDataType INTEGER_T = TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER); + public static final RelDataType LONG_T = TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT); + public static final RelDataType FLOAT_T = TYPE_FACTORY.createSqlType(SqlTypeName.REAL); + public static final RelDataType DOUBLE_T = TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE); + public static final RelDataType STRING_T = TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR); + public static final RelDataType BOOLEAN_T = TYPE_FACTORY.createSqlType(SqlTypeName.BOOLEAN); + /** List of all scalar type signatures (single parameter each) */ - private static final java.util.List> - SCALAR_TYPES = - java.util.List.of( - // Numeric types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BYTE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.SHORT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.INTEGER), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.LONG), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.FLOAT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DOUBLE), - // String type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.STRING), - // Boolean type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BOOLEAN), - // Temporal types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DATE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIME), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP), - // Special scalar types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.IP), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BINARY)); + private static final List> SCALAR_TYPES = + List.of( + // Numeric types + List.of(BYTE_T), + List.of(SHORT_T), + List.of(INTEGER_T), + List.of(LONG_T), + List.of(FLOAT_T), + List.of(DOUBLE_T), + // String type + List.of(STRING_T), + // Boolean type + List.of(BOOLEAN_T), + // Temporal types + List.of(DATE_UDT), + List.of(TIME_UDT), + List.of(TIMESTAMP_UDT), + // Special scalar types + List.of(IP_UDT), + List.of(BINARY_UDT)); /** Helper method to create scalar types with optional integer parameter */ - private static java.util.List> - createScalarWithOptionalInteger() { - java.util.List> result = - new java.util.ArrayList<>(SCALAR_TYPES); + private static List> createScalarWithOptionalInteger() { + List> result = new ArrayList<>(SCALAR_TYPES); // Add scalar + integer combinations - SCALAR_TYPES.forEach( - scalarType -> - result.add( - java.util.List.of( - scalarType.get(0), org.opensearch.sql.data.type.ExprCoreType.INTEGER))); + SCALAR_TYPES.forEach(scalarType -> result.add(List.of(scalarType.get(0), INTEGER_T))); return result; } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index f619d966cc8..0b365fc98e6 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -94,9 +94,32 @@ public static SqlUserDefinedAggFunction createUserDefinedAggFunction( String functionName, SqlReturnTypeInference returnType, @Nullable UDFOperandMetadata operandMetadata) { + return createReflectiveAggFunction(udafClass, functionName, returnType, operandMetadata); + } + + /** Creates an aggregate function from a class following Calcite's reflective UDAF convention. */ + public static SqlUserDefinedAggFunction createReflectiveAggFunction( + Class udafClass, + String functionName, + SqlReturnTypeInference returnType, + @Nullable UDFOperandMetadata operandMetadata) { + return createReflectiveAggFunction( + udafClass, functionName, SqlKind.OTHER_FUNCTION, returnType, operandMetadata); + } + + /** + * Creates an aggregate function whose kind remains visible to planner rules while execution uses + * the supplied reflective UDAF. + */ + public static SqlUserDefinedAggFunction createReflectiveAggFunction( + Class udafClass, + String functionName, + SqlKind kind, + SqlReturnTypeInference returnType, + @Nullable UDFOperandMetadata operandMetadata) { return new SqlUserDefinedAggFunction( new SqlIdentifier(functionName, SqlParserPos.ZERO), - SqlKind.OTHER_FUNCTION, + kind, returnType, null, operandMetadata, diff --git a/core/src/main/java/org/opensearch/sql/executor/AnalyzeResponse.java b/core/src/main/java/org/opensearch/sql/executor/AnalyzeResponse.java new file mode 100644 index 00000000000..d8e0b12a8e7 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/AnalyzeResponse.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor; + +import java.util.List; +import lombok.Builder; +import lombok.Data; +import org.opensearch.sql.monitor.profile.QueryProfile; + +@Data +@Builder +public class AnalyzeResponse { + + private final String query; + private final List querySegments; + // private final String ast; + private final List logicalPlan; + private final List physicalPlan; + private final QueryProfile profile; + private final List operator_tree; + private final List recommendations; + private final List schema; + private final Object[][] datarows; + private final long total; + private final long size; + + @Data + @Builder + public static class SchemaColumn { + private final String name; + private final String type; + } + + @Data + @Builder + public static class QuerySegment { + private final String nodeType; + private final String source; + } + + @Data + @Builder + public static class OperatorNode { + private final String source; + private final List node_type; + private final List description; + private final String estimated_cost; + private final Long estimated_rows; + private final String actual_time_ms; + private final Long actual_rows; + private final Boolean is_pushed_down; + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/DirectExecutionDispatcher.java b/core/src/main/java/org/opensearch/sql/executor/DirectExecutionDispatcher.java new file mode 100644 index 00000000000..5af110fa7e5 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/DirectExecutionDispatcher.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor; + +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.response.ResponseListener; + +/** + * Default no-op dispatcher that executes inline on the current thread. Used when complex-pool + * routing is disabled or as a fallback. + */ +public class DirectExecutionDispatcher implements ExecutionDispatcher { + + @Override + public void dispatch( + RelNode plan, + CalcitePlanContext context, + ResponseListener listener, + ExecutionEngine engine) { + engine.execute(plan, context, listener); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionDispatcher.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionDispatcher.java new file mode 100644 index 00000000000..ec7fa193852 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionDispatcher.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor; + +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.response.ResponseListener; + +/** + * Dispatches query execution to an appropriate thread pool based on plan characteristics. After + * query analysis and optimization, the dispatcher inspects the plan and routes execution to either + * the fast worker pool (for queries fully pushed to OpenSearch) or the complex worker pool (for + * queries requiring scripts/table scans). + */ +public interface ExecutionDispatcher { + + /** + * Dispatch execution of the given plan via the standard ExecutionEngine. + * + * @param plan the optimized Calcite plan + * @param context the plan context + * @param listener response listener for query results + * @param engine the execution engine to invoke + */ + void dispatch( + RelNode plan, + CalcitePlanContext context, + ResponseListener listener, + ExecutionEngine engine); + + /** + * Dispatch a task to the appropriate thread pool based on plan characteristics. Use this when the + * execution path differs from the standard ExecutionEngine interface (e.g., analytics engine). + * + * @param plan the optimized Calcite plan used for routing decisions + * @param context the plan context + * @param task the execution task to run + */ + default void dispatchTask(RelNode plan, CalcitePlanContext context, Runnable task) { + task.run(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index b97a679cbd3..858ba0598e6 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -5,25 +5,36 @@ package org.opensearch.sql.executor; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; -import lombok.AllArgsConstructor; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitDef; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.runtime.Hook; import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParser; @@ -42,6 +53,8 @@ import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit; import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.utils.CalciteClassLoaderHelper; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.common.error.QueryProcessingStage; import org.opensearch.sql.common.error.StageErrorHandler; @@ -54,6 +67,7 @@ import org.opensearch.sql.monitor.profile.MetricName; import org.opensearch.sql.monitor.profile.ProfileContext; import org.opensearch.sql.monitor.profile.ProfileMetric; +import org.opensearch.sql.monitor.profile.QueryProfile; import org.opensearch.sql.monitor.profile.QueryProfiling; import org.opensearch.sql.planner.PlanContext; import org.opensearch.sql.planner.Planner; @@ -64,7 +78,6 @@ /** The low level interface of core engine. */ @RequiredArgsConstructor -@AllArgsConstructor @Log4j2 public class QueryService { private final Analyzer analyzer; @@ -72,6 +85,37 @@ public class QueryService { private final Planner planner; private DataSourceService dataSourceService; private Settings settings; + private ExecutionDispatcher executionDispatcher = new DirectExecutionDispatcher(); + + public QueryService( + Analyzer analyzer, + ExecutionEngine executionEngine, + Planner planner, + DataSourceService dataSourceService, + Settings settings) { + this( + analyzer, + executionEngine, + planner, + dataSourceService, + settings, + new DirectExecutionDispatcher()); + } + + public QueryService( + Analyzer analyzer, + ExecutionEngine executionEngine, + Planner planner, + DataSourceService dataSourceService, + Settings settings, + ExecutionDispatcher executionDispatcher) { + this.analyzer = analyzer; + this.executionEngine = executionEngine; + this.planner = planner; + this.dataSourceService = dataSourceService; + this.settings = settings; + this.executionDispatcher = executionDispatcher; + } @Getter(lazy = true) private final CalciteRelNodeVisitor relNodeVisitor = new CalciteRelNodeVisitor(dataSourceService); @@ -185,9 +229,7 @@ public void executeWithCalcite( convertToCalcitePlan(relNode, context), context), "while converting the query to an executable plan"); - analyzeMetric.set(System.nanoTime() - analyzeStart); - - executeCalcitePlan(calcitePlan, context, listener); + executeCalcitePlan(calcitePlan, context, listener, analyzeMetric, analyzeStart); }, QueryService.class); } catch (Throwable t) { @@ -205,11 +247,20 @@ public void executeWithCalcite( private void executeCalcitePlan( RelNode calcitePlan, CalcitePlanContext context, - ResponseListener listener) { + ResponseListener listener, + ProfileMetric analyzeMetric, + long analyzeStart) { try { + // Optimize before dispatch so the dispatcher's ScriptDetector + // sees the post-optimization plan for accurate routing. + RelNode optimizedPlan = CalciteToolsHelper.optimize(calcitePlan, context); + analyzeMetric.set(System.nanoTime() - analyzeStart); + + // Wrap execution with EXECUTING stage tracking — dispatch via + // ExecutionDispatcher which may route to a complex worker pool StageErrorHandler.executeStageVoid( QueryProcessingStage.EXECUTING, - () -> executionEngine.execute(calcitePlan, context, listener), + () -> executionDispatcher.dispatch(optimizedPlan, context, listener, executionEngine), "while running the query"); } catch (RuntimeException e) { ArithmeticException overflow = findArithmeticOverflow(e); @@ -273,6 +324,471 @@ public void explainWithCalcite( settings); } + public void analyzeWithCalcite( + String query, + List querySegments, + UnresolvedPlan plan, + QueryType queryType, + ResponseListener listener) { + if (!shouldUseCalcite(queryType)) { + listener.onFailure( + new UnsupportedOperationException( + "Analyze requires the Calcite engine to be enabled" + + " (plugins.calcite.enabled=true) and a PPL query type")); + return; + } + // Phase 1: Execute via the exact same path as executeWithCalcite + executionEngine.execute + // to get identical profile timings. Use a latch to synchronize the async callback. + // Force profiling on so executeWithCalcite activates QueryProfiling. + QueryContext.setProfile(true); + AtomicReference queryResponseRef = new AtomicReference<>(); + AtomicReference profileRef = new AtomicReference<>(); + AtomicReference errorRef = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + + executeWithCalcite( + plan, + queryType, + null, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.QueryResponse response) { + ProfileMetric formatMetric = + QueryProfiling.current().getOrCreateMetric(MetricName.FORMAT); + long formatStart = System.nanoTime(); + int resultSize = response.getResults().size(); + for (var exprValue : response.getResults()) { + exprValue.tupleValue().entrySet().stream() + .map(e -> e.getValue().value()) + .toArray(Object[]::new); + } + formatMetric.set(System.nanoTime() - formatStart); + profileRef.set(QueryProfiling.current().finish()); + queryResponseRef.set(response); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + errorRef.set(e); + latch.countDown(); + } + }); + + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + listener.onFailure(new RuntimeException("Interrupted while waiting for query execution", e)); + return; + } + + if (errorRef.get() != null) { + listener.onFailure(errorRef.get()); + return; + } + + ExecutionEngine.QueryResponse queryResponse = queryResponseRef.get(); + QueryProfile profile = profileRef.get(); + + // If the profile plan tree has branching (any node with >1 child), our linear + // operator tree logic won't work. Return a response that 'fallsback' on `profile` + // by only including fields mirroring the `profile` endpoint. + if (profile != null && profile.getPlan() != null && !isLinearPlanTree(profile)) { + List schema = new ArrayList<>(); + if (queryResponse.getSchema() != null) { + for (ExecutionEngine.Schema.Column col : queryResponse.getSchema().getColumns()) { + schema.add( + AnalyzeResponse.SchemaColumn.builder() + .name(col.getName()) + .type(col.getExprType().typeName()) + .build()); + } + } + Object[][] datarows = new Object[queryResponse.getResults().size()][]; + int rowIdx = 0; + for (var exprValue : queryResponse.getResults()) { + datarows[rowIdx++] = + exprValue.tupleValue().entrySet().stream() + .map(e -> e.getValue().value()) + .toArray(Object[]::new); + } + listener.onResponse( + AnalyzeResponse.builder() + // .query(query) + .profile(profile) + .schema(schema) + .datarows(datarows) + .total(datarows.length) + .size(datarows.length) + .build()); + return; + } + + // Phase 2: Re-run with tracking to capture logical/physical plans and node mappings. + // This run benefits from warm caches but we don't report its timings. + CalcitePlanContext.run( + () -> { + try { + QueryProfiling.noop(); + CalciteClassLoaderHelper.withCalciteClassLoader( + () -> { + CalcitePlanContext context = + CalcitePlanContext.create( + buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType); + context.setTrackingEnabled(true); + RelNode relNode = analyze(plan, context); + RelNode calcitePlan = convertToCalcitePlan(relNode, context); + + AtomicReference physicalPlanRef = new AtomicReference<>(); + AtomicReference physicalRelRef = new AtomicReference<>(); + try (Hook.Closeable closeable = + Hook.PLAN_BEFORE_IMPLEMENTATION.addThread( + obj -> { + RelRoot relRoot = (RelRoot) obj; + physicalRelRef.set(relRoot.rel); + physicalPlanRef.set( + RelOptUtil.toString(relRoot.rel, SqlExplainLevel.ALL_ATTRIBUTES)); + })) { + try (java.sql.PreparedStatement ignored = + OpenSearchRelRunners.run(context, calcitePlan)) { + } catch (java.sql.SQLException e) { + throw new RuntimeException(e); + } + } + + String logicalPlanStr = + RelOptUtil.toString(calcitePlan, SqlExplainLevel.ALL_ATTRIBUTES); + List logicalPlanNodes = + java.util.Arrays.stream(logicalPlanStr.split("\n")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toList(); + List physicalPlanNodes = + java.util.Arrays.stream(physicalPlanRef.get().split("\n")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toList(); + + // Build operator tree using phase 2's tracking data + phase 1's profile. + List operatorTree = + buildOperatorTree( + querySegments, + logicalPlanNodes, + context.getNodeIdMappings(), + calcitePlan, + physicalRelRef.get(), + profile); + + // Convert QueryResponse results to analyze format. + List schema = new ArrayList<>(); + if (queryResponse.getSchema() != null) { + for (ExecutionEngine.Schema.Column col : + queryResponse.getSchema().getColumns()) { + schema.add( + AnalyzeResponse.SchemaColumn.builder() + .name(col.getName()) + .type(col.getExprType().typeName()) + .build()); + } + } + + Object[][] datarows = new Object[queryResponse.getResults().size()][]; + int rowIdx = 0; + for (var exprValue : queryResponse.getResults()) { + datarows[rowIdx++] = + exprValue.tupleValue().entrySet().stream() + .map(e -> e.getValue().value()) + .toArray(Object[]::new); + } + + AnalyzeResponse response = + AnalyzeResponse.builder() + .query(query) + .querySegments(querySegments) + .logicalPlan(logicalPlanNodes) + .physicalPlan(physicalPlanNodes) + .operator_tree(operatorTree) + .recommendations(List.of()) + .profile(profile) + .schema(schema) + .datarows(datarows) + .total(datarows.length) + .size(datarows.length) + .build(); + listener.onResponse(response); + }, + QueryService.class); + } catch (Throwable t) { + if (t instanceof Exception) { + listener.onFailure((Exception) t); + } else { + listener.onFailure(new RuntimeException(t)); + } + } + }, + settings); + } + + private List buildOperatorTree( + List querySegments, + List logicalPlanNodes, + List nodeIdMappings, + RelNode logicalPlan, + RelNode physicalPlan, + QueryProfile profile) { + // Build a map from RelNode id to its logical plan description string. + Map idToDescription = new HashMap<>(); + for (String node : logicalPlanNodes) { + int idIdx = node.lastIndexOf("id = "); + if (idIdx >= 0) { + String idStr = node.substring(idIdx + 5).trim(); + try { + int id = Integer.parseInt(idStr); + idToDescription.put(id, node); + } catch (NumberFormatException ignored) { + } + } + } + + // Compute exclusive ids per mapping by subtracting the previous mapping's ids. + // Mappings are recorded bottom-up: [Relation:[0], Filter:[0,1], Project:[0,1,2]] + // Exclusive: Relation=[0], Filter=[1], Project=[2] + List> exclusiveIds = new ArrayList<>(); + Set previousIds = new HashSet<>(); + for (CalcitePlanContext.NodeIdMapping mapping : nodeIdMappings) { + Set current = new HashSet<>(mapping.relNodeIds()); + Set exclusive = new HashSet<>(current); + exclusive.removeAll(previousIds); + exclusiveIds.add(exclusive); + previousIds = current; + } + + // Determine how many segments from the bottom were pushed into the physical scan. + // The physical plan's leaf node (the scan) absorbs logical nodes from the bottom up. + // Physical depth tells us how many separate physical operators exist; everything else + // was pushed down. We count segments bottom-up until we've covered all pushed logical nodes. + int physicalDepth = getLinearDepth(physicalPlan); + int logicalDepth = getLinearDepth(logicalPlan); + int pushedNodeCount = logicalDepth - physicalDepth; + + // log.info( + // "buildOperatorTree: logicalDepth={}, physicalDepth={}, pushedNodeCount={}," + // + " segments={}, exclusiveIds={}", + // logicalDepth, + // physicalDepth, + // pushedNodeCount, + // querySegments.size(), + // exclusiveIds); + + // Walk segments bottom-up (they're already in bottom-up order) and greedily assign + // them to the pushed group until we've accounted for all pushed logical nodes. + // The LogicalSystemLimit added by convertToCalcitePlan counts toward the logical depth + // but has no segment, so we only count nodes that appear in exclusiveIds. + long pushedLogicalNodes = 0; + int pushedSegments = 0; + for (int idx = 0; idx < querySegments.size() && pushedLogicalNodes < pushedNodeCount; idx++) { + Set ids = idx < exclusiveIds.size() ? exclusiveIds.get(idx) : Set.of(); + long planNodeCount = ids.stream().filter(idToDescription::containsKey).count(); + pushedLogicalNodes += planNodeCount; + pushedSegments++; + } + + // log.info( + // "buildOperatorTree: pushedSegments={}, pushedLogicalNodes={}", + // pushedSegments, + // pushedLogicalNodes); + + // Compute estimated row counts from the logical plan using RelMetadataQuery. + // Walk the logical plan bottom-up to get rowcount per node by id. + org.apache.calcite.rel.metadata.RelMetadataQuery mq = + logicalPlan.getCluster().getMetadataQuery(); + Map idToRowCount = new HashMap<>(); + collectRowCounts(logicalPlan, mq, idToRowCount); + + // Compute exclusive time and rows per physical node from the profile plan tree. + // The plan tree is top-down; we flatten it bottom-up to match operator tree order. + List physicalTimings = new ArrayList<>(); + if (profile != null && profile.getPlan() != null) { + List planNodes = new ArrayList<>(); + QueryProfile.PlanNode current = (QueryProfile.PlanNode) profile.getPlan(); + while (current != null) { + planNodes.add(current); + current = + (current.getChildren() != null && !current.getChildren().isEmpty()) + ? current.getChildren().get(0) + : null; + } + // planNodes is top-down; reverse to bottom-up + java.util.Collections.reverse(planNodes); + for (int p = 0; p < planNodes.size(); p++) { + double inclusive = planNodes.get(p).getTimeMillis(); + double childInclusive = (p > 0) ? planNodes.get(p - 1).getTimeMillis() : 0; + double exclusive = Math.max(0, inclusive - childInclusive); + long rows = planNodes.get(p).getRows(); + physicalTimings.add(new double[] {exclusive, rows}); + } + } + + List operators = new ArrayList<>(); + int physicalIdx = 0; + + // Build the pushed-down merged entry (first pushedSegments segments) + if (pushedSegments > 1) { + List mergedSegments = querySegments.subList(0, pushedSegments); + List descriptions = new ArrayList<>(); + for (int idx = 0; idx < pushedSegments; idx++) { + Set ids = idx < exclusiveIds.size() ? exclusiveIds.get(idx) : Set.of(); + ids.stream() + .sorted() + .map(idToDescription::get) + .filter(Objects::nonNull) + .forEach(descriptions::add); + } + String combinedSource = + mergedSegments.stream() + .map(AnalyzeResponse.QuerySegment::getSource) + .reduce((a, b) -> a + " | " + b) + .orElse(""); + List nodeTypes = + mergedSegments.stream().map(AnalyzeResponse.QuerySegment::getNodeType).toList(); + // Collect all plan node ids in the pushed group for estimated_rows + Set allPushedPlanIds = new HashSet<>(); + for (int i = 0; i < pushedSegments; i++) { + Set ids = i < exclusiveIds.size() ? exclusiveIds.get(i) : Set.of(); + ids.stream().filter(idToDescription::containsKey).forEach(allPushedPlanIds::add); + } + double[] timing = + physicalIdx < physicalTimings.size() ? physicalTimings.get(physicalIdx) : null; + physicalIdx++; + operators.add( + AnalyzeResponse.OperatorNode.builder() + .source(combinedSource) + .node_type(nodeTypes) + .description(descriptions.isEmpty() ? null : descriptions) + .is_pushed_down(true) + .estimated_rows(getEstimatedRows(allPushedPlanIds, idToRowCount)) + .actual_time_ms(timing != null ? String.format("%.2f ms", timing[0]) : null) + .actual_rows(timing != null ? (long) timing[1] : null) + .build()); + } else if (pushedSegments == 1) { + AnalyzeResponse.QuerySegment seg = querySegments.get(0); + Set ids = !exclusiveIds.isEmpty() ? exclusiveIds.get(0) : Set.of(); + Set planIds = + ids.stream() + .filter(idToDescription::containsKey) + .collect(java.util.stream.Collectors.toSet()); + List descriptions = + ids.stream().sorted().map(idToDescription::get).filter(Objects::nonNull).toList(); + double[] timing = + physicalIdx < physicalTimings.size() ? physicalTimings.get(physicalIdx) : null; + physicalIdx++; + operators.add( + AnalyzeResponse.OperatorNode.builder() + .source(seg.getSource()) + .node_type(List.of(seg.getNodeType())) + .description(descriptions.isEmpty() ? null : descriptions) + .estimated_rows(getEstimatedRows(planIds, idToRowCount)) + .actual_time_ms(timing != null ? String.format("%.2f ms", timing[0]) : null) + .actual_rows(timing != null ? (long) timing[1] : null) + .build()); + } + + // Remaining segments map to non-scan physical nodes (physicalDepth - 1 of them). + // Each physical node corresponds to one logical plan node. Group segments so that each + // group covers exactly one logical plan node; segments with 0 plan nodes merge into the + // next group that has one. + int idx = pushedSegments; + while (idx < querySegments.size()) { + List group = new ArrayList<>(); + List descriptions = new ArrayList<>(); + Set groupPlanIds = new HashSet<>(); + long logicalNodesInGroup = 0; + while (idx < querySegments.size() && logicalNodesInGroup < 1) { + group.add(querySegments.get(idx)); + Set ids = idx < exclusiveIds.size() ? exclusiveIds.get(idx) : Set.of(); + ids.stream() + .sorted() + .map(idToDescription::get) + .filter(Objects::nonNull) + .forEach(descriptions::add); + ids.stream().filter(idToDescription::containsKey).forEach(groupPlanIds::add); + logicalNodesInGroup += ids.stream().filter(idToDescription::containsKey).count(); + idx++; + } + String combinedSource = + group.stream() + .map(AnalyzeResponse.QuerySegment::getSource) + .reduce((a, b) -> a + " | " + b) + .orElse(""); + List nodeTypes = + group.stream().map(AnalyzeResponse.QuerySegment::getNodeType).toList(); + double[] timing = + physicalIdx < physicalTimings.size() ? physicalTimings.get(physicalIdx) : null; + physicalIdx++; + operators.add( + AnalyzeResponse.OperatorNode.builder() + .source(combinedSource) + .node_type(nodeTypes) + .description(descriptions.isEmpty() ? null : descriptions) + .estimated_rows(getEstimatedRows(groupPlanIds, idToRowCount)) + .actual_time_ms(timing != null ? String.format("%.2f ms", timing[0]) : null) + .actual_rows(timing != null ? (long) timing[1] : null) + .build()); + } + + return operators; + } + + private static boolean isLinearPlanTree(QueryProfile profile) { + QueryProfile.PlanNode current = (QueryProfile.PlanNode) profile.getPlan(); + while (current != null) { + if (current.getChildren() != null && current.getChildren().size() > 1) { + return false; + } + current = + (current.getChildren() != null && !current.getChildren().isEmpty()) + ? current.getChildren().get(0) + : null; + } + return true; + } + + private static int getLinearDepth(RelNode node) { + int depth = 0; + RelNode current = node; + while (current != null) { + depth++; + List inputs = current.getInputs(); + current = inputs.isEmpty() ? null : inputs.get(0); + } + return depth; + } + + private void collectRowCounts( + RelNode node, + org.apache.calcite.rel.metadata.RelMetadataQuery mq, + Map idToRowCount) { + try { + Double rowCount = mq.getRowCount(node); + if (rowCount != null) { + idToRowCount.put(node.getId(), rowCount); + } + } catch (Exception ignored) { + } + for (RelNode input : node.getInputs()) { + collectRowCounts(input, mq, idToRowCount); + } + } + + private Long getEstimatedRows(Set ids, Map idToRowCount) { + return ids.stream() + .filter(idToRowCount::containsKey) + .max(Integer::compareTo) + .map(id -> Math.round(idToRowCount.get(id))) + .orElse(null); + } + public void executeWithLegacy( UnresolvedPlan plan, QueryType queryType, diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/AnalyzePlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/AnalyzePlan.java new file mode 100644 index 00000000000..a43bc32792e --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/execution/AnalyzePlan.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.execution; + +import java.util.List; +import org.opensearch.sql.ast.statement.ExplainMode; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.AnalyzeResponse; +import org.opensearch.sql.executor.AnalyzeResponse.QuerySegment; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.executor.QueryType; + +/** Plan that produces an AnalyzeResponse (AST + logical plan). */ +public class AnalyzePlan extends AbstractPlan { + + private final String query; + private final List querySegments; + private final UnresolvedPlan plan; + private final QueryService queryService; + private final ResponseListener listener; + + public AnalyzePlan( + QueryId queryId, + QueryType queryType, + String query, + List querySegments, + UnresolvedPlan plan, + QueryService queryService, + ResponseListener listener) { + super(queryId, queryType); + this.query = query; + this.querySegments = querySegments; + this.plan = plan; + this.queryService = queryService; + this.listener = listener; + } + + @Override + public void execute() { + queryService.analyzeWithCalcite(query, querySegments, plan, getQueryType(), listener); + } + + @Override + public void explain( + ResponseListener listener, ExplainMode mode) { + throw new UnsupportedOperationException("Explain is not supported for analyze plan"); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java index 93c73a2315b..0e44dd02e7e 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -7,6 +7,7 @@ import static java.util.Objects.requireNonNull; +import java.util.List; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -19,6 +20,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.exception.UnsupportedCursorRequestException; +import org.opensearch.sql.executor.AnalyzeResponse; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; import org.opensearch.sql.executor.QueryService; @@ -147,4 +149,15 @@ public AbstractPlan visitExplain( node.getFormat(), context.getRight()); } + + /** Create an AnalyzePlan that produces AST node and logical plan RelNode. */ + public AbstractPlan createAnalyzePlan( + String query, + List querySegments, + UnresolvedPlan plan, + QueryType queryType, + ResponseListener listener) { + return new AnalyzePlan( + QueryId.queryId(), queryType, query, querySegments, plan, queryService, listener); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java index 7562dac74d8..3ec4911821c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java @@ -37,7 +37,14 @@ public final class CoercionUtils { */ public static @Nullable List castArguments( RexBuilder builder, PPLTypeChecker typeChecker, List arguments) { - List> paramTypeCombinations = typeChecker.getParameterTypes(); + List> paramTypeCombinations = + typeChecker.getParameterTypes().stream() + .map( + types -> + types.stream() + .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) + .toList()) + .toList(); List sourceTypes = arguments.stream() diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index d64f04bb9ad..812b94967f7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptExprMethodToUDF; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptExprMethodWithPropertiesToUDF; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptMathFunctionToUDF; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.createReflectiveAggFunction; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.createUserDefinedAggFunction; import com.google.common.base.Suppliers; @@ -29,6 +30,8 @@ import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.util.BuiltInMethod; +import org.opensearch.sql.calcite.udf.udaf.BigintAvgAggFunction; +import org.opensearch.sql.calcite.udf.udaf.CheckedLongSumAggFunction; import org.opensearch.sql.calcite.udf.udaf.DistinctCountApproxLogicalAggFunction; import org.opensearch.sql.calcite.udf.udaf.FirstAggFunction; import org.opensearch.sql.calcite.udf.udaf.LastAggFunction; @@ -246,11 +249,12 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator SECOND = new DatePartFunction(TimeUnit.SECOND).toUDF("SECOND"); public static final SqlOperator MICROSECOND = new DatePartFunction(TimeUnit.MICROSECOND).toUDF("MICROSECOND"); - public static final SqlOperator NOW = new CurrentFunction(ExprCoreType.TIMESTAMP).toUDF("NOW"); + public static final SqlOperator NOW = + new CurrentFunction(CurrentFunction.Kind.TIMESTAMP).toUDF("NOW"); public static final SqlOperator CURRENT_TIME = - new CurrentFunction(ExprCoreType.TIME).toUDF("CURRENT_TIME"); + new CurrentFunction(CurrentFunction.Kind.TIME).toUDF("CURRENT_TIME"); public static final SqlOperator CURRENT_DATE = - new CurrentFunction(ExprCoreType.DATE).toUDF("CURRENT_DATE"); + new CurrentFunction(CurrentFunction.Kind.DATE).toUDF("CURRENT_DATE"); public static final SqlOperator DATE_FORMAT = new FormatFunction(ExprCoreType.DATE).toUDF("DATE_FORMAT"); public static final SqlOperator TIME_FORMAT = @@ -448,7 +452,6 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); public static final SqlOperator TONUMBER = new ToNumberFunction().toUDF("TONUMBER"); public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); - // PPL Convert command functions public static final SqlOperator AUTO = new AutoConvertFunction().toUDF("AUTO"); public static final SqlOperator NUM = new NumConvertFunction().toUDF("NUM"); @@ -487,6 +490,20 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { new NullableSqlAvgAggFunction(SqlKind.VAR_POP); public static final SqlAggFunction VAR_SAMP_NULLABLE = new NullableSqlAvgAggFunction(SqlKind.VAR_SAMP); + public static final SqlAggFunction CHECKED_LONG_SUM = + createReflectiveAggFunction( + CheckedLongSumAggFunction.class, + "CHECKED_LONG_SUM", + SqlKind.SUM, + ReturnTypes.BIGINT_FORCE_NULLABLE, + PPLOperandTypes.NUMERIC); + public static final SqlAggFunction BIGINT_AVG = + createReflectiveAggFunction( + BigintAvgAggFunction.class, + "AVG", + SqlKind.AVG, + ReturnTypes.DOUBLE_NULLABLE, + PPLOperandTypes.NUMERIC); public static final SqlAggFunction TAKE = createUserDefinedAggFunction( TakeAggFunction.class, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 151c4a96655..a4f05dd675e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -283,6 +283,7 @@ import java.util.Optional; import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; @@ -309,6 +310,7 @@ import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.calcite.tools.RelBuilder; @@ -319,8 +321,6 @@ import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.CollectionUDF.MVIndexFunctionImp; @@ -450,7 +450,7 @@ public boolean requiresNumericArgument(String functionName, int argumentIndex) { return false; } try { - List> signatures = + List> signatures = checker.getParameterTypes().stream() .filter(parameters -> argumentIndex < parameters.size()) .toList(); @@ -458,12 +458,12 @@ public boolean requiresNumericArgument(String functionName, int argumentIndex) { return false; } foundArgument = true; - List acceptedTypes = + List acceptedTypes = signatures.stream().map(parameters -> parameters.get(argumentIndex)).toList(); - if (acceptedTypes.stream().allMatch(ExprCoreType.numberTypes()::contains)) { + if (acceptedTypes.stream().allMatch(SqlTypeUtil::isNumeric)) { continue; } - if (acceptedTypes.stream().anyMatch(type -> type != ExprCoreType.UNKNOWN) + if (acceptedTypes.stream().anyMatch(type -> type.getSqlTypeName() != SqlTypeName.ANY) || !requiresNumericByValidation(checker, signatures, argumentIndex)) { return false; } @@ -475,7 +475,7 @@ public boolean requiresNumericArgument(String functionName, int argumentIndex) { } private boolean requiresNumericByValidation( - PPLTypeChecker checker, List> signatures, int argumentIndex) { + PPLTypeChecker checker, List> signatures, int argumentIndex) { RelDataType numericType = TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE); RelDataType stringType = TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR); return signatures.stream() @@ -1596,7 +1596,14 @@ void register( } void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFunction) { - SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(aggFunction); + registerOperator(functionName, aggFunction, field -> aggFunction); + } + + void registerOperator( + BuiltinFunctionName functionName, + SqlAggFunction typeCheckerSource, + Function aggFunctionSelector) { + SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(typeCheckerSource); PPLTypeChecker typeChecker = wrapSqlOperandTypeChecker(innerTypeChecker, functionName.name(), true); AggHandler handler = @@ -1604,15 +1611,30 @@ void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFuncti List newArgList = argList.stream().map(PlanUtils::derefMapCall).collect(Collectors.toList()); return UserDefinedFunctionUtils.makeAggregateCall( - aggFunction, List.of(field), newArgList, ctx.relBuilder); + aggFunctionSelector.apply(field), List.of(field), newArgList, ctx.relBuilder); }; register(functionName, handler, typeChecker); } + /** Registers checked integral sums while retaining standard SUM behavior for other types. */ + void registerSumOperator() { + registerOperator( + SUM, + SqlStdOperatorTable.SUM, + field -> + isIntegral(field.getType().getSqlTypeName()) + ? PPLBuiltinOperators.CHECKED_LONG_SUM + : SqlStdOperatorTable.SUM); + } + + private static boolean isIntegral(SqlTypeName typeName) { + return SqlTypeName.INT_TYPES.contains(typeName); + } + void populate() { registerOperator(MAX, SqlStdOperatorTable.MAX); registerOperator(MIN, SqlStdOperatorTable.MIN); - registerOperator(SUM, SqlStdOperatorTable.SUM); + registerSumOperator(); registerOperator(VARSAMP, PPLBuiltinOperators.VAR_SAMP_NULLABLE); registerOperator(VARPOP, PPLBuiltinOperators.VAR_POP_NULLABLE); registerOperator(STDDEV_SAMP, PPLBuiltinOperators.STDDEV_SAMP_NULLABLE); @@ -1631,7 +1653,14 @@ void populate() { register( AVG, - (distinct, field, argList, ctx) -> ctx.relBuilder.avg(distinct, null, field), + (distinct, field, argList, ctx) -> { + if (field.getType().getSqlTypeName() == SqlTypeName.BIGINT) { + return ctx.relBuilder + .aggregateCall(PPLBuiltinOperators.BIGINT_AVG, field) + .distinct(distinct); + } + return ctx.relBuilder.avg(distinct, null, field); + }, wrapSqlOperandTypeChecker( SqlStdOperatorTable.AVG.getOperandTypeChecker(), AVG.name(), false)); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java index 521764ba7bb..4925b35b649 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java @@ -27,8 +27,10 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.Pair; +import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.type.ExprIPType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -61,12 +63,9 @@ public interface PPLTypeChecker { /** * Get a list of all possible parameter type combinations for the function. * - *

This method is used to generate the allowed signatures for the function based on the - * parameter types. - * * @return a list of lists, where each inner list represents an allowed parameter type combination */ - List> getParameterTypes(); + List> getParameterTypes(); private static boolean validateOperands( List funcTypeFamilies, List operandTypes) { @@ -111,8 +110,8 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { - return PPLTypeChecker.getExprSignatures(families); + public List> getParameterTypes() { + return PPLTypeChecker.getRelDataTypeSignatures(families); } @Override @@ -150,17 +149,17 @@ public boolean checkOperandTypes(List types) { @Override public String getAllowedSignatures() { if (innerTypeChecker instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - var allowedExprSignatures = getExprSignatures(familyOperandTypeChecker); - return PPLTypeChecker.formatExprSignatures(allowedExprSignatures); + var allowedSignatures = getRelDataTypeSignatures(familyOperandTypeChecker); + return PPLTypeChecker.formatSignatures(allowedSignatures); } else { return ""; } } @Override - public List> getParameterTypes() { + public List> getParameterTypes() { if (innerTypeChecker instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - return getExprSignatures(familyOperandTypeChecker); + return getRelDataTypeSignatures(familyOperandTypeChecker); } else { // If the inner type checker is not a FamilyOperandTypeChecker, we cannot provide // parameter types. @@ -232,11 +231,11 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { - List> parameterTypes = new ArrayList<>(); + public List> getParameterTypes() { + List> parameterTypes = new ArrayList<>(); for (SqlOperandTypeChecker rule : allowedRules) { if (rule instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - parameterTypes.addAll(getExprSignatures(familyOperandTypeChecker)); + parameterTypes.addAll(getRelDataTypeSignatures(familyOperandTypeChecker)); } else { throw new IllegalArgumentException( "Currently only compositions of FamilyOperandTypeChecker are supported"); @@ -337,9 +336,10 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { - // Should not be used - return List.of(List.of(ExprCoreType.UNKNOWN, ExprCoreType.UNKNOWN)); + public List> getParameterTypes() { + // Should not be used by coercion since comparable operators don't drive type widening here. + RelDataType anyType = OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY); + return List.of(List.of(anyType, anyType)); } } @@ -397,23 +397,23 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { + public List> getParameterTypes() { if (internal instanceof FamilyOperandTypeChecker familyChecker) { - return getExprSignatures(familyChecker); + return getRelDataTypeSignatures(familyChecker); } else { - // For unknown type checkers, return UNKNOWN types + // For unknown type checkers, return ANY-typed signatures. int min = internal.getOperandCountRange().getMin(); int max = internal.getOperandCountRange().getMax(); + RelDataType anyType = OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY); if (min == -1 || max == -1) { - // Variable arguments - return a single signature with UNKNOWN - return List.of(List.of(ExprCoreType.UNKNOWN)); + return List.of(List.of(anyType)); } else { - List> parameterTypes = new ArrayList<>(); + List> parameterTypes = new ArrayList<>(); final int MAX_ARGS = 10; max = Math.min(MAX_ARGS, max); for (int i = min; i <= max; i++) { - parameterTypes.add(Collections.nCopies(i, ExprCoreType.UNKNOWN)); + parameterTypes.add(Collections.nCopies(i, anyType)); } return parameterTypes; } @@ -513,28 +513,27 @@ static PPLDefaultTypeChecker wrapDefault(SqlOperandTypeChecker typeChecker) { } /** - * Create a {@link PPLTypeChecker} from a list of allowed signatures consisted of {@link - * ExprType}. This is useful to validate arguments against user-defined types (UDT) that does not - * match any Calcite {@link SqlTypeFamily}. + * Create a {@link PPLTypeChecker} from a list of allowed signatures composed of {@link + * RelDataType}. This is used for functions whose argument types include user-defined types (UDTs) + * that don't fit any standard {@link SqlTypeFamily}. * * @param allowedSignatures a list of allowed signatures, where each signature is a list of {@link - * ExprType} representing the expected types of the function arguments. + * RelDataType} representing the expected types of the function arguments. * @return a {@link PPLTypeChecker} that checks if the operand types match any of the allowed * signatures */ - static PPLTypeChecker wrapUDT(List> allowedSignatures) { + static PPLTypeChecker wrapUDT(List> allowedSignatures) { return new PPLTypeChecker() { @Override public boolean checkOperandTypes(List types) { - List argExprTypes = - types.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList(); for (var allowedSignature : allowedSignatures) { if (allowedSignature.size() != types.size()) { continue; // Skip signatures that do not match the operand count } - // Check if the argument types match the allowed signature + // Match each operand against the allowed signature using nullability-insensitive + // equality with UDT-class-aware comparison. if (IntStream.range(0, allowedSignature.size()) - .allMatch(i -> allowedSignature.get(i).equals(argExprTypes.get(i)))) { + .allMatch(i -> typesMatch(allowedSignature.get(i), types.get(i)))) { return true; } } @@ -543,17 +542,68 @@ public boolean checkOperandTypes(List types) { @Override public String getAllowedSignatures() { - return PPLTypeChecker.formatExprSignatures(allowedSignatures); + return PPLTypeChecker.formatSignatures(allowedSignatures); } @Override - public List> getParameterTypes() { + public List> getParameterTypes() { return allowedSignatures; } }; } + /** + * Compares two RelDataTypes for signature matching. Two UDTs match if they share the same {@link + * ExprUDT} tag — comparing {@code getClass()} is unsafe because addCharsetAndCollation collapses + * ExprDateType/ExprTimeType/ExprTimeStampType/ExprBinaryType down to ExprSqlType, so different + * UDTs would appear equal. Plain types match by SqlTypeName. + * + *

A UDT signature also accepts the equivalent plain Calcite type. Signatures such as {@code + * PPLOperandTypes.ANY_SCALAR} declare temporal/IP/BINARY operands as UDTs, but the analytics + * engine builds its row types from plain Calcite types plus its own markers, which extend + * Calcite's {@code AbstractSqlType} rather than {@link AbstractExprRelDataType}. Matching on + * class identity alone made {@code list()} fail with an error that listed the very + * type it had just rejected. + */ + private static boolean typesMatch(RelDataType expected, RelDataType actual) { + if (expected instanceof AbstractExprRelDataType expUdt + && actual instanceof AbstractExprRelDataType actUdt) { + return expUdt.getUdt() == actUdt.getUdt(); + } + if (expected instanceof AbstractExprRelDataType expUdt) { + return matchesPlainType(expUdt.getUdt(), actual); + } + if (actual instanceof AbstractExprRelDataType actUdt) { + return matchesPlainType(actUdt.getUdt(), expected); + } + return expected.getSqlTypeName() == actual.getSqlTypeName(); + } + + /** + * Whether a plain Calcite type is the non-UDT spelling of {@code udt}. The UDTs themselves are + * all VARCHAR-backed, so this maps the tag to the {@link SqlTypeName}s a backend would produce + * for the same logical type instead of comparing backing types. + */ + private static boolean matchesPlainType(ExprUDT udt, RelDataType plain) { + return switch (udt) { + case EXPR_DATE -> plain.getSqlTypeName() == SqlTypeName.DATE; + case EXPR_TIME -> + switch (plain.getSqlTypeName()) { + case TIME, TIME_TZ, TIME_WITH_LOCAL_TIME_ZONE -> true; + default -> false; + }; + case EXPR_TIMESTAMP -> + switch (plain.getSqlTypeName()) { + case TIMESTAMP, TIMESTAMP_TZ, TIMESTAMP_WITH_LOCAL_TIME_ZONE -> true; + default -> false; + }; + // ip and binary both land as VARBINARY. + case EXPR_IP, EXPR_BINARY -> SqlTypeName.BINARY_TYPES.contains(plain.getSqlTypeName()); + }; + } + // Util Functions + /** * Generates a list of allowed function signatures based on the provided {@link * FamilyOperandTypeChecker}. The signatures are generated by iterating through the operand count @@ -563,14 +613,15 @@ public List> getParameterTypes() { * to 10 to avoid excessive enumeration. * * @param typeChecker the {@link FamilyOperandTypeChecker} to use for generating signatures - * @return a list of allowed function signatures + * @return a string representation of allowed function signatures */ private static String getFamilySignatures(FamilyOperandTypeChecker typeChecker) { - var allowedExprSignatures = getExprSignatures(typeChecker); - return formatExprSignatures(allowedExprSignatures); + var allowedSignatures = getRelDataTypeSignatures(typeChecker); + return formatSignatures(allowedSignatures); } - private static List> getExprSignatures(FamilyOperandTypeChecker typeChecker) { + private static List> getRelDataTypeSignatures( + FamilyOperandTypeChecker typeChecker) { var operandCountRange = typeChecker.getOperandCountRange(); int min = operandCountRange.getMin(); int max = operandCountRange.getMax(); @@ -578,91 +629,79 @@ private static List> getExprSignatures(FamilyOperandTypeChecker t for (int i = 0; i < min; i++) { families.add(typeChecker.getOperandSqlTypeFamily(i)); } - List> allowedSignatures = new ArrayList<>(getExprSignatures(families)); + List> allowedSignatures = new ArrayList<>(getRelDataTypeSignatures(families)); // Avoid enumerating signatures for infinite args final int MAX_ARGS = 10; max = Math.min(max, MAX_ARGS); for (int i = min; i < max; i++) { families.add(typeChecker.getOperandSqlTypeFamily(i)); - allowedSignatures.addAll(getExprSignatures(families)); + allowedSignatures.addAll(getRelDataTypeSignatures(families)); } return allowedSignatures; } /** - * Converts a {@link SqlTypeFamily} to a list of {@link ExprType}. This method is used to display - * the allowed signatures for functions based on their type families. - * - * @param family the {@link SqlTypeFamily} to convert - * @return a list of {@link ExprType} corresponding to the concrete types of the family + * Converts a {@link SqlTypeFamily} to a list of concrete {@link RelDataType} representatives. + * Used to enumerate allowed signatures and to drive widening in PPL coercion. */ - private static List getExprTypes(SqlTypeFamily family) { - List concreteTypes = - switch (family) { - case DATETIME -> - List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DATE), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIME)); - case NUMERIC -> - List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE)); - // Integer is mapped to BIGINT in family.getDefaultConcreteType - case INTEGER -> - List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER)); - case ANY, IGNORE -> - List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY)); - case DATETIME_INTERVAL -> - SqlTypeName.INTERVAL_TYPES.stream() - .map( - type -> - OpenSearchTypeFactory.TYPE_FACTORY.createSqlIntervalType( + private static List getRelDataTypes(SqlTypeFamily family) { + OpenSearchTypeFactory tf = OpenSearchTypeFactory.TYPE_FACTORY; + return switch (family) { + case DATETIME -> + List.of( + tf.createUDT(ExprUDT.EXPR_TIMESTAMP), + tf.createUDT(ExprUDT.EXPR_DATE), + tf.createUDT(ExprUDT.EXPR_TIME)); + case TIMESTAMP -> List.of(tf.createUDT(ExprUDT.EXPR_TIMESTAMP)); + case DATE -> List.of(tf.createUDT(ExprUDT.EXPR_DATE)); + case TIME -> List.of(tf.createUDT(ExprUDT.EXPR_TIME)); + case NUMERIC -> + List.of(tf.createSqlType(SqlTypeName.INTEGER), tf.createSqlType(SqlTypeName.DOUBLE)); + // Integer is mapped to BIGINT in family.getDefaultConcreteType + case INTEGER -> List.of(tf.createSqlType(SqlTypeName.INTEGER)); + case ANY, IGNORE -> List.of(tf.createSqlType(SqlTypeName.ANY)); + // ARRAY of nullable ANY, matching convertExprTypeToRelDataType(ARRAY, nullable=true). + // Calcite's default concrete type for ARRAY has a NOT NULL element, which diverges from + // PPL semantics (see #5175: null literals must remain nullable inside an array). + case ARRAY -> List.of(tf.createArrayType(tf.createSqlType(SqlTypeName.ANY, true), -1)); + case BINARY -> List.of(tf.createUDT(ExprUDT.EXPR_BINARY)); + case DATETIME_INTERVAL -> + SqlTypeName.INTERVAL_TYPES.stream() + .map( + type -> + (RelDataType) + tf.createSqlIntervalType( new SqlIntervalQualifier( type.getStartUnit(), type.getEndUnit(), SqlParserPos.ZERO))) - .collect(Collectors.toList()); - default -> { - RelDataType type = family.getDefaultConcreteType(OpenSearchTypeFactory.TYPE_FACTORY); - if (type == null) { - yield List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.OTHER)); - } - yield List.of(type); - } - }; - return concreteTypes.stream() - .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) - .distinct() - .collect(Collectors.toList()); + .collect(Collectors.toList()); + default -> { + RelDataType type = family.getDefaultConcreteType(tf); + if (type == null) { + yield List.of(tf.createSqlType(SqlTypeName.OTHER)); + } + yield List.of(type); + } + }; } /** - * Generates a list of all possible {@link ExprType} signatures based on the provided {@link - * SqlTypeFamily} list. - * - * @param families the list of {@link SqlTypeFamily} to generate signatures for - * @return a list of lists, where each inner list contains {@link ExprType} signatures + * Generates a list of all possible {@link RelDataType} signatures based on the provided {@link + * SqlTypeFamily} list (cartesian product per-position). */ - private static List> getExprSignatures(List families) { - List> exprTypes = - families.stream().map(PPLTypeChecker::getExprTypes).collect(Collectors.toList()); - - // Do a cartesian product of all ExprTypes in the family - return Lists.cartesianProduct(exprTypes); + private static List> getRelDataTypeSignatures(List families) { + List> perPosition = + families.stream().map(PPLTypeChecker::getRelDataTypes).collect(Collectors.toList()); + return Lists.cartesianProduct(perPosition); } /** * Generates a string representation of the function signature based on the provided type * families. The format is a list of type families enclosed in square brackets, e.g.: "[INTEGER, * STRING]". - * - * @param families the list of type families to include in the signature - * @return a string representation of the function signature */ private static String getFamilySignature(List families) { - List> signatures = getExprSignatures(families); - // Convert each signature to a string representation and then concatenate them - return formatExprSignatures(signatures); + return formatSignatures(getRelDataTypeSignatures(families)); } /** @@ -686,13 +725,20 @@ private static boolean isCompositionOr(CompositeOperandTypeChecker typeChecker) return composition == CompositeOperandTypeChecker.Composition.OR; } - private static String formatExprSignatures(List> signatures) { + /** + * Renders a list of {@link RelDataType} signatures as a pipe-separated string of bracketed + * signatures, e.g. {@code [INTEGER,STRING]|[DOUBLE,STRING]}. Each type is rendered through {@link + * OpenSearchTypeFactory#convertRelDataTypeToExprType} so plain SQL types come out as their PPL + * names ({@code STRING}, {@code LONG}, ...) and UDTs as their {@code ExprCoreType} names; {@code + * UNDEFINED} (Calcite {@code NULL}/{@code ANY}) is displayed as {@code ANY}. + */ + static String formatSignatures(List> signatures) { return signatures.stream() .map( types -> "[" + types.stream() - // Display ExprCoreType.UNDEFINED as "ANY" for better interpretability + .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) .map(t -> t == ExprCoreType.UNDEFINED ? "ANY" : t.toString()) .collect(Collectors.joining(",")) + "]") diff --git a/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java b/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java index dc4761b26e7..d67b612e0ea 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java @@ -17,7 +17,6 @@ import org.apache.calcite.sql.type.SqlOperandMetadata; import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; -import org.opensearch.sql.data.type.ExprType; /** * This class is created for the compatibility with {@link SqlUserDefinedFunction} constructors when @@ -106,11 +105,12 @@ public String getAllowedSignatures(SqlOperator op, String opName) { }; } - static UDFOperandMetadata wrapUDT(List> allowSignatures) { + static UDFOperandMetadata wrapUDT(List> allowSignatures) { return new UDTOperandMetadata(allowSignatures); } - record UDTOperandMetadata(List> allowedParamTypes) implements UDFOperandMetadata { + record UDTOperandMetadata(List> allowedParamTypes) + implements UDFOperandMetadata { @Override public SqlOperandTypeChecker getInnerTypeChecker() { return this; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java index 28dff66a8ae..df8783a311e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java @@ -10,7 +10,6 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonSyntaxException; -import java.math.BigDecimal; import java.util.List; import java.util.stream.StreamSupport; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -20,6 +19,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; @@ -97,14 +97,19 @@ private static Object cast(Object value, SqlTypeName elementType) { if (value == null) { return null; } - return switch (elementType) { - case DOUBLE -> ((Number) value).doubleValue(); - case VARCHAR -> - value instanceof List || value instanceof java.util.Map - ? gson.toJson(value) - : String.valueOf(value); - case DECIMAL -> BigDecimal.valueOf(((Number) value).doubleValue()); - default -> value; - }; + // Match Calcite SAFE_CAST semantics for runtime values whose type is unknown during planning. + try { + return switch (elementType) { + case DOUBLE -> SqlFunctions.toDouble(value); + case VARCHAR -> + value instanceof List || value instanceof java.util.Map + ? gson.toJson(value) + : String.valueOf(value); + case DECIMAL -> SqlFunctions.toBigDecimal(value); + default -> value; + }; + } catch (RuntimeException e) { + return null; + } } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java index 49e06afa3d6..f620ffd21c5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java @@ -20,8 +20,6 @@ import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.function.FunctionProperties; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -39,21 +37,28 @@ *

It returns the current date, time, or timestamp based on the specified return type. */ public class CurrentFunction extends ImplementorUDF { - private final ExprType returnType; - public CurrentFunction(ExprType returnType) { - super(new CurrentFunctionImplementor(returnType), NullPolicy.NONE); - this.returnType = returnType; + /** Discriminates the temporal flavour at registration time, decoupled from {@code ExprType}. */ + public enum Kind { + DATE, + TIME, + TIMESTAMP + } + + private final Kind kind; + + public CurrentFunction(Kind kind) { + super(new CurrentFunctionImplementor(kind), NullPolicy.NONE); + this.kind = kind; } @Override public SqlReturnTypeInference getReturnTypeInference() { return opBinding -> - switch (returnType) { - case ExprCoreType.DATE -> UserDefinedFunctionUtils.NULLABLE_DATE_UDT; - case ExprCoreType.TIME -> UserDefinedFunctionUtils.NULLABLE_TIME_UDT; - case ExprCoreType.TIMESTAMP -> UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT; - default -> throw new IllegalArgumentException("Unsupported return type: " + returnType); + switch (kind) { + case DATE -> UserDefinedFunctionUtils.NULLABLE_DATE_UDT; + case TIME -> UserDefinedFunctionUtils.NULLABLE_TIME_UDT; + case TIMESTAMP -> UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT; }; } @@ -64,18 +69,17 @@ public UDFOperandMetadata getOperandMetadata() { @RequiredArgsConstructor public static class CurrentFunctionImplementor implements NotNullImplementor { - private final ExprType returnType; + private final Kind kind; @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { String functionName = - switch (returnType) { - case ExprCoreType.DATE -> "currentDate"; - case ExprCoreType.TIME -> "currentTime"; - case ExprCoreType.TIMESTAMP -> "currentTimestamp"; - default -> throw new IllegalArgumentException("Unsupported return type: " + returnType); + switch (kind) { + case DATE -> "currentDate"; + case TIME -> "currentTime"; + case TIMESTAMP -> "currentTimestamp"; }; Expression properties = diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java index 109bad16bf1..a53c733de66 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java @@ -17,11 +17,9 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprDateValue; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -65,9 +63,6 @@ public PeriodNameFunctionImplementor(TimeUnit periodUnit) { @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { - ExprType dateType = - OpenSearchTypeFactory.convertRelDataTypeToExprType( - call.getOperands().getFirst().getType()); return Expressions.call( PeriodNameFunctionImplementor.class, "name", diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java index 11fdd7947af..b213181038b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java @@ -14,10 +14,10 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.sql.expression.ip.IPFunctions; @@ -51,9 +51,9 @@ public UDFOperandMetadata getOperandMetadata() { // We use a specific type checker to serve return UDFOperandMetadata.wrapUDT( List.of( - List.of(ExprCoreType.IP, ExprCoreType.STRING), - List.of(ExprCoreType.STRING, ExprCoreType.STRING), - List.of(ExprCoreType.BINARY, ExprCoreType.STRING))); + List.of(PPLOperandTypes.IP_UDT, PPLOperandTypes.STRING_T), + List.of(PPLOperandTypes.STRING_T, PPLOperandTypes.STRING_T), + List.of(PPLOperandTypes.BINARY_UDT, PPLOperandTypes.STRING_T))); } public static class CidrMatchImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java index ce200323f60..34d5fec2f3e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java @@ -25,8 +25,8 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprIpValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -120,7 +120,8 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrapUDT(List.of(List.of(ExprCoreType.IP, ExprCoreType.IP))); + return UDFOperandMetadata.wrapUDT( + List.of(List.of(PPLOperandTypes.IP_UDT, PPLOperandTypes.IP_UDT))); } public static class CompareImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java index baf6b8a37e1..13dccfee6ae 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java @@ -11,13 +11,14 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.calcite.type.ExprIPType; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprIpValue; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -41,7 +42,7 @@ public IPFunction() { @Override public UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrapUDT( - List.of(List.of(ExprCoreType.IP), List.of(ExprCoreType.STRING))); + List.of(List.of(PPLOperandTypes.IP_UDT), List.of(PPLOperandTypes.STRING_T))); } @Override @@ -57,12 +58,10 @@ public Expression implement( if (call.getOperands().size() != 1) { throw new IllegalArgumentException("IP function requires exactly one operand"); } - ExprType argType = - OpenSearchTypeFactory.convertRelDataTypeToExprType( - call.getOperands().getFirst().getType()); - if (argType == ExprCoreType.IP) { + RelDataType argType = call.getOperands().getFirst().getType(); + if (argType instanceof ExprIPType) { return translatedOperands.getFirst(); - } else if (argType == ExprCoreType.STRING) { + } else if (SqlTypeUtil.isCharacter(argType)) { return Expressions.new_(ExprIpValue.class, translatedOperands); } else { throw new ExpressionEvaluationException( diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java b/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java index 63327c2d6dd..f3df41356f2 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import org.opensearch.sql.calcite.CalcitePlanContext; /** Default implementation that records profiling metrics. */ public class DefaultProfileContext implements ProfileContext { @@ -63,7 +64,8 @@ public synchronized QueryProfile finish() { double totalMillis = ProfileUtils.roundToMillis(endNanos - startNanos); Object planSnapshot = enginePlan != null ? enginePlan : (planRoot == null ? null : planRoot.snapshot()); - profile = new QueryProfile(totalMillis, snapshot, planSnapshot); + String threadPool = CalcitePlanContext.executionPool.get(); + profile = new QueryProfile(totalMillis, snapshot, planSnapshot, threadPool); return profile; } } diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java index d9d2a785868..71454951557 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java @@ -24,6 +24,9 @@ public final class QueryProfile { /** Execution-engine-specific plan profile: a {@link PlanNode} tree, or a pre-rendered object. */ private final Object plan; + @SerializedName("thread_pool") + private final String threadPool; + /** * Create a new query profile snapshot. * @@ -31,7 +34,7 @@ public final class QueryProfile { * @param phases metric values keyed by {@link MetricName} */ public QueryProfile(double totalTimeMillis, Map phases) { - this(totalTimeMillis, phases, null); + this(totalTimeMillis, phases, null, null); } /** @@ -42,9 +45,23 @@ public QueryProfile(double totalTimeMillis, Map phases) { * @param plan plan tree profiling output */ public QueryProfile(double totalTimeMillis, Map phases, Object plan) { + this(totalTimeMillis, phases, plan, null); + } + + /** + * Create a new query profile snapshot. + * + * @param totalTimeMillis total elapsed milliseconds for the query (rounded to two decimals) + * @param phases metric values keyed by {@link MetricName} + * @param plan plan tree profiling output + * @param threadPool thread pool name that executed the query + */ + public QueryProfile( + double totalTimeMillis, Map phases, Object plan, String threadPool) { this.summary = new Summary(totalTimeMillis); this.phases = buildPhases(phases); this.plan = plan; + this.threadPool = threadPool; } private Map buildPhases(Map phases) { diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfiling.java b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfiling.java index 3ef32dac748..900be175050 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfiling.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfiling.java @@ -57,6 +57,16 @@ public static void clear() { CURRENT.remove(); } + /** + * Set the profiling context for the current thread. Used when propagating context across thread + * boundaries. + * + * @param ctx profiling context to bind + */ + public static void set(ProfileContext ctx) { + CURRENT.set(Objects.requireNonNull(ctx, "ctx")); + } + /** * Run a supplier with the provided profiling context bound to the current thread. * diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 7589cf522f6..059bb8238d0 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -6,6 +6,7 @@ package org.opensearch.sql.utils; import java.nio.charset.StandardCharsets; +import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.Map; import lombok.Getter; @@ -117,25 +118,14 @@ public static RestSpec decodeRestSpec(String indexName) { } private static String toHex(String s) { - StringBuilder h = new StringBuilder(); - for (byte b : s.getBytes(StandardCharsets.UTF_8)) { - h.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); - } - return h.toString(); + return HexFormat.of().formatHex(s.getBytes(StandardCharsets.UTF_8)); } private static String fromHex(String h) { if (h.length() % 2 != 0) { throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); } - byte[] bytes = new byte[h.length() / 2]; - for (int i = 0; i < bytes.length; i++) { - bytes[i] = - (byte) - ((Character.digit(h.charAt(2 * i), 16) << 4) - + Character.digit(h.charAt(2 * i + 1), 16)); - } - return new String(bytes, StandardCharsets.UTF_8); + return new String(HexFormat.of().parseHex(h), StandardCharsets.UTF_8); } /** diff --git a/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java new file mode 100644 index 00000000000..ab70f912211 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.calcite.sql.SqlKind; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +class BigintAvgAggFunctionTest { + + @Test + void retainsAvgKindForPushdownRules() { + assertEquals(SqlKind.AVG, PPLBuiltinOperators.BIGINT_AVG.getKind()); + } + + @Test + void averagesWithoutLongOverflow() { + BigintAvgAggFunction.Accumulator accumulator = BigintAvgAggFunction.init(); + accumulator = BigintAvgAggFunction.add(accumulator, Long.MAX_VALUE); + accumulator = BigintAvgAggFunction.add(accumulator, Long.MAX_VALUE); + + assertEquals((double) Long.MAX_VALUE, BigintAvgAggFunction.result(accumulator)); + } + + @Test + void ignoresNulls() { + BigintAvgAggFunction.Accumulator accumulator = BigintAvgAggFunction.init(); + accumulator = BigintAvgAggFunction.add(accumulator, null); + accumulator = BigintAvgAggFunction.add(accumulator, 10L); + + assertEquals(10D, BigintAvgAggFunction.result(accumulator)); + } + + @Test + void returnsNullForEmptyInput() { + assertNull(BigintAvgAggFunction.result(BigintAvgAggFunction.init())); + } +} diff --git a/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java new file mode 100644 index 00000000000..0128cb4aa67 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.calcite.sql.SqlKind; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +class CheckedLongSumAggFunctionTest { + + @Test + void retainsSumKindForPlannerRules() { + assertEquals(SqlKind.SUM, PPLBuiltinOperators.CHECKED_LONG_SUM.getKind()); + } + + @Test + void sumsExactly() { + long accumulator = CheckedLongSumAggFunction.init(); + accumulator = CheckedLongSumAggFunction.add(accumulator, 1L << 62); + accumulator = CheckedLongSumAggFunction.add(accumulator, 1L); + + assertEquals((1L << 62) + 1L, CheckedLongSumAggFunction.result(accumulator)); + } + + @Test + void throwsOnPositiveOverflow() { + assertThrows( + ArithmeticException.class, () -> CheckedLongSumAggFunction.add(Long.MAX_VALUE, 1L)); + } + + @Test + void throwsOnNegativeOverflow() { + assertThrows( + ArithmeticException.class, () -> CheckedLongSumAggFunction.add(Long.MIN_VALUE, -1L)); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java index 5533fd90915..2727710fc18 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java @@ -24,7 +24,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; class CoercionUtilsTest { @@ -88,7 +87,8 @@ void widenArgumentsUnifiesPlainTimestampWithDateUdtBounds() { @Test void castArgumentsReturnsExactMatchWhenAvailable() { - PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(INTEGER), List.of(DOUBLE))); + PPLTypeChecker typeChecker = + new StubTypeChecker(List.of(List.of(sqlType(INTEGER)), List.of(sqlType(DOUBLE)))); List arguments = List.of(nullLiteral(INTEGER)); List result = CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments); @@ -102,7 +102,7 @@ void castArgumentsReturnsExactMatchWhenAvailable() { @Test void castArgumentsFallsBackToWidestCandidate() { PPLTypeChecker typeChecker = - new StubTypeChecker(List.of(List.of(ExprCoreType.LONG), List.of(DOUBLE))); + new StubTypeChecker(List.of(List.of(sqlType(ExprCoreType.LONG)), List.of(sqlType(DOUBLE)))); List arguments = List.of(nullLiteral(STRING)); List result = CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments); @@ -114,16 +114,23 @@ void castArgumentsFallsBackToWidestCandidate() { @Test void castArgumentsReturnsNullWhenNoCompatibleSignatureExists() { - PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(ExprCoreType.GEO_POINT))); + PPLTypeChecker typeChecker = + new StubTypeChecker( + List.of( + List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.GEOMETRY)))); List arguments = List.of(nullLiteral(INTEGER)); assertNull(CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments)); } + private static RelDataType sqlType(ExprCoreType type) { + return OpenSearchTypeFactory.convertExprTypeToRelDataType(type); + } + private static class StubTypeChecker implements PPLTypeChecker { - private final List> signatures; + private final List> signatures; - private StubTypeChecker(List> signatures) { + private StubTypeChecker(List> signatures) { this.signatures = signatures; } @@ -138,7 +145,7 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { + public List> getParameterTypes() { return signatures; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java index e3c3724e0aa..5a633a1a9bb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java @@ -40,6 +40,13 @@ public void testMalformedJsonArrayIsEmpty() { assertEquals(List.of(), ForeachJsonArrayFunctionImpl.eval("not-json", "VARCHAR")); } + @Test + public void testJsonArraySafelyCoercesNumericElements() { + assertEquals( + Arrays.asList(10.0, 20.0, null), + ForeachJsonArrayFunctionImpl.eval("[10,\"20\",\"not-a-number\"]", "DOUBLE")); + } + @Test public void testStatePreservesHeterogeneousAndNullSlots() { assertEquals( diff --git a/core/src/test/java/org/opensearch/sql/expression/function/PPLComparableTypeCheckerTest.java b/core/src/test/java/org/opensearch/sql/expression/function/PPLComparableTypeCheckerTest.java new file mode 100644 index 00000000000..ba03a8b6280 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/PPLComparableTypeCheckerTest.java @@ -0,0 +1,141 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SameOperandTypeChecker; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.expression.function.PPLTypeChecker.PPLComparableTypeChecker; + +/** + * Exercises {@link PPLComparableTypeChecker#checkOperandTypes} against representative type pairs. + */ +class PPLComparableTypeCheckerTest { + + private static final OpenSearchTypeFactory TF = OpenSearchTypeFactory.TYPE_FACTORY; + private static final PPLComparableTypeChecker CHECKER = + new PPLComparableTypeChecker((SameOperandTypeChecker) OperandTypes.SAME_SAME); + + private static RelDataType sql(SqlTypeName name) { + return TF.createSqlType(name); + } + + private static RelDataType udt(ExprUDT udt) { + return TF.createUDT(udt); + } + + private static RelDataType udt(ExprUDT udt, boolean nullable) { + return TF.createUDT(udt, nullable); + } + + private static RelDataType interval(TimeUnit unit) { + return TF.createSqlIntervalType(new SqlIntervalQualifier(unit, unit, SqlParserPos.ZERO)); + } + + private static boolean comparable(RelDataType a, RelDataType b) { + return CHECKER.checkOperandTypes(List.of(a, b)); + } + + @Test + void numericAndNumericAreComparable() { + assertTrue(comparable(sql(SqlTypeName.INTEGER), sql(SqlTypeName.DOUBLE))); + assertTrue(comparable(sql(SqlTypeName.TINYINT), sql(SqlTypeName.BIGINT))); + } + + @Test + void sameUdtIsComparable() { + assertTrue(comparable(udt(ExprUDT.EXPR_DATE), udt(ExprUDT.EXPR_DATE))); + assertTrue(comparable(udt(ExprUDT.EXPR_TIMESTAMP, true), udt(ExprUDT.EXPR_TIMESTAMP, false))); + } + + @Test + void plainBinaryVsBinaryUdtAreComparable() { + // Guardrail: any future refactor of isComparable that classifies via SqlTypeFamily or Java + // class would break this pair — VARBINARY (family=BINARY) and EXPR_BINARY (VARCHAR-backed + // UDT) look unrelated at that level, but both map to ExprCoreType.BINARY and must compare. + assertTrue(comparable(sql(SqlTypeName.VARBINARY), udt(ExprUDT.EXPR_BINARY))); + assertTrue(comparable(udt(ExprUDT.EXPR_BINARY), sql(SqlTypeName.VARBINARY))); + } + + @Test + void dayTimeAndYearMonthIntervalsAreComparable() { + // Guardrail: Calcite splits INTERVAL into day-time and year-month SqlTypeFamilies, so a + // family-based check would reject this pair. convertRelDataTypeToExprType collapses every + // interval SqlTypeName to ExprCoreType.INTERVAL, so shouldCast is false and both compare. + assertTrue(comparable(interval(TimeUnit.DAY), interval(TimeUnit.YEAR))); + assertTrue(comparable(interval(TimeUnit.HOUR), interval(TimeUnit.MONTH))); + } + + @Test + void plainTemporalVsMatchingTemporalUdtIsComparable() { + assertTrue(comparable(sql(SqlTypeName.TIMESTAMP), udt(ExprUDT.EXPR_TIMESTAMP))); + assertTrue(comparable(sql(SqlTypeName.DATE), udt(ExprUDT.EXPR_DATE))); + assertTrue(comparable(sql(SqlTypeName.TIME), udt(ExprUDT.EXPR_TIME))); + } + + @Test + void udtVsUnrelatedPlainTypeIsNotComparable() { + assertFalse(comparable(udt(ExprUDT.EXPR_DATE), sql(SqlTypeName.VARCHAR))); + assertFalse(comparable(udt(ExprUDT.EXPR_TIMESTAMP), sql(SqlTypeName.INTEGER))); + } + + @Test + void stringVsNumericIsNotComparable() { + assertFalse(comparable(sql(SqlTypeName.VARCHAR), sql(SqlTypeName.INTEGER))); + } + + @Test + void anyIsComparableWithAnything() { + assertTrue(comparable(sql(SqlTypeName.ANY), sql(SqlTypeName.INTEGER))); + assertTrue(comparable(sql(SqlTypeName.ANY), udt(ExprUDT.EXPR_DATE))); + } + + @Test + void structVsNonStructIsNotComparable() { + RelDataType struct = + TF.createStructType( + List.of(sql(SqlTypeName.INTEGER), sql(SqlTypeName.VARCHAR)), List.of("a", "b")); + assertFalse(comparable(struct, sql(SqlTypeName.INTEGER))); + } + + @Test + void structsWithMatchingFieldsAreComparable() { + RelDataType s1 = + TF.createStructType( + List.of(sql(SqlTypeName.INTEGER), sql(SqlTypeName.VARCHAR)), List.of("a", "b")); + RelDataType s2 = + TF.createStructType( + List.of(sql(SqlTypeName.BIGINT), sql(SqlTypeName.CHAR)), List.of("x", "y")); + assertTrue(comparable(s1, s2)); + } + + @Test + void structsWithMismatchedFieldCountsAreNotComparable() { + RelDataType s1 = TF.createStructType(List.of(sql(SqlTypeName.INTEGER)), List.of("a")); + RelDataType s2 = + TF.createStructType( + List.of(sql(SqlTypeName.INTEGER), sql(SqlTypeName.VARCHAR)), List.of("a", "b")); + assertFalse(comparable(s1, s2)); + } + + @Test + void ipTypesAreRejectedByOuterChecker() { + // IP UDTs are explicitly filtered out in PPLComparableTypeChecker.checkOperandTypes so that + // built-in comparable functions (COALESCE, NULLIF, IFNULL, IF) cannot accept them. + assertFalse(comparable(udt(ExprUDT.EXPR_IP), udt(ExprUDT.EXPR_IP))); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/PPLUdtSignatureMatchTest.java b/core/src/test/java/org/opensearch/sql/expression/function/PPLUdtSignatureMatchTest.java new file mode 100644 index 00000000000..2388721435f --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/PPLUdtSignatureMatchTest.java @@ -0,0 +1,87 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; + +/** + * Exercises {@link PPLTypeChecker#wrapUDT} against plain Calcite types. Signatures declare + * temporal/IP/BINARY operands as UDTs, but the analytics engine builds row types from plain Calcite + * types, so a UDT signature must still accept the equivalent plain type. + */ +class PPLUdtSignatureMatchTest { + + private static final OpenSearchTypeFactory TF = OpenSearchTypeFactory.TYPE_FACTORY; + + /** The checker behind {@code list()}. */ + private static final PPLTypeChecker ANY_SCALAR = + PPLTypeChecker.wrapUDT( + ((UDFOperandMetadata.UDTOperandMetadata) PPLOperandTypes.ANY_SCALAR).allowedParamTypes()); + + private static RelDataType nullable(RelDataType type) { + return TF.createTypeWithNullability(type, true); + } + + private static boolean accepts(RelDataType type) { + return ANY_SCALAR.checkOperandTypes(List.of(type)); + } + + @Test + void plainTimestampMatchesTimestampUdt() { + // date -> TIMESTAMP(3), date_nanos -> TIMESTAMP(9) on the analytics route. + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.TIMESTAMP, 3)))); + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.TIMESTAMP, 9)))); + } + + @Test + void plainDateAndTimeMatchTheirUdts() { + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.DATE)))); + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.TIME)))); + } + + @Test + void plainVarbinaryMatchesBinaryUdt() { + // ip and binary both map to VARBINARY on the analytics route. + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.VARBINARY)))); + } + + @Test + void udtOperandsStillMatch() { + assertTrue(accepts(TF.createUDT(ExprUDT.EXPR_TIMESTAMP))); + assertTrue(accepts(TF.createUDT(ExprUDT.EXPR_DATE))); + assertTrue(accepts(TF.createUDT(ExprUDT.EXPR_TIME))); + assertTrue(accepts(TF.createUDT(ExprUDT.EXPR_IP))); + } + + @Test + void plainScalarsStillMatch() { + assertTrue(accepts(TF.createSqlType(SqlTypeName.INTEGER))); + assertTrue(accepts(TF.createSqlType(SqlTypeName.BIGINT))); + assertTrue(accepts(TF.createSqlType(SqlTypeName.VARCHAR))); + assertTrue(accepts(TF.createSqlType(SqlTypeName.BOOLEAN))); + } + + @Test + void nonScalarsAreStillRejected() { + assertFalse( + accepts(TF.createArrayType(TF.createSqlType(SqlTypeName.INTEGER), -1)), + "ANY_SCALAR must not accept arrays"); + assertFalse( + accepts( + TF.createMapType( + TF.createSqlType(SqlTypeName.VARCHAR), TF.createSqlType(SqlTypeName.INTEGER))), + "ANY_SCALAR must not accept maps"); + } +} diff --git a/docs/dev/ppl-lint-analytics-engine-ci-validation.md b/docs/dev/ppl-lint-analytics-engine-ci-validation.md new file mode 100644 index 00000000000..ba7632b2d54 --- /dev/null +++ b/docs/dev/ppl-lint-analytics-engine-ci-validation.md @@ -0,0 +1,818 @@ +# Analytics Engine Coverage for PPL Lint CI Validation + +- **Status:** Deferred; not part of PPL lint pull-request or multi-version CI +- **Last updated:** 2026-07-28 +- **Scope:** PPL lint contract validation in + `.github/workflows/ppl-lint-rule-validation.yml` and + `.github/workflows/ppl-lint-multiversion-validation.yml` + +> **Decision update (2026-08-04):** Analytics-engine lint validation is +> deferred because the feature build and composite/Parquet fixture surface are +> not stable enough for this compatibility workflow. The active design is +> [PPL Lint Runtime Compatibility CI](ppl-lint-runtime-compatibility-ci-design.md), +> which covers standard runtime-bundle engines only. This document is retained +> as future design context and is not an implementation commitment. + +## 1. Summary + +The PPL lint CI contract currently compares OpenSearch Dashboards (OSD) +detectors with the standard SQL execution route only. It does not prove that +the same lint diagnostics are correct when a query is routed through the +analytics engine and executed by DataFusion over composite/Parquet storage. + +This design adds analytics-engine coverage by: + +1. Running the existing `PplLintRuleValidationIT` corpus against a dedicated, + full-stack analytics-engine test cluster. +2. Making execution backend an explicit contract and artifact dimension, + separate from OpenSearch version, Calcite applicability, and grammar + surface. +3. Failing if the analytics lane silently falls back to the standard route. +4. Running the OSD detector comparison against both standard and analytics + backend reports while bootstrapping OSD only once. +5. Shipping the lane as non-enforcing observation first, then adding it to the + stable required result after its artifacts, expectations, and reliability + meet the promotion criteria in this document. + +The initial implementation covers the SQL pull request build on one shard. It +does not add a Cartesian product of analytics backends, released OpenSearch +versions, grammar surfaces, and shard counts. + +## 2. Current State + +### 2.1 Required PPL lint validation + +`.github/workflows/ppl-lint-rule-validation.yml` is a three-job pipeline: + +```text +backend-validation + -> detector-validation + -> validation-result +``` + +- `backend-validation` runs `PplLintRuleValidationIT` against the ordinary + Gradle `integTest` cluster. That cluster installs SQL, Job Scheduler, and + Geospatial, but not the analytics-engine stack. +- The integration test executes every scheduled trigger and control query + against `POST /_plugins/_ppl`, then exports: + - `ppl-grammar-bundle.json` + - `target.json` + - `backend-report.json` +- `detector-validation` bootstraps OSD, runs its production headless PPL lint + API against the exported grammar, and compares detector output with the + backend report. +- `validation-result` uses `if: always()` and fails unless both producer jobs + succeeded. This is the stable branch-protection check. + +The multi-version companion workflow repeats the same contract against +released standard engines and the pull request build. Its current dimensions +are OpenSearch version and grammar surface. + +### 2.2 Existing analytics-engine support + +The repository already contains most of the required test infrastructure: + +- `integ-test/build.gradle` can download the analytics engine, Arrow, + composite engine, Parquet data format, and Lucene/DataFusion backend plugin + ZIPs. +- The full analytics stack is already configured for + `analyticsEngineProfileIT` and `analyticsEngineSecurityIT`. +- `-Dtests.analytics.parquet_indices=true` makes helper-created fixtures use + composite/Parquet storage. +- `SQLIntegTestCase` applies the corresponding cluster defaults before fixture + creation. +- `PPLIntegTestCase.isAnalyticsParquetIndicesEnabled()` exposes the active + route to tests. +- `integTestRemote` already forwards the analytics fixture properties. +- `CalciteAnalyticsDatetimeWireFormatIT` demonstrates route attestation using + explain output: analytics plans contain + `LogicalTableScan(table=[[opensearch,` and not + `CalciteLogicalIndexScan`. + +### 2.3 Gap in the existing analytics workflow + +`.github/workflows/analytics-engine-compat.yml` runs only +`AnalyticsEngineCompatIT`. Its purpose is plugin coexistence. Its PPL assertion +uses the `rest` row source, which is explicitly excluded from analytics +routing. The workflow can therefore pass without executing a PPL query through +DataFusion. + +The `analyticsEngineCompat` cluster is also intentionally smaller than the +stack required for real analytics execution. It does not install the composite +engine, Parquet data format, or both analytics backends. + +### 2.4 Terminology + +The following dimensions must remain independent: + +| Dimension | Examples | Meaning | +| --- | --- | --- | +| Engine version | `3.7.0`, `3.8.0-SNAPSHOT` | OpenSearch/SQL product version | +| Grammar surface | `runtime-bundle`, `compiled-simplified` | Grammar used by OSD lint | +| Lint/planner applicability | `engine: "calcite"` | Existing OSD rule applicability | +| Execution backend | `standard`, `analytics` | SQL execution route selected at runtime | +| Storage | `lucene`, `composite-parquet` | Fixture storage that drives routing | + +Analytics uses Calcite planning, so treating `analytics` as another value of +the existing `engine` field would be incorrect. Treating it as another engine +version would also cause the drift analyzer to recommend version scoping for a +backend-specific difference. + +## 3. Problem Statement + +A lint rule is presented to users before query execution. OSD currently has no +analytics-route signal in the lint context, so the same detector result applies +whether the selected index later uses the standard or analytics route. + +The current CI can miss these failures: + +1. A detector reports an error for a query that the analytics backend accepts. + This is a false positive for analytics users. +2. A detector is silent for a query rejected only by the analytics route. This + is a false negative for analytics users. +3. A control query passes on the standard route but fails on analytics. +4. An analytics test is configured incorrectly and silently executes on the + standard route, producing a vacuous green result. +5. Standard and analytics observations are stored under the same product + version, causing aggregation to overwrite or misclassify one of them. +6. A required job consumes mutable `feature-datafusion/latest` artifacts, so a + rerun can test a different stack without recording that change. + +## 4. Goals and Non-Goals + +### 4.1 Goals + +- Run every scheduled PPL lint trigger and control against the pull request's + analytics route. +- Reuse the existing contract corpus and Java integration-test oracle. +- Use byte-identical query text, the same SQL commit, the same runtime grammar, + the same OSD commit, and the same frontend lint context for both backends. +- Represent execution backend in contracts, reports, manifests, summaries, and + aggregation keys. +- Prove that the analytics plugin stack is installed, fixtures are + composite/Parquet, routing selected analytics, and DataFusion executed a + canary query. +- Distinguish backend-route divergence from version drift. +- Fail closed on missing reports, missing expectations, route fallback, + incomplete matrices, or inconsistent grammar identity. +- Produce enough artifacts to reproduce infrastructure and semantic failures. +- Keep pull request wall-clock growth bounded by running backend jobs in + parallel and bootstrapping OSD once. + +### 4.2 Non-goals + +- Replacing the existing broad analytics compatibility, security, or profile + suites. +- Running the entire PPL integration-test suite in the lint validation job. +- Adding browser, Monaco, or a running OSD server. +- Performance or benchmark validation. +- Testing every released OpenSearch version with every analytics stack in the + first release. +- Adding multi-shard analytics coverage to the required lint check. +- Automatically accepting known analytics limitations through broad Gradle + exclusions or JUnit assumptions. +- Changing production routing solely to make the test easier. + +## 5. Design Invariants + +The implementation must preserve these invariants: + +1. **Same SQL candidate:** both backend lanes build the same checked-out SQL + commit. +2. **Same grammar:** both lanes export a runtime bundle. Their engine version + and grammar hash must match before detector validation starts. +3. **Same OSD candidate:** both detector comparisons use one resolved OSD SHA + and one OSD bootstrap. +4. **Same queries:** standard, analytics, and detector passes read the same + contract files and substitute the same index names. +5. **Explicit identity:** every target and report names its execution backend. + Missing or conflicting identity is an infrastructure failure. +6. **Proven route:** setting `tests.analytics.parquet_indices=true` is not + sufficient evidence. The analytics lane must attest the installed plugins, + index settings, explain plan, and a profiled execution. +7. **No semantic retry:** downloads and cluster startup may be retried within + bounded limits. Contract queries and assertions are executed once. +8. **No vacuous pass:** missing queries, reports, detector rows, route evidence, + or planned matrix legs fail or become an explicit non-applicable result. +9. **No implicit fallback:** the analytics lane must never count a standard + route result as analytics coverage. +10. **One detector oracle:** detector count and severity remain route + independent until OSD exposes an execution-backend lint context. +11. **Complete contracts:** every selected expectation names exactly the same + query keys as the contract's top-level `queries` map. Duplicate or missing + report rows are infrastructure failures. +12. **Strict artifacts:** requested targets and reports must exist, parse, and + agree on execution identity. Writers and consumers fail rather than degrade + to an identity-free or differential-free run. + +## 6. Target Identity + +`target.json` currently records only engine version, grammar hash, and bundle +name. It will move to schema version 2 and include execution identity: + +```json +{ + "schemaVersion": 2, + "sqlSha": "...", + "engineVersion": "3.8.0-SNAPSHOT", + "grammarHash": "sha256:...", + "grammarBundle": "ppl-grammar-bundle.json", + "executionBackend": "analytics", + "storage": "composite-parquet", + "shardCount": 1, + "analyticsStack": { + "source": "immutable feature-build URL", + "buildId": "...", + "components": [ + { + "name": "analytics-engine", + "version": "3.8.0-SNAPSHOT", + "sha256": "..." + } + ] + }, + "routeAttestation": { + "pluginsVerified": true, + "clusterSettingsVerified": true, + "fixtureIndicesVerified": true, + "explainVerified": true, + "profiledExecutionVerified": true + } +} +``` + +For the standard route: + +```json +{ + "executionBackend": "standard", + "storage": "lucene", + "shardCount": 1 +} +``` + +The backend report, detector report, drift report, and run manifest will also +carry `executionBackend`. Aggregation keys become: + +```text +(leg label, engine version, grammar surface, execution backend) +``` + +The leg label remains the presentation key because multiple legs can share the +same engine version. + +## 7. Contract Schema + +### 7.1 Schema version 4 + +Detector expectations are shared, while backend oracles are keyed by execution +backend: + +```json +{ + "schemaVersion": 4, + "ruleId": "union-min-datasets", + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { + "role": "trigger", + "query": "union [ source={{index}} ]" + }, + "union-two-datasets-control": { + "role": "control", + "query": "union [ source={{index}} ] [ source={{index}} ]" + } + }, + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException" + } + } + } + } + }, + "union-two-datasets-control": { + "detectorCount": 0, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} +``` + +The existing `engine` field keeps its current meaning. It is not renamed to +avoid mixing this work with an unrelated contract migration. + +### 7.2 Compatibility and migration + +- A schema version 3 `backend` object is read as `backends.standard`. It is not + used as an implicit analytics oracle. +- Observation can begin before every analytics oracle is reviewed. In + observation mode, a missing analytics oracle executes the query once and + records `coverage-missing` plus the raw backend result; it does not score that + result against the standard oracle. Infrastructure and route-attestation + failures still fail the lane. +- Enforcement requires `backends.analytics` for every selected query. +- An unknown execution backend is a contract error. +- An unknown schema version is a contract error. +- More than one version/planner expectation match remains an error. +- A selected expectation must name exactly the top-level contract query set. +- The Java test and Node runner must implement identical selection behavior. + +An explicit non-applicable form is permitted only when the fixture cannot +meaningfully exercise analytics: + +```json +{ + "kind": "not-applicable", + "reason": "Fixture field type cannot be represented by composite/Parquet storage", + "owner": "@analytics-team", + "issue": "https://github.com/opensearch-project/sql/issues/..." +} +``` + +Rules in the required `defaultError` set cannot be promoted while their +analytics oracle is non-applicable. For other rules, non-applicable entries +remain visible in the report and require an owner and issue. + +### 7.3 Differential policy + +| Case | Detector requirement | Backend requirement | +| --- | --- | --- | +| Control | Zero diagnostics | Every applicable backend accepts | +| Rejection trigger | Expected diagnostic count and severity | Every applicable backend rejects with its reviewed error shape | +| Advisory trigger | Expected diagnostic count and severity | Backend matches its reviewed acceptance/result-shape oracle | +| Missing backend oracle | Not scored | Coverage failure | +| Backend transport error | Not scored | Infrastructure/inconclusive failure, never acceptance | + +If an error rule fires while analytics accepts the trigger, the result is +`execution-backend-divergence`. The remediation must not recommend changing an +OpenSearch version range. Because OSD currently lacks backend context, the +choices are to make the rule valid for both routes, narrow the detector to +behavior common to both, disable it, or first add a reliable backend signal to +the OSD lint context. + +## 8. Analytics Test Cluster and Gradle Task + +### 8.1 Chosen approach + +Add a dedicated Gradle-managed cluster and task: + +```text +testClusters.analyticsEnginePplLint +:integ-test:analyticsEnginePplLintIT +``` + +The cluster will install: + +- Job Scheduler +- Arrow Base +- Arrow Flight RPC +- Analytics Engine +- Composite Engine +- Parquet Data Format +- Analytics Backend Lucene +- Analytics Backend DataFusion +- The SQL plugin built from the current checkout + +It will reuse the native-access, Netty, and experimental feature settings used +by the existing full-stack profile/security clusters. Shared cluster +configuration should be extracted into a small Gradle helper if that can be +done without changing those tasks' behavior. + +The task will: + +- Depend on all analytics plugin downloads and SQL `bundlePlugin`. +- Filter to `PplLintRuleValidationIT`. +- Set `tests.analytics.parquet_indices=true`. +- Set `tests.analytics.num_shards=1`. +- Set `ppl.lint.execution_backend=analytics`. +- Forward the existing `ppl.lint.*` paths and schedule. +- Run as a non-root user in CI. + +Example invocation: + +```bash +./gradlew :integ-test:analyticsEnginePplLintIT \ + -Dppl.lint.execution_backend=analytics \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report="$PWD/leg/backend-report.json" \ + -Dppl.lint.grammar.bundle="$PWD/leg/ppl-grammar-bundle.json" \ + -Dppl.lint.target="$PWD/leg/target.json" +``` + +The task sets the analytics fixture properties itself so a caller cannot +accidentally request an analytics report while creating Lucene fixtures. + +### 8.2 Why not the alternatives + +**Reuse `analyticsEngineCompatIT`:** rejected because its cluster lacks the full +execution stack and its test intentionally avoids analytics routing. + +**Provision an external cluster and use `integTestRemote`:** the remote task is +a valid future path for released analytics stacks, but it requires separate +cluster lifecycle, plugin installation, and SQL-plugin provenance checks. A +managed cluster is simpler and guarantees that the SQL plugin comes from the +current checkout. + +**Run the full PPL integration suite:** rejected for the required lint check. +It adds unrelated capability exclusions, runtime, and flakiness without +improving the detector contract. + +## 9. Route Attestation + +The analytics integration test will perform attestation before scoring any +contract: + +1. Query `/_cat/plugins?format=json` and require every plugin in the full stack. +2. Verify plugin versions are compatible with the OpenSearch/SQL version. +3. Read cluster settings and require composite data format defaults. +4. Read settings for every fixture index and require: + - `index.pluggable.dataformat.enabled=true` + - `index.pluggable.dataformat=composite` + - `index.composite.primary_data_format=parquet` +5. Run one valid explain canary per fixture index: + + ```text + source= | head 1 + ``` + + Require `LogicalTableScan(table=[[opensearch,` and reject + `CalciteLogicalIndexScan`. +6. Run the same canary with `profile=true` and require at least one successful + execution stage. Record every `execution_type`. Before required promotion, + pin and require the exact DataFusion-specific marker exposed by the locked + analytics stack. A generic non-empty profile is sufficient only for the + observation lane. +7. Write the attestation outcome into `target.json`. + +Invalid trigger queries may fail before DataFusion execution. Such results are +observations from an analytics-configured, route-attested environment, not +claims that DataFusion executed the invalid query. The canary proves that each +fixture is capable of analytics execution. A static +`cluster.pluggable.dataformat=composite` startup setting is also required so +query-initial parse failures see the same routing configuration as valid +queries. + +Attestation uses assertions, not JUnit assumptions. A missing plugin or legacy +explain plan fails the lane. + +## 10. CI Workflow + +### 10.1 Final required topology + +```text +Get-CI-Image-Tag + |-------------------------------| + v v +standard-backend-validation analytics-backend-validation + | | + |---- standard artifacts |---- analytics artifacts + \ / + v v + detector-validation + (one OSD checkout/bootstrap, + two backend comparisons) + | + v + validation-result +``` + +The backend jobs run in parallel. The analytics job uses JDK 25 to match the +existing analytics compatibility workflow; the standard job keeps its current +JDK. + +Artifacts use distinct names and directories: + +```text +ppl-lint-backend-standard/ +ppl-lint-backend-analytics/ +ppl-lint-backend-standard-logs/ +ppl-lint-backend-analytics-logs/ +``` + +Before linting, `detector-validation` verifies: + +- Both target manifests exist. +- Both backend reports are non-empty. +- Both targets report the expected execution backend. +- Both targets report the same engine version and grammar hash. +- Analytics route attestation is complete. +- Every requested report exists, is non-empty, contains no duplicate identities, + and agrees with its target's execution backend. + +It then invokes `run-frontend-contract.mjs` twice against the same OSD checkout +and runtime grammar: + +```text +standard backend report -> detector-standard-report.json +analytics backend report -> detector-analytics-report.json +``` + +The duplicate detector pass costs seconds; the OSD bootstrap dominates the +job. Two explicit invocations are lower risk than redesigning the runner to +accept an arbitrary report collection. The result job also compares normalized +detector rows: rule/query identity, count, severity, and any asserted message +match. Equal counts alone are not sufficient parity. + +`validation-result` continues to use `if: always()` and becomes red unless all +three validation jobs succeeded. A skipped detector caused by either backend +failure therefore cannot appear green. + +### 10.2 Multi-version workflow + +The first analytics leg is `pr-build-analytics`. It is not added to every +released version: + +| Leg | Version | Grammar surface | Execution backend | +| --- | --- | --- | --- | +| Existing released legs | Released matrix | Runtime/compiled as configured | Standard | +| `pr-build` | Pull request build | Runtime bundle | Standard | +| `pr-build-analytics` | Pull request build | Runtime bundle | Analytics | + +`aggregate-versions.mjs` must understand the backend dimension before this leg +is added. It reports backend divergence separately and never turns an +analytics-only difference into version-scoping advice. + +The discovery corpus remains standard-only in the initial implementation. It +has no reviewed oracle and should not expand the analytics rollout's cost or +diagnostic surface. + +### 10.3 Local entry point + +`scripts/ppl-lint-rule-validation.sh` will gain an opt-in analytics mode, for +example `RUN_ANALYTICS=1`. It will support the existing local ZIP override +properties. Standard local behavior remains unchanged. + +## 11. Artifact Provenance + +The current Gradle default uses a mutable +`feature-datafusion/latest/linux/x64` URL. This is acceptable for early +observation but not for a required check. + +Before promotion: + +1. Add a checked-in compatibility lock describing the immutable analytics + feature build for the current OpenSearch line. +2. Add a Gradle property such as `analyticsFeatureBuildBase` so CI can pass the + immutable base while local development can retain the current default. +3. Verify SHA-256 for every downloaded plugin ZIP before cluster startup. +4. Record the immutable source, build ID, component versions, and hashes in + `target.json`. +5. Fail if installed plugin versions do not match the locked tuple. + +If an immutable artifact source cannot be provided, the analytics lane remains +non-enforcing. + +## 12. Failure Semantics + +| Failure | Classification | CI behavior | +| --- | --- | --- | +| Plugin download or checksum failure | Infrastructure | Retry download at most three times, then fail lane | +| Cluster does not become healthy | Infrastructure | Fail and upload cluster logs/thread dump | +| Required plugin absent or wrong version | Infrastructure | Fail before contracts | +| Fixture is not composite/Parquet | Route attestation | Fail before contracts | +| Explain/profile canary uses standard route | Route attestation | Fail before contracts | +| Standard and analytics grammar hashes differ | Candidate identity | Fail detector job | +| Missing/empty backend or detector report | Incomplete run | Fail; never aggregate survivors only | +| Missing analytics expectation | Coverage hole | Fail once analytics enforcement is enabled | +| Contract query transport timeout | Inconclusive run | Fail; never treat as backend acceptance | +| Trigger/control behavior differs from oracle | Semantic drift | Report backend, query, observed status/type, and remediation | +| Standard and analytics behavior differ | Execution-backend divergence | Report separately; do not suggest version scoping | +| Detector output differs between backend passes | Harness/context defect | Fail detector job | + +Semantic assertions are never retried. A retry could hide a nondeterministic +backend or detector defect. + +## 13. Diagnostics and Resource Bounds + +The analytics job will use: + +- A 30-minute GitHub job timeout. +- A bounded OpenSearch heap consistent with current workflows. +- One Netty direct arena and the existing native-access flags. +- One primary shard for required contract coverage. +- No credentials or fork secrets. +- `permissions: contents: read`. + +Always upload on failure: + +- `target.json` and analytics stack identity. +- Backend and detector reports. +- JUnit XML and HTML reports. +- Gradle test reports. +- Installed plugin list. +- Effective cluster and fixture index settings. +- Fixture mapping hashes and any fields stripped by the analytics fixture + helper. +- OpenSearch and test-cluster logs. +- Detector logs. +- Thread dumps for startup or query timeout. + +Reports must distinguish `accepted`, `rejected`, `error`, and +`not-applicable`. An absent `rejected` field is not equivalent to acceptance. + +## 14. Test Plan + +### 14.1 Harness unit tests + +Add Node tests for: + +- Schema version 3 compatibility and schema version 4 backend selection. +- Unknown or missing execution backend. +- Missing analytics oracle. +- Duplicate target identities. +- Same version with standard and analytics legs. +- Standard/analytics grammar mismatch. +- Backend transport error not being read as acceptance. +- Analytics divergence producing backend remediation, not version scoping. +- Detector parity between standard and analytics passes. +- Non-applicable handling and required-rule coverage holes. +- Summary and annotation output naming the execution backend. + +### 14.2 Java integration coverage + +Verify: + +- The standard `PplLintRuleValidationIT` behavior is unchanged. +- The analytics task installs the full stack. +- ACCOUNT and FLAT_OBJECT fixtures are composite/Parquet or fail explicitly. +- Explain and profile canaries attest the analytics route. +- Every scheduled contract emits one backend result per expected query. +- Report entries include `executionBackend`. +- A forced missing-plugin or standard-route configuration fails attestation. + +### 14.3 Workflow validation + +Use `workflow_dispatch` to validate: + +- Canonical OSD `main`. +- An explicit OSD branch/SHA. +- A successful dual-backend run. +- An intentionally wrong analytics oracle. +- An intentionally missing analytics artifact. +- A backend failure that skips detector work but still makes the final result + red. + +No production branch-protection change is made during this validation. + +## 15. Rollout + +### Phase 1: Identity and observation + +- Add execution-backend identity to targets and reports. +- Add the schema version 4 reader with version 3 compatibility. +- Make artifact consumers fail closed on missing, malformed, duplicate, or + conflicting identities. +- Add backend-aware aggregation and divergence remediation before introducing + an analytics leg. +- Add the managed analytics Gradle task and route attestation. +- Add `pr-build-analytics` to the non-required multi-version workflow. +- Missing analytics oracles are recorded as unscored coverage gaps during + observation. Infrastructure, identity, completeness, and attestation failures + remain red. Do not use `continue-on-error` inside the producer lane. + +### Phase 2: Baseline and review + +- Capture real analytics observations for the full contract corpus. +- Add reviewed analytics oracles. +- Resolve every default-error non-applicable case. +- Pin immutable analytics artifacts and verify their checksums. +- Pin the DataFusion-specific profile execution marker. +- Measure runtime and infrastructure reliability. + +Promotion requires: + +- Every scheduled contract has a reviewed analytics oracle. +- No `defaultError` contract is non-applicable. +- No unexplained semantic divergence remains. +- At least 25 consecutive green observation runs. +- At least 50 total runs with less than 1% infrastructure failure. +- Analytics job p95 runtime is at most 15 minutes. +- Artifact provenance is immutable and recorded. + +### Phase 3: Required check + +- Add `analytics-backend-validation` to the required single-version workflow. +- Make detector validation require both backend artifacts. +- Make `validation-result` require standard backend, analytics backend, and + detector success. +- Update the run manifest and PR summary to show both routes. + +There is no silent repository-variable bypass after promotion. An emergency +rollback requires an explicit workflow/branch-protection change and a tracking +issue. + +### Phase 4: Optional expansion + +After the required lane is stable, evaluate: + +- Matching released analytics stacks. +- A scheduled three-shard analytics leg. +- Analytics execution for the discovery corpus. +- Consolidating or retiring redundant parts of + `analytics-engine-compat.yml`. + +These are separate changes and are not prerequisites for initial enforcement. + +## 16. Planned File Changes + +| File | Change | +| --- | --- | +| `integ-test/build.gradle` | Add the full-stack analytics lint cluster/task and artifact lock inputs | +| `PplLintRuleValidationIT.java` | Select backend-specific oracles, attest route, and emit backend identity | +| `integ-test/src/test/resources/ppl-lint/contracts/*.spec.json` | Migrate to schema version 4 and add analytics oracles | +| `integ-test/src/test/resources/ppl-lint/contracts/manifest.json` | Bump schema metadata and document analytics coverage | +| `scripts/ppl-lint/run-frontend-contract.mjs` | Select the active backend oracle and emit backend identity | +| `scripts/ppl-lint/contract-schema.mjs` | Share strict Node schema, identity, and backend-oracle selection | +| `scripts/ppl-lint/aggregate-versions.mjs` | Key and render legs by execution backend | +| `scripts/ppl-lint/drift.mjs` | Add execution-backend divergence and remediation | +| `scripts/ppl-lint/annotate.mjs` | Attach backend-specific findings to contract declarations | +| `scripts/ppl-lint/assemble-run-manifest.mjs` | Record both targets and job results | +| `scripts/ppl-lint/__tests__/*` | Cover schema, identity, aggregation, and remediation changes | +| `.github/workflows/ppl-lint-multiversion-validation.yml` | Add the observation leg | +| `.github/workflows/ppl-lint-rule-validation.yml` | Add the required lane after promotion | +| `scripts/ppl-lint-rule-validation.sh` | Add opt-in local analytics reproduction | +| `scripts/ppl-lint/README.md` | Document backend-aware contracts and commands | +| Analytics compatibility lock (path TBD) | Pin immutable plugin URLs, versions, and SHA-256 values before required promotion | + +## 17. Success Criteria + +The work is complete when: + +1. A pull request can produce standard and analytics observations from the same + SQL commit and grammar. +2. CI proves the analytics route instead of relying on a configuration flag. +3. Every scheduled contract has an explicit analytics result. +4. Reports cannot confuse backend divergence with version drift. +5. Missing analytics coverage cannot pass as agreement. +6. The required result fails when either backend or the OSD detector contract + fails. +7. A failed run includes enough immutable identity and logs to reproduce the + target that was tested. + +## 18. Open Questions + +1. Which system owns publishing and retaining immutable analytics feature-build + tuples for required CI? +2. Should the artifact compatibility lock live in this repository or be + generated by the OpenSearch feature-build pipeline? +3. Which current contract queries produce intentional analytics behavior + differences once the first observation run is available? +4. Will OSD eventually expose a reliable execution-backend signal to lint + context? If so, detector expectations may later become backend-aware. +5. After the full semantic lane is required, does the smaller coexistence smoke + workflow still provide enough independent value to keep? diff --git a/docs/dev/ppl-lint-runtime-compatibility-ci-design.md b/docs/dev/ppl-lint-runtime-compatibility-ci-design.md new file mode 100644 index 00000000000..e2b3375f25b --- /dev/null +++ b/docs/dev/ppl-lint-runtime-compatibility-ci-design.md @@ -0,0 +1,200 @@ +# PPL Lint Runtime Compatibility CI + +- **Status:** Draft implementation design +- **Last updated:** 2026-08-04 +- **Scope:** `.github/workflows/ppl-lint-multiversion-validation.yml` + +## 1. Decision + +The multi-surface workflow validates the 12 active PPL lint detectors against +exactly three standard-engine configurations: + +```text +OpenSearch 2.19.6 + OSD compiled-simplified fallback grammar --+ +Latest eligible GA + its exported runtime grammar -------------+--> Aggregate rule compatibility +SQL pull request build + its exported runtime grammar ----------+ +``` + +The fixed `2.19.6` leg covers the checked-in grammar OSD uses when an engine +cannot export a runtime grammar bundle. The planner reads the SQL pull request's +default `opensearch.version`, normalizes prerelease/build suffixes, and selects +the highest exact-semver OpenSearch release tag at or below that target for the +GA runtime leg. The PR leg validates the candidate runtime grammar built by the +change under review. + +The workflow does not run: + +- the analytics engine or composite/Parquet storage; +- syntax-channel features; +- AI action tests. + +Analytics coverage is deferred until that engine and its fixtures provide a +stable CI contract. The required single-version workflow remains responsible +for proving that all active detectors agree with the standard SQL pull request +build. The multi-surface workflow explains compatibility across shipping grammar +surfaces and fails its final aggregation job when declared support drifts. + +## 2. Rule Inventory + +The active inventory contains **12 detector rules**, not 13. +`command-suggestion` is not a lint rule and must not be silently reintroduced. +The final table is generated from `manifest.json`, and CI asserts the exact +inventory so adding a future reviewed rule requires an intentional guard and +test update. + +| Rule | Grammar surface | Declared scope | +| --- | --- | --- | +| `agg-on-text` | Both | Calcite, OpenSearch >= 3.7 | +| `division-by-zero` | Both | All versions and engine modes | +| `enabled-false-object` | Both | Calcite, OpenSearch >= 3.7 | +| `field-validation` | Both | All versions and engine modes | +| `invalid-capture-group-name` | Runtime bundle | OpenSearch >= 3.4 | +| `multisearch-min-subsearch` | Runtime bundle | OpenSearch >= 3.4 | +| `replace-wildcard-asymmetry` | Runtime bundle | Calcite, OpenSearch >= 3.4 | +| `rex-scan-cost` | Both | All versions and engine modes | +| `type-mismatch-numeric` | Both | Calcite, OpenSearch >= 3.7 | +| `union-min-datasets` | Runtime bundle | Calcite, OpenSearch >= 3.7 | +| `unsupported-window-function-in-eventstats` | Both | OpenSearch >= 3.4 | +| `wildcard-source-zero-match` | Both | All versions and engine modes | + +Four preserved default-off contracts remain in `dormantContracts`. They do not +count toward the 12-rule active inventory. + +## 3. Workflow Shape + +### 3.1 Plan configurations + +`Plan compatibility matrix` resolves: + +- the fixed compiled-fallback target, `2.19.6`; +- the highest official GA release at or below the normalized PR target; +- the raw and normalized PR target from `build.gradle`; +- the OSD repository and revision; +- immutable configuration IDs, surfaces, engine modes, and artifact names. + +Only exact `X.Y.Z` release tags are eligible. The plan is uploaded as +`compatibility-plan.json` and drives the released-engine matrix. + +### 3.2 Observe released configurations + +One `Observe engine ()` matrix job runs for each released +configuration: + +1. start the matching official OpenSearch distribution, which includes its SQL + plugin; +2. run the same contract corpus in observe-only mode; +3. record `target.json` and `backend-report.json`; +4. export `ppl-grammar-bundle.json` only for the runtime-bundle configuration; +5. upload the observation even when a semantic mismatch is found. + +The `2.19.6` configuration records backend behavior but intentionally has no +runtime bundle. Its detector pass uses OSD's compiled-simplified fallback +grammar. The selected GA configuration exports and uses that release's runtime +bundle. + +### 3.3 Observe the pull request build + +`Observe engine pr-build (runtime)` runs the same corpus against the standard +Gradle test cluster built from the pull request. It exports the candidate +runtime grammar and the same target/backend artifact shape as the GA runtime +leg. + +### 3.4 Aggregate rule compatibility + +`Aggregate rule compatibility` is the only fan-in job. It: + +1. waits for the plan and all three backend observations; +2. downloads every `ppl-lint-observation-*` artifact; +3. bootstraps OSD once at the resolved revision; +4. runs production headless lint against the compiled fallback or each runtime + bundle, as specified by the plan; +5. applies surface, version, then engine-mode exclusions before detector + execution; +6. compares declared compatibility with detector and backend evidence; +7. writes the complete 12 x 3 `drift-report.json`; +8. publishes the Markdown compatibility table and file-aware annotations; +9. uploads the mandatory report and supplemental evidence before enforcement; +10. fails if the recorded result contains supported-configuration drift or an + enforced inconclusive cell. + +The display name is intentionally explicit. A reader should not have to infer +that this fan-in is the final compatibility decision. + +## 4. Expected Versus Actual Compatibility + +The aggregate summary has one row per active rule: + +| Rule | Expected compatibility | 2.19.6 compiled | Latest GA runtime | PR runtime | +| --- | --- | --- | --- | --- | +| `agg-on-text` | Both surfaces, Calcite >= 3.7 | expected n/a | compatible | compatible | +| `division-by-zero` | Both surfaces, all versions | compatible | compatible | compatible | + +Each actual cell uses one of these states: + +| State | Meaning | +| --- | --- | +| `compatible` | Detector output and backend behavior match the contract. | +| `expected n/a` | Surface, version, or engine mode is outside `wiring.appliesTo`. | +| `drift` | The configuration is declared compatible but observed behavior differs. | +| `inconclusive` | A fixture, query, artifact, or detector execution did not produce a trustworthy verdict. | + +Applicability is part of the expected result, not a workaround applied after +observation. For example, a Calcite-only rule is expected n/a on the legacy +compiled configuration and does not count as drift there. + +The JSON report retains query-level evidence and remediation details. The +Markdown table is the concise compatibility view, not a replacement for the +machine-readable report. + +## 5. Failure Semantics + +Compatibility aggregation is write-first and then enforcing: + +- observation jobs record detector and backend mismatches without failing; +- expected out-of-scope configurations do not fail the workflow; +- one rule cannot prevent results for the remaining rules; +- detector execution errors become complete inconclusive cells; +- the fan-in writes the complete table and `drift-report.json`; +- artifact upload runs before the enforcement step; +- only after those outputs exist does supported drift or an enforced + inconclusive result fail the job. + +Structural failures remain errors because no truthful table can be produced: + +- a planned observation uploads no usable artifacts; +- JSON artifacts are malformed; +- target and report identities conflict; +- the contract manifest is malformed; +- a runtime configuration has no grammar bundle; +- `drift-report.json` cannot be written. + +An artifact named `ppl-lint-multiversion-drift` must contain +`drift-report.json`; raw target files alone are not a compatibility report. + +## 6. Outputs + +Every run produces: + +- `compatibility-plan.json`; +- a GitHub step-summary table with expected and actual compatibility; +- `drift-report.json`; +- one detector report and detector log per configuration; +- target manifests that identify exact SQL, engine, surface, backend, and + grammar identities. + +The required PPL lint workflow remains the stable branch-protection signal. The +multi-surface aggregation is also red on declared-supported drift, with the +table and artifacts providing evidence for adjusting applicability, narrowing a +detector, or updating a backend oracle after review. + +## 7. Deferred Coverage + +Analytics-engine validation may return only after: + +- its feature build is immutable for the duration of a run; +- all required fixtures can be represented or explicitly scoped; +- route attestation is stable; +- a rule-specific analytics limitation cannot invalidate unrelated rules. + +Syntax-channel and AI-action behavior require separate contracts and are not +part of this detector compatibility matrix. diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index 3e115b249ec..adefd59397b 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -87,6 +87,8 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da +-----------------+---------------------+-----------+ | keyword | keyword | VARCHAR | +-----------------+---------------------+-----------+ +| constant_keyword| keyword | VARCHAR | ++-----------------+---------------------+-----------+ | text | text | VARCHAR | +-----------------+---------------------+-----------+ | date* | timestamp | TIMESTAMP | diff --git a/docs/user/ppl/cmd/foreach.md b/docs/user/ppl/cmd/foreach.md index 45dcb939f8a..ba6b758fa5d 100644 --- a/docs/user/ppl/cmd/foreach.md +++ b/docs/user/ppl/cmd/foreach.md @@ -41,9 +41,9 @@ Placeholders renamed via `itemstr`/`iterstr` may be written without the `<<...>> The following considerations apply when using the `foreach` command: * In collection modes, the bracketed `eval` acts as an accumulator: each target field must already exist (typically initialized with a preceding `eval`), and the expressions are applied once per element. Multiple assignments run from left to right, so a later assignment in the same iteration sees an earlier assignment's updated value. -* In `json_array` mode the element type is inferred: a `json_array(...)` call or JSON string literal is inspected at plan time; for a field holding JSON text, elements are treated as numbers when `<>` is used in arithmetic and as strings otherwise. Mixed string/number JSON arrays are rejected. +* In `json_array` mode the element type is inferred: a `json_array(...)` call or JSON string literal is inspected at plan time; for a field holding JSON text, elements are treated as numbers when `<>` is used in arithmetic and as strings otherwise. Plan-time arrays with mixed string/number elements are rejected. When numeric use is inferred for field-backed JSON text, elements that cannot be converted to numbers evaluate to `null`. * Placeholders are also substituted inside string literals. For example, `eval <> = '<>'` replaces each selected field value with that field's name. -* As in Splunk, `multivalue` mode applied to a non-array value and `json_array` mode applied to a native array are no-ops. A field whose mapping is scalar cannot be identified as multivalue at plan time even when a document stores several values, so `multivalue` mode also no-ops for that mapping. +* `multivalue` mode applied to a non-array value and `json_array` mode applied to a native array are no-ops. A field whose mapping is scalar cannot be identified as multivalue at plan time even when a document stores several values, so `multivalue` mode also no-ops for that mapping. ## Example 1: Apply the same calculation to multiple fields diff --git a/docs/user/ppl/cmd/makeresults.md b/docs/user/ppl/cmd/makeresults.md new file mode 100644 index 00000000000..370129f1673 --- /dev/null +++ b/docs/user/ppl/cmd/makeresults.md @@ -0,0 +1,88 @@ + +# makeresults + +The `makeresults` command generates in-memory rows. With no arguments it produces a single row containing only the `@timestamp` field, set to the query time. It is commonly used as a seed for `eval` and to generate test data. The time column is named `@timestamp` (OpenSearch's implicit time field) so it is recognized by the time-aware commands such as `timechart`, `reverse`, and `span`. + +> **Note**: The `makeresults` command is a leading command (it opens a query) and is executed only on the coordinating node. It has no backing index. It requires the Calcite engine (`plugins.calcite.enabled=true`). + +## Syntax + +The `makeresults` command has the following syntax: + +```syntax +makeresults [count=] [format=csv|json data=] +``` + +## Parameters + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `count` | Optional | The number of rows to generate. Must be a non-negative integer up to 5000. A negative value produces zero rows. Each row has a single `@timestamp` (timestamp) column. Default is `1`. | +| `format` + `data` | Optional | Generate rows from an inline `csv` or `json` literal instead (up to 5000 cells, where cells = rows x columns, and no single cell value may exceed 60000 characters). When provided, `count` is ignored. | + +### Inline data typing + +Column types for `data=` follow OpenSearch dynamic-mapping semantics: + +- JSON: an integer becomes `long`, a decimal becomes `float`, `true`/`false` becomes `boolean`, and a string becomes `string`. A nested object or array is serialized to its compact JSON string and typed as `string`; use `spath` or the `json_extract` function to re-parse it downstream. +- CSV: a header token of the form `name:type` declares the column type using the same vocabulary as `cast` (for example `age:int`); a bare header token defaults to `string`. + +The `date`, `time`, `timestamp`, `ip`, and `json` inline types are not yet supported on this path; declare the column as `string` and `cast` it downstream, for example `makeresults format=csv data='addr\n192.168.1.1' | eval addr = cast(addr as ip)`. + +### Implicit `@timestamp` column + +`format=json data=` treats each JSON object as an event and prepends an implicit `@timestamp` +(timestamp) column set to the query time, in addition to the object's own fields. If the JSON data +already defines an `@timestamp` field, that value is kept and no implicit column is added. +`format=csv data=` is a pure table and does not add an `@timestamp` column. + +## Example 1: Generate rows for testing + +The following query generates five rows: + +```ppl +makeresults count=5 +``` + +## Example 2: Seed a row for eval + +```ppl +makeresults +| eval message="hello" +``` + +## Example 3: Generate typed rows from JSON + +```ppl +makeresults format=json data='[{"name":"John","age":35},{"name":"Sarah","age":39}]' +``` + +The query returns two rows with an `@timestamp` (timestamp, query time) column followed by a `name` (string) column and an `age` (bigint) column. A JSON integer is typed as a long value; because makeresults rows have no index mapping, the column reports its Calcite type name `bigint` in the response schema. + +## Example 4: Generate typed rows from CSV + +```ppl +makeresults format=csv data='name:string,age:int +John,35 +Sarah,39' +``` + +The query returns two rows with a `name` (string) column and an `age` (int) column. + +## Limitations + +A global aggregate that references no input column, applied directly to `makeresults`, is not +currently supported and raises an error: + +```ppl +makeresults count=5 | stats count() as c +``` + +This is due to an upstream Apache Calcite field-trimming defect on zero-column relations, not a +`makeresults`-specific issue. Use any of the following equivalent forms instead: + +```ppl +makeresults count=5 | stats count(1) as c +makeresults count=5 | stats count() as c by @timestamp +makeresults count=5 | eval g=1 | stats count() as c by g +``` diff --git a/docs/user/ppl/cmd/rare.md b/docs/user/ppl/cmd/rare.md index 993e1a9c987..3f85144e5d0 100644 --- a/docs/user/ppl/cmd/rare.md +++ b/docs/user/ppl/cmd/rare.md @@ -23,7 +23,7 @@ The `rare` command supports the following parameters. | --- | --- | --- | | `` | Required | A comma-delimited list of field names. | | `` | Optional | One or more fields to group the results by. | -| `rare-options` | Optional | Additional options for controlling output:
- `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`.
- `countfield`: The name of the field that contains the count. Default is `count`.
- `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | +| `rare-options` | Optional | Additional options for controlling output:
- `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`.
- `countfield`: The name of the field that contains the count. Default is `count`.
- `percentfield`: The name of the field that contains the percentage. Default is `percent`.
- `showperc`: Whether to create a field in the output containing the percentage of each value's count relative to the total. Default is `false`.
- `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | ## Example 1: Finding the least common values without showing counts @@ -125,7 +125,52 @@ fetched rows / total rows = 4/4 +--------------+-----+ ``` -## Example 5: Specifying null value handling +## Example 5: Displaying percentages + + The following query finds the least common severity levels and shows what percentage each represents: + +```ppl +source=otellogs +| rare showperc=true severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+---------+ +| severityText | count | percent | +|--------------+-------+---------| +| DEBUG | 3 | 15.0 | +| WARN | 4 | 20.0 | +| INFO | 6 | 30.0 | +| ERROR | 7 | 35.0 | ++--------------+-------+---------+ +``` + +## Example 6: Customizing the percent field name +The following query uses `percentfield` to rename the percentage column from the default `percent` to `pct`: + +```ppl +source=otellogs +| rare showperc=true percentfield='pct' severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+------+ +| severityText | count | pct | +|--------------+-------+------| +| DEBUG | 3 | 15.0 | +| WARN | 4 | 20.0 | +| INFO | 6 | 30.0 | +| ERROR | 7 | 35.0 | ++--------------+-------+------+ +``` + +## Example 7: Specifying null value handling The following query uses `usenull=false` to exclude null values: @@ -166,4 +211,4 @@ fetched rows / total rows = 4/4 | @opentelemetry/instrumentation-http | 2 | | null | 16 | +-----------------------------------------------------------------------------+-------+ -``` \ No newline at end of file +``` diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md index a9761aa87e8..6860a619b97 100644 --- a/docs/user/ppl/cmd/rest.md +++ b/docs/user/ppl/cmd/rest.md @@ -1,39 +1,34 @@ # rest -The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. +The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. -> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. + +The `rest` command is a generic, extensible framework: a plugin contributes additional read-only endpoints through the `RestEndpointProvider` extension point without changing the grammar. This first version ships a single built-in endpoint, `/_cluster/health`. Additional endpoints (for example `/_cat/nodes`, `/_cat/shards`, `/_cluster/state`, `/_cluster/settings`) can be added in follow-ups, with any response redaction handled inside the provider's own handler. ## Enabling the command -The `rest` command is **disabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to an empty list, so every endpoint is rejected until a deployment explicitly opts in. Enable specific endpoints by setting the allow-list (a node-level setting, so it is applied at node startup and cannot be changed at runtime): +`/_cluster/health` is **enabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to `["/_cluster/health"]`. Any other endpoint is rejected until a deployment adds it to the allow-list (a node-level setting, applied at node startup and not changeable at runtime): ```yaml -plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_cat/nodes"] +plugins.ppl.rest.allowed_endpoints: ["/_cluster/health"] ``` -Use `["*"]` to allow every endpoint in the curated list below. An empty list (the default) disables the command entirely. - -The `rest` command also supports optional response redaction of network identifiers (IPv4/IPv6 addresses, `inet[...]` forms, EC2-style host names, and availability-zone names) in `/_cat/*` and `/_cluster/settings` cell values, controlled by `plugins.ppl.rest.redaction.enabled` (a node-level setting, default `false`). Managed deployments that must not expose host topology should set it to `true`. +Every endpoint must be listed explicitly by name; there is no wildcard, so a newly installed or upgraded provider is never enabled without an explicit allow-list change. Set an empty list to disable the command entirely. ## Syntax -The `rest` command has the following syntax: - ```syntax -rest [count=] [timeout=] [= ...] +rest [count=] [= ...] ``` ## Parameters -The `rest` command supports the following parameters. - | Parameter | Required/Optional | Description | | --- | --- | --- | | `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | | `count=` | Optional | Caps the number of emitted rows. | -| `timeout=` | Optional | Reserved for forward compatibility. It is currently rejected with a clear error, because a single uniform timeout does not map cleanly across the different endpoints. | -| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`, `health=green` for `/_cat/indices`, `expand_wildcards=open` for `/_resolve/index`). | +| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`). | ## Allow-list @@ -41,54 +36,28 @@ The `rest` command supports the following parameters. | Endpoint | Output columns | Accepted args | | --- | --- | --- | -| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | `local` | -| `/_cluster/state` | `cluster_name` (string), `state_uuid` (string), `version` (long), `cluster_manager_node` (string) | (none) | -| `/_cluster/settings` | `setting` (string), `value` (string), `tier` (string) | (none) | -| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | `health` | -| `/_cat/nodes` | `name` (string), `ip` (string), `node_role` (string), `heap_percent` (integer), `ram_percent` (integer), `cpu` (integer) | (none) | -| `/_cat/cluster_manager` | `id` (string), `host` (string), `ip` (string), `node` (string) | (none) | -| `/_cat/plugins` | `name` (string), `component` (string), `version` (string) | (none) | -| `/_cat/shards` | `index` (string), `shard` (integer), `prirep` (string), `state` (string), `node` (string) | (none) | -| `/_resolve/index` | `name` (string), `type` (string) | `expand_wildcards` | - -## Example 1: Counting the nodes in the cluster - -The following query reads cluster health and projects a column that is deterministic on a single-node cluster: - -```ppl -| rest '/_cluster/health' | fields number_of_nodes -``` - -The query returns the following results: - -```text -fetched rows / total rows = 1/1 -+-----------------+ -| number_of_nodes | -|-----------------| -| 1 | -+-----------------+ -``` - -`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list, which you can project and filter the same way. +| `/_cluster/health` | `response` (string): the full cluster-health response as JSON. Extract fields with `json_extract` or the `spath` command (see the example below). | `local` | -## Example 2: Composing downstream commands over a cat endpoint +## Example: Reading fields from the response -The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan. The following query reads `/_cat/cluster_manager` and counts the rows: +`/_cluster/health` returns the full health response in a single `response` column as JSON. Extract the fields you need with `json_extract` (or the `spath` command): -```ppl -| rest '/_cat/cluster_manager' | stats count() as managers +```ppl ignore +| rest '/_cluster/health' +| eval status = json_extract(response, 'status'), + number_of_nodes = json_extract(response, 'number_of_nodes') +| fields status, number_of_nodes ``` The query returns the following results: ```text fetched rows / total rows = 1/1 -+----------+ -| managers | -|----------| -| 1 | -+----------+ ++--------+-----------------+ +| status | number_of_nodes | +|--------+-----------------| +| green | 1 | ++--------+-----------------+ ``` -For example, `| rest '/_cat/indices' | where health = 'green' | sort index | fields index, health, pri` lists green indexes; the projected columns come from the endpoint's fixed schema. +Because the whole response is available, a query can read any field it exposes (for example `active_shards`, `active_primary_shards`, `unassigned_shards`) without the endpoint pre-declaring a column for it. The extracted columns then compose with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan, for example `| rest '/_cluster/health' | spath input=response path=status output=status | where status = 'green'`. diff --git a/docs/user/ppl/cmd/top.md b/docs/user/ppl/cmd/top.md index 678b62354a0..162ad56e905 100644 --- a/docs/user/ppl/cmd/top.md +++ b/docs/user/ppl/cmd/top.md @@ -20,7 +20,7 @@ The `top` command supports the following parameters. | Parameter | Required/Optional | Description | | --- | --- | --- | | `` | Optional | The number of results to return. Default is `10`. | -| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.
`countfield`: The name of the field that contains the count. Default is `count`.
`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | +| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.
`countfield`: The name of the field that contains the count. Default is `count`.
`percentfield`: The name of the field that contains the percentage. Default is `percent`.
`showperc`: Whether to create a field in the output that represents the percentage of the count relative to the total. Default is `false`.
`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | | `` | Required | A comma-delimited list of field names. | | `` | Optional | One or more fields to group the results by. | @@ -139,7 +139,52 @@ fetched rows / total rows = 7/7 +----------------------------------+--------------+ ``` -## Example 6: Specifying null value handling +## Example 6: Displaying percentages + +The following query finds the most common severity levels and shows the percentage each represents: + +```ppl +source=otellogs +| top showperc=true severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+---------+ +| severityText | count | percent | +|--------------+-------+---------| +| ERROR | 7 | 35.0 | +| INFO | 6 | 30.0 | +| WARN | 4 | 20.0 | +| DEBUG | 3 | 15.0 | ++--------------+-------+---------+ +``` + +## Example 7: Customizing the percent field name +The following query uses `percentfield` to rename the percentage column from the default `percent` to `pct`: + +```ppl +source=otellogs +| top showperc=true percentfield='pct' severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+------+ +| severityText | count | pct | +|--------------+-------+------| +| ERROR | 7 | 35.0 | +| INFO | 6 | 30.0 | +| WARN | 4 | 20.0 | +| DEBUG | 3 | 15.0 | ++--------------+-------+------+ +``` + +## Example 8: Specifying null value handling The following query specifies `usenull=false` to exclude null values: diff --git a/docs/user/ppl/cmd/xyseries.md b/docs/user/ppl/cmd/xyseries.md new file mode 100644 index 00000000000..a81ea7d604c --- /dev/null +++ b/docs/user/ppl/cmd/xyseries.md @@ -0,0 +1,155 @@ +# xyseries + +## Description + +The `xyseries` command converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. + +## Syntax + +```syntax +xyseries [sep=] [format=] in (, , ...) [, , ...] +``` + +## Parameters + +| Parameter | Required/Optional | Description | Default | +| --- | --- | --- | --- | +| `` | Required | The field used as the row key in the output. Results are grouped and sorted by this field. | N/A | +| `` | Required | The field whose values are used to generate output column names. Only the values listed in the `in` clause are pivoted into columns. | N/A | +| `in (, , ...)` | Required | Explicit list of pivot values to select from ``. Each value generates one output column per ``. Values must be quoted strings. | N/A | +| `` | Required (at least one) | One or more fields containing the data to pivot. If multiple fields are specified, separate them with commas. | N/A | +| `sep` | Optional | Separator between the `` name and the pivot value in output column names. Ignored if `format` is specified. | `": "` | +| `format` | Optional | Naming template for output column names. Use `$AGG$` as a placeholder for the `` name and `$VAL$` as a placeholder for the pivot value. When specified, overrides `sep`. | N/A | + +## Notes + +The following considerations apply when using the `xyseries` command: + +* The `xyseries` command is typically used after a `stats` command that groups results by both the `` and ``. +* Output column names follow the pattern `` by default (for example, `host_cnt: 200`). Use the `format` option to customize this pattern. +* When a pivot value has no matching data for a given `` row, the output cell is `null`. +* The `` values are compared as strings. Non-string fields are cast to string automatically. +* Results are sorted by `` in ascending order. +* This command requires the Calcite engine to be enabled (`plugins.calcite.enabled: true`). + +## Example 1: Basic xyseries with a single data field + +This example pivots HTTP response codes into columns for a count of hosts per URL: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries url response in ("200", "404", "500") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+---------------+---------------+---------------+ +| url | host_cnt: 200 | host_cnt: 404 | host_cnt: 500 | +|--------+---------------+---------------+---------------| +| /page1 | 3 | 1 | null | +| /page2 | 5 | null | 2 | ++--------+---------------+---------------+---------------+ +``` + +## Example 2: Multiple data fields + +This example pivots multiple aggregated fields at once: + +```ppl +source=weblogs +| stats count(host) as host_cnt, count(method) as method_cnt by url, response +| xyseries url response in ("200", "404", "500") host_cnt, method_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+---------------+---------------+---------------+------------------+------------------+------------------+ +| url | host_cnt: 200 | host_cnt: 404 | host_cnt: 500 | method_cnt: 200 | method_cnt: 404 | method_cnt: 500 | +|--------+---------------+---------------+---------------+------------------+------------------+------------------| +| /page1 | 3 | 1 | null | 3 | 1 | null | +| /page2 | 5 | null | 2 | 5 | null | 2 | ++--------+---------------+---------------+---------------+------------------+------------------+------------------+ +``` + +## Example 3: Custom separator + +This example uses a custom separator between the data field name and pivot value in column names: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries sep="-" url response in ("200", "404") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+--------------+--------------+ +| url | host_cnt-200 | host_cnt-404 | +|--------+--------------+--------------| +| /page1 | 3 | 1 | +| /page2 | 5 | null | ++--------+--------------+--------------+ +``` + +## Example 4: Format template + +This example uses a format template to customize output column names. `$VAL$` is replaced with the pivot value and `$AGG$` is replaced with the data field name: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries format="$VAL$_$AGG$" url response in ("200", "404") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+--------------+--------------+ +| url | 200_host_cnt | 404_host_cnt | +|--------+--------------+--------------| +| /page1 | 3 | 1 | +| /page2 | 5 | null | ++--------+--------------+--------------+ +``` + +## Example 5: Partial pivot values + +When only a subset of values is specified in the `in` clause, rows with unmatched `` values produce `null` for the corresponding `` rows: + +```ppl +source=accounts +| stats avg(balance) as avg_balance by gender, state +| xyseries state gender in ("F") avg_balance +``` + +The query returns the following results: + +```text +fetched rows / total rows = 7/7 ++-------+-----------------+ +| state | avg_balance: F | +|-------+-----------------| +| IL | null | +| IN | 48086.0 | +| MD | null | +| PA | 40540.0 | +| TN | null | +| VA | 32838.0 | +| WA | null | ++-------+-----------------+ +``` + +## Limitations + +The `xyseries` command has the following limitations: + +* Pivot values must be explicitly provided in the `in` clause. Dynamic pivot (deriving column names from data at runtime) is not supported. +* This command is only available when the Calcite engine is enabled. diff --git a/docs/user/ppl/functions/expressions.md b/docs/user/ppl/functions/expressions.md index 427a0334b58..14d9606a0c2 100644 --- a/docs/user/ppl/functions/expressions.md +++ b/docs/user/ppl/functions/expressions.md @@ -13,7 +13,14 @@ Arithmetic expressions are formed by combining numeric literals and binary arith ### Overflow behavior -Integer and long arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. For example, `eval x = int_field + 1` where `int_field` is `2147483647` (integer max) returns an error rather than `-2147483648`. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. +Long (`BIGINT`) arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. Narrower integer operands are widened before arithmetic, so crossing the 32-bit integer boundary does not overflow. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. + +The accumulator used by `stats sum(integral_field)` depends on whether Calcite pushdown is enabled: + +- With `plugins.calcite.pushdown.enabled=true` (the default), OpenSearch uses its native double-based `sum`, then the result is checked and narrowed to `BIGINT`. Large in-range sums can lose low-order precision. A small overflow that rounds to a signed `BIGINT` boundary can also be indistinguishable from an in-range sum and may saturate at the boundary instead of returning an error. +- With `plugins.calcite.pushdown.enabled=false`, Calcite uses an exact `BIGINT` accumulator and `Math.addExact` for every addition. It returns an error as soon as the running sum exceeds the `BIGINT` range. + +For example, summing `4611686018427387904` (`2^62`) and `1` returns the exact `4611686018427387905` without pushdown. With pushdown enabled, the double accumulator cannot represent the low-order `1`, so the result is `4611686018427387904`. ### Precedence @@ -189,4 +196,3 @@ fetched rows / total rows = 2/2 | 28 | +-----+ ``` - \ No newline at end of file diff --git a/docs/user/ppl/general/datatypes.md b/docs/user/ppl/general/datatypes.md index 85c24bd7dec..b2f38418ea8 100644 --- a/docs/user/ppl/general/datatypes.md +++ b/docs/user/ppl/general/datatypes.md @@ -42,6 +42,7 @@ The table below list the mapping between OpenSearch Data Type, PPL Data Type and | scaled_float | float | DOUBLE | | double | double | DOUBLE | | keyword | string | VARCHAR | +| constant_keyword | string | VARCHAR | | text | string | VARCHAR | | match_only_text | string | VARCHAR | | date | timestamp | TIMESTAMP | diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 939684c0ecc..3e5a08d990d 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -43,6 +43,7 @@ source=accounts | [fields command](cmd/fields.md) | 1.0 | stable (since 1.0) | Keep or remove fields from the search result. | | [rename command](cmd/rename.md) | 1.0 | stable (since 1.0) | Rename one or more fields in the search result. | | [eval command](cmd/eval.md) | 1.0 | stable (since 1.0) | Evaluate an expression and append the result to the search result. | +| [foreach command](cmd/foreach.md) | 3.8 | experimental (since 3.8) | Run a templated evaluation for each selected field or collection element. | | [convert command](cmd/convert.md) | 3.5 | experimental (since 3.5) | Transform field values to numeric values using specialized conversion functions. | | [replace command](cmd/replace.md) | 3.4 | experimental (since 3.4) | Replace text in one or more fields in the search result | | [fillnull command](cmd/fillnull.md) | 3.0 | experimental (since 3.0) | Fill null with provided value in one or more fields in the search result. | @@ -74,13 +75,14 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | -| [rest command](cmd/rest.md) | 3.8 | experimental (since 3.8) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | +| [rest command](cmd/rest.md) | 3.9 | experimental (since 3.9) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | | [describe command](cmd/describe.md) | 2.1 | stable (since 2.1) | Query the metadata of an index. | | [explain command](cmd/explain.md) | 3.1 | stable (since 3.1) | Explain the plan of query. | | [show datasources command](cmd/showdatasources.md) | 2.4 | stable (since 2.4) | Query datasources configured in the PPL engine. | +| [makeresults command](cmd/makeresults.md) | 3.8 | experimental (since 3.8) | Generate in-memory rows for testing and seeding, optionally from inline CSV/JSON data. | | [addtotals command](cmd/addtotals.md) | 3.5 | stable (since 3.5) | Adds row and column values and appends a totals column and row. | | [addcoltotals command](cmd/addcoltotals.md) | 3.5 | stable (since 3.5) | Adds column values and appends a totals row. | | [transpose command](cmd/transpose.md) | 3.5 | stable (since 3.5) | Transpose rows to columns. | @@ -88,7 +90,8 @@ source=accounts | [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | | [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | | [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm.| - +| [xyseries command](cmd/xyseries.md) | 3.8 | experimental (since 3.8) | Converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. | + - [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting * **Functions** - [Aggregation Functions](functions/aggregations.md) diff --git a/docs/user/ppl/interfaces/endpoint.md b/docs/user/ppl/interfaces/endpoint.md index 9360d5198ca..6704cc0cd48 100644 --- a/docs/user/ppl/interfaces/endpoint.md +++ b/docs/user/ppl/interfaces/endpoint.md @@ -152,8 +152,122 @@ calcite: physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, state_country]], PushDownContext=[[PROJECT->[name, country, state, month, year, age], FILTER->>($5, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["name","country","state","month","year","age"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) ``` +## Analyze (Experimental) -## Profile (Experimental) +You can enable analysis on the PPL endpoint to capture query execution details including per-stage timings, logical and physical plans, operator tree with pushdown visibility, and optimization recommendations. Analysis is returned only for regular query execution (not explain) and only when using the default `format=jdbc`. + +### Example + +```bash ppl ignore +curl -sS -H 'Content-Type: application/json' \ + -X POST localhost:9200/_plugins/_ppl \ + -d '{ + "query": "source=accounts | where age < 30 | eval full_name = firstname + \" \" + lastname | fields full_name, email, age", + "analyze": true + }' +``` + +Expected output (trimmed): + +```json +{ + "query": "source=accounts | where age < 30 | eval full_name = firstname + \" \" + lastname | fields full_name, email, age", + "querySegments": [ + {"nodeType": "SearchFrom", "source": "source=accounts"}, + {"nodeType": "WhereCommand", "source": "where age < 30"}, + ], + "logicalPlan": [ + "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 5000.0, cumulative cost = {114000.0 rows, 145000.0 cpu, 0.0 io}, id = 4229", + "LogicalProject(full_name=[||(||($0, ' '), $4)], email=[$3], age=[$2]): rowcount = 5000.0, cumulative cost = {109000.0 rows, 25000.0 cpu, 0.0 io}, id = 4228", + "LogicalFilter(condition=[<($2, 30)]): rowcount = 5000.0, cumulative cost = {104000.0 rows, 10000.0 cpu, 0.0 io}, id = 4226", + "CalciteLogicalIndexScan(table=[[OpenSearch, accounts]]): rowcount = 10000.0, cumulative cost = {99000.0 rows, 0.0 cpu, 0.0 io}, id = 4225" + ], + "physicalPlan": [ + "EnumerableCalc(expr#0..3=[{inputs}], expr#4=[' '], expr#5=[||($t0, $t4)], expr#6=[||($t5, $t3)], full_name=[$t6], email=[$t2], age=[$t1]): rowcount = 5000.0, cumulative cost = {22996.4 rows, 50000.0 cpu, 0.0 io}, id = 4319", + "CalciteEnumerableIndexScan(table=[[OpenSearch, accounts]], PushDownContext=[[PROJECT->[firstname, age, email, lastname], FILTER-><($1, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{ + ], + "profile": { + "summary": { + "total_time_ms": 37.13 + }, + "phases": { + "analyze": { "time_ms": 7.06 }, + "optimize": { "time_ms": 25.29 }, + "execute": { "time_ms": 4.73 }, + "format": { "time_ms": 0.03 } + }, + "plan": { + "node": "EnumerableCalc", + "time_ms": 3.44, + "rows": 3, + "children": [ + { "node": "CalciteEnumerableIndexScan", "time_ms": 3.31, "rows": 3 } + ] + } + }, + "operator_tree": [ + { + "source": "source=accounts | where age < 30", + "node_type": [ + "SearchFrom", + "WhereCommand" + ], + "description": [ + "CalciteLogicalIndexScan(table=[[OpenSearch, accounts]]): rowcount = 10000.0, cumulative cost = {99000.0 rows, 0.0 cpu, 0.0 io}, id = 4225", + "LogicalFilter(condition=[<($2, 30)]): rowcount = 5000.0, cumulative cost = {104000.0 rows, 10000.0 cpu, 0.0 io}, id = 4226" + ], + "estimated_rows": 5000, + "actual_time_ms": "3.31 ms", + "actual_rows": 3, + "is_pushed_down": true + }, + ], + "recommendations": [] +} +``` + +### Response fields + +| Field | Type | Description | +|-------|------|-------------| +| `query` | String | The original PPL query. | +| `querySegments` | Array | Breakdown of the query into AST segments with `nodeType` and `source`. | +| `logicalPlan` | Array | Calcite logical plan nodes (top-down). | +| `physicalPlan` | Array | Calcite physical plan nodes after optimization. | +| `operator_tree` | Array | Per-stage execution details linking query segments to plan operators. | +| `recommendations` | Array | Optimization suggestions generated from the execution profile. | +| `profile` | Object | Per-phase timing breakdown (same format as the profile endpoint). | +| `schema` | Array | Column names and types of the query result. | +| `datarows` | Array | Query result rows. | +| `total` | Integer | Total number of result rows. | +| `size` | Integer | Number of result rows returned. | + +### Operator tree fields + +| Field | Type | Description | +|-------|------|-------------| +| `source` | String | The PPL query fragment(s) that produced this operator. | +| `node_type` | Array | AST node type(s) (e.g. `Relation`, `Filter`, `Project`). | +| `description` | Array | Logical plan node descriptions. | +| `estimated_rows` | Long | Estimated row count from Calcite metadata. | +| `actual_time_ms` | String | Exclusive wall-clock time for this operator. | +| `actual_rows` | Long | Actual rows produced by this operator. | +| `is_pushed_down` | Boolean | Whether the operator was pushed down to the storage engine. | + + + +### Notes +- Analyze output is only returned when the query finishes successfully. +- Analyze requires the Calcite engine to be enabled (`plugins.calcite.enabled=true`). +- Operator tree nodes with `is_pushed_down: true` were executed within the OpenSearch storage engine (single network round-trip). Remaining operators ran in-memory on the coordinating node. +- This endpoint is meant to replace/override the existing `profile` endpoint. As a result, any POST requests with either `"analyze": true` or `"profile": true` (or both) will be routed to this endpoint. + - The `profile` section uses the same format as the previous `profile` endpoint. This means current consumers of `profile` should not face any breaking changes. +- The logic for `analyze` doesn't hold for queries that produce non-linear physical plan trees (for example, JOINs). In this scenario, `analyze` will return an output identical to the previous `profile` endpoint. + + +## Profile (Experimental) (Deprecated) + +**This endpoint is outdated, see the `analyze` section above.** You can enable profiling on the PPL endpoint to capture per-stage timings in milliseconds. Profiling is returned only for regular query execution (not explain) and only when using the default `format=jdbc`. @@ -189,7 +303,8 @@ Expected output (trimmed): "children": [ { "node": "CalciteEnumerableIndexScan", "time_ms": 4.12, "rows": 2 } ] - } + }, + "thread_pool": "sql-worker" } } ``` @@ -201,6 +316,7 @@ Expected output (trimmed): - Plan node names use Calcite physical operator names (for example, `EnumerableCalc` or `CalciteEnumerableIndexScan`). - Plan `time_ms` is inclusive of child operators and represents wall-clock time; overlapping work can make summed plan times exceed `summary.total_time_ms`. - Scan nodes reflect operator wall-clock time; background prefetch can make scan time smaller than total request latency. +- `thread_pool` indicates which thread pool executed the query. Possible values are `sql-worker` (default, pushdown-only queries) and `sql-complex-worker` (queries requiring in-memory evaluation such as scripted fields). ## Highlight diff --git a/doctest/build.gradle b/doctest/build.gradle index 1ac658457dc..45f4b768aef 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,11 +205,8 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' - // The rest command is disabled by default (empty allow-list). Opt the doctest cluster into - // the registered endpoints so the rest.md examples run against an enabled command. A literal - // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' + // Only /_cluster/health is registered; pin the allow-list to it so the rest.md examples run. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index c18fa6e37f6..4676f594042 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -168,6 +168,28 @@ tasks.withType(licenseHeaders.class) { additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")' } +// Forward the PPL lint rule validation contract knobs to every integ test JVM +// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), +// whether to observe rather than assert (the multi-version matrix), an optional +// path to write the observed backend report, and — while the cluster is alive — +// optional paths to export the candidate runtime grammar bundle and its target +// manifest for the detector-validation job. Applied globally so every +// RestIntegTestTask that runs the class picks it up without per-task edits. +// +// Forwarded by PREFIX rather than by an explicit list. A hand-maintained list is +// a silent trap: a property the IT reads but the list omits simply never reaches +// the test JVM, with no error anywhere. That already cost one CI run — +// `ppl.lint.observe.only` was added to the IT and the workflow but not to the +// list, so a multi-version leg asserted expectations pinned for a DIFFERENT +// engine version and failed instead of observing. Forwarding every `ppl.lint.*` +// property the invoker set means adding a knob to the IT is enough. +tasks.withType(Test).configureEach { + systemProperty "ppl.lint.schedule", System.getProperty("ppl.lint.schedule", "pr") + System.properties.stringPropertyNames() + .findAll { it.startsWith("ppl.lint.") && it != "ppl.lint.schedule" } + .each { prop -> systemProperty prop, System.getProperty(prop) } +} + validateNebulaPom.enabled = false loggerUsageCheck.enabled = false @@ -272,9 +294,13 @@ def getGeoSpatialPlugin() { } } -// fetch from the feature-build artifact for now (linux/x64 only; for local dev pass -PanalyticsEngineZip=/path instead). +// Fetch from the mutable feature-build artifact for observation (linux/x64 only). CI can +// select a specific build with -PanalyticsFeatureBuildBase, and local development can pass +// individual plugin ZIP overrides such as -PanalyticsEngineZip=/path. ext.pluginVersion = opensearch_version.tokenize('-')[0] -ext.featureBuildBase = "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.featureBuildBase = project.findProperty('analyticsFeatureBuildBase') ?: + "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.featureBuildArtifactRoot = featureBuildBase.replaceFirst('/plugins/?$', '') ext.analyticsEngineZipDest = "${buildDir}/distributions/analytics-engine-${pluginVersion}-SNAPSHOT.zip" ext.arrowFlightRpcZipDest = "${buildDir}/distributions/arrow-flight-rpc-${pluginVersion}-SNAPSHOT.zip" ext.arrowBaseZipDest = "${buildDir}/distributions/arrow-base-${pluginVersion}-SNAPSHOT.zip" @@ -283,6 +309,16 @@ ext.analyticsBackendLuceneZipDest = "${buildDir}/distributions/analytics-backend ext.parquetDataFormatZipDest = "${buildDir}/distributions/parquet-data-format-${pluginVersion}-SNAPSHOT.zip" ext.compositeEngineZipDest = "${buildDir}/distributions/composite-engine-${pluginVersion}-SNAPSHOT.zip" ext.analyticsBackendDatafusionZipDest = "${buildDir}/distributions/analytics-backend-datafusion-${pluginVersion}-SNAPSHOT.zip" +ext.analyticsNativeLibUrl = project.findProperty('analyticsNativeLibUrl') ?: + "${featureBuildArtifactRoot}/dist/libopensearch_native.so" +ext.analyticsNativeLibDest = "${buildDir}/native/${pluginVersion}/release/libopensearch_native.so" +ext.analyticsNativeLibDir = project.findProperty('nativeLibPath') ? + rootProject.file(project.findProperty('nativeLibPath')).canonicalFile : + file(analyticsNativeLibDest).parentFile.canonicalFile +ext.analyticsJavaLibraryPath = [ + analyticsNativeLibDir.absolutePath, + System.getProperty('java.library.path') +].findAll { it != null && !it.isEmpty() }.join(File.pathSeparator) task downloadAnalyticsEngineZip(type: Download) { src "${featureBuildBase}/1-analytics-engine-${pluginVersion}.zip" @@ -317,7 +353,7 @@ task downloadTestPplFrontendZip(type: Download) { } task downloadAnalyticsBackendLuceneZip(type: Download) { - src "${featureBuildBase}/1-analytics-backend-lucene-${pluginVersion}.zip" + src "${featureBuildBase}/analytics-backend-lucene-${pluginVersion}.zip" dest analyticsBackendLuceneZipDest overwrite false onlyIfModified true @@ -325,7 +361,7 @@ task downloadAnalyticsBackendLuceneZip(type: Download) { } task downloadParquetDataFormatZip(type: Download) { - src "${featureBuildBase}/1-parquet-data-format-${pluginVersion}.zip" + src "${featureBuildBase}/parquet-data-format-${pluginVersion}.zip" dest parquetDataFormatZipDest overwrite false onlyIfModified true @@ -333,7 +369,7 @@ task downloadParquetDataFormatZip(type: Download) { } task downloadCompositeEngineZip(type: Download) { - src "${featureBuildBase}/1-composite-engine-${pluginVersion}.zip" + src "${featureBuildBase}/2-composite-engine-${pluginVersion}.zip" dest compositeEngineZipDest overwrite false onlyIfModified true @@ -341,13 +377,57 @@ task downloadCompositeEngineZip(type: Download) { } task downloadAnalyticsBackendDatafusionZip(type: Download) { - src "${featureBuildBase}/1-analytics-backend-datafusion-${pluginVersion}.zip" + src "${featureBuildBase}/analytics-backend-datafusion-${pluginVersion}.zip" dest analyticsBackendDatafusionZipDest overwrite false onlyIfModified true onlyIf { !project.findProperty('analyticsBackendDatafusionZip') } } +task downloadAnalyticsNativeLib(type: Download) { + src analyticsNativeLibUrl + dest analyticsNativeLibDest + // The mutable observation URL can publish another build under the same + // product version. Revalidate an existing file and never expose a partial + // download to the test cluster. + overwrite true + onlyIfModified true + tempAndMove true + retries 3 + onlyIf { !project.findProperty('nativeLibPath') } + doFirst { + def osName = System.getProperty('os.name', '').toLowerCase(Locale.ROOT) + def osArch = System.getProperty('os.arch', '').toLowerCase(Locale.ROOT) + if (!osName.contains('linux') || !(osArch in ['amd64', 'x86_64'])) { + throw new GradleException( + "The default analytics native artifact is Linux/x64 only " + + "(detected ${osName}/${osArch}); pass -PnativeLibPath=.") + } + } +} + +task validateAnalyticsNativeLib { + dependsOn downloadAnalyticsNativeLib + doLast { + File nativeLib = new File(analyticsNativeLibDir, 'libopensearch_native.so') + if (!nativeLib.isFile() || !nativeLib.canRead() || nativeLib.length() == 0) { + throw new GradleException( + "Expected a readable non-empty native library at ${nativeLib}. " + + "Pass -PnativeLibPath= " + + "or -PanalyticsNativeLibUrl=.") + } + byte[] magic = new byte[4] + int bytesRead + nativeLib.withInputStream { stream -> bytesRead = stream.read(magic) } + if (bytesRead != magic.length || + (magic[0] & 0xff) != 0x7f || (magic[1] & 0xff) != 0x45 || + (magic[2] & 0xff) != 0x4c || (magic[3] & 0xff) != 0x46) { + throw new GradleException( + "Analytics native library ${nativeLib} is not an ELF shared object.") + } + } +} + def getAnalyticsEnginePlugin() { provider { (RegularFile) (() -> file(project.findProperty('analyticsEngineZip') ?: analyticsEngineZipDest)) } } @@ -387,11 +467,8 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" - // The rest command is disabled by default (empty allow-list). Opt this cluster into the - // registered endpoints so the rest integration tests exercise the enabled path. A literal - // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' + // Only /_cluster/health is registered; pin the allow-list to it for the rest ITs. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } yamlRestTest { testDistribution = 'archive' @@ -410,9 +487,8 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" - // Opt into the rest endpoints (disabled by default) so RestCommandSecurityIT runs. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' + // Only /_cluster/health is registered; pin the allow-list to it for RestCommandSecurityIT. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } remoteIntegTestWithSecurity { testDistribution = 'archive' @@ -430,6 +506,34 @@ testClusters { // Composite-default cluster: PPL queries route to the analytics engine unless excluded. setting 'cluster.pluggable.dataformat', 'composite' } + analyticsEnginePplLintIT { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin(getArrowBasePlugin()) + plugin(getArrowFlightRpcPlugin()) + plugin(getAnalyticsEnginePlugin()) + plugin(getCompositeEnginePlugin()) + plugin(getParquetDataFormatPlugin()) + plugin(getAnalyticsBackendLucenePlugin()) + plugin(getAnalyticsBackendDatafusionPlugin()) + plugin ":opensearch-sql-plugin" + setting 'cluster.pluggable.dataformat.enabled', 'true' + setting 'cluster.pluggable.dataformat', 'composite' + setting 'cluster.composite.primary_data_format', 'parquet' + setting 'cluster.composite.secondary_data_formats', '[lucene]' + // Arrow Flight / streaming transport requirements + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' + systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true' + // DataFusion/parquet loads libopensearch_native.so at cluster startup. Use the + // matching feature-build artifact unless a local release directory is supplied. + systemProperty 'java.library.path', analyticsJavaLibraryPath + } } def isPrometheusRunning() { @@ -489,6 +593,26 @@ task analyticsEngineCompatIT(type: RestIntegTestTask) { } } +task analyticsEnginePplLintIT(type: RestIntegTestTask) { + useCluster testClusters.analyticsEnginePplLintIT + dependsOn downloadArrowBaseZip, downloadArrowFlightRpcZip, downloadAnalyticsEngineZip, + downloadCompositeEngineZip, downloadParquetDataFormatZip, + downloadAnalyticsBackendLuceneZip, downloadAnalyticsBackendDatafusionZip, + validateAnalyticsNativeLib + dependsOn ':opensearch-sql-plugin:bundlePlugin' + + systemProperty 'tests.analytics.parquet_indices', 'true' + systemProperty 'tests.analytics.num_shards', '1' + systemProperty 'ppl.lint.execution_backend', 'analytics' + systemProperty 'ppl.lint.analytics.stack.source', featureBuildBase + systemProperty 'tests.security.manager', 'false' + systemProperty 'project.root', project.projectDir.absolutePath + + filter { + includeTestsMatching 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' + } +} + task analyticsEngineSecurityIT(type: RestIntegTestTask) { dependsOn downloadAnalyticsEngineZip, downloadArrowFlightRpcZip, downloadArrowBaseZip, downloadAnalyticsBackendLuceneZip, downloadParquetDataFormatZip, downloadCompositeEngineZip, downloadAnalyticsBackendDatafusionZip dependsOn ':opensearch-sql-plugin:bundlePlugin' diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteComplexPoolIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteComplexPoolIT.java new file mode 100644 index 00000000000..b419c8043d3 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteComplexPoolIT.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import java.util.Locale; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for queries dispatched to the complex worker pool. Verifies that queries + * containing scripts (e.g., parse command) are correctly routed and that profile responses include + * thread_pool metadata. + */ +public class CalciteComplexPoolIT extends PPLIntegTestCase { + + @Override + protected void init() throws Exception { + super.init(); + loadIndex(Index.BANK); + enableCalcite(); + enableComplexPool(); + } + + private void enableComplexPool() throws IOException { + updateClusterSettings( + new ClusterSetting(PERSISTENT, "plugins.sql.complex_worker_pool.enabled", "true")); + } + + @Test + public void testParseCommandDispatchesToComplexPool() throws IOException { + // parse creates script nodes, triggering complex pool dispatch + JSONObject result = + executeQuery( + String.format( + "source=%s | parse address '(?\\\\d+) (?.*)'" + + " | fields number, street | head 1", + TEST_INDEX_BANK)); + + verifyDataRows(result, rows("880", "Holmes Lane")); + } + + @Test + public void testComplexPoolProfileIncludesThreadPool() throws IOException { + // Query with parse to trigger complex pool + String query = + String.format( + "source=%s | parse address '(?\\\\d+)' | fields num | head 1", TEST_INDEX_BANK); + + JSONObject result = executeQueryWithProfile(query); + + assertTrue("Response has profile", result.has("profile")); + JSONObject profile = result.getJSONObject("profile"); + + assertTrue("Profile has thread_pool", profile.has("thread_pool")); + String threadPool = profile.getString("thread_pool"); + assertEquals("Thread pool is sql-complex-worker", "sql-complex-worker", threadPool); + + // Verify profile structure is intact + assertTrue("Profile has summary", profile.has("summary")); + assertTrue("Profile has phases", profile.has("phases")); + } + + @Test + public void testSimpleQueryUsesWorkerPool() throws IOException { + // Query without scripts — should use sql-worker pool + String query = String.format("source=%s | fields account_number | head 1", TEST_INDEX_BANK); + + JSONObject result = executeQueryWithProfile(query); + + assertTrue("Response has profile", result.has("profile")); + JSONObject profile = result.getJSONObject("profile"); + + assertTrue("Profile has thread_pool", profile.has("thread_pool")); + String threadPool = profile.getString("thread_pool"); + assertEquals("Thread pool is sql-worker", "sql-worker", threadPool); + } + + private JSONObject executeQueryWithProfile(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"%s\", \"profile\": true}", query)); + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response, true)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 801d56fd49d..e7c024e49fa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -38,6 +38,7 @@ CalciteExpandCommandIT.class, CalciteFieldFormatCommandIT.class, CalciteForeachCommandIT.class, + ForeachFieldJsonIT.class, CalciteFieldsCommandIT.class, CalciteFillNullCommandIT.class, CalciteFlattenCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyzeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyzeIT.java new file mode 100644 index 00000000000..df2e1b88e9a --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyzeIT.java @@ -0,0 +1,350 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; + +import java.io.IOException; +import java.util.Locale; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteAnalyzeIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.ACCOUNT); + } + + // === Helper === + + private JSONObject executeAnalyze(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"%s\", \"analyze\": true}", query)); + RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); + opts.addHeader("Content-Type", "application/json"); + request.setOptions(opts); + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response, true)); + } + + private JSONObject executeProfile(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"%s\", \"profile\": true}", query)); + RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); + opts.addHeader("Content-Type", "application/json"); + request.setOptions(opts); + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response, true)); + } + + // === A. Query result correctness === + + @Test + public void analyzeResultsMatchNormalExecution() throws IOException { + String query = "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname, age"; + JSONObject normal = executeQuery(query); + JSONObject analyzed = executeAnalyze(query); + + // Schema should match + assertEquals(normal.getJSONArray("schema").length(), analyzed.getJSONArray("schema").length()); + // Row counts should match + assertEquals(normal.getInt("total"), analyzed.getInt("total")); + assertEquals(normal.getInt("size"), analyzed.getInt("size")); + // Datarows should have same length + assertEquals( + normal.getJSONArray("datarows").length(), analyzed.getJSONArray("datarows").length()); + } + + @Test + public void analyzeResultsMatchWithAggregation() throws IOException { + String query = "source=" + TEST_INDEX_ACCOUNT + " | stats count() by gender"; + JSONObject normal = executeQuery(query); + JSONObject analyzed = executeAnalyze(query); + + assertEquals(normal.getInt("total"), analyzed.getInt("total")); + assertEquals( + normal.getJSONArray("datarows").length(), analyzed.getJSONArray("datarows").length()); + } + + // === B. Operator tree — all pushed down === + + @Test + public void operatorTreeAllPushedDown() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname, age"); + JSONArray tree = result.getJSONArray("operator_tree"); + + // Single physical node → all segments merged into one entry + assertEquals(1, tree.length()); + JSONObject node = tree.getJSONObject(0); + assertTrue(node.getBoolean("is_pushed_down")); + + JSONArray nodeTypes = node.getJSONArray("node_type"); + assertTrue(nodeTypes.toString().contains("SearchFrom")); + assertTrue(nodeTypes.toString().contains("WhereCommand")); + assertTrue(nodeTypes.toString().contains("FieldsCommand")); + } + + @Test + public void operatorTreeAllPushedDownWithStats() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | stats count() by gender"); + JSONArray tree = result.getJSONArray("operator_tree"); + + assertEquals(1, tree.length()); + JSONObject node = tree.getJSONObject(0); + assertTrue(node.getBoolean("is_pushed_down")); + + JSONArray nodeTypes = node.getJSONArray("node_type"); + assertTrue(nodeTypes.toString().contains("SearchFrom")); + assertTrue(nodeTypes.toString().contains("WhereCommand")); + assertTrue(nodeTypes.toString().contains("StatsCommand")); + } + + // === C. Operator tree — partial pushdown === + + @Test + public void operatorTreePartialPushdown() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + + TEST_INDEX_ACCOUNT + + " | where age > 30 | eval name = firstname | fields name, age"); + JSONArray tree = result.getJSONArray("operator_tree"); + + // At least 2 entries: pushed-down group + non-pushed group + assertTrue(tree.length() >= 2); + // First entry should be pushed down + assertTrue(tree.getJSONObject(0).optBoolean("is_pushed_down", false)); + // Last entry should NOT be pushed down + assertFalse(tree.getJSONObject(tree.length() - 1).optBoolean("is_pushed_down", false)); + } + + // === D. Profile structure === + + @Test + public void analyzeIncludesProfileWithAllPhases() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + assertTrue(result.has("profile")); + + JSONObject profile = result.getJSONObject("profile"); + assertTrue(profile.has("summary")); + assertTrue(profile.has("phases")); + assertTrue(profile.has("plan")); + + JSONObject phases = profile.getJSONObject("phases"); + assertTrue(phases.has("analyze")); + assertTrue(phases.has("optimize")); + assertTrue(phases.has("execute")); + assertTrue(phases.has("format")); + + // All phase times should be non-negative + assertTrue(phases.getJSONObject("analyze").getDouble("time_ms") >= 0); + assertTrue(phases.getJSONObject("optimize").getDouble("time_ms") >= 0); + assertTrue(phases.getJSONObject("execute").getDouble("time_ms") >= 0); + assertTrue(phases.getJSONObject("format").getDouble("time_ms") >= 0); + } + + @Test + public void analyzeProfilePlanHasNodeInfo() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + JSONObject plan = result.getJSONObject("profile").getJSONObject("plan"); + + assertTrue(plan.has("node")); + assertTrue(plan.has("time_ms")); + assertTrue(plan.has("rows")); + assertTrue(plan.getDouble("time_ms") >= 0); + assertTrue(plan.getLong("rows") >= 0); + } + + // === E. Timing correctness === + + @Test + public void operatorTreeHasTimings() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + JSONArray tree = result.getJSONArray("operator_tree"); + + for (int i = 0; i < tree.length(); i++) { + JSONObject node = tree.getJSONObject(i); + assertTrue("node " + i + " has actual_time_ms", node.has("actual_time_ms")); + assertTrue("node " + i + " has actual_rows", node.has("actual_rows")); + assertTrue(node.getLong("actual_rows") >= 0); + } + } + + @Test + public void operatorTreeTimingsSumApproximatesPlanRoot() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + + TEST_INDEX_ACCOUNT + + " | where age > 30 | eval x = age * 2 | fields x, firstname"); + JSONArray tree = result.getJSONArray("operator_tree"); + JSONObject profile = result.getJSONObject("profile"); + + double totalOperatorTime = 0; + for (int i = 0; i < tree.length(); i++) { + String timeStr = tree.getJSONObject(i).getString("actual_time_ms"); + totalOperatorTime += Double.parseDouble(timeStr.replace(" ms", "")); + } + double planRootTime = profile.getJSONObject("plan").getDouble("time_ms"); + + // Exclusive times should sum to roughly the root inclusive time. + // Allow generous tolerance for off-spine subtree time not captured. + assertTrue( + "operator times (" + totalOperatorTime + ") roughly match plan root (" + planRootTime + ")", + totalOperatorTime <= planRootTime * 2.0 && totalOperatorTime >= planRootTime * 0.1); + } + + // === F. Estimated rows === + + @Test + public void operatorTreeHasEstimatedRows() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + JSONArray tree = result.getJSONArray("operator_tree"); + + for (int i = 0; i < tree.length(); i++) { + JSONObject node = tree.getJSONObject(i); + assertTrue("node " + i + " has estimated_rows", node.has("estimated_rows")); + assertTrue(node.getLong("estimated_rows") > 0); + } + } + + // === G. Logical and physical plan presence === + + @Test + public void analyzeIncludesLogicalAndPhysicalPlan() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + + assertTrue(result.has("logicalPlan")); + assertTrue(result.has("physicalPlan")); + + JSONArray logicalPlan = result.getJSONArray("logicalPlan"); + JSONArray physicalPlan = result.getJSONArray("physicalPlan"); + + assertTrue(logicalPlan.length() > 0); + assertTrue(physicalPlan.length() > 0); + + // Logical plan should contain known node types + String logicalStr = logicalPlan.toString(); + assertTrue(logicalStr.contains("LogicalFilter") || logicalStr.contains("LogicalProject")); + } + + // === H. Degenerate cases === + + @Test + public void analyzeEmptyResults() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 99999 | fields firstname"); + + assertEquals(0, result.getInt("total")); + assertEquals(0, result.getJSONArray("datarows").length()); + // Profile and operator tree should still be present + assertTrue(result.has("profile")); + assertTrue(result.has("operator_tree")); + assertTrue(result.getJSONArray("operator_tree").length() > 0); + } + + @Test + public void analyzeNonexistentIndexReturnsError() { + assertThrows( + ResponseException.class, () -> executeAnalyze("source=nonexistent_index_xyz | fields a")); + } + + @Test + public void analyzeSyntaxErrorReturnsError() { + assertThrows(ResponseException.class, () -> executeAnalyze("this is not valid ppl")); + } + + // === I. Schema correctness === + + @Test + public void analyzeSchemaMatchesQueryFields() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname, age"); + JSONArray schema = result.getJSONArray("schema"); + + assertEquals(2, schema.length()); + assertEquals("firstname", schema.getJSONObject(0).getString("name")); + assertEquals("age", schema.getJSONObject(1).getString("name")); + } + + // === J. Profile timing similarity to standalone profile endpoint === + + @Test + public void analyzeTimingsInSameOrderOfMagnitudeAsProfile() throws IOException { + String query = "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | stats count() by gender"; + + JSONObject profileResult = executeProfile(query); + JSONObject analyzeResult = executeAnalyze(query); + + double profileTotal = + profileResult.getJSONObject("profile").getJSONObject("summary").getDouble("total_time_ms"); + double analyzeTotal = + analyzeResult.getJSONObject("profile").getJSONObject("summary").getDouble("total_time_ms"); + + // Should be within 5x of each other (generous for CI environments) + assertTrue( + "analyze total (" + analyzeTotal + ") within 5x of profile total (" + profileTotal + ")", + analyzeTotal < profileTotal * 5 && analyzeTotal > profileTotal / 5); + } + + // === K. Pushdown disabled === + + @Test + public void analyzeWithPushdownDisabledShowsNoPushdown() throws IOException { + // Disable pushdown + updateClusterSettings( + new ClusterSetting( + "transient", + org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ENABLED.getKeyValue(), + "false")); + try { + JSONObject result = + executeAnalyze( + "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname, age"); + JSONArray tree = result.getJSONArray("operator_tree"); + + // With pushdown disabled, nothing should be marked as pushed down + // (or it should have multiple nodes since operations stay separate) + if (tree.length() == 1) { + // If still 1 node, it shouldn't be marked pushed_down + assertFalse(tree.getJSONObject(0).optBoolean("is_pushed_down", false)); + } else { + // Multiple nodes means operations weren't merged + assertTrue(tree.length() > 1); + } + } finally { + // Re-enable pushdown + updateClusterSettings( + new ClusterSetting( + "transient", + org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ENABLED.getKeyValue(), + "true")); + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 4ab1d40a5af..d3589fcf138 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -69,7 +69,7 @@ public void init() throws Exception { // Only for Calcite: the rest row source explains as a CalciteScannableCatalogScan. @Test public void explainRestCommand() throws IOException { - String result = explainQueryToString("| rest '/_cluster/health' | fields status"); + String result = explainQueryToString("| rest '/_cluster/health' | fields response"); Assert.assertTrue( "Expected a rest scan node in the explain output, got: " + result, result.contains("CatalogScan")); @@ -2375,9 +2375,11 @@ public void testDedupWithExpr() throws IOException { } @Test - public void testDedupTextTypeNotPushdown() throws IOException { + public void testDedupTextTypePushdown() throws IOException { + // A text field with no .keyword sub-field is aggregated by reading its value from _source via + // a Calcite script; dedup therefore pushes down as a composite terms + top_hits aggregation. enabledOnlyWhenPushdownIsEnabled(); - String expected = loadExpectedPlan("explain_dedup_text_type_no_push.yaml"); + String expected = loadExpectedPlan("explain_dedup_text_type_push.yaml"); assertYamlEqualsIgnoreId( expected, explainQueryYaml(String.format("source=%s | dedup email", TEST_INDEX_BANK))); } @@ -3037,6 +3039,48 @@ public void testHighlightOsdObjectFormatExplain() throws IOException { assertYamlEqualsIgnoreId(expected, result); } + @Test + public void testXyseriesExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testXyseriesMultipleDataFieldsExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance, count() as cnt by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance, cnt", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries_multiple_data_fields.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testXyseriesWithFormatExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance by gender, state | xyseries" + + " format=\"$VAL$_$AGG$\" state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries_with_format.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + @Test public void testExplainConsecutiveSortsAfterAggIssue5125() throws IOException { enabledOnlyWhenPushdownIsEnabled(); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index a2ab93b6599..f043d9b3afc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -20,12 +20,16 @@ import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; +import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; +import static org.opensearch.sql.util.TestUtils.isIndexExist; +import static org.opensearch.sql.util.TestUtils.performRequest; import java.io.IOException; import java.util.Arrays; import java.util.List; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -93,6 +97,173 @@ public void testSumAvg() throws IOException { verifyDataRows(actual, rows(186973)); } + @Test + public void testSumAllIntegralTypes() throws IOException { + String stats = + "stats sum(byte_number), sum(short_number), sum(integer_number), sum(long_number)"; + String query = String.format("source=%s | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + + JSONObject actual = executeQuery(query); + verifySchema( + actual, + schema("sum(byte_number)", "bigint"), + schema("sum(short_number)", "bigint"), + schema("sum(integer_number)", "bigint"), + schema("sum(long_number)", "bigint")); + verifyDataRows(actual, rows(4L, 3L, 2L, 1L)); + + String explain = explainQueryToString(query); + assertAllIntegralSumsAreChecked(explain); + + // HEAD prevents aggregation pushdown while preserving each field's integral input type. + String fallbackQuery = + String.format("source=%s | head 1 | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + verifyDataRows(executeQuery(fallbackQuery), rows(4L, 3L, 2L, 1L)); + + String fallbackExplain = explainQueryToString(fallbackQuery); + assertTrue(fallbackExplain.contains("EnumerableAggregate")); + assertAllIntegralSumsAreChecked(fallbackExplain); + } + + private static void assertAllIntegralSumsAreChecked(String explain) { + assertTrue(explain.contains("sum(byte_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(short_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(integer_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(long_number)=[CHECKED_LONG_SUM(")); + } + + @Test + public void testSumAvgLongOverflow() throws IOException { + String overflowIndex = "test_sum_long_overflow"; + createLongIndex(overflowIndex, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); + String inRangeIndex = "test_sum_long_in_range"; + createLongIndex(inRangeIndex, 1000000000000L, 2000000000000L, 3000000000000L); + String boundaryIndex = "test_sum_long_boundary"; + createLongIndex(boundaryIndex, Long.MAX_VALUE); + String exactIndex = "test_sum_long_exact"; + createLongIndex(exactIndex, 4611686018427387904L, 1L); + + // SUM overflows the BIGINT range (3 * (2^63 - 1)); surfaced as a client error rather than + // silently wrapping to a negative value. + assertSumOverflow(String.format("source=%s | stats sum(v)", overflowIndex)); + + // HEAD forces enumerable execution even when global pushdown is enabled. + assertSumOverflow(String.format("source=%s | head 3 | stats sum(v)", overflowIndex)); + + // AVG is averaged in DOUBLE, so it holds the true average (the shared value) without wrapping. + JSONObject avg = executeQuery(String.format("source=%s | stats avg(v)", overflowIndex)); + verifySchema(avg, schema("avg(v)", "double")); + verifyDataRows(avg, rows(9.223372036854776e18)); + + JSONObject fallbackAvg = + executeQuery(String.format("source=%s | head 3 | stats avg(v)", overflowIndex)); + verifySchema(fallbackAvg, schema("avg(v)", "double")); + verifyDataRows(fallbackAvg, rows(9.223372036854776e18)); + + JSONObject expressionAvg = + executeQuery(String.format("source=%s | head 3 | stats avg(v + 0)", overflowIndex)); + verifySchema(expressionAvg, schema("avg(v + 0)", "double")); + verifyDataRows(expressionAvg, rows(9.223372036854776e18)); + + String pushedExpressionQuery = String.format("source=%s | stats avg(v + 0)", overflowIndex); + JSONObject pushedExpressionAvg = executeQuery(pushedExpressionQuery); + verifySchema(pushedExpressionAvg, schema("avg(v + 0)", "double")); + verifyDataRows(pushedExpressionAvg, rows(9.223372036854776e18)); + if (!isPushdownDisabled() && !isAnalyticsParquetIndicesEnabled()) { + assertTrue(explainQueryToString(pushedExpressionQuery).contains("AGGREGATION->")); + } + + // A sum well within the BIGINT range returns the exact value with no error. + JSONObject inRange = executeQuery(String.format("source=%s | stats sum(v)", inRangeIndex)); + verifySchema(inRange, schema("sum(v)", "bigint")); + verifyDataRows(inRange, rows(6000000000000L)); + + // A single Long.MAX_VALUE is a valid (non-overflowing) sum and must not error. + JSONObject boundary = executeQuery(String.format("source=%s | stats sum(v)", boundaryIndex)); + verifySchema(boundary, schema("sum(v)", "bigint")); + verifyDataRows(boundary, rows(9223372036854775807L)); + + // Native sum pushdown uses double and loses the low-order bit; the fallback and analytics + // backends retain it. + JSONObject exact = executeQuery(String.format("source=%s | stats sum(v)", exactIndex)); + verifySchema(exact, schema("sum(v)", "bigint")); + long expectedExact = + (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) + ? 4611686018427387905L + : 4611686018427387904L; + verifyDataRows(exact, rows(expectedExact)); + + // HEAD prevents pushdown, so the checked long accumulator retains the low-order bit. + JSONObject exactFallback = + executeQuery(String.format("source=%s | head 2 | stats sum(v)", exactIndex)); + verifySchema(exactFallback, schema("sum(v)", "bigint")); + verifyDataRows(exactFallback, rows(4611686018427387905L)); + } + + @Test + public void testNegativeLongSumOverflowAndBoundary() throws IOException { + String overflowIndex = "test_sum_long_negative_overflow"; + createLongIndex(overflowIndex, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + String boundaryIndex = "test_sum_long_negative_boundary"; + createLongIndex(boundaryIndex, Long.MIN_VALUE); + + assertSumOverflow(String.format("source=%s | stats sum(v)", overflowIndex)); + assertSumOverflow(String.format("source=%s | head 3 | stats sum(v)", overflowIndex)); + + JSONObject boundary = executeQuery(String.format("source=%s | stats sum(v)", boundaryIndex)); + verifySchema(boundary, schema("sum(v)", "bigint")); + verifyDataRows(boundary, rows(Long.MIN_VALUE)); + + JSONObject avg = executeQuery(String.format("source=%s | stats avg(v)", overflowIndex)); + verifySchema(avg, schema("avg(v)", "double")); + verifyDataRows(avg, rows((double) Long.MIN_VALUE)); + } + + @Test + public void testFallbackLongSumRejectsIntermediateOverflow() throws IOException { + String index = "test_sum_long_intermediate_overflow"; + createLongIndex(index, Long.MAX_VALUE, 1L, -1L); + + // The final mathematical result fits, but Math.addExact rejects the intermediate MAX + 1. + assertSumOverflow(String.format("source=%s | sort - v | head 3 | stats sum(v)", index)); + } + + @Test + public void testFloatingSumsDoNotUseCheckedLongAccumulator() throws IOException { + String stats = + "stats sum(double_number), sum(float_number)," + + " sum(half_float_number), sum(scaled_float_number)"; + String query = String.format("source=%s | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + String fallbackQuery = + String.format("source=%s | head 1 | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + + assertEquals(1, executeQuery(query).getInt("total")); + assertFalse(explainQueryToString(query).contains("CHECKED_LONG_SUM")); + assertEquals(1, executeQuery(fallbackQuery).getInt("total")); + assertFalse(explainQueryToString(fallbackQuery).contains("CHECKED_LONG_SUM")); + } + + private void createLongIndex(String index, long... values) throws IOException { + if (isIndexExist(client(), index)) { + return; + } + + createIndexByRestClient( + client(), index, "{\"mappings\":{\"properties\":{\"v\":{\"type\":\"long\"}}}}"); + StringBuilder body = new StringBuilder(); + for (long value : values) { + body.append("{\"index\":{}}\n").append("{\"v\":").append(value).append("}\n"); + } + Request bulk = new Request("POST", "/" + index + "/_bulk?refresh=true"); + bulk.setJsonEntity(body.toString()); + performRequest(client(), bulk); + } + + private void assertSumOverflow(String query) throws IOException { + Throwable error = assertThrowsWithReplace(RuntimeException.class, () -> executeQuery(query)); + verifyErrorMessageContains(error, "verflow"); + } + @Test public void testAsExistedField() throws IOException { JSONObject actual = @@ -993,9 +1164,7 @@ public void testSumGroupByNullValue() throws IOException { String.format( "source=%s | stats sum(balance) as a by age", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("a", null, "bigint"), schema("age", null, "int")); - // SUM of an all-null bucket is null per the SQL spec. The DSL-pushdown path returns 0 instead - // (a known pushdown quirk); the analytics-engine backend (DataFusion) follows the spec like - // Calcite-no-pushdown and returns null. See testSumNull and #3408. + // Native sum returns 0 for an all-null bucket; fallback and analytics backends return null. Object emptySum = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0; verifyDataRows( response, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java index 70ee4ef0de6..ae16f839e9c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -13,14 +13,14 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; -import org.opensearch.client.Request; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; /** - * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code - * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also - * verifies that a non-allow-listed / mutating endpoint is refused. + * Integration tests for the {@code rest} leading command on the Calcite path. This first version + * ships a single built-in endpoint, {@code /_cluster/health} (a deterministic single-row endpoint + * that carries no network identifiers and needs no redaction). These tests exercise it end to end + * and verify the allow-list and per-arg gates. */ public class CalcitePPLRestIT extends PPLIntegTestCase { @@ -32,179 +32,81 @@ public void init() throws Exception { @Test public void testRestClusterHealthSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); - verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); + JSONObject result = executeQuery("| rest '/_cluster/health' | fields response"); + verifySchema(result, schema("response", "string")); } @Test - public void testRestClusterHealthDataRows() throws IOException { - // Single-node test cluster: exactly one node, status is green or yellow. - JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestRejectsNonAllowListedEndpoint() throws IOException { - assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); - } - - @Test - public void testRestRejectsEmptyEndpoint() throws IOException { - assertRestBadRequest("| rest ''", "non-empty path"); - } - - @Test - public void testRestRejectsDisallowedArg() throws IOException { - assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); - } - - @Test - public void testRestRejectsNegativeCount() throws IOException { - assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); - } - - @Test - public void testRestRejectsTimeoutArg() throws IOException { - assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); - } - - /** - * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) - * with the given substring in the response body. Covers allow-list and bad-argument rejection. - */ - private void assertRestBadRequest(String query, String expectedSubstring) { - ResponseException e = - org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); - org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); - org.junit.Assert.assertTrue( - "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), - e.getMessage().contains(expectedSubstring)); - } - - @Test - public void testRestCatIndicesSchema() throws IOException { - // Schema is fixed by the registry, independent of how many indices exist. - JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); - verifySchema(result, schema("index", "string"), schema("health", "string")); - } - - @Test - public void testRestCatIndicesReturnsCreatedIndex() throws IOException { - // Create a known index, then confirm rest surfaces it and downstream where/fields compose. - client().performRequest(new Request("PUT", "/rest_cat_test")); + public void testRestClusterHealthResponseIsValidJson() throws IOException { JSONObject result = - executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); - verifyDataRows(result, rows("rest_cat_test")); - } - - @Test - public void testRestCatNodesSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); - verifySchema(result, schema("name", "string"), schema("cpu", "int")); - } - - @Test - public void testRestCatNodesSingleNode() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatClusterManagerSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); - verifySchema(result, schema("node", "string"), schema("id", "string")); - } - - @Test - public void testRestCatClusterManagerSingleRow() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatPluginsSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); - verifySchema(result, schema("component", "string"), schema("version", "string")); + executeQuery("| rest '/_cluster/health' | eval ok = json_valid(response) | fields ok"); + verifyDataRows(result, rows(true)); } @Test - public void testRestCatShardsSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); - verifySchema( - result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); - } - - @Test - public void testRestClusterStateSchema() throws IOException { - // Assert the string columns; version is the LONG epoch column. + public void testRestClusterHealthJsonExtract() throws IOException { JSONObject result = - executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); - verifySchema( - result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); + executeQuery( + "| rest '/_cluster/health' | eval status = json_extract(response, 'status')" + + " | fields status"); + verifySchema(result, schema("status", "string")); } @Test - public void testRestClusterStateSingleRow() throws IOException { - JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); + public void testRestClusterHealthSpath() throws IOException { + JSONObject result = + executeQuery( + "| rest '/_cluster/health' | spath input=response path=status output=status" + + " | where status = 'green' or status = 'yellow' | stats count() as cnt"); verifyDataRows(result, rows(1)); } @Test - public void testRestClusterSettingsSchema() throws IOException { - // Schema is registry-fixed regardless of how many settings are configured. - JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); - verifySchema( - result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); - } - - @Test - public void testRestResolveIndexSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); - verifySchema(result, schema("name", "string"), schema("type", "string")); - } - - @Test - public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { - client().performRequest(new Request("PUT", "/rest_resolve_test")); - JSONObject result = - executeQuery( - "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); - verifyDataRows(result, rows("rest_resolve_test", "index")); + public void testRestClusterHealthComposesDownstream() throws IOException { + // The rest row source composes with downstream stats exactly like an index scan. + JSONObject result = executeQuery("| rest '/_cluster/health' | stats count() as cnt"); + verifyDataRows(result, rows(1)); } - // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- - @Test public void testRestClusterHealthLocalArg() throws IOException { // local=true reads health from the local node; on a single-node cluster the row is unchanged. JSONObject result = - executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); + executeQuery("| rest '/_cluster/health' local='true' | stats count() as cnt"); verifyDataRows(result, rows(1)); } @Test - public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { - // health filters rows server-side; a healthy cluster has no red indices, so count is 0. - JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); - verifyDataRows(result, rows(0)); + public void testRestRejectsNonAllowListedEndpoint() { + // /_cat/nodes is not registered in this version; it is refused before any transport call. + assertRestBadRequest("| rest '/_cat/nodes'", "allow-list"); } @Test - public void testRestResolveIndexExpandWildcardsArg() throws IOException { - // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. - JSONObject result = - executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); - verifySchema(result, schema("name", "string"), schema("type", "string")); + public void testRestRejectsEmptyEndpoint() { + assertRestBadRequest("| rest ''", "non-empty path"); } @Test - public void testRestRejectsDroppedLevelArg() throws IOException { - // level was dropped from the allow-list (no-op against the fixed health schema). - assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); + public void testRestRejectsDisallowedArg() { + assertRestBadRequest("| rest '/_cluster/health' h='name'", "does not accept arg"); } @Test - public void testRestRejectsBadArgValue() throws IOException { - assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); + public void testRestRejectsNegativeCount() { + assertRestBadRequest("| rest '/_cluster/health' count=-1", "non-negative"); + } + + /** + * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) + * with the given substring in the response body. Covers allow-list and bad-argument rejection. + */ + private void assertRestBadRequest(String query, String expectedSubstring) { + ResponseException e = + org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); + org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertTrue( + "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), + e.getMessage().contains(expectedSubstring)); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java index 73396ab31b9..7bd32bb2af2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java @@ -85,6 +85,26 @@ public void testTimechartWithCustomTimeField() throws IOException { verifyDataRows(result, rows("2017-01-01 00:00:00", 2), rows("2018-01-01 00:00:00", 5)); } + @Test + public void testTimechartAvgByTextHost() throws IOException { + // `host` is mapped as text with no .keyword sub-field, so this timechart pushes down via the + // text-field aggregation path (composite terms(script over _source) + date_histogram, with + // avg(cpu_usage) as a numeric metric on the composite). + JSONObject result = executeQuery("source=events | timechart span=1m avg(cpu_usage) by host"); + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("avg(cpu_usage)", "double")); + verifyDataRows( + result, + rows("2024-07-01 00:00:00", "web-01", 45.2), + rows("2024-07-01 00:01:00", "web-02", 38.7), + rows("2024-07-01 00:02:00", "web-01", 55.3), + rows("2024-07-01 00:03:00", "db-01", 42.1), + rows("2024-07-01 00:04:00", "web-02", 41.8)); + } + @Test public void testTimechartWithMinuteSpanNoGroupBy() throws IOException { JSONObject result = executeQuery("source=events | timechart span=1m avg(cpu_usage)"); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java index e555576a9cd..78cc4cbee73 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java @@ -6,7 +6,9 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; @@ -41,6 +43,43 @@ public void testTopCommandUseNullFalse() throws IOException { verifyNumOfRows(result, 5); } + @Test + public void testTopCommandShowPerc() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | top showperc=true age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder( + result, schema("age", "int"), schema("count", "bigint"), schema("percent", "double")); + verifyNumOfRows(result, 6); + verifyDataRows( + result, + rows(36, 2, 28.571429), + rows(28, 1, 14.285714), + rows(32, 1, 14.285714), + rows(33, 1, 14.285714), + rows(34, 1, 14.285714), + rows(null, 1, 14.285714)); + } + + @Test + public void testTopCommandShowPercWithoutShowCount() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | top showperc=true showcount=false age", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder(result, schema("age", "int"), schema("percent", "double")); + verifyNumOfRows(result, 6); + verifyDataRows( + result, + rows(36, 28.571429), + rows(28, 14.285714), + rows(32, 14.285714), + rows(33, 14.285714), + rows(34, 14.285714), + rows(null, 14.285714)); + } + @Test public void testTopCommandLegacyFalse() throws IOException { withSettings( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java index 2fea0858abd..08243595fd8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java @@ -17,7 +17,7 @@ import org.opensearch.sql.legacy.TestUtils; import org.opensearch.sql.ppl.PPLIntegTestCase; -/** Foreach collection modes over index fields (Splunk-parity scenarios). */ +/** Foreach collection modes over index fields. */ public class ForeachFieldJsonIT extends PPLIntegTestCase { @Override @@ -42,7 +42,6 @@ public void init() throws Exception { @Test public void testJsonArrayModeOnFieldWithNumericContent() throws IOException { - // Splunk: field holding "[10,20,30]" with foreach mode=json_array sums to 60. JSONObject result = executeQuery( "source=test_foreach_field2 | eval total = 0 | foreach mode=json_array jsonfield [" @@ -81,6 +80,16 @@ public void testJsonArrayModeOnFieldWithStringContent() throws IOException { verifyDataRows(result, rows("ab")); } + @Test + public void testJsonArrayFieldSafelyCoercesNonNumericItems() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval total = 0 | foreach mode=json_array jsonstrs [" + + " eval total = total + <> ] | fields total"); + verifySchema(result, schema("total", "double")); + verifyDataRows(result, rows((Object) null)); + } + /** * Native OpenSearch array fields (a long field holding [1,2,3]) are typed as scalar BIGINT at * plan time because OpenSearch mappings do not distinguish scalars from arrays. foreach @@ -111,7 +120,7 @@ public void testNestedFieldMultivalueIterates() throws IOException { verifyDataRows(result, rows(2)); } - /** Splunk silently no-ops when a mode is fed the wrong collection shape. */ + /** Collection modes are no-ops when the input has a different collection shape. */ @Test public void testJsonArrayModeOnRealArrayIsNoOp() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java new file mode 100644 index 00000000000..7410eb32023 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -0,0 +1,1993 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.legacy.TestUtils; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Backend half of the schema-v3/schema-v4 PPL frontend validation contract. + * + *

This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from + * the current checkout. For every contract (see {@code + * src/test/resources/ppl-lint/contracts/*.spec.json}) it selects the single {@code expectations[]} + * entry that matches the candidate backend version (exactly one must match, or the contract fails + * before any query runs), applies the contract's cluster settings, and asserts the oracle selected + * for {@code ppl.lint.execution_backend}: + * + *

+ * + *

The contract files are shared verbatim with the SQL-owned OSD frontend runner ({@code + * scripts/ppl-lint/run-frontend-contract.mjs}) so the same reviewed cases pin both the OSD analyzer + * output and the SQL backend behavior; neither side can drift without a red build. The + * rejection-body parsing mirrors {@link + * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT}; the Calcite setup follows {@link + * org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. + * + *

While the ephemeral cluster is alive, the test also exports the candidate runtime grammar + * bundle it built ({@code GET /_plugins/_ppl/_grammar}) and a small target manifest pairing the + * bundle with the backend version and grammar hash. These become workflow artifacts that the + * detector-validation job injects into OSD's headless lint API, so both halves validate against the + * SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code + * -Dppl.lint.grammar.bundle} is set (CI); local runs without it are unaffected. + * + *

The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): a PR run skips + * contracts declaring {@code schedule: "nightly"}, while nightly runs all 12 active detector + * contracts. The filter holds new detector contracts back from PR runs while their standard and + * analytics oracles are still settling. + * + *

Note that a contract which RUNS also ASSERTS. This class does not consult the manifest's + * {@code enforced} list — that list records oracle quality and review status, not blocking + * behavior. Adding a contract, or moving one onto the PR schedule, makes it capable of failing the + * required check. + */ +public class PplLintRuleValidationIT extends PPLIntegTestCase { + + private static final String CONTRACT_DIR = "src/test/resources/ppl-lint/contracts"; + private static final String MANIFEST = CONTRACT_DIR + "/manifest.json"; + private static final String GRAMMAR_API_ENDPOINT = "/_plugins/_ppl/_grammar"; + private static final String EXECUTION_BACKEND_PROPERTY = "ppl.lint.execution_backend"; + private static final String ANALYTICS_SHARD_COUNT_PROPERTY = "tests.analytics.num_shards"; + private static final String[] REQUIRED_ANALYTICS_PLUGIN_COMPONENTS = { + "job-scheduler", + "arrow-base", + "arrow-flight-rpc", + "analytics-engine", + "analytics-backend-lucene", + "analytics-backend-datafusion", + "parquet-data-format", + "composite-engine", + "opensearch-sql" + }; + + /** Which contracts to run this session; PR is the fast blocking subset. */ + private final String schedule = System.getProperty("ppl.lint.schedule", "pr"); + + /** Execution route whose backend oracle and artifact identity this run represents. */ + private final ExecutionBackend executionBackend = + ExecutionBackend.parse(System.getProperty(EXECUTION_BACKEND_PROPERTY, "standard")); + + /** + * Observe-only mode, used by the multi-version workflow ({@code + * .github/workflows/ppl-lint-multiversion-validation.yml}). + * + *

The default mode asserts each case against the expectation pinned for the cluster's version, + * which is right when the cluster IS the build under test. The multi-version matrix instead + * points this suite at OLDER released engines, where a disagreement is the very signal being + * collected — a 3.6 engine that accepts what the contract pins as rejected is a finding for the + * drift classifier, not a broken test run. + * + *

So in observe-only mode the suite still runs every query and records the true observed + * behavior in the report, but does not fail on an expectation mismatch, and does not require an + * expectation to exist for this version at all. Failures that mean the RUN itself is broken (no + * grammar bundle, an unreachable cluster, a malformed contract) still fail, because those would + * otherwise produce an empty report that reads as agreement. + */ + private final boolean observeOnly = Boolean.getBoolean("ppl.lint.observe.only"); + + private int[] clusterVersion; + private String engineVersionRaw; + private final JSONObject analyticsRouteAttestation = + new JSONObject() + .put("pluginsVerified", false) + .put("clusterSettingsVerified", false) + .put("fixtureIndicesVerified", false) + .put("explainVerified", false) + .put("profiledExecutionVerified", false); + + /** + * Whether this cluster recognizes the Calcite settings at all. False on a pre-Calcite (2.x) + * engine, where every {@code plugins.calcite.*} write is rejected as "not recognized". + * Established once in {@code init()} and honored by {@code applyClusterSettings}, so the whole + * family is skipped rather than retried and failed once per contract. + */ + private boolean calciteSettingsSupported = true; + + /** + * Index fixtures that could not be created on this engine (observe-only mode only). Contracts + * that need one are reported as {@code outcome: "error"} instead of as engine behavior, because + * an IndexNotFoundException from a missing fixture is not a verdict about the query. + */ + private final Set unseededIndices = new LinkedHashSet<>(); + + @Override + public void init() throws Exception { + // Calcite is a 3.x engine feature: on 2.x the cluster rejects + // `plugins.calcite.enabled` outright with "not recognized", and the base class's + // init() sets it unconditionally. In observe-only mode (the multi-version + // matrix) that must not abort the leg — a pre-Calcite engine is a legitimate + // thing to observe, and the contracts' own `frontendContext.isCalcite` already + // describes what the linter should assume there. + // + // Asserting mode keeps the strict behavior: the required check runs against the + // PR's own build, where a missing Calcite setting is a real problem. + try { + super.init(); + enableCalcite(); + } catch (Exception e) { + if (!observeOnly || !isUnrecognizedCalciteSetting(e)) { + throw e; + } + calciteSettingsSupported = false; + System.err.println( + "[ppl-lint] engine does not support the Calcite settings; observing without them: " + + e.getMessage()); + // super.init() aborted partway, so redo the part that is version-independent. + increaseMaxCompilationsRate(); + } + clusterVersion = fetchClusterVersion(); + + // Fall through to fixture seeding either way. + // Seed the union of every index every scheduled contract needs, once. + for (String indexEnum : requiredIndexEnums()) { + try { + loadIndex(Index.valueOf(indexEnum)); + } catch (Exception e) { + if (!observeOnly) { + throw e; + } + // In the multi-version matrix an older engine may not support a field type + // a fixture uses (a mapping that only exists in a later release). Losing + // that one index must not abort the whole leg — every other rule is still + // validated against this engine. + // + // But it must not be silent either: without the index, every query against + // it fails with IndexNotFoundException, which looks exactly like a real + // engine verdict. Left unmarked, the drift report would advise pinning the + // contract to IndexNotFoundException, or "extending the detector" for a + // control the engine only rejected because its data was missing. Record the + // failure so those cases are reported as unusable rather than as behavior. + unseededIndices.add(indexEnum); + System.err.println( + "[ppl-lint] could not seed index " + indexEnum + " on this engine: " + e.getMessage()); + } + } + } + + @Test + public void testValidatesLintRuleContracts() throws IOException { + List contracts = loadScheduledContracts(); + List failures = new ArrayList<>(); + JSONArray report = new JSONArray(); + if (contracts.isEmpty()) { + failures.add("[contracts] no contracts were selected for schedule \"" + schedule + "\""); + } + + boolean routeAttested = + executionBackend != ExecutionBackend.ANALYTICS || attestAnalyticsRoute(failures); + + // Export the candidate grammar bundle + target manifest while the cluster is + // alive. Runs before the contract loop so the artifacts are emitted even if a + // contract later fails. + exportGrammarArtifacts(failures); + + // A failed route attestation is infrastructure failure, not backend behavior. + // Do not score any contract against a route that was not proven. + if (routeAttested) { + for (JSONObject contract : contracts) { + String ruleId = contract.getString("ruleId"); + runContract(contract, ruleId, failures, report); + } + } else { + recordUnattestedRouteContracts(contracts, report); + } + + try { + writeReport(report); + } catch (IOException e) { + failures.add("[report] failed to write backend report: " + e.getMessage()); + } + + if (!failures.isEmpty()) { + fail( + "PPL lint backend contract failures (" + + failures.size() + + "):\n- " + + String.join("\n- ", failures)); + } + } + + private void runContract( + JSONObject contract, String ruleId, List failures, JSONArray report) + throws IOException { + int schemaVersion = contract.getInt("schemaVersion"); + if (schemaVersion != 3 && schemaVersion != 4) { + failures.add( + "[" + ruleId + "] unsupported schemaVersion " + schemaVersion + " (expected 3 or 4)"); + return; + } + + String index = contract.getString("index"); + JSONObject queries = contract.getJSONObject("queries"); + JSONArray expectations = contract.getJSONArray("expectations"); + JSONObject fixture = contract.optJSONObject("backendFixture"); + boolean calciteOn = fixtureCalciteEnabled(fixture); + + if (expectations.length() == 0) { + failures.add("[" + ruleId + "] expectations must not be empty"); + return; + } + + if (!validateAllExpectations(ruleId, queries, expectations, schemaVersion, failures)) { + return; + } + + List matches = matchingExpectations(expectations, calciteOn); + if (matches.size() > 1) { + failures.add( + "[" + + ruleId + + "] " + + matches.size() + + " expectations match backend version " + + backendVersionLabel() + + " (exactly one required)"); + return; + } + + JSONObject selected = matches.isEmpty() ? null : matches.get(0); + if (selected == null && !observeOnly) { + failures.add( + "[" + + ruleId + + "] no version expectation matches backend version " + + backendVersionLabel()); + return; + } + + JSONObject expectedQueries = selected == null ? null : selected.getJSONObject("queries"); + + // A contract whose fixture index never got created cannot produce a meaningful + // observation: every query would fail with IndexNotFoundException regardless of + // the rule. Report each case as an error so the aggregator counts it as + // inconclusive rather than as the engine's verdict. + String missingIndex = missingFixtureIndex(fixture); + if (missingIndex != null) { + recordUnusableContract(ruleId, index, queries, report, missingIndex); + return; + } + + if (!observeOnly + && recordEnforcementCoverageGaps( + ruleId, index, queries, expectedQueries, schemaVersion, failures, report)) { + return; + } + + List applied = applyClusterSettings(fixture); + try { + if (selected == null) { + // Record the raw behavior of every query and let the aggregator decide + // whether the gap matters (out-of-scope rule vs a real coverage hole). + observeAllQueries(ruleId, index, queries, failures, report); + return; + } + + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject expected = expectedQueries.getJSONObject(queryName); + JSONObject backend = resolveBackendOracle(schemaVersion, expected); + if (backend == null) { + recordMissingOracle(ruleId, queryName, role, query, schemaVersion, failures, report); + continue; + } + + String kind = backend.getString("kind"); + + JSONObject entry = reportEntry(ruleId, queryName, role, query, kind); + if ("not-applicable".equals(kind)) { + recordNotApplicable(ruleId, queryName, backend, entry, report); + continue; + } + + try { + verifyCase(kind, queryName, query, backend, entry); + entry.put("outcome", "pass"); + log(ruleId, queryName, "PASS (" + kind + ", " + role + ")"); + } catch (IOException e) { + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend query transport failed: " + + String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR (" + kind + "): " + e.getMessage()); + } catch (AssertionError | RuntimeException e) { + entry.put("outcome", observeOnly ? "observed-mismatch" : "fail"); + entry.put("error", String.valueOf(e.getMessage())); + if (observeOnly) { + // Not a failure here: the observation is the deliverable, and the + // drift classifier turns it into a remediation. + log(ruleId, queryName, "OBSERVED MISMATCH (" + kind + "): " + e.getMessage()); + } else { + failures.add("[" + ruleId + "/" + queryName + "] " + e.getMessage()); + log(ruleId, queryName, "FAIL (" + kind + "): " + e.getMessage()); + } + } + report.put(entry); + } + } finally { + resetClusterSettings(applied); + } + } + + /** + * Route attestation failure prevents query execution, but it must not produce a misleadingly + * empty report. Emit one non-verdict row per query; explicit, complete non-applicable rows remain + * non-applicable because they do not depend on the unavailable fixture. + */ + private void recordUnattestedRouteContracts(List contracts, JSONArray report) { + for (JSONObject contract : contracts) { + String ruleId = contract.getString("ruleId"); + String index = contract.getString("index"); + JSONObject queries = contract.getJSONObject("queries"); + JSONObject fixture = contract.optJSONObject("backendFixture"); + List matches = + matchingExpectations( + contract.getJSONArray("expectations"), fixtureCalciteEnabled(fixture)); + JSONObject expectedQueries = + matches.size() == 1 ? matches.get(0).optJSONObject("queries") : null; + + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject expected = + expectedQueries == null ? null : expectedQueries.optJSONObject(queryName); + JSONObject backend = null; + int schemaVersion = contract.optInt("schemaVersion"); + if (expected != null && (schemaVersion == 3 || schemaVersion == 4)) { + try { + backend = resolveBackendOracle(schemaVersion, expected); + } catch (RuntimeException ignored) { + // Malformed contracts are still failures; keep this report row as a non-verdict. + } + } + JSONObject entry = reportEntry(ruleId, queryName, role, query, "route-attestation-failed"); + if (isCompleteNotApplicableOracle(backend)) { + entry.put("kind", "not-applicable"); + recordNotApplicable(ruleId, queryName, backend, entry, report); + } else { + report.put( + entry + .put("outcome", "error") + .put("error", "analytics route attestation failed before contract execution")); + } + } + } + } + + /** + * The first index fixture this contract needs that failed to seed, or null when every index it + * declares is present. Only ever non-null in observe-only mode, where a seeding failure is + * tolerated instead of aborting the leg. + */ + private String missingFixtureIndex(JSONObject fixture) { + if (unseededIndices.isEmpty() || fixture == null) { + return null; + } + JSONArray declared = fixture.optJSONArray("indices"); + if (declared == null) { + return unseededIndices.contains("ACCOUNT") ? "ACCOUNT" : null; + } + for (int i = 0; i < declared.length(); i++) { + String name = declared.getString(i); + if (unseededIndices.contains(name)) { + return name; + } + } + return null; + } + + /** + * Record every case of a contract whose fixture index is missing as {@code outcome: "error"}, so + * the multi-version aggregator treats them as inconclusive. Writing nothing at all would be + * worse: absent rows are indistinguishable from a detector that never ran. + */ + private void recordUnusableContract( + String ruleId, String index, JSONObject queries, JSONArray report, String missingIndex) { + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + report.put( + reportEntry(ruleId, queryName, role, query, "observe-only") + .put("outcome", "error") + .put( + "error", + "fixture index " + missingIndex + " could not be created on this engine")); + log(ruleId, queryName, "SKIPPED (fixture index " + missingIndex + " unavailable)"); + } + } + + /** + * Observe-only helper: run every query a contract declares and record what the engine actually + * did, without comparing against any expectation. Used when this engine version has no matching + * {@code expectations[]} entry, so the multi-version report still shows real behavior instead of + * a blank row that would read as agreement. + */ + private void observeAllQueries( + String ruleId, String index, JSONObject queries, List failures, JSONArray report) { + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject entry = reportEntry(ruleId, queryName, role, query, "observe-only"); + try { + BackendObservation obs = observeBackend(query); + entry + .put("rejected", obs.rejected) + .put("observed", obs.toJson()) + .put("outcome", "observed"); + log(ruleId, queryName, "OBSERVED (" + (obs.rejected ? "rejected" : "accepted") + ")"); + } catch (IOException | RuntimeException e) { + // A transport-level problem is a broken run, not an engine verdict; mark it + // so the aggregator does not read the absence of a rejection as acceptance. + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend observation failed: " + + String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR: " + e.getMessage()); + } + report.put(entry); + } + } + + /** + * A missing backend oracle is a coverage result, not an invitation to borrow another backend's + * expectation. Observation executes the query exactly once and records its raw behavior; + * enforcement records the gap without executing or scoring the query. + */ + private void recordMissingOracle( + String ruleId, + String queryName, + String role, + String query, + int schemaVersion, + List failures, + JSONArray report) { + String reason = + schemaVersion == 3 + ? "schema v3 provides only a standard backend oracle" + : "schema v4 has no " + executionBackend.id + " entry in expected query backends"; + JSONObject entry = + reportEntry(ruleId, queryName, role, query, "coverage-missing") + .put("coverage", "missing") + .put("reason", reason); + + if (!observeOnly) { + entry.put("outcome", "coverage-missing"); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] missing " + + executionBackend.id + + " backend oracle: " + + reason); + report.put(entry); + log(ruleId, queryName, "COVERAGE MISSING (" + executionBackend.id + ")"); + return; + } + + try { + BackendObservation obs = observeBackend(query); + entry + .put("rejected", obs.rejected) + .put("observed", obs.toJson()) + .put("outcome", "coverage-missing"); + log( + ruleId, + queryName, + "COVERAGE MISSING; OBSERVED (" + (obs.rejected ? "rejected" : "accepted") + ")"); + } catch (IOException | RuntimeException e) { + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend observation failed: " + + String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR: " + e.getMessage()); + } + report.put(entry); + } + + /** + * Enforcement must establish complete backend-oracle coverage before executing any query in the + * contract. This avoids producing partially scored evidence when a later query has no oracle. + */ + private boolean recordEnforcementCoverageGaps( + String ruleId, + String index, + JSONObject queries, + JSONObject expectedQueries, + int schemaVersion, + List failures, + JSONArray report) { + boolean missing = false; + for (String queryName : queries.keySet()) { + JSONObject expected = expectedQueries.getJSONObject(queryName); + if (resolveBackendOracle(schemaVersion, expected) != null) { + continue; + } + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + recordMissingOracle(ruleId, queryName, role, query, schemaVersion, failures, report); + missing = true; + } + return missing; + } + + /** Record an explicit schema-v4 non-applicable oracle without executing the query. */ + private void recordNotApplicable( + String ruleId, String queryName, JSONObject backend, JSONObject entry, JSONArray report) { + String reason = backend.getString("reason"); + entry + .put("outcome", "not-applicable") + .put("reason", reason) + .put("owner", backend.getString("owner")) + .put("issue", backend.getString("issue")); + report.put(entry); + log(ruleId, queryName, "NOT APPLICABLE (" + executionBackend.id + ")"); + } + + /** + * Resolve the execution backend oracle without fallback. Schema v3 is standard-only; schema v4 + * requires an explicit entry in {@code backends}. + */ + private JSONObject resolveBackendOracle(int schemaVersion, JSONObject expected) { + if (schemaVersion == 3) { + return executionBackend == ExecutionBackend.STANDARD + ? expected.getJSONObject("backend") + : null; + } + if (schemaVersion == 4) { + JSONObject backends = expected.optJSONObject("backends"); + return backends != null && backends.has(executionBackend.id) + ? backends.getJSONObject(executionBackend.id) + : null; + } + throw new IllegalArgumentException("unsupported contract schemaVersion " + schemaVersion); + } + + /** + * Validate every expectation before version selection or query execution. Observation mode may + * tolerate a missing route oracle, but it must never turn a malformed oracle into observed drift. + */ + private boolean validateAllExpectations( + String ruleId, + JSONObject declaredQueries, + JSONArray expectations, + int schemaVersion, + List failures) { + Set declared = new LinkedHashSet<>(declaredQueries.keySet()); + boolean valid = true; + if (declared.isEmpty()) { + failures.add("[" + ruleId + "] queries must not be empty"); + valid = false; + } + for (int i = 0; i < expectations.length(); i++) { + String expectationPath = "expectations[" + i + "]"; + Object expectationValue = expectations.opt(i); + if (!(expectationValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + expectationPath + " must be an object"); + valid = false; + continue; + } + JSONObject expectation = (JSONObject) expectationValue; + Object expectationQueriesValue = expectation.opt("queries"); + if (!(expectationQueriesValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + expectationPath + ".queries must be an object"); + valid = false; + continue; + } + JSONObject expectationQueries = (JSONObject) expectationQueriesValue; + Set expected = new LinkedHashSet<>(expectationQueries.keySet()); + if (!declared.equals(expected)) { + Set missingFromExpectation = new LinkedHashSet<>(declared); + missingFromExpectation.removeAll(expected); + Set unknownInExpectation = new LinkedHashSet<>(expected); + unknownInExpectation.removeAll(declared); + failures.add( + "[" + + ruleId + + "] " + + expectationPath + + " query keys must exactly match top-level queries" + + "; missing from expectation=" + + missingFromExpectation + + "; unknown in expectation=" + + unknownInExpectation); + valid = false; + } + + for (String queryName : expected) { + String queryPath = expectationPath + ".queries." + queryName; + Object queryExpectationValue = expectationQueries.opt(queryName); + if (!(queryExpectationValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + queryPath + " must be an object"); + valid = false; + continue; + } + JSONObject queryExpectation = (JSONObject) queryExpectationValue; + if (schemaVersion == 3) { + Object backendValue = queryExpectation.opt("backend"); + if (!(backendValue instanceof JSONObject)) { + failures.add( + "[" + ruleId + "] " + queryPath + ".backend must be a schema-v3 oracle object"); + valid = false; + continue; + } + valid &= + validateBackendOracle( + ruleId, queryPath + ".backend", (JSONObject) backendValue, failures); + continue; + } + + if (!queryExpectation.has("backends")) { + continue; + } + Object backendsValue = queryExpectation.opt("backends"); + if (!(backendsValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + queryPath + ".backends must be an object"); + valid = false; + continue; + } + JSONObject backends = (JSONObject) backendsValue; + for (String backend : backends.keySet()) { + String backendPath = queryPath + ".backends." + backend; + if (!"standard".equals(backend) && !"analytics".equals(backend)) { + failures.add( + "[" + + ruleId + + "] " + + queryPath + + " declares unknown execution backend \"" + + backend + + "\""); + valid = false; + continue; + } + Object oracleValue = backends.opt(backend); + if (!(oracleValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + backendPath + " must be an oracle object"); + valid = false; + continue; + } + valid &= validateBackendOracle(ruleId, backendPath, (JSONObject) oracleValue, failures); + } + } + } + return valid; + } + + private boolean validateBackendOracle( + String ruleId, String path, JSONObject oracle, List failures) { + int initialFailureCount = failures.size(); + String kind = requireNonBlankString(ruleId, path + ".kind", oracle.opt("kind"), failures); + if (kind == null) { + return false; + } + + if ("not-applicable".equals(kind)) { + requireNonBlankString(ruleId, path + ".reason", oracle.opt("reason"), failures); + requireNonBlankString(ruleId, path + ".owner", oracle.opt("owner"), failures); + requireNonBlankString(ruleId, path + ".issue", oracle.opt("issue"), failures); + return failures.size() == initialFailureCount; + } + + Integer httpStatus = + requireInteger(ruleId, path + ".httpStatus", oracle.opt("httpStatus"), 100, 599, failures); + switch (kind) { + case "rejection": + validateRejectionOracle(ruleId, path, oracle, httpStatus, failures); + break; + case "result-shape": + requireHttpOk(ruleId, path, httpStatus, failures); + validateResultShapeOracle(ruleId, path, oracle, failures); + break; + case "advisory": + requireHttpOk(ruleId, path, httpStatus, failures); + validateAdvisoryOracle(ruleId, path, oracle, failures); + break; + default: + failures.add("[" + ruleId + "] " + path + ".kind is unknown: \"" + kind + "\""); + break; + } + return failures.size() == initialFailureCount; + } + + private void validateRejectionOracle( + String ruleId, String path, JSONObject oracle, Integer httpStatus, List failures) { + Object bodyValue = oracle.opt("body"); + if (!(bodyValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".body must be an object"); + return; + } + JSONObject body = (JSONObject) bodyValue; + Integer bodyStatus = + requireInteger(ruleId, path + ".body.status", body.opt("status"), 100, 599, failures); + if (httpStatus != null && bodyStatus != null && !httpStatus.equals(bodyStatus)) { + failures.add("[" + ruleId + "] " + path + ".httpStatus must equal " + path + ".body.status"); + } + + if (!body.has("error")) { + return; + } + Object errorValue = body.opt("error"); + if (!(errorValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".body.error must be an object"); + return; + } + JSONObject error = (JSONObject) errorValue; + if (error.has("type")) { + requireNonBlankString(ruleId, path + ".body.error.type", error.opt("type"), failures); + } + if (error.has("reason")) { + requireNonBlankString(ruleId, path + ".body.error.reason", error.opt("reason"), failures); + } + } + + private void validateResultShapeOracle( + String ruleId, String path, JSONObject oracle, List failures) { + if (!oracle.has("expect")) { + return; + } + Object expectValue = oracle.opt("expect"); + if (!(expectValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".expect must be an object"); + return; + } + JSONObject expect = (JSONObject) expectValue; + if (expect.has("datarowsNonEmpty") && !(expect.opt("datarowsNonEmpty") instanceof Boolean)) { + failures.add("[" + ruleId + "] " + path + ".expect.datarowsNonEmpty must be a boolean"); + } + if (expect.has("datarowsCount")) { + requireInteger( + ruleId, + path + ".expect.datarowsCount", + expect.opt("datarowsCount"), + 0, + Integer.MAX_VALUE, + failures); + } + if (expect.has("columnAllNull")) { + requireNonBlankString( + ruleId, path + ".expect.columnAllNull", expect.opt("columnAllNull"), failures); + } + } + + private void validateAdvisoryOracle( + String ruleId, String path, JSONObject oracle, List failures) { + if (!oracle.has("expect")) { + return; + } + Object expectValue = oracle.opt("expect"); + if (!(expectValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".expect must be an object"); + return; + } + JSONObject expect = (JSONObject) expectValue; + if (expect.has("accepted") && !Boolean.TRUE.equals(expect.opt("accepted"))) { + failures.add("[" + ruleId + "] " + path + ".expect.accepted must be true"); + } + } + + private void requireHttpOk( + String ruleId, String path, Integer httpStatus, List failures) { + if (httpStatus != null && httpStatus != 200) { + failures.add("[" + ruleId + "] " + path + ".httpStatus must be 200"); + } + } + + private String requireNonBlankString( + String ruleId, String path, Object value, List failures) { + if (!(value instanceof String) || ((String) value).trim().isEmpty()) { + failures.add("[" + ruleId + "] " + path + " must be a non-blank string"); + return null; + } + return (String) value; + } + + private Integer requireInteger( + String ruleId, String path, Object value, int minimum, int maximum, List failures) { + if (!(value instanceof Number)) { + failures.add("[" + ruleId + "] " + path + " must be an integer"); + return null; + } + double numeric = ((Number) value).doubleValue(); + if (!Double.isFinite(numeric) + || numeric != Math.rint(numeric) + || numeric < minimum + || numeric > maximum) { + failures.add( + "[" + + ruleId + + "] " + + path + + " must be an integer from " + + minimum + + " through " + + maximum); + return null; + } + return ((Number) value).intValue(); + } + + /** + * Find the expectations that apply to the candidate version and planner. The caller treats zero + * matches as raw-observation-only and multiple matches as fatal in every mode. + */ + private List matchingExpectations(JSONArray expectations, boolean calciteOn) { + List matches = new ArrayList<>(); + for (int i = 0; i < expectations.length(); i++) { + JSONObject exp = expectations.getJSONObject(i); + if (!versionMatchesRange(exp.optString("version", null))) { + continue; + } + String engine = exp.optString("engine", ""); + if ("calcite".equals(engine) && !calciteOn) { + continue; + } + matches.add(exp); + } + return matches; + } + + private String backendVersionLabel() { + return engineVersionRaw == null ? "unknown" : engineVersionRaw; + } + + private void verifyCase( + String kind, String queryName, String query, JSONObject backend, JSONObject entry) + throws IOException { + BackendObservation obs = observeBackend(query); + entry.put("rejected", obs.rejected); + entry.put("observed", obs.toJson()); + switch (kind) { + case "rejection": + assertRejection( + queryName, query, obs, backend.getInt("httpStatus"), backend.getJSONObject("body")); + break; + case "result-shape": + assertResultShape(queryName, query, obs, backend.optJSONObject("expect")); + break; + case "advisory": + assertAdvisory(queryName, query, obs); + break; + default: + throw new IllegalArgumentException( + "case \"" + queryName + "\": unknown backend.kind \"" + kind + "\""); + } + } + + /** + * Run the query once and categorize the observed backend behavior independently of the + * expectation, so the report carries the true behavior even when a case fails (e.g. a trigger the + * backend unexpectedly accepted). A non-2xx surfaces as a {@link ResponseException} from the REST + * client, which is the rejection signal. + */ + private BackendObservation observeBackend(String query) throws IOException { + try { + JSONObject response = runPplQuery(query); + return BackendObservation.accepted(response); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + JSONObject body; + try { + body = new JSONObject(getResponseBody(e.getResponse(), true)); + } catch (IOException ioe) { + throw new IOException("failed to read rejection response body for query: " + query, ioe); + } + return BackendObservation.rejected(status, body); + } + } + + /** A rejected query must have thrown with the contracted status and structured error fields. */ + private void assertRejection( + String queryName, + String query, + BackendObservation obs, + int expectedStatus, + JSONObject expectedBody) { + assertTrue( + "case \"" + + queryName + + "\": expected the backend to REJECT the query but it was accepted: " + + query, + obs.rejected); + assertEquals( + "case \"" + queryName + "\": unexpected HTTP status for query: " + query, + expectedStatus, + obs.status); + assertEquals( + "case \"" + queryName + "\": unexpected top-level status field for query: " + query, + expectedBody.getInt("status"), + obs.body.getInt("status")); + + // A contract may omit `error` entirely to assert only THAT the engine rejects, + // without pinning wording that has not been observed live on that version. That + // is weaker than a full oracle but honest; inventing a type/reason would either + // fail spuriously or get "fixed" by pinning whatever CI first happened to see. + if (!expectedBody.has("error")) { + return; + } + JSONObject expectedError = expectedBody.getJSONObject("error"); + JSONObject actualError = obs.body.getJSONObject("error"); + if (expectedError.has("type")) { + assertEquals( + "case \"" + queryName + "\": unexpected error.type for query: " + query, + expectedError.getString("type"), + actualError.getString("type")); + } + if (expectedError.has("reason")) { + assertEquals( + "case \"" + queryName + "\": unexpected error.reason for query: " + query, + expectedError.getString("reason"), + actualError.getString("reason")); + } + } + + /** A result-shape case returns 200 whose datarows match the declared expectations. */ + private void assertResultShape( + String queryName, String query, BackendObservation obs, JSONObject expect) { + assertTrue( + "case \"" + + queryName + + "\": expected a 200 result but the backend rejected the query: " + + query, + !obs.rejected); + JSONObject response = obs.response; + assertTrue( + "case \"" + + queryName + + "\": expected a datarows array in the 200 response for query: " + + query, + response.has("datarows")); + if (expect == null) { + return; + } + JSONArray datarows = response.getJSONArray("datarows"); + + if (expect.optBoolean("datarowsNonEmpty", false)) { + assertTrue( + "case \"" + queryName + "\": expected non-empty datarows for query: " + query, + datarows.length() > 0); + } + if (expect.has("datarowsCount")) { + assertEquals( + "case \"" + queryName + "\": unexpected datarows count for query: " + query, + expect.getInt("datarowsCount"), + datarows.length()); + } + if (expect.has("columnAllNull")) { + String column = expect.getString("columnAllNull"); + int columnIndex = schemaColumnIndex(response, column); + assertTrue( + "case \"" + + queryName + + "\": column \"" + + column + + "\" not found in schema for query: " + + query, + columnIndex >= 0); + assertTrue( + "case \"" + + queryName + + "\": expected non-empty datarows to check null column for query: " + + query, + datarows.length() > 0); + for (int r = 0; r < datarows.length(); r++) { + JSONArray row = datarows.getJSONArray(r); + assertTrue( + "case \"" + + queryName + + "\": expected column \"" + + column + + "\" to be null in every row but row " + + r + + " was " + + row.get(columnIndex) + + " for query: " + + query, + row.isNull(columnIndex)); + } + } + } + + /** An advisory case only requires the query to be accepted (HTTP 200 with data). */ + private void assertAdvisory(String queryName, String query, BackendObservation obs) { + assertTrue( + "case \"" + + queryName + + "\": expected the query to be accepted (advisory) but it was " + + "rejected: " + + query, + !obs.rejected); + assertTrue( + "case \"" + + queryName + + "\": expected a datarows array in the 200 response for query: " + + query, + obs.response.has("datarows")); + } + + /** + * POST a PPL query to {@code /_plugins/_ppl} with a JSON-escaped body. The inherited {@code + * executeQuery} raw-interpolates the query into {@code {"query":"%s"}}, so a contract query that + * contains a double quote (e.g. {@code grok field=body "%{WORD:w}"}) would break the request + * payload and surface a spurious core-REST parse error instead of the real engine behavior. Build + * the body with a JSON serializer so any query is sent faithfully. Asserts HTTP 200 (a non-200 + * surfaces as a ResponseException, which the rejection path expects). + */ + private JSONObject runPplQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(new JSONObject().put("query", query).toString()); + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + options.addHeader("Content-Type", "application/json"); + request.setOptions(options); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return new JSONObject(getResponseBody(response, true)); + } + + private int schemaColumnIndex(JSONObject response, String column) { + if (!response.has("schema")) { + return -1; + } + JSONArray schema = response.getJSONArray("schema"); + for (int i = 0; i < schema.length(); i++) { + JSONObject col = schema.getJSONObject(i); + String name = col.optString("alias", col.optString("name", "")); + if (column.equals(name) || column.equals(col.optString("name", ""))) { + return i; + } + } + return -1; + } + + /** Observed backend behavior for one query, captured before asserting the expectation. */ + private static final class BackendObservation { + final boolean rejected; + final int status; + final JSONObject body; // rejection body, or null when accepted + final JSONObject response; // accepted 200 response, or null when rejected + + private BackendObservation(boolean rejected, int status, JSONObject body, JSONObject response) { + this.rejected = rejected; + this.status = status; + this.body = body; + this.response = response; + } + + static BackendObservation accepted(JSONObject response) { + return new BackendObservation(false, 200, null, response); + } + + static BackendObservation rejected(int status, JSONObject body) { + return new BackendObservation(true, status, body, null); + } + + JSONObject toJson() { + JSONObject o = new JSONObject().put("httpStatus", status).put("rejected", rejected); + if (body != null) { + o.put("body", body); + JSONObject err = body.optJSONObject("error"); + if (err != null) { + o.put("type", err.opt("type")).put("reason", err.opt("reason")); + } + } + if (response != null) { + o.put("response", response); + } + return o; + } + } + + // --- analytics route attestation ------------------------------------------ + + /** + * Prove the analytics route before any contract is scored. Each check is retained in the target + * manifest, including failures, so a missing route cannot be mistaken for backend coverage. + */ + private boolean attestAnalyticsRoute(List failures) { + boolean plugins = + runAnalyticsAttestationCheck( + "pluginsVerified", "required plugins", this::verifyAnalyticsPlugins, failures); + boolean clusterSettings = + runAnalyticsAttestationCheck( + "clusterSettingsVerified", + "cluster settings", + this::verifyAnalyticsClusterSettings, + failures); + boolean fixtureIndices = + runAnalyticsAttestationCheck( + "fixtureIndicesVerified", + "fixture index settings", + this::verifyAnalyticsFixtureIndices, + failures); + boolean explain = + runAnalyticsAttestationCheck( + "explainVerified", "explain route", this::verifyAnalyticsExplainCanaries, failures); + boolean profile = + runAnalyticsAttestationCheck( + "profiledExecutionVerified", + "profiled execution", + this::verifyAnalyticsProfileCanaries, + failures); + return plugins && clusterSettings && fixtureIndices && explain && profile; + } + + private boolean runAnalyticsAttestationCheck( + String targetField, String label, AttestationCheck check, List failures) { + try { + check.run(); + analyticsRouteAttestation.put(targetField, true); + log("route-attestation", label, "PASS"); + return true; + } catch (Exception | AssertionError e) { + analyticsRouteAttestation.put(targetField, false); + failures.add( + "[route-attestation/" + + label + + "] " + + (e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage())); + log("route-attestation", label, "FAIL: " + e.getMessage()); + return false; + } + } + + private void verifyAnalyticsPlugins() throws IOException { + Response response = + client() + .performRequest(new Request("GET", "/_cat/plugins?format=json&h=component,version")); + JSONArray plugins = new JSONArray(getResponseBody(response, true)); + List installed = new ArrayList<>(); + for (int i = 0; i < plugins.length(); i++) { + installed.add(plugins.getJSONObject(i).getString("component")); + } + analyticsRouteAttestation.put("plugins", plugins); + + requireAttestation( + engineVersionRaw != null && !engineVersionRaw.trim().isEmpty(), + "cluster engine version is unavailable for plugin compatibility checks"); + String expectedVersionPrefix = engineVersionRaw.split("-")[0]; + for (String required : REQUIRED_ANALYTICS_PLUGIN_COMPONENTS) { + JSONObject matched = null; + for (int i = 0; i < plugins.length(); i++) { + JSONObject plugin = plugins.getJSONObject(i); + if (pluginComponentMatches(plugin.getString("component"), required)) { + matched = plugin; + break; + } + } + requireAttestation( + matched != null, + "required plugin component matching \"" + + required + + "\" is missing; installed=" + + installed); + String version = matched.optString("version", ""); + requireAttestation( + version.equals(expectedVersionPrefix) + || version.startsWith(expectedVersionPrefix + ".") + || version.startsWith(expectedVersionPrefix + "-"), + "plugin " + + matched.getString("component") + + " version " + + version + + " is incompatible with engine " + + engineVersionRaw); + } + } + + private boolean pluginComponentMatches(String component, String required) { + return component.equals(required) || component.endsWith("-" + required); + } + + private void verifyAnalyticsClusterSettings() throws IOException { + Response nodesResponse = + client().performRequest(new Request("GET", "/_nodes/settings?flat_settings=true")); + JSONObject nodesBody = new JSONObject(getResponseBody(nodesResponse, true)); + analyticsRouteAttestation.put("nodeSettings", nodesBody); + JSONObject nodes = nodesBody.getJSONObject("nodes"); + requireAttestation(nodes.length() > 0, "node settings response contained no nodes"); + for (String nodeId : nodes.keySet()) { + String startupDataFormat = + nodes + .getJSONObject(nodeId) + .getJSONObject("settings") + .optString("cluster.pluggable.dataformat", ""); + String startupEnabled = + nodes + .getJSONObject(nodeId) + .getJSONObject("settings") + .optString("cluster.pluggable.dataformat.enabled", ""); + requireAttestation( + "composite".equals(startupDataFormat), + "node " + + nodeId + + " startup cluster.pluggable.dataformat must be composite but was \"" + + startupDataFormat + + "\""); + requireAttestation( + "true".equals(startupEnabled), + "node " + + nodeId + + " startup cluster.pluggable.dataformat.enabled must be true but was \"" + + startupEnabled + + "\""); + } + + Response response = + client() + .performRequest( + new Request("GET", "/_cluster/settings?flat_settings=true&include_defaults=true")); + JSONObject settings = new JSONObject(getResponseBody(response, true)); + analyticsRouteAttestation.put("clusterSettings", settings); + + requireEffectiveSetting(settings, "cluster.pluggable.dataformat", "composite"); + requireEffectiveSetting(settings, "cluster.pluggable.dataformat.enabled", "true"); + requireEffectiveSetting(settings, "cluster.composite.primary_data_format", "parquet"); + requireEffectiveSettingContains(settings, "cluster.composite.secondary_data_formats", "lucene"); + } + + private void verifyAnalyticsFixtureIndices() throws IOException { + int expectedShards = analyticsShardCount(); + JSONObject documentCounts = new JSONObject(); + JSONObject fixtureIndices = new JSONObject(); + analyticsRouteAttestation + .put("fixtureDocumentCounts", documentCounts) + .put("fixtureIndices", fixtureIndices); + for (String indexEnum : requiredIndexEnums()) { + String indexName = Index.valueOf(indexEnum).getName(); + Response response = + client() + .performRequest( + new Request( + "GET", + "/" + indexName + "/_settings?flat_settings=true&include_defaults=true")); + JSONObject body = new JSONObject(getResponseBody(response, true)); + JSONObject settings = body.getJSONObject(indexName).getJSONObject("settings"); + JSONObject fixtureEvidence = new JSONObject().put("settings", settings); + fixtureIndices.put(indexName, fixtureEvidence); + + Response mappingResponse = + client().performRequest(new Request("GET", "/" + indexName + "/_mapping")); + JSONObject mappingBody = new JSONObject(getResponseBody(mappingResponse, true)); + JSONObject mapping = mappingBody.getJSONObject(indexName).getJSONObject("mappings"); + fixtureEvidence.put("mappingHash", sha256(canonicalJson(mapping))).put("mapping", mapping); + + requireIndexSetting(indexName, settings, "index.pluggable.dataformat.enabled", "true"); + requireIndexSetting(indexName, settings, "index.pluggable.dataformat", "composite"); + requireIndexSetting(indexName, settings, "index.composite.primary_data_format", "parquet"); + requireIndexSettingContains( + indexName, settings, "index.composite.secondary_data_formats", "lucene"); + requireIndexSetting( + indexName, settings, "index.number_of_shards", Integer.toString(expectedShards)); + + long count = analyticsDocumentCount(indexName); + documentCounts.put(indexName, count); + fixtureEvidence.put("documentCount", count); + requireAttestation( + count > 0, + "fixture " + indexName + " contains no documents; fixture ingestion did not complete"); + } + } + + private long analyticsDocumentCount(String indexName) throws IOException { + JSONObject response = + executeQuery("source=" + indexName + " | stats count() as document_count"); + JSONArray rows = response.getJSONArray("datarows"); + requireAttestation(rows.length() == 1, "fixture " + indexName + " count returned " + rows); + JSONArray row = rows.getJSONArray(0); + requireAttestation( + row.length() == 1 && row.get(0) instanceof Number, + "fixture " + indexName + " count did not return one numeric value: " + rows); + return ((Number) row.get(0)).longValue(); + } + + private String canonicalJson(Object value) { + if (value == null || value == JSONObject.NULL) { + return "null"; + } + if (value instanceof JSONObject) { + JSONObject object = (JSONObject) value; + List keys = new ArrayList<>(object.keySet()); + Collections.sort(keys); + StringBuilder canonical = new StringBuilder("{"); + for (int i = 0; i < keys.size(); i++) { + if (i > 0) { + canonical.append(','); + } + String key = keys.get(i); + canonical.append(JSONObject.quote(key)).append(':').append(canonicalJson(object.get(key))); + } + return canonical.append('}').toString(); + } + if (value instanceof JSONArray) { + JSONArray array = (JSONArray) value; + StringBuilder canonical = new StringBuilder("["); + for (int i = 0; i < array.length(); i++) { + if (i > 0) { + canonical.append(','); + } + canonical.append(canonicalJson(array.get(i))); + } + return canonical.append(']').toString(); + } + if (value instanceof String) { + return JSONObject.quote((String) value); + } + if (value instanceof Number || value instanceof Boolean) { + return value.toString(); + } + throw new IllegalArgumentException( + "unsupported JSON value type in fixture mapping: " + value.getClass().getName()); + } + + private String sha256(String value) { + try { + byte[] digest = + MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder("sha256:"); + for (byte octet : digest) { + hex.append(String.format(Locale.ROOT, "%02x", octet & 0xff)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 digest is unavailable", e); + } + } + + private void verifyAnalyticsExplainCanaries() throws IOException { + JSONObject explainPlans = new JSONObject(); + analyticsRouteAttestation.put("explainPlans", explainPlans); + for (String indexEnum : requiredIndexEnums()) { + String query = analyticsCanaryQuery(indexEnum); + String explained = explainQueryToString(query); + explainPlans.put(indexEnum, explained); + requireAttestation( + explained.contains("LogicalTableScan(table=[[opensearch,"), + "fixture " + indexEnum + " did not use LogicalTableScan(opensearch): " + explained); + requireAttestation( + !explained.contains("CalciteLogicalIndexScan"), + "fixture " + indexEnum + " fell back to CalciteLogicalIndexScan: " + explained); + } + } + + private void verifyAnalyticsProfileCanaries() throws IOException { + JSONArray executionTypes = new JSONArray(); + JSONObject profiles = new JSONObject(); + analyticsRouteAttestation + .put("profileExecutionTypes", executionTypes) + .put("profiles", profiles); + for (String indexEnum : requiredIndexEnums()) { + JSONObject response = runProfiledPplQuery(analyticsCanaryQuery(indexEnum)); + profiles.put(indexEnum, response); + JSONObject profile = response.getJSONObject("profile"); + JSONArray stages = profile.getJSONObject("plan").getJSONArray("stages"); + requireAttestation( + stages.length() > 0, "fixture " + indexEnum + " profile returned no execution stages"); + for (int i = 0; i < stages.length(); i++) { + JSONObject stage = stages.getJSONObject(i); + requireAttestation( + "SUCCEEDED".equals(stage.optString("state")), + "fixture " + indexEnum + " profile stage " + i + " was not successful: " + stage); + requireAttestation( + !stage.optString("execution_type", "").trim().isEmpty(), + "fixture " + indexEnum + " profile stage " + i + " has no execution_type: " + stage); + executionTypes.put(stage.getString("execution_type")); + } + } + } + + private String analyticsCanaryQuery(String indexEnum) { + String indexName = Index.valueOf(indexEnum).getName(); + switch (indexEnum) { + case "ACCOUNT": + return "source=" + indexName + " | fields account_number, firstname | head 1"; + case "FLAT_OBJECT": + return "source=" + indexName + " | fields name, status | head 1"; + default: + throw new IllegalArgumentException( + "no fixture-safe analytics canary projection is defined for " + indexEnum); + } + } + + private JSONObject runProfiledPplQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(new JSONObject().put("query", query).put("profile", true).toString()); + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + options.addHeader("Content-Type", "application/json"); + request.setOptions(options); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return new JSONObject(getResponseBody(response, true)); + } + + private void requireEffectiveSetting(JSONObject settings, String key, String expected) { + String actual = effectiveSetting(settings, key); + requireAttestation( + expected.equals(actual), + "effective " + key + " must be " + expected + " but was \"" + actual + "\""); + } + + private void requireEffectiveSettingContains(JSONObject settings, String key, String expected) { + String actual = effectiveSetting(settings, key); + requireAttestation( + actual.contains(expected), + "effective " + key + " must contain " + expected + " but was \"" + actual + "\""); + } + + private String effectiveSetting(JSONObject settings, String key) { + String transientValue = settingInSection(settings, "transient", key); + if (!transientValue.isEmpty()) { + return transientValue; + } + String persistentValue = settingInSection(settings, "persistent", key); + if (!persistentValue.isEmpty()) { + return persistentValue; + } + return settingInSection(settings, "defaults", key); + } + + private String settingInSection(JSONObject settings, String section, String key) { + JSONObject values = settings.optJSONObject(section); + return values == null ? "" : values.optString(key, ""); + } + + private void requireIndexSetting( + String indexName, JSONObject settings, String key, String expected) { + String actual = settings.optString(key, ""); + requireAttestation( + expected.equals(actual), + "fixture " + + indexName + + " setting " + + key + + " must be " + + expected + + " but was \"" + + actual + + "\""); + } + + private void requireIndexSettingContains( + String indexName, JSONObject settings, String key, String expected) { + String actual = settings.optString(key, ""); + requireAttestation( + actual.contains(expected), + "fixture " + + indexName + + " setting " + + key + + " must contain " + + expected + + " but was \"" + + actual + + "\""); + } + + private int analyticsShardCount() { + int shardCount = Integer.parseInt(System.getProperty(ANALYTICS_SHARD_COUNT_PROPERTY, "1")); + requireAttestation(shardCount > 0, ANALYTICS_SHARD_COUNT_PROPERTY + " must be positive"); + return shardCount; + } + + private static void requireAttestation(boolean condition, String message) { + if (!condition) { + throw new IllegalStateException(message); + } + } + + @FunctionalInterface + private interface AttestationCheck { + void run() throws Exception; + } + + // --- grammar bundle export ------------------------------------------------- + + /** + * Fetch the candidate runtime grammar bundle and write it plus a target manifest, so the + * detector-validation job can lint against the SAME grammar this backend built. Best-effort by + * design: a run without {@code -Dppl.lint.grammar.bundle} (local dev) exports nothing; in CI a + * fetch/write failure is a real failure — a missing bundle means the detector half cannot run. + */ + private void exportGrammarArtifacts(List failures) { + String bundlePath = System.getProperty("ppl.lint.grammar.bundle"); + if (bundlePath == null || bundlePath.isEmpty()) { + // No bundle requested. That is a compiled-surface leg (an engine predating + // GET /_plugins/_ppl/_grammar) or a local run. The target manifest still has + // to be written: it carries the engine version every consumer keys on, and + // the multi-version aggregator treats a leg without one as fatal. Writing it + // only alongside the bundle silently produced legs the aggregator could not + // read. + writeTargetManifest("", failures); + return; + } + String grammarHash = ""; + String bundleName = ""; + try { + Response response = client().performRequest(new Request("GET", GRAMMAR_API_ENDPOINT)); + String bundleBody = getResponseBody(response, true); + JSONObject bundle = new JSONObject(bundleBody); + grammarHash = bundle.optString("grammarHash", ""); + Files.write(Paths.get(bundlePath), bundleBody.getBytes(StandardCharsets.UTF_8)); + bundleName = Paths.get(bundlePath).getFileName().toString(); + log("_grammar", "export", "wrote candidate bundle (" + grammarHash + ") to " + bundlePath); + } catch (Exception e) { + failures.add( + "[grammar-export] failed to fetch/write " + GRAMMAR_API_ENDPOINT + ": " + e.getMessage()); + } finally { + // Route and attestation identity remain available even when the grammar + // endpoint or bundle write fails. + writeTargetManifest(grammarHash, bundleName, failures); + } + } + + /** + * True when a failure is the cluster rejecting {@code plugins.calcite.enabled} because it does + * not know that setting — i.e. a pre-Calcite (2.x) engine. + * + *

Deliberately narrow: matched on the setting name plus "not recognized" rather than on any + * 400, so a genuinely broken settings call on a Calcite-capable engine still fails the run + * instead of being waved through as "old engine". + */ + private static boolean isUnrecognizedCalciteSetting(Throwable error) { + for (Throwable current = error; current != null; current = current.getCause()) { + String message = current.getMessage(); + if (message != null + && message.contains(Settings.Key.CALCITE_ENGINE_ENABLED.getKeyValue()) + && message.contains("not recognized")) { + return true; + } + if (current.getCause() == current) { + break; + } + } + return false; + } + + private void writeTargetManifest(String grammarHash, List failures) { + writeTargetManifest(grammarHash, "", failures); + } + + /** + * Write target schema v2 with engine, grammar, execution route, storage, shard count, and (for + * analytics) route attestation identity. + */ + private void writeTargetManifest(String grammarHash, String bundleName, List failures) { + String targetPath = System.getProperty("ppl.lint.target"); + if (targetPath == null || targetPath.isEmpty()) { + return; + } + try { + JSONObject target = + new JSONObject() + .put("schemaVersion", 2) + .put("sqlSha", System.getProperty("ppl.lint.sql_sha", "")) + .put("engineVersion", engineVersionRaw == null ? "" : engineVersionRaw) + .put("grammarHash", grammarHash) + .put("grammarBundle", bundleName) + .put("executionBackend", executionBackend.id) + .put("storage", executionBackend.storage) + .put( + "shardCount", + executionBackend == ExecutionBackend.ANALYTICS ? analyticsShardCount() : 1); + if (executionBackend == ExecutionBackend.ANALYTICS) { + target + .put( + "analyticsStack", + new JSONObject() + .put("source", System.getProperty("ppl.lint.analytics.stack.source", ""))) + .put("routeAttestation", analyticsRouteAttestation); + } + Files.write(Paths.get(targetPath), target.toString(2).getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + failures.add("[grammar-export] failed to write " + targetPath + ": " + e.getMessage()); + } + } + + // --- cluster settings ------------------------------------------------------ + + /** True when the contract's fixture leaves Calcite enabled (the default). */ + private boolean fixtureCalciteEnabled(JSONObject fixture) { + if (fixture == null) { + return true; + } + JSONObject settings = fixture.optJSONObject("clusterSettings"); + if (settings == null || !settings.has("calcite")) { + return true; + } + return settings.getBoolean("calcite"); + } + + /** + * Apply the contract's cluster settings and return the list of settings changed so the caller can + * reset them afterwards. Grouped per-contract (not global) because contracts disagree: eventstats + * needs {@code calciteFallback=false} to force rejection, while dedup-consecutive needs it {@code + * true} to succeed via V2 fallback. + */ + private List applyClusterSettings(JSONObject fixture) throws IOException { + List applied = new ArrayList<>(); + if (fixture == null) { + return applied; + } + JSONObject settings = fixture.optJSONObject("clusterSettings"); + if (settings == null) { + return applied; + } + // Every setting below is Calcite-family, and a pre-Calcite engine rejects all of + // them the same way. `init()` already established whether this cluster knows + // them, so skip the whole block rather than fail per contract — otherwise the + // tolerance added there is undone here, once per contract. + if (!calciteSettingsSupported) { + return applied; + } + if (settings.has("calcite")) { + if (settings.getBoolean("calcite")) { + enableCalcite(); + } else { + disableCalcite(); + } + } + if (settings.has("calciteFallback")) { + if (settings.getBoolean("calciteFallback")) { + allowCalciteFallback(); + } else { + disallowCalciteFallback(); + } + } + if (settings.has("allJoinTypesAllowed")) { + String key = Settings.Key.CALCITE_SUPPORT_ALL_JOIN_TYPES.getKeyValue(); + String value = Boolean.toString(settings.getBoolean("allJoinTypesAllowed")); + updateClusterSettings(new PPLIntegTestCase.ClusterSetting("persistent", key, value)); + applied.add(key); + } + return applied; + } + + /** + * Reset each explicitly-applied dynamic setting to its cluster default by writing a null value. + * (calcite/calciteFallback are toggled via the inherited helpers and re-set explicitly by each + * contract, so only the persistent settings applied here are reset.) + */ + private void resetClusterSettings(List appliedKeys) { + for (String key : appliedKeys) { + try { + updateClusterSettings(new PPLIntegTestCase.ClusterSetting("persistent", key, null)); + } catch (IOException e) { + // Best-effort reset; the next contract sets what it needs explicitly, so + // keep the failure visible without failing the suite. + System.err.println("[ppl-lint] failed to reset a cluster setting: " + e.getMessage()); + } + } + } + + // --- version gating -------------------------------------------------------- + + private int[] fetchClusterVersion() { + try { + Response response = client().performRequest(new Request("GET", "/")); + JSONObject body = new JSONObject(getResponseBody(response, false)); + String number = body.getJSONObject("version").getString("number"); + engineVersionRaw = number; + return parseVersion(number); + } catch (Exception e) { + // Unknown version → do not skip anything. + return null; + } + } + + /** + * Test a space-separated semver range (e.g. {@code ">=3.6.0 <3.8.0"}) against the candidate + * backend version. An empty/absent range or an unknown cluster version matches (do not + * over-filter). Supports the {@code >= > <= < =} comparators the design uses. + */ + private boolean versionMatchesRange(String range) { + if (range == null || range.trim().isEmpty()) { + return true; + } + if (clusterVersion == null) { + return true; + } + for (String token : range.trim().split("\\s+")) { + if (!satisfiesComparator(token)) { + return false; + } + } + return true; + } + + private boolean satisfiesComparator(String token) { + String op; + String ver; + if (token.startsWith(">=")) { + op = ">="; + ver = token.substring(2); + } else if (token.startsWith("<=")) { + op = "<="; + ver = token.substring(2); + } else if (token.startsWith(">")) { + op = ">"; + ver = token.substring(1); + } else if (token.startsWith("<")) { + op = "<"; + ver = token.substring(1); + } else if (token.startsWith("=")) { + op = "="; + ver = token.substring(1); + } else { + op = "="; + ver = token; + } + int cmp = compareVersion(clusterVersion, parseVersion(ver)); + switch (op) { + case ">=": + return cmp >= 0; + case "<=": + return cmp <= 0; + case ">": + return cmp > 0; + case "<": + return cmp < 0; + default: + return cmp == 0; + } + } + + private int compareVersion(int[] a, int[] b) { + for (int i = 0; i < 3; i++) { + if (a[i] != b[i]) { + return Integer.compare(a[i], b[i]); + } + } + return 0; + } + + private int[] parseVersion(String raw) { + String cleaned = raw.split("-")[0]; + String[] parts = cleaned.split("\\."); + int[] v = new int[] {0, 0, 0}; + for (int i = 0; i < 3 && i < parts.length; i++) { + try { + v[i] = Integer.parseInt(parts[i]); + } catch (NumberFormatException ignored) { + v[i] = 0; + } + } + return v; + } + + // --- contract loading ------------------------------------------------------ + + private List loadScheduledContracts() throws IOException { + List result = new ArrayList<>(); + Set ruleIds = new LinkedHashSet<>(); + for (String fileName : manifestContractNames()) { + JSONObject contract = loadContractFile(CONTRACT_DIR + "/" + fileName); + String ruleId = contract.getString("ruleId"); + if (!ruleIds.add(ruleId)) { + throw new IOException("contract manifest contains duplicate ruleId \"" + ruleId + "\""); + } + String contractSchedule = contract.optString("schedule", "pr"); + if ("pr".equals(schedule) && !"pr".equals(contractSchedule)) { + continue; // PR runs only PR-scheduled contracts; nightly runs all. + } + result.add(contract); + } + return result; + } + + private List manifestContractNames() throws IOException { + JSONObject manifest = loadContractFile(MANIFEST); + JSONArray contracts = manifest.getJSONArray("contracts"); + List names = new ArrayList<>(); + Set unique = new LinkedHashSet<>(); + for (int i = 0; i < contracts.length(); i++) { + String name = contracts.getString(i); + if (!unique.add(name)) { + throw new IOException("contract manifest contains duplicate file \"" + name + "\""); + } + names.add(name); + } + return names; + } + + /** + * Union of index enums required by the contracts scheduled to run this session. + * + *

A schema-v4 contract whose selected analytics oracle marks every query explicitly + * non-applicable does not need its unrepresentable fixture. Missing or malformed oracles remain + * fixture-requiring so they cannot turn into an implicit skip. + */ + private Set requiredIndexEnums() throws IOException { + Set indices = new LinkedHashSet<>(); + for (JSONObject contract : loadScheduledContracts()) { + if (!contractRequiresFixture(contract)) { + continue; + } + JSONObject fixture = contract.optJSONObject("backendFixture"); + if (fixture == null) { + continue; + } + JSONArray declared = fixture.optJSONArray("indices"); + if (declared == null) { + continue; + } + for (int i = 0; i < declared.length(); i++) { + indices.add(declared.getString(i)); + } + } + if (indices.isEmpty()) { + indices.add("ACCOUNT"); + } + return indices; + } + + private boolean contractRequiresFixture(JSONObject contract) { + if (executionBackend != ExecutionBackend.ANALYTICS || contract.optInt("schemaVersion") != 4) { + return true; + } + + JSONObject fixture = contract.optJSONObject("backendFixture"); + List matches = + matchingExpectations(contract.getJSONArray("expectations"), fixtureCalciteEnabled(fixture)); + if (matches.size() != 1) { + return true; + } + + JSONObject declaredQueries = contract.optJSONObject("queries"); + JSONObject expectedQueries = matches.get(0).optJSONObject("queries"); + if (declaredQueries == null + || declaredQueries.length() == 0 + || expectedQueries == null + || !declaredQueries.keySet().equals(expectedQueries.keySet())) { + return true; + } + + for (String queryName : declaredQueries.keySet()) { + JSONObject expected = expectedQueries.optJSONObject(queryName); + JSONObject backend = expected == null ? null : resolveBackendOracle(4, expected); + if (!isCompleteNotApplicableOracle(backend)) { + return true; + } + } + return false; + } + + private boolean isCompleteNotApplicableOracle(JSONObject backend) { + return backend != null + && "not-applicable".equals(backend.optString("kind")) + && hasNonBlankString(backend, "reason") + && hasNonBlankString(backend, "owner") + && hasNonBlankString(backend, "issue"); + } + + private boolean hasNonBlankString(JSONObject object, String key) { + Object value = object.opt(key); + return value instanceof String && !((String) value).trim().isEmpty(); + } + + private JSONObject loadContractFile(String resourcePath) throws IOException { + String path = TestUtils.getResourceFilePath(resourcePath); + return new JSONObject(new String(Files.readAllBytes(Paths.get(path)))); + } + + // --- reporting ------------------------------------------------------------- + + private JSONObject reportEntry( + String ruleId, String queryName, String role, String query, String kind) { + return new JSONObject() + .put("ruleId", ruleId) + .put("queryName", queryName) + .put("role", role) + .put("query", query) + .put("kind", kind) + .put("executionBackend", executionBackend.id); + } + + private void writeReport(JSONArray report) throws IOException { + String target = System.getProperty("ppl.lint.report"); + if (target == null || target.isEmpty()) { + return; + } + Files.write(Paths.get(target), report.toString(2).getBytes(StandardCharsets.UTF_8)); + } + + private void log(String ruleId, String caseId, String message) { + System.out.println( + String.format( + Locale.ROOT, "[ppl-lint-backend-contract] %s/%s: %s", ruleId, caseId, message)); + } + + private enum ExecutionBackend { + STANDARD("standard", "lucene"), + ANALYTICS("analytics", "composite-parquet"); + + private final String id; + private final String storage; + + ExecutionBackend(String id, String storage) { + this.id = id; + this.storage = storage; + } + + private static ExecutionBackend parse(String value) { + for (ExecutionBackend backend : values()) { + if (backend.id.equals(value)) { + return backend; + } + } + throw new IllegalArgumentException( + EXECUTION_BACKEND_PROPERTY + " must be standard or analytics but was \"" + value + "\""); + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java new file mode 100644 index 00000000000..517503cc148 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -0,0 +1,290 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.util.Timeout; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.test.OpenSearchTestCase; + +/** + * Connectivity probe for {@code tests.rest.cluster}, used to diagnose why the REST test client + * cannot reach some engine versions that plain HTTP clients reach fine. + * + *

Context: on the PPL lint multi-version matrix the 2.19.0 observation leg fails with {@code + * SocketTimeoutException} after the full 60s response timeout, thrown from {@code + * OpenSearchRestTestCase.initClient} on its {@code GET _nodes/plugins} call — before any test body + * runs. From the same runner, {@code curl} against the same endpoint on the same address returns + * HTTP 200 in 0s. The 3.5.0 leg, with byte-identical Gradle args, network topology, publish address + * and task graph, passes. Seven hypotheses (index-wipe race, Gradle cluster fallback, HTTP/2 + * negotiation, FIPS, plugin set, port collision, address family) have each been refuted by + * observation. + * + *

It extends {@link OpenSearchTestCase}, NOT the REST base class. Two constraints meet here: + * {@code integTestRemote} runs the default JUnit 4 runner (only {@code integJdbcTest} calls {@code + * useJUnitPlatform()}), and Gradle only discovers an IT that inherits a runner from a framework + * base class — a standalone class is silently collected as ZERO tests, which is how the first + * version of this probe "passed" while reporting nothing. {@code OpenSearchTestCase} supplies that + * runner but builds no REST client, so discovery works without inheriting the hang under + * investigation. Instead of the framework's client setup, this walks up the stack one layer at a + * time against the same address, so a single run says exactly which layer stops working: + * + *

    + *
  1. raw TCP connect — is the port reachable from this JVM at all? + *
  2. {@code HttpURLConnection} — does the JDK's own HTTP stack get a response? + *
  3. {@code RestClient} with default settings — does the OpenSearch async client work? + *
  4. {@code RestClient} on the endpoints the framework itself calls, timed individually. + *
+ * + *

Every step is time-bounded and reports rather than asserts, because the point is to collect + * evidence from a leg that is already failing. The one assertion is that step 1 succeeded: if the + * JVM cannot open a socket, nothing below it means anything. + * + *

Run with: {@code ./gradlew :integ-test:integTestRemote --tests + * '*RestClientConnectivityProbeIT' -Dtests.rest.cluster=localhost:9200} + */ +public class RestClientConnectivityProbeIT extends OpenSearchTestCase { + + /** Bound well below the framework's 60s so a hang is visibly a hang, not a wait. */ + private static final Timeout PROBE_TIMEOUT = Timeout.ofSeconds(15); + + private static final String[] FRAMEWORK_ENDPOINTS = { + // The exact call OpenSearchRestTestCase.initClient makes, and the one that hangs. + "_nodes/plugins", + // What the wipe in OpenSearchSQLRestTestCase.wipeAllOpenSearchIndices calls next. + "_cat/indices?format=json&expand_wildcards=all", + // A trivial response, to separate "any request" from "this request". + "_cluster/health", + // The PPL endpoint the contract actually needs, so a pass here means the leg could work. + "_plugins/_ppl/_grammar", + }; + + @Test + public void probeConnectivity() { + String cluster = System.getProperty("tests.rest.cluster"); + if (cluster == null || cluster.isEmpty()) { + log("SKIP: -Dtests.rest.cluster not set"); + return; + } + String hostPort = cluster.split(",")[0]; + int sep = hostPort.lastIndexOf(':'); + String host = hostPort.substring(0, sep); + int port = Integer.parseInt(hostPort.substring(sep + 1)); + log("probing " + host + ":" + port); + + boolean tcpOk = probeRawSocket(host, port); + probeHttpUrlConnection(host, port); + probeRestClient(host, port); + probeClientVariants(host, port); + + // Only a hard failure here is fatal: without a socket the rest is noise. + if (!tcpOk) { + throw new AssertionError("could not open a TCP connection to " + host + ":" + port); + } + } + + /** Layer 1: can this JVM open a socket to the published port? */ + private boolean probeRawSocket(String host, int port) { + long start = System.nanoTime(); + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), (int) PROBE_TIMEOUT.toMilliseconds()); + log("tcp connect OK in " + millis(start) + "ms (localAddr=" + socket.getLocalAddress() + ")"); + return true; + } catch (Exception e) { + log("tcp connect FAILED after " + millis(start) + "ms: " + describe(e)); + return false; + } + } + + /** + * Layer 2: the JDK's own blocking HTTP stack. If this works while {@code RestClient} does not, + * the problem is in the async client rather than in the network or the engine. + */ + private void probeHttpUrlConnection(String host, int port) { + long start = System.nanoTime(); + try { + URL url = new URL("http://" + host + ":" + port + "/_nodes/plugins"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setConnectTimeout((int) PROBE_TIMEOUT.toMilliseconds()); + connection.setReadTimeout((int) PROBE_TIMEOUT.toMilliseconds()); + int status = connection.getResponseCode(); + long bytes = drain(connection); + log( + "HttpURLConnection _nodes/plugins OK in " + + millis(start) + + "ms: HTTP " + + status + + ", " + + bytes + + " bytes"); + connection.disconnect(); + } catch (Exception e) { + log("HttpURLConnection _nodes/plugins FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + + /** + * Layer 3: the real {@code RestClient}, built the way the framework builds it (defaults only, no + * credentials or TLS since these legs are plain HTTP), then each framework endpoint in turn. + * + *

Timed per endpoint: a uniform failure means the client cannot talk to this engine at all, + * while one slow endpoint among fast ones means the response itself is the problem. + */ + private void probeRestClient(String host, int port) { + RestClientBuilder builder = + RestClient.builder(new HttpHost("http", host, port)) + .setRequestConfigCallback( + config -> config.setConnectTimeout(PROBE_TIMEOUT).setResponseTimeout(PROBE_TIMEOUT)) + // The framework sets this too; without it a deprecation warning header can turn into a + // failure and confuse the diagnosis. + .setStrictDeprecationMode(false); + + try (RestClient client = builder.build()) { + for (String endpoint : FRAMEWORK_ENDPOINTS) { + long start = System.nanoTime(); + try { + Response response = client.performRequest(new Request("GET", "/" + endpoint)); + long bytes = response.getEntity() == null ? 0 : response.getEntity().getContentLength(); + log( + "RestClient " + + endpoint + + " OK in " + + millis(start) + + "ms: HTTP " + + response.getStatusLine().getStatusCode() + + ", " + + bytes + + " bytes"); + } catch (Exception e) { + log("RestClient " + endpoint + " FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + } catch (IOException e) { + log("RestClient could not be built/closed: " + describe(e)); + } + } + + private static long drain(HttpURLConnection connection) throws IOException { + byte[] buffer = new byte[8192]; + long total = 0; + try (var stream = connection.getInputStream()) { + int read; + while ((read = stream.read(buffer)) != -1) { + total += read; + } + } + return total; + } + + /** Full cause chain: the outer message alone hides which layer actually gave up. */ + private static String describe(Throwable error) { + List chain = new ArrayList<>(); + for (Throwable current = error; current != null; current = current.getCause()) { + chain.add(current.getClass().getSimpleName() + "(" + current.getMessage() + ")"); + if (current.getCause() == current) { + break; + } + } + return String.join(" <- ", chain); + } + + /** + * Layer 4: candidate fixes, each a one-line change from the default client, all against the same + * endpoint on the same engine. + * + *

The default async client times out on EVERY endpoint here — including a 459-byte {@code + * _cluster/health} — while {@code HttpURLConnection} against the same URL returns 200 in 27ms. So + * the fault is in how the async client speaks to this engine, not in the network, the engine, the + * response size, or any one endpoint. Each variant isolates one suspect; whichever succeeds names + * the fix, and if none do, the client cannot be configured around it. + */ + private void probeClientVariants(String host, int port) { + // Forcing HTTP/1.1 up front, rather than letting HttpClient 5.x negotiate h2. + variant( + host, + port, + "FORCE_HTTP_1", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.FORCE_HTTP_1))); + // Same, but negotiating h2 explicitly, to tell "policy matters" from "1.1 specifically works". + variant( + host, + port, + "NEGOTIATE", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.NEGOTIATE))); + // FORCE_HTTP_2, to confirm the direction of any protocol effect rather than assume it. + variant( + host, + port, + "FORCE_HTTP_2", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.FORCE_HTTP_2))); + // A fresh connection manager: rules out connection reuse/pooling against this engine. + variant( + host, + port, + "fresh-conn-manager", + b -> + b.setHttpClientConfigCallback( + c -> + c.setConnectionManager( + org.apache.hc.client5.http.impl.nio + .PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnPerRoute(1) + .setMaxConnTotal(1) + .build()))); + } + + /** Run one client variant against {@code _cluster/health} — the smallest response available. */ + private void variant( + String host, + int port, + String name, + java.util.function.UnaryOperator tune) { + RestClientBuilder builder = + RestClient.builder(new HttpHost("http", host, port)) + .setRequestConfigCallback( + config -> config.setConnectTimeout(PROBE_TIMEOUT).setResponseTimeout(PROBE_TIMEOUT)) + .setStrictDeprecationMode(false); + long start = System.nanoTime(); + try (RestClient client = tune.apply(builder).build()) { + Response response = client.performRequest(new Request("GET", "/_cluster/health")); + log( + "variant " + + name + + " OK in " + + millis(start) + + "ms: HTTP " + + response.getStatusLine().getStatusCode()); + } catch (Exception e) { + log("variant " + name + " FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + + private static long millis(long startNanos) { + return (System.nanoTime() - startNanos) / 1_000_000; + } + + private static void log(String message) { + // stdout so it lands in the Gradle test output the CI job already prints. + System.out.println("[rest-connectivity-probe] " + message); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLMakeResultsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLMakeResultsIT.java new file mode 100644 index 00000000000..ba623c649e0 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLMakeResultsIT.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for the makeresults leading command. + * + *

The count path output ({@code @timestamp = now()}) is non-deterministic, so it asserts schema + * + row count only. The data= path is deterministic and asserts schema + datarows. + */ +public class CalcitePPLMakeResultsIT extends CalcitePPLIntegTestCase { + @Override + public void init() throws IOException { + super.init(); + enableCalcite(); + } + + @Test + public void testCount() throws IOException { + JSONObject result = executeQuery("makeresults count=5"); + verifySchema(result, schema("@timestamp", "timestamp")); + assertEquals(5, result.getInt("total")); + } + + @Test + public void testBare() throws IOException { + JSONObject result = executeQuery("makeresults"); + verifySchema(result, schema("@timestamp", "timestamp")); + assertEquals(1, result.getInt("total")); + } + + @Test + public void testJson() throws IOException { + String data = + "makeresults format=json data='[{\"name\":\"John\",\"age\":35,\"score\":3.5}," + + "{\"name\":\"Sarah\",\"age\":39,\"score\":4.0}]'"; + JSONObject result = executeQuery(data); + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("name", "string"), + schema("age", "bigint"), + schema("score", "float")); + JSONObject projected = executeQuery(data + " | fields name, age, score"); + verifyDataRows(projected, rows("John", 35, 3.5), rows("Sarah", 39, 4.0)); + } + + @Test + public void testNestedJsonSerializesToString() throws IOException { + String data = + "makeresults format=json data='[{\"name\":\"John\"," + + "\"addr\":{\"city\":\"NYC\",\"zip\":10001},\"tags\":[\"a\",\"b\"]}]'"; + JSONObject result = executeQuery(data); + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("name", "string"), + schema("addr", "string"), + schema("tags", "string")); + JSONObject projected = executeQuery(data + " | fields name, addr, tags"); + verifyDataRows(projected, rows("John", "{\"city\":\"NYC\",\"zip\":10001}", "[\"a\",\"b\"]")); + } + + @Test + public void testNestedJsonSpathRoundTrip() throws IOException { + JSONObject result = + executeQuery( + "makeresults format=json data='[{\"addr\":{\"city\":\"NYC\"}}]'" + + " | spath input=addr output=city path=city | fields addr, city"); + verifyDataRows(result, rows("{\"city\":\"NYC\"}", "NYC")); + } + + @Test + public void testTypedCsv() throws IOException { + JSONObject result = + executeQuery("makeresults format=csv data='name:string,age:int\nJohn,35\nSarah,39'"); + verifySchema(result, schema("name", "string"), schema("age", "int")); + verifyDataRows(result, rows("John", 35), rows("Sarah", 39)); + } + + @Test + public void testBareCsv() throws IOException { + JSONObject result = executeQuery("makeresults format=csv data='name,age\nJohn,35\nSarah,39'"); + verifySchema(result, schema("name", "string"), schema("age", "string")); + verifyDataRows(result, rows("John", "35"), rows("Sarah", "39")); + } + + @Test + public void testComposesAsSource() throws IOException { + JSONObject result = executeQuery("makeresults count=3 | eval n=1"); + verifySchema(result, schema("@timestamp", "timestamp"), schema("n", "int")); + assertEquals(3, result.getInt("total")); + } + + @Test + public void testBareGlobalCountIsUnsupported() { + assertThrows(Exception.class, () -> executeQuery("makeresults count=5 | stats count() as c")); + } + + @Test + public void testBareGlobalCountWorkaroundCountArg() throws IOException { + JSONObject result = executeQuery("makeresults count=5 | stats count(1) as c"); + verifySchema(result, schema("c", "bigint")); + verifyDataRows(result, rows(5)); + } + + @Test + public void testBareGlobalCountWorkaroundByTimestamp() throws IOException { + JSONObject result = executeQuery("makeresults count=5 | stats count() as c by @timestamp"); + verifyDataRows(result, rows(5, result.getJSONArray("datarows").getJSONArray(0).get(1))); + } + + @Test + public void testBareGlobalCountWorkaroundEvalGroup() throws IOException { + JSONObject result = executeQuery("makeresults count=5 | eval g=1 | stats count() as c by g"); + verifyDataRows(result, rows(5, 1)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java index 9c7d8c90ac3..744b9ec124b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java @@ -25,6 +25,7 @@ import org.apache.calcite.tools.RelBuilder; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.SysLimit; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.executor.QueryType; @@ -87,7 +88,8 @@ protected RexNode createStringArray(RexBuilder rexBuilder, String... values) { protected void executeRelNodeAndVerify( CalcitePlanContext planContext, RelNode relNode, ResultVerifier verifier) throws SQLException { - try (PreparedStatement statement = OpenSearchRelRunners.run(planContext, relNode)) { + try (PreparedStatement statement = + OpenSearchRelRunners.run(planContext, CalciteToolsHelper.optimize(relNode, planContext))) { ResultSet resultSet = statement.executeQuery(); verifier.verify(resultSet); } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java index 36d2c4ed82a..0fcedfee093 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java @@ -7,6 +7,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.util.List; @@ -168,7 +169,7 @@ public void mappingStrip_noopWhenDisabled() { public void bulkStrip_removesDroppedPathsFromSourceLinesOnly() { enable(); String bulk = - "{\"index\":{\"_id\":\"1\"}}\n" + "{\"index\":{\"_index\":\"one\",\"_id\":\"1\",\"routing\":\"r1\"}}\n" + "{\"keep_text\":\"x\",\"geo_point_value\":{\"lat\":1,\"lon\":2},\"geo_shape_value\":\"POINT(1" + " 2)\"}\n" + "{\"index\":{\"_id\":\"2\"}}\n" @@ -178,9 +179,12 @@ public void bulkStrip_removesDroppedPathsFromSourceLinesOnly() { bulk, Set.of(path("geo_point_value"), path("geo_shape_value"), path("nested_value"))); String[] lines = out.split("\n"); - // action lines untouched - assertTrue(lines[0].contains("\"index\"")); - assertTrue(lines[2].contains("\"index\"")); + // Append-only analytics indices require generated IDs, but all other action metadata remains. + JSONObject firstAction = new JSONObject(lines[0]).getJSONObject("index"); + assertFalse(firstAction.has("_id")); + assertEquals("one", firstAction.getString("_index")); + assertEquals("r1", firstAction.getString("routing")); + assertFalse(new JSONObject(lines[2]).getJSONObject("index").has("_id")); // source lines stripped, supported field retained JSONObject doc1 = new JSONObject(lines[1]); assertTrue(doc1.has("keep_text")); @@ -213,12 +217,52 @@ public void bulkStrip_leavesUntouchedSourceLinesByteForByte() { } @Test - public void bulkStrip_noopWhenDisabledOrEmptyDropSet() { - String bulk = "{\"index\":{}}\n{\"geo_point_value\":{\"lat\":1}}\n"; + public void bulkStrip_emptyDropSet_onlyRemovesAnalyticsCustomIds() { + String indexSource = "{\"index\":\"source-value\",\"spacing\": 2}"; + String createSource = "{\"delete\":\"also-a-source-value\"}"; + String bulk = + "{\"index\":{\"_index\":\"fixture\",\"_id\":\"1\"}}\n" + + indexSource + + "\n\n" + + "{\"create\":{\"_index\":\"fixture\",\"_id\":\"2\",\"routing\":\"r2\"}}\n" + + createSource + + "\n"; // disabled -> unchanged even with a drop set assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("geo_point_value")))); - // enabled but empty drop set -> unchanged + + // enabled with no dropped fields -> generated IDs for append-only writes, source unchanged + enable(); + String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of()); + String[] lines = out.split("\n", -1); + JSONObject index = new JSONObject(lines[0]).getJSONObject("index"); + assertFalse(index.has("_id")); + assertEquals("fixture", index.getString("_index")); + assertEquals(indexSource, lines[1]); + assertEquals("", lines[2]); + JSONObject create = new JSONObject(lines[3]).getJSONObject("create"); + assertEquals("2", create.getString("_id")); + assertEquals("fixture", create.getString("_index")); + assertEquals("r2", create.getString("routing")); + assertEquals(createSource, lines[4]); + // split(..., -1) proves the original terminal newline survived. + assertEquals("", lines[5]); + } + + @Test + public void bulkStrip_rejectsActionsThatAppendOnlyStorageCannotRepresent() { enable(); - assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of())); + IllegalArgumentException update = + assertThrows( + IllegalArgumentException.class, + () -> + AnalyticsIndexConfig.stripBulkFields( + "{\"update\":{\"_id\":\"1\"}}\n{\"doc\":{\"value\":1}}\n", Set.of())); + assertTrue(update.getMessage().contains("does not support update")); + + IllegalArgumentException delete = + assertThrows( + IllegalArgumentException.class, + () -> AnalyticsIndexConfig.stripBulkFields("{\"delete\":{\"_id\":\"1\"}}\n", Set.of())); + assertTrue(delete.getMessage().contains("does not support delete")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 91584fb45cf..267adea43c3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -213,9 +213,19 @@ protected static void wipeAllOpenSearchIndices(RestClient client) throws IOExcep String indexName = jsonObject.getString("index"); try { // System index, mostly named .opensearch-xxx or .opendistro-xxx, are not allowed to - // delete + // delete. + // + // `.plugins-` covers the system indices of bundled plugins (ML Commons' + // `.plugins-ml-config`, and friends). Deleting those is never the point of a + // test wipe, and it is actively harmful: on an engine whose plugins are + // still initializing, the DELETE blocks until the client's socket timeout + // and fails the suite before any test runs. That is what broke the 2.19 + // observation leg of the PPL lint multi-version matrix — `_cluster/health` + // reports GREEN before ML Commons finishes creating its config index, so + // the wipe raced initialization. if (!indexName.startsWith(".opensearch") && !indexName.startsWith(".opendistro") + && !indexName.startsWith(".plugins-") && !indexName.startsWith(".ql")) { client.performRequest(new Request("DELETE", "/" + indexName)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index fc15c908c63..97db1640a5e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -869,6 +869,19 @@ public enum Index { "flattened_value", null, "src/test/resources/flattened_value.json"), + // An index with a `flat_object` field, which PPL cannot reference at all — + // neither the root nor a dotted subfield. Backs the flat-object-subfield lint + // contract; see ppl-lint/contracts/flat-object-subfield.spec.json. + FLAT_OBJECT( + TestsConstants.TEST_INDEX_FLAT_OBJECT, + "flat_object", + getFlatObjectIndexMapping(), + "src/test/resources/flat_object.json"), + PPL_LINT_DISABLED_OBJECT( + TestsConstants.TEST_INDEX_PPL_LINT_DISABLED_OBJECT, + "ppl_lint_disabled_object", + getPplLintDisabledObjectIndexMapping(), + "src/test/resources/ppl_lint_disabled_object.json"), DUPLICATION_NULLABLE( TestsConstants.TEST_INDEX_DUPLICATION_NULLABLE, "duplication_nullable", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index c478165bf07..1d92f807249 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -248,40 +248,67 @@ private static void collectAndRemoveUnsupported( } /** - * Strip the given dropped paths from every source document of a bulk NDJSON payload. - * Bulk format alternates an action line ({@code {"index":{...}}}) with a source line; only - * source lines (those without a bulk action key) are rewritten. No-op when disabled or {@code - * droppedPaths} is empty. + * Prepare a bulk NDJSON payload for an analytics-engine append-only index. + * + *

Custom document IDs are not supported for {@code index} operations when {@code + * index.append_only.enabled} is active, so {@code _id} is removed from that action metadata + * while preserving metadata such as {@code _index} and {@code routing}. Create actions retain + * their semantics; update/delete actions fail locally because they are incompatible with + * append-only storage. + * + *

The given dropped paths are also removed from every source document. Bulk format + * alternates an action line ({@code {"index":{...}}}) with a source line; only source lines + * (those without a bulk action key) have mapped fields removed. No-op when analytics mode is + * disabled. * *

Each path is removed recursively: it descends through nested objects and arrays of * objects (so a {@code nested}/object array has the field stripped from every element), * leaving unaffected siblings intact. A source line is re-serialized only when removing a - * path actually changed it; every other line (action lines and docs that never had the - * dropped path) is appended byte-for-byte unchanged, so untouched docs match the fixture - * exactly. + * path actually changed it. Action lines are re-serialized only when removing {@code _id}; + * every other line is appended byte-for-byte unchanged. */ static String stripBulkFields(String bulkBody, Set> droppedPaths) { - if (!isEnabled() || droppedPaths.isEmpty()) { + if (!isEnabled()) { return bulkBody; } String[] lines = bulkBody.split("\n", -1); StringBuilder out = new StringBuilder(bulkBody.length()); + boolean expectSource = false; for (int i = 0; i < lines.length; i++) { String line = lines[i]; String trimmed = line.trim(); - if (!trimmed.isEmpty() && trimmed.charAt(0) == '{') { - JSONObject doc = new JSONObject(trimmed); - boolean isActionLine = - doc.has("index") || doc.has("create") || doc.has("update") || doc.has("delete"); - if (!isActionLine) { - boolean removedAny = false; - for (List path : droppedPaths) { - removedAny |= removePath(doc, path, 0); + boolean terminalNewline = i == lines.length - 1 && trimmed.isEmpty(); + if (!terminalNewline) { + if (trimmed.isEmpty()) { + if (expectSource) { + throw new IllegalArgumentException( + "analytics bulk action is missing its source document"); } - // Only rewrite the line if we actually removed something; otherwise leave it verbatim - // so untouched docs stay byte-for-byte identical to the fixture. - if (removedAny) { - line = doc.toString(); + } else { + JSONObject json = new JSONObject(trimmed); + if (expectSource) { + boolean removedAny = false; + for (List path : droppedPaths) { + removedAny |= removePath(json, path, 0); + } + // Only rewrite the line if we actually removed something; otherwise leave it + // verbatim so untouched docs stay byte-for-byte identical to the fixture. + if (removedAny) { + line = json.toString(); + } + expectSource = false; + } else { + String operation = bulkOperation(json); + if ("update".equals(operation) || "delete".equals(operation)) { + throw new IllegalArgumentException( + "analytics append-only bulk payload does not support " + + operation + + " actions"); + } + if ("index".equals(operation) && removeCustomDocumentId(json, operation)) { + line = json.toString(); + } + expectSource = true; } } } @@ -290,9 +317,40 @@ static String stripBulkFields(String bulkBody, Set> droppedPaths) { out.append('\n'); } } + if (expectSource) { + throw new IllegalArgumentException( + "analytics bulk payload ended before the final action's source document"); + } return out.toString(); } + private static String bulkOperation(JSONObject action) { + List operations = + List.of("index", "create", "update", "delete").stream() + .filter(action::has) + .collect(Collectors.toList()); + if (operations.size() != 1 || action.length() != 1) { + throw new IllegalArgumentException( + "analytics bulk action line must contain exactly one index/create/update/delete" + + " action"); + } + String operation = operations.get(0); + if (!(action.opt(operation) instanceof JSONObject)) { + throw new IllegalArgumentException( + "analytics bulk " + operation + " action metadata must be an object"); + } + return operation; + } + + private static boolean removeCustomDocumentId(JSONObject action, String operation) { + JSONObject metadata = action.optJSONObject(operation); + if (metadata == null || !metadata.has("_id")) { + return false; + } + metadata.remove("_id"); + return true; + } + /** * Remove {@code path[idx..]} from {@code node}, descending through objects and arrays of * objects. Returns true if anything was removed. At the last path part the key is deleted from @@ -429,8 +487,9 @@ public static void loadDataByRestClient( /** * Same as {@link #loadDataByRestClient(RestClient, String, String)} but strips {@code * droppedPaths} (the exact field paths removed from the mapping on the analytics-engine route) - * from every bulk source doc, so the index mapping and the data agree. When AE is disabled or - * {@code droppedPaths} is empty this is byte-for-byte identical to the 3-arg form. + * from every bulk source doc, so the index mapping and the data agree. Analytics append-only + * {@code index} operations also discard custom document IDs. When analytics mode is disabled this + * is byte-for-byte identical to the 3-arg form. */ public static void loadDataByRestClient( RestClient client, String indexName, String dataSetFilePath, Set> droppedPaths) @@ -523,6 +582,16 @@ public static String getAccountExtendedIndexMapping() { return getMappingFile(mappingFile); } + public static String getFlatObjectIndexMapping() { + String mappingFile = "flat_object_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getPplLintDisabledObjectIndexMapping() { + String mappingFile = "ppl_lint_disabled_object_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getPhraseIndexMapping() { String mappingFile = "phrase_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 5d7eeb328af..357b5d37ef4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -76,6 +76,9 @@ public class TestsConstants { public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test"; public static final String TEST_INDEX_ALIAS = TEST_INDEX + "_alias"; public static final String TEST_INDEX_FLATTENED_VALUE = TEST_INDEX + "_flattened_value"; + public static final String TEST_INDEX_FLAT_OBJECT = TEST_INDEX + "_flat_object"; + public static final String TEST_INDEX_PPL_LINT_DISABLED_OBJECT = + TEST_INDEX + "_ppl_lint_disabled_object"; public static final String TEST_INDEX_GEOIP = TEST_INDEX + "_geoip"; public static final String DATASOURCES = ".ql-datasources"; public static final String TEST_INDEX_STATE_COUNTRY = TEST_INDEX + "_state_country"; diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index b5c08cc8ad0..9e3fe563d41 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -64,27 +64,24 @@ public void testClusterStarted() { /** * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its - * fixed schema and correct data unchanged when the analytics-engine plugin is present. + * single-column schema and correct data unchanged when the analytics-engine plugin is present. */ @Test public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { Request request = new Request("POST", "/_plugins/_ppl"); - request.setJsonEntity( - "{\"query\": \"| rest '/_cluster/health' | fields status, number_of_nodes\"}"); + request.setJsonEntity("{\"query\": \"| rest '/_cluster/health' | fields response\"}"); Response response = client().performRequest(request); assertEquals(200, response.getStatusLine().getStatusCode()); JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); JSONArray schema = result.getJSONArray("schema"); - assertEquals(2, schema.length()); - assertEquals("status", schema.getJSONObject(0).getString("name")); + assertEquals(1, schema.length()); + assertEquals("response", schema.getJSONObject(0).getString("name")); assertEquals("string", schema.getJSONObject(0).getString("type")); - assertEquals("number_of_nodes", schema.getJSONObject(1).getString("name")); - assertEquals("int", schema.getJSONObject(1).getString("type")); JSONArray datarows = result.getJSONArray("datarows"); assertEquals(1, datarows.length()); - assertTrue(datarows.getJSONArray(0).getInt(1) >= 1); + assertTrue(datarows.getJSONArray(0).getString(0).contains("number_of_nodes")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 1af872a8ab6..5b4181d4e33 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -207,6 +207,35 @@ public void testBooleanFieldFromNumberAcrossWildcardIndices() throws Exception { } } + @Test + public void test_constant_keyword_data_type() throws Exception { + String index = "test_constant_keyword"; + try { + Request createIndex = new Request("PUT", "/" + index); + createIndex.setJsonEntity( + "{\"mappings\":{\"properties\":{" + + "\"tenant\":{\"type\":\"constant_keyword\",\"value\":\"acme\"}," + + "\"message\":{\"type\":\"text\"}}}}"); + client().performRequest(createIndex); + + Request insertDoc = new Request("PUT", "/" + index + "/_doc/1?refresh=true"); + insertDoc.setJsonEntity("{\"tenant\":\"acme\",\"message\":\"hello\"}"); + client().performRequest(insertDoc); + + JSONObject result = executeQuery(String.format("source=%s | fields tenant, message", index)); + verifySchema(result, schema("tenant", "string"), schema("message", "string")); + verifyDataRows(result, rows("acme", "hello")); + + // constant_keyword should filter like a regular string. + JSONObject filtered = + executeQuery( + String.format("source=%s | where tenant='acme' | fields tenant, message", index)); + verifyDataRows(filtered, rows("acme", "hello")); + } finally { + client().performRequest(new Request("DELETE", "/" + index)); + } + } + @Test @RequiresCapability( value = DOC_MUTATION, diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java index 45ecf02af8f..1507764b020 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java @@ -58,6 +58,27 @@ public void testAllowMoreDuplicates() throws IOException { verifyDataRows(result, rows(true), rows(true), rows(false), rows(false)); } + @Test + public void testDedupOnTextField() throws IOException { + // `email` is mapped as text with no .keyword sub-field, so dedup runs via the text-field + // aggregation pushdown path (composite terms + top_hits with the field read from _source). + // Assert not just the dedup key set but also the associated projected columns per row, so + // the top_hits round-trip is exercised end-to-end. + JSONObject result = + executeQuery( + String.format( + "source=%s | dedup email | fields email, firstname, balance", TEST_INDEX_BANK)); + verifyDataRows( + result, + rows("amberduke@pyrami.com", "Amber JOHnny", 39225), + rows("hattiebond@netagy.com", "Hattie", 5686), + rows("nanettebates@quility.com", "Nanette", 32838), + rows("daleadams@boink.com", "Dale", 4180), + rows("elinorratliff@scentric.com", "Elinor", 16418), + rows("virginiaayala@filodyne.com", "Virginia", 40540), + rows("dillardmcpherson@quailcom.com", "Dillard", 48086)); + } + @Test public void testKeepEmptyDedup() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 2ccda31eea7..6a6c0bc222c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; import java.io.IOException; +import org.apache.commons.text.StringEscapeUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -36,7 +37,7 @@ public void init() throws Exception { public void testRest() throws IOException { JSONObject result; try { - result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + result = executeQuery("| rest '/_cluster/health' | fields response"); } catch (ResponseException e) { result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); } @@ -331,6 +332,27 @@ public void testNoMvUnsupportedInV2() throws IOException { verifyQuery(result); } + @Test + public void testMakeResults() throws IOException { + JSONObject result; + try { + result = executeQuery("makeresults count=2"); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + + if (isCalciteEnabled()) { + assertThat(result.getJSONArray("datarows").length(), equalTo(2)); + } else { + JSONObject error = result.getJSONObject("error"); + assertThat( + error.getString("details"), + containsString( + "is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true")); + assertThat(error.getString("type"), equalTo("UnsupportedOperationException")); + } + } + @Test public void testMvExpandCommandBasicExpansion() throws IOException { JSONObject result; @@ -559,4 +581,78 @@ public void testUnionUnsupportedInV2() throws IOException { } verifyQuery(result); } + + @Test + public void testXyseriesCommand() throws IOException { + + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK))); + + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandMultipleDataFields() throws IOException { + + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance, count() as cnt by" + + " gender, state | xyseries state gender in (\"F\", \"M\") avg_balance," + + " cnt", + TEST_INDEX_BANK))); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandWithSep() throws IOException { + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries sep=\"-\" state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK))); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandWithFormat() throws IOException { + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries format=\"$VAL$_$AGG$\" state gender in (\"F\", \"M\")" + + " avg_balance", + TEST_INDEX_BANK))); + + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index 7417fd112ec..4fd8aa7f852 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -512,11 +512,6 @@ public void testSumWithNull() throws IOException { "source=%s | where age = 36 | stats sum(balance)", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("sum(balance)", null, "bigint")); - // TODO: Fix -- temporary workaround for the pushdown issue: - // The current pushdown implementation will return 0 for sum when getting null values as input. - // Returning null should be the expected behavior. - // The analytics-engine backend (DataFusion) follows the SQL spec like Calcite-no-pushdown — - // SUM of all-null is null, not 0. Integer expectedValue = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0; verifyDataRows(response, rows(expectedValue)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java index 936e728bd10..8121ce0ea6b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java @@ -58,4 +58,20 @@ public void testTopNWithGroup() throws IOException { verifyDataRows(result, rows("F", "TX"), rows("M", "MD")); } } + + @Test + public void testTopWithShowPerc() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | top showperc=true gender", TEST_INDEX_ACCOUNT)); + if (isCalciteEnabled()) { + verifySchemaInOrder( + result, + schema("gender", "string"), + schema("count", "bigint"), + schema("percent", "double")); + verifyDataRows(result, rows("M", 507, 50.7), rows("F", 493, 49.3)); + } else { + verifyDataRows(result, rows("M"), rows("F")); + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java index 0029921c1fc..86e98c1523f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java @@ -237,4 +237,28 @@ public void testCrossClusterConvertWithAlias() throws IOException { disableCalcite(); } + + @Test + public void testCrossClusterXyseries() throws IOException { + enableCalcite(); + + JSONObject result = + executeQuery( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in ('F', 'M') avg_balance", + TEST_INDEX_BANK_REMOTE)); + verifyColumn(result, columnName("state"), columnName("F"), columnName("M")); + verifyDataRows( + result, + rows("IL", null, 39225.0), + rows("IN", 48086.0, null), + rows("MD", null, 4180.0), + rows("PA", 40540.0, null), + rows("TN", null, 5686.0), + rows("VA", 32838.0, null), + rows("WA", null, 16418.0)); + + disableCalcite(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java b/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java index 22c93591241..97559b6032c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java @@ -656,4 +656,88 @@ public void testRowLevelSecurity(boolean useCalcite) throws IOException { expectedPublicDocs, totalDocs); } + + /** + * Verifies that document-level security is enforced when queries are dispatched to the complex + * worker pool. Queries containing window functions (eventstats) are routed to sql-complex-worker; + * this test ensures the OpenSearch ThreadContext (which carries DLS filters) is correctly + * propagated across that thread pool boundary. + */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testRowLevelSecurityEnforcedOnComplexPool(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + String engineLabel = useCalcite ? "V3" : "V2"; + + // eventstats creates a Window node, which ScriptDetector flags as expensive, + // routing the query to the sql-complex-worker pool. + String query = + String.format( + "search source=%s | eventstats count() as total_count by security_level" + + " | stats count() by security_level", + SECURE_LOGS); + JSONObject result = executeQueryAsUser(query, LIMITED_USER); + + var datarows = result.getJSONArray("datarows"); + var schema = result.getJSONArray("schema"); + int levelIdx = -1; + for (int i = 0; i < schema.length(); i++) { + String name = schema.getJSONObject(i).getString("name"); + if ("security_level".equals(name)) { + levelIdx = i; + } + } + assertTrue("Expected security_level in schema", levelIdx >= 0); + + for (int i = 0; i < datarows.length(); i++) { + var row = datarows.getJSONArray(i); + String securityLevel = row.getString(levelIdx); + assertFalse( + String.format( + "[%s] SECURITY VIOLATION on complex pool: limited_user saw '%s' documents. " + + "DLS ThreadContext may not be propagated to sql-complex-worker pool.", + engineLabel, securityLevel), + "confidential".equals(securityLevel) || "internal".equals(securityLevel)); + } + } + + /** + * Verifies that field-level security is enforced when queries are dispatched to the slow worker + * pool. The eventstats command creates a Window node that triggers complex pool dispatch; this + * test ensures the restricted field (ssn) remains invisible. + */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testFieldLevelSecurityEnforcedOnSlowPool(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + + // eventstats creates a Window node, which ScriptDetector flags as expensive, + // routing the query to the sql-complex-worker pool. + // manager_user should still NOT see ssn. + String query = + String.format( + "search source=%s | eventstats avg(salary) as avg_salary by department" + + " | fields name, department, salary, avg_salary | head 10", + EMPLOYEE_RECORDS); + JSONObject result = executeQueryAsUser(query, MANAGER_USER); + + var resultSchema = result.getJSONArray("schema"); + boolean hasSSN = false; + boolean hasName = false; + boolean hasAvgSalary = false; + + for (int i = 0; i < resultSchema.length(); i++) { + String fieldName = resultSchema.getJSONObject(i).getString("name"); + if ("ssn".equals(fieldName)) hasSSN = true; + if ("name".equals(fieldName)) hasName = true; + if ("avg_salary".equals(fieldName)) hasAvgSalary = true; + } + + assertTrue("manager_user should see 'name' field on complex pool", hasName); + assertTrue("manager_user should see computed 'avg_salary' field on complex pool", hasAvgSalary); + assertFalse( + "SECURITY VIOLATION on complex pool: manager_user saw 'ssn' field. " + + "FLS ThreadContext may not be propagated to sql-complex-worker pool.", + hasSSN); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java index 042a3aefbf7..c4b5e726a54 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java @@ -9,28 +9,21 @@ import static org.opensearch.sql.util.MatcherUtils.verifyColumn; import java.io.IOException; -import java.util.HashSet; -import java.util.Set; -import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; -import org.opensearch.client.Request; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.TestUtils; /** - * Integration tests that verify the rest command is subject to the security plugin fine grained - * access control. The command dispatches standard transport actions under the caller identity, so - * the security ActionFilter authorizes each one by action name. A caller without the required - * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege - * can run them, and the resolve index endpoint requires the resolve index privilege because the - * command resolves all indices. + * Integration tests that the rest command is subject to the security plugin fine grained access + * control. The command dispatches a standard transport action under the caller identity, so the + * security ActionFilter authorizes it by action name. This version ships only {@code + * /_cluster/health}, which requires the {@code cluster:monitor/health} privilege: a caller holding + * cluster monitor can run it, a caller without it is denied. The command therefore grants no access + * beyond calling the endpoint natively. */ public class RestCommandSecurityIT extends SecurityTestBase { - private static final String ALPHA_INDEX = "rest_sec_alpha"; - private static final String BETA_INDEX = "rest_sec_beta"; - private static final String MONITOR_USER = "rest_monitor_user"; private static final String MONITOR_ROLE = "rest_monitor_role"; @@ -40,71 +33,37 @@ public class RestCommandSecurityIT extends SecurityTestBase { @Override protected void init() throws Exception { super.init(); - setupRolesUsersAndIndices(); + setupRolesAndUsers(); enableCalcite(); // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported // command error. Disable fallback so the denial reason surfaces to the caller. disallowCalciteFallback(); } - private void setupRolesUsersAndIndices() throws IOException { - createIndexIfAbsent(ALPHA_INDEX); - createIndexIfAbsent(BETA_INDEX); - + private void setupRolesAndUsers() throws IOException { createRoleWithPermissions( MONITOR_ROLE, "*", - new String[] { - "cluster:admin/opensearch/ppl", - "cluster:monitor/health", - "cluster:monitor/state", - "cluster:monitor/nodes/stats", - "cluster:monitor/nodes/info" - }, - new String[] {"indices:admin/resolve/index"}); + new String[] {"cluster:admin/opensearch/ppl", "cluster:monitor/health"}, + new String[] {}); createUser(MONITOR_USER, MONITOR_ROLE); createRoleWithPermissions( - NO_MONITOR_ROLE, - ALPHA_INDEX, - new String[] {"cluster:admin/opensearch/ppl"}, - new String[] {"indices:data/read/search*"}); + NO_MONITOR_ROLE, "*", new String[] {"cluster:admin/opensearch/ppl"}, new String[] {}); createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); } @Test - public void monitorUserCanRunCatNodes() throws IOException { - JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); - verifyColumn(result, columnName("name")); - } - - @Test - public void monitorUserCanResolveIndex() throws IOException { + public void monitorUserCanRunClusterHealth() throws IOException { JSONObject result = - executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); - Set names = resolvedNames(result); - assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); - assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); - } - - @Test - public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { - assertDenied( - "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); - } - - @Test - public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { - assertDenied( - "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); + executeQueryAsUser("| rest '/_cluster/health' | fields response", MONITOR_USER); + verifyColumn(result, columnName("response")); } @Test - public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { + public void userWithoutClusterMonitorCannotRunClusterHealth() throws IOException { assertDenied( - "| rest '/_resolve/index' | fields name, type", - NO_MONITOR_USER, - "indices:admin/resolve/index"); + "| rest '/_cluster/health' | fields response", NO_MONITOR_USER, "cluster:monitor/health"); } /** @@ -128,27 +87,4 @@ private void assertDenied(String query, String user, String deniedAction) throws || body.contains(deniedAction)); } } - - private void createIndexIfAbsent(String name) throws IOException { - Request request = new Request("PUT", "/" + name); - request.setJsonEntity( - "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); - try { - client().performRequest(request); - } catch (ResponseException e) { - String body = TestUtils.getResponseBody(e.getResponse(), false); - if (!body.contains("resource_already_exists_exception")) { - throw e; - } - } - } - - private Set resolvedNames(JSONObject result) { - Set names = new HashSet<>(); - JSONArray datarows = result.getJSONArray("datarows"); - for (int i = 0; i < datarows.length(); i++) { - names.add(datarows.getJSONArray(i).getString(0)); - } - return names; - } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index 726eeedc429..9776f5dafc1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -25,15 +25,11 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["gender"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age0, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["gender"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml index f900b2ccbec..4e024b0c12d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(sum(AdvEngineID)=[$1], c=[$2], avg(ResolutionWidth)=[$3], dc(UserID)=[$4], RegionID=[$0]) - LogicalAggregate(group=[{0}], sum(AdvEngineID)=[SUM($1)], c=[COUNT()], avg(ResolutionWidth)=[AVG($2)], dc(UserID)=[COUNT(DISTINCT $3)]) + LogicalAggregate(group=[{0}], sum(AdvEngineID)=[CHECKED_LONG_SUM($1)], c=[COUNT()], avg(ResolutionWidth)=[AVG($2)], dc(UserID)=[COUNT(DISTINCT $3)]) LogicalProject(RegionID=[$68], AdvEngineID=[$19], ResolutionWidth=[$80], UserID=[$84]) LogicalFilter(condition=[IS NOT NULL($68)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=CHECKED_LONG_SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml index ef93b63ee80..24ddecd3881 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{}], sum(AdvEngineID)=[SUM($0)], count()=[COUNT()], avg(ResolutionWidth)=[AVG($1)]) + LogicalAggregate(group=[{}], sum(AdvEngineID)=[CHECKED_LONG_SUM($0)], count()=[COUNT()], avg(ResolutionWidth)=[AVG($1)]) LogicalProject(AdvEngineID=[$19], ResolutionWidth=[$80]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=CHECKED_LONG_SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml index d50a9ec47ce..189ab6349e1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml @@ -1,11 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + LogicalAggregate(group=[{}], sum(ResolutionWidth)=[CHECKED_LONG_SUM($0)], sum(ResolutionWidth+1)=[CHECKED_LONG_SUM($1)], sum(ResolutionWidth+2)=[CHECKED_LONG_SUM($2)], sum(ResolutionWidth+3)=[CHECKED_LONG_SUM($3)], sum(ResolutionWidth+4)=[CHECKED_LONG_SUM($4)], sum(ResolutionWidth+5)=[CHECKED_LONG_SUM($5)], sum(ResolutionWidth+6)=[CHECKED_LONG_SUM($6)], sum(ResolutionWidth+7)=[CHECKED_LONG_SUM($7)], sum(ResolutionWidth+8)=[CHECKED_LONG_SUM($8)], sum(ResolutionWidth+9)=[CHECKED_LONG_SUM($9)], sum(ResolutionWidth+10)=[CHECKED_LONG_SUM($10)], sum(ResolutionWidth+11)=[CHECKED_LONG_SUM($11)], sum(ResolutionWidth+12)=[CHECKED_LONG_SUM($12)], sum(ResolutionWidth+13)=[CHECKED_LONG_SUM($13)], sum(ResolutionWidth+14)=[CHECKED_LONG_SUM($14)], sum(ResolutionWidth+15)=[CHECKED_LONG_SUM($15)], sum(ResolutionWidth+16)=[CHECKED_LONG_SUM($16)], sum(ResolutionWidth+17)=[CHECKED_LONG_SUM($17)], sum(ResolutionWidth+18)=[CHECKED_LONG_SUM($18)], sum(ResolutionWidth+19)=[CHECKED_LONG_SUM($19)], sum(ResolutionWidth+20)=[CHECKED_LONG_SUM($20)], sum(ResolutionWidth+21)=[CHECKED_LONG_SUM($21)], sum(ResolutionWidth+22)=[CHECKED_LONG_SUM($22)], sum(ResolutionWidth+23)=[CHECKED_LONG_SUM($23)], sum(ResolutionWidth+24)=[CHECKED_LONG_SUM($24)], sum(ResolutionWidth+25)=[CHECKED_LONG_SUM($25)], sum(ResolutionWidth+26)=[CHECKED_LONG_SUM($26)], sum(ResolutionWidth+27)=[CHECKED_LONG_SUM($27)], sum(ResolutionWidth+28)=[CHECKED_LONG_SUM($28)], sum(ResolutionWidth+29)=[CHECKED_LONG_SUM($29)], sum(ResolutionWidth+30)=[CHECKED_LONG_SUM($30)], sum(ResolutionWidth+31)=[CHECKED_LONG_SUM($31)], sum(ResolutionWidth+32)=[CHECKED_LONG_SUM($32)], sum(ResolutionWidth+33)=[CHECKED_LONG_SUM($33)], sum(ResolutionWidth+34)=[CHECKED_LONG_SUM($34)], sum(ResolutionWidth+35)=[CHECKED_LONG_SUM($35)], sum(ResolutionWidth+36)=[CHECKED_LONG_SUM($36)], sum(ResolutionWidth+37)=[CHECKED_LONG_SUM($37)], sum(ResolutionWidth+38)=[CHECKED_LONG_SUM($38)], sum(ResolutionWidth+39)=[CHECKED_LONG_SUM($39)], sum(ResolutionWidth+40)=[CHECKED_LONG_SUM($40)], sum(ResolutionWidth+41)=[CHECKED_LONG_SUM($41)], sum(ResolutionWidth+42)=[CHECKED_LONG_SUM($42)], sum(ResolutionWidth+43)=[CHECKED_LONG_SUM($43)], sum(ResolutionWidth+44)=[CHECKED_LONG_SUM($44)], sum(ResolutionWidth+45)=[CHECKED_LONG_SUM($45)], sum(ResolutionWidth+46)=[CHECKED_LONG_SUM($46)], sum(ResolutionWidth+47)=[CHECKED_LONG_SUM($47)], sum(ResolutionWidth+48)=[CHECKED_LONG_SUM($48)], sum(ResolutionWidth+49)=[CHECKED_LONG_SUM($49)], sum(ResolutionWidth+50)=[CHECKED_LONG_SUM($50)], sum(ResolutionWidth+51)=[CHECKED_LONG_SUM($51)], sum(ResolutionWidth+52)=[CHECKED_LONG_SUM($52)], sum(ResolutionWidth+53)=[CHECKED_LONG_SUM($53)], sum(ResolutionWidth+54)=[CHECKED_LONG_SUM($54)], sum(ResolutionWidth+55)=[CHECKED_LONG_SUM($55)], sum(ResolutionWidth+56)=[CHECKED_LONG_SUM($56)], sum(ResolutionWidth+57)=[CHECKED_LONG_SUM($57)], sum(ResolutionWidth+58)=[CHECKED_LONG_SUM($58)], sum(ResolutionWidth+59)=[CHECKED_LONG_SUM($59)], sum(ResolutionWidth+60)=[CHECKED_LONG_SUM($60)], sum(ResolutionWidth+61)=[CHECKED_LONG_SUM($61)], sum(ResolutionWidth+62)=[CHECKED_LONG_SUM($62)], sum(ResolutionWidth+63)=[CHECKED_LONG_SUM($63)], sum(ResolutionWidth+64)=[CHECKED_LONG_SUM($64)], sum(ResolutionWidth+65)=[CHECKED_LONG_SUM($65)], sum(ResolutionWidth+66)=[CHECKED_LONG_SUM($66)], sum(ResolutionWidth+67)=[CHECKED_LONG_SUM($67)], sum(ResolutionWidth+68)=[CHECKED_LONG_SUM($68)], sum(ResolutionWidth+69)=[CHECKED_LONG_SUM($69)], sum(ResolutionWidth+70)=[CHECKED_LONG_SUM($70)], sum(ResolutionWidth+71)=[CHECKED_LONG_SUM($71)], sum(ResolutionWidth+72)=[CHECKED_LONG_SUM($72)], sum(ResolutionWidth+73)=[CHECKED_LONG_SUM($73)], sum(ResolutionWidth+74)=[CHECKED_LONG_SUM($74)], sum(ResolutionWidth+75)=[CHECKED_LONG_SUM($75)], sum(ResolutionWidth+76)=[CHECKED_LONG_SUM($76)], sum(ResolutionWidth+77)=[CHECKED_LONG_SUM($77)], sum(ResolutionWidth+78)=[CHECKED_LONG_SUM($78)], sum(ResolutionWidth+79)=[CHECKED_LONG_SUM($79)], sum(ResolutionWidth+80)=[CHECKED_LONG_SUM($80)], sum(ResolutionWidth+81)=[CHECKED_LONG_SUM($81)], sum(ResolutionWidth+82)=[CHECKED_LONG_SUM($82)], sum(ResolutionWidth+83)=[CHECKED_LONG_SUM($83)], sum(ResolutionWidth+84)=[CHECKED_LONG_SUM($84)], sum(ResolutionWidth+85)=[CHECKED_LONG_SUM($85)], sum(ResolutionWidth+86)=[CHECKED_LONG_SUM($86)], sum(ResolutionWidth+87)=[CHECKED_LONG_SUM($87)], sum(ResolutionWidth+88)=[CHECKED_LONG_SUM($88)], sum(ResolutionWidth+89)=[CHECKED_LONG_SUM($89)]) LogicalProject(ResolutionWidth=[$80], $f90=[+(CAST($80):BIGINT, 1)], $f91=[+(CAST($80):BIGINT, 2)], $f92=[+(CAST($80):BIGINT, 3)], $f93=[+(CAST($80):BIGINT, 4)], $f94=[+(CAST($80):BIGINT, 5)], $f95=[+(CAST($80):BIGINT, 6)], $f96=[+(CAST($80):BIGINT, 7)], $f97=[+(CAST($80):BIGINT, 8)], $f98=[+(CAST($80):BIGINT, 9)], $f99=[+(CAST($80):BIGINT, 10)], $f100=[+(CAST($80):BIGINT, 11)], $f101=[+(CAST($80):BIGINT, 12)], $f102=[+(CAST($80):BIGINT, 13)], $f103=[+(CAST($80):BIGINT, 14)], $f104=[+(CAST($80):BIGINT, 15)], $f105=[+(CAST($80):BIGINT, 16)], $f106=[+(CAST($80):BIGINT, 17)], $f107=[+(CAST($80):BIGINT, 18)], $f108=[+(CAST($80):BIGINT, 19)], $f109=[+(CAST($80):BIGINT, 20)], $f110=[+(CAST($80):BIGINT, 21)], $f111=[+(CAST($80):BIGINT, 22)], $f112=[+(CAST($80):BIGINT, 23)], $f113=[+(CAST($80):BIGINT, 24)], $f114=[+(CAST($80):BIGINT, 25)], $f115=[+(CAST($80):BIGINT, 26)], $f116=[+(CAST($80):BIGINT, 27)], $f117=[+(CAST($80):BIGINT, 28)], $f118=[+(CAST($80):BIGINT, 29)], $f119=[+(CAST($80):BIGINT, 30)], $f120=[+(CAST($80):BIGINT, 31)], $f121=[+(CAST($80):BIGINT, 32)], $f122=[+(CAST($80):BIGINT, 33)], $f123=[+(CAST($80):BIGINT, 34)], $f124=[+(CAST($80):BIGINT, 35)], $f125=[+(CAST($80):BIGINT, 36)], $f126=[+(CAST($80):BIGINT, 37)], $f127=[+(CAST($80):BIGINT, 38)], $f128=[+(CAST($80):BIGINT, 39)], $f129=[+(CAST($80):BIGINT, 40)], $f130=[+(CAST($80):BIGINT, 41)], $f131=[+(CAST($80):BIGINT, 42)], $f132=[+(CAST($80):BIGINT, 43)], $f133=[+(CAST($80):BIGINT, 44)], $f134=[+(CAST($80):BIGINT, 45)], $f135=[+(CAST($80):BIGINT, 46)], $f136=[+(CAST($80):BIGINT, 47)], $f137=[+(CAST($80):BIGINT, 48)], $f138=[+(CAST($80):BIGINT, 49)], $f139=[+(CAST($80):BIGINT, 50)], $f140=[+(CAST($80):BIGINT, 51)], $f141=[+(CAST($80):BIGINT, 52)], $f142=[+(CAST($80):BIGINT, 53)], $f143=[+(CAST($80):BIGINT, 54)], $f144=[+(CAST($80):BIGINT, 55)], $f145=[+(CAST($80):BIGINT, 56)], $f146=[+(CAST($80):BIGINT, 57)], $f147=[+(CAST($80):BIGINT, 58)], $f148=[+(CAST($80):BIGINT, 59)], $f149=[+(CAST($80):BIGINT, 60)], $f150=[+(CAST($80):BIGINT, 61)], $f151=[+(CAST($80):BIGINT, 62)], $f152=[+(CAST($80):BIGINT, 63)], $f153=[+(CAST($80):BIGINT, 64)], $f154=[+(CAST($80):BIGINT, 65)], $f155=[+(CAST($80):BIGINT, 66)], $f156=[+(CAST($80):BIGINT, 67)], $f157=[+(CAST($80):BIGINT, 68)], $f158=[+(CAST($80):BIGINT, 69)], $f159=[+(CAST($80):BIGINT, 70)], $f160=[+(CAST($80):BIGINT, 71)], $f161=[+(CAST($80):BIGINT, 72)], $f162=[+(CAST($80):BIGINT, 73)], $f163=[+(CAST($80):BIGINT, 74)], $f164=[+(CAST($80):BIGINT, 75)], $f165=[+(CAST($80):BIGINT, 76)], $f166=[+(CAST($80):BIGINT, 77)], $f167=[+(CAST($80):BIGINT, 78)], $f168=[+(CAST($80):BIGINT, 79)], $f169=[+(CAST($80):BIGINT, 80)], $f170=[+(CAST($80):BIGINT, 81)], $f171=[+(CAST($80):BIGINT, 82)], $f172=[+(CAST($80):BIGINT, 83)], $f173=[+(CAST($80):BIGINT, 84)], $f174=[+(CAST($80):BIGINT, 85)], $f175=[+(CAST($80):BIGINT, 86)], $f176=[+(CAST($80):BIGINT, 87)], $f177=[+(CAST($80):BIGINT, 88)], $f178=[+(CAST($80):BIGINT, 89)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + EnumerableAggregate(group=[{}], sum(ResolutionWidth)=[CHECKED_LONG_SUM($0)], sum(ResolutionWidth+1)=[CHECKED_LONG_SUM($1)], sum(ResolutionWidth+2)=[CHECKED_LONG_SUM($2)], sum(ResolutionWidth+3)=[CHECKED_LONG_SUM($3)], sum(ResolutionWidth+4)=[CHECKED_LONG_SUM($4)], sum(ResolutionWidth+5)=[CHECKED_LONG_SUM($5)], sum(ResolutionWidth+6)=[CHECKED_LONG_SUM($6)], sum(ResolutionWidth+7)=[CHECKED_LONG_SUM($7)], sum(ResolutionWidth+8)=[CHECKED_LONG_SUM($8)], sum(ResolutionWidth+9)=[CHECKED_LONG_SUM($9)], sum(ResolutionWidth+10)=[CHECKED_LONG_SUM($10)], sum(ResolutionWidth+11)=[CHECKED_LONG_SUM($11)], sum(ResolutionWidth+12)=[CHECKED_LONG_SUM($12)], sum(ResolutionWidth+13)=[CHECKED_LONG_SUM($13)], sum(ResolutionWidth+14)=[CHECKED_LONG_SUM($14)], sum(ResolutionWidth+15)=[CHECKED_LONG_SUM($15)], sum(ResolutionWidth+16)=[CHECKED_LONG_SUM($16)], sum(ResolutionWidth+17)=[CHECKED_LONG_SUM($17)], sum(ResolutionWidth+18)=[CHECKED_LONG_SUM($18)], sum(ResolutionWidth+19)=[CHECKED_LONG_SUM($19)], sum(ResolutionWidth+20)=[CHECKED_LONG_SUM($20)], sum(ResolutionWidth+21)=[CHECKED_LONG_SUM($21)], sum(ResolutionWidth+22)=[CHECKED_LONG_SUM($22)], sum(ResolutionWidth+23)=[CHECKED_LONG_SUM($23)], sum(ResolutionWidth+24)=[CHECKED_LONG_SUM($24)], sum(ResolutionWidth+25)=[CHECKED_LONG_SUM($25)], sum(ResolutionWidth+26)=[CHECKED_LONG_SUM($26)], sum(ResolutionWidth+27)=[CHECKED_LONG_SUM($27)], sum(ResolutionWidth+28)=[CHECKED_LONG_SUM($28)], sum(ResolutionWidth+29)=[CHECKED_LONG_SUM($29)], sum(ResolutionWidth+30)=[CHECKED_LONG_SUM($30)], sum(ResolutionWidth+31)=[CHECKED_LONG_SUM($31)], sum(ResolutionWidth+32)=[CHECKED_LONG_SUM($32)], sum(ResolutionWidth+33)=[CHECKED_LONG_SUM($33)], sum(ResolutionWidth+34)=[CHECKED_LONG_SUM($34)], sum(ResolutionWidth+35)=[CHECKED_LONG_SUM($35)], sum(ResolutionWidth+36)=[CHECKED_LONG_SUM($36)], sum(ResolutionWidth+37)=[CHECKED_LONG_SUM($37)], sum(ResolutionWidth+38)=[CHECKED_LONG_SUM($38)], sum(ResolutionWidth+39)=[CHECKED_LONG_SUM($39)], sum(ResolutionWidth+40)=[CHECKED_LONG_SUM($40)], sum(ResolutionWidth+41)=[CHECKED_LONG_SUM($41)], sum(ResolutionWidth+42)=[CHECKED_LONG_SUM($42)], sum(ResolutionWidth+43)=[CHECKED_LONG_SUM($43)], sum(ResolutionWidth+44)=[CHECKED_LONG_SUM($44)], sum(ResolutionWidth+45)=[CHECKED_LONG_SUM($45)], sum(ResolutionWidth+46)=[CHECKED_LONG_SUM($46)], sum(ResolutionWidth+47)=[CHECKED_LONG_SUM($47)], sum(ResolutionWidth+48)=[CHECKED_LONG_SUM($48)], sum(ResolutionWidth+49)=[CHECKED_LONG_SUM($49)], sum(ResolutionWidth+50)=[CHECKED_LONG_SUM($50)], sum(ResolutionWidth+51)=[CHECKED_LONG_SUM($51)], sum(ResolutionWidth+52)=[CHECKED_LONG_SUM($52)], sum(ResolutionWidth+53)=[CHECKED_LONG_SUM($53)], sum(ResolutionWidth+54)=[CHECKED_LONG_SUM($54)], sum(ResolutionWidth+55)=[CHECKED_LONG_SUM($55)], sum(ResolutionWidth+56)=[CHECKED_LONG_SUM($56)], sum(ResolutionWidth+57)=[CHECKED_LONG_SUM($57)], sum(ResolutionWidth+58)=[CHECKED_LONG_SUM($58)], sum(ResolutionWidth+59)=[CHECKED_LONG_SUM($59)], sum(ResolutionWidth+60)=[CHECKED_LONG_SUM($60)], sum(ResolutionWidth+61)=[CHECKED_LONG_SUM($61)], sum(ResolutionWidth+62)=[CHECKED_LONG_SUM($62)], sum(ResolutionWidth+63)=[CHECKED_LONG_SUM($63)], sum(ResolutionWidth+64)=[CHECKED_LONG_SUM($64)], sum(ResolutionWidth+65)=[CHECKED_LONG_SUM($65)], sum(ResolutionWidth+66)=[CHECKED_LONG_SUM($66)], sum(ResolutionWidth+67)=[CHECKED_LONG_SUM($67)], sum(ResolutionWidth+68)=[CHECKED_LONG_SUM($68)], sum(ResolutionWidth+69)=[CHECKED_LONG_SUM($69)], sum(ResolutionWidth+70)=[CHECKED_LONG_SUM($70)], sum(ResolutionWidth+71)=[CHECKED_LONG_SUM($71)], sum(ResolutionWidth+72)=[CHECKED_LONG_SUM($72)], sum(ResolutionWidth+73)=[CHECKED_LONG_SUM($73)], sum(ResolutionWidth+74)=[CHECKED_LONG_SUM($74)], sum(ResolutionWidth+75)=[CHECKED_LONG_SUM($75)], sum(ResolutionWidth+76)=[CHECKED_LONG_SUM($76)], sum(ResolutionWidth+77)=[CHECKED_LONG_SUM($77)], sum(ResolutionWidth+78)=[CHECKED_LONG_SUM($78)], sum(ResolutionWidth+79)=[CHECKED_LONG_SUM($79)], sum(ResolutionWidth+80)=[CHECKED_LONG_SUM($80)], sum(ResolutionWidth+81)=[CHECKED_LONG_SUM($81)], sum(ResolutionWidth+82)=[CHECKED_LONG_SUM($82)], sum(ResolutionWidth+83)=[CHECKED_LONG_SUM($83)], sum(ResolutionWidth+84)=[CHECKED_LONG_SUM($84)], sum(ResolutionWidth+85)=[CHECKED_LONG_SUM($85)], sum(ResolutionWidth+86)=[CHECKED_LONG_SUM($86)], sum(ResolutionWidth+87)=[CHECKED_LONG_SUM($87)], sum(ResolutionWidth+88)=[CHECKED_LONG_SUM($88)], sum(ResolutionWidth+89)=[CHECKED_LONG_SUM($89)]) EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):BIGINT], expr#2=[1:BIGINT], expr#3=[+($t1, $t2)], expr#4=[2:BIGINT], expr#5=[+($t1, $t4)], expr#6=[3:BIGINT], expr#7=[+($t1, $t6)], expr#8=[4:BIGINT], expr#9=[+($t1, $t8)], expr#10=[5:BIGINT], expr#11=[+($t1, $t10)], expr#12=[6:BIGINT], expr#13=[+($t1, $t12)], expr#14=[7:BIGINT], expr#15=[+($t1, $t14)], expr#16=[8:BIGINT], expr#17=[+($t1, $t16)], expr#18=[9:BIGINT], expr#19=[+($t1, $t18)], expr#20=[10:BIGINT], expr#21=[+($t1, $t20)], expr#22=[11:BIGINT], expr#23=[+($t1, $t22)], expr#24=[12:BIGINT], expr#25=[+($t1, $t24)], expr#26=[13:BIGINT], expr#27=[+($t1, $t26)], expr#28=[14:BIGINT], expr#29=[+($t1, $t28)], expr#30=[15:BIGINT], expr#31=[+($t1, $t30)], expr#32=[16:BIGINT], expr#33=[+($t1, $t32)], expr#34=[17:BIGINT], expr#35=[+($t1, $t34)], expr#36=[18:BIGINT], expr#37=[+($t1, $t36)], expr#38=[19:BIGINT], expr#39=[+($t1, $t38)], expr#40=[20:BIGINT], expr#41=[+($t1, $t40)], expr#42=[21:BIGINT], expr#43=[+($t1, $t42)], expr#44=[22:BIGINT], expr#45=[+($t1, $t44)], expr#46=[23:BIGINT], expr#47=[+($t1, $t46)], expr#48=[24:BIGINT], expr#49=[+($t1, $t48)], expr#50=[25:BIGINT], expr#51=[+($t1, $t50)], expr#52=[26:BIGINT], expr#53=[+($t1, $t52)], expr#54=[27:BIGINT], expr#55=[+($t1, $t54)], expr#56=[28:BIGINT], expr#57=[+($t1, $t56)], expr#58=[29:BIGINT], expr#59=[+($t1, $t58)], expr#60=[30:BIGINT], expr#61=[+($t1, $t60)], expr#62=[31:BIGINT], expr#63=[+($t1, $t62)], expr#64=[32:BIGINT], expr#65=[+($t1, $t64)], expr#66=[33:BIGINT], expr#67=[+($t1, $t66)], expr#68=[34:BIGINT], expr#69=[+($t1, $t68)], expr#70=[35:BIGINT], expr#71=[+($t1, $t70)], expr#72=[36:BIGINT], expr#73=[+($t1, $t72)], expr#74=[37:BIGINT], expr#75=[+($t1, $t74)], expr#76=[38:BIGINT], expr#77=[+($t1, $t76)], expr#78=[39:BIGINT], expr#79=[+($t1, $t78)], expr#80=[40:BIGINT], expr#81=[+($t1, $t80)], expr#82=[41:BIGINT], expr#83=[+($t1, $t82)], expr#84=[42:BIGINT], expr#85=[+($t1, $t84)], expr#86=[43:BIGINT], expr#87=[+($t1, $t86)], expr#88=[44:BIGINT], expr#89=[+($t1, $t88)], expr#90=[45:BIGINT], expr#91=[+($t1, $t90)], expr#92=[46:BIGINT], expr#93=[+($t1, $t92)], expr#94=[47:BIGINT], expr#95=[+($t1, $t94)], expr#96=[48:BIGINT], expr#97=[+($t1, $t96)], expr#98=[49:BIGINT], expr#99=[+($t1, $t98)], expr#100=[50:BIGINT], expr#101=[+($t1, $t100)], expr#102=[51:BIGINT], expr#103=[+($t1, $t102)], expr#104=[52:BIGINT], expr#105=[+($t1, $t104)], expr#106=[53:BIGINT], expr#107=[+($t1, $t106)], expr#108=[54:BIGINT], expr#109=[+($t1, $t108)], expr#110=[55:BIGINT], expr#111=[+($t1, $t110)], expr#112=[56:BIGINT], expr#113=[+($t1, $t112)], expr#114=[57:BIGINT], expr#115=[+($t1, $t114)], expr#116=[58:BIGINT], expr#117=[+($t1, $t116)], expr#118=[59:BIGINT], expr#119=[+($t1, $t118)], expr#120=[60:BIGINT], expr#121=[+($t1, $t120)], expr#122=[61:BIGINT], expr#123=[+($t1, $t122)], expr#124=[62:BIGINT], expr#125=[+($t1, $t124)], expr#126=[63:BIGINT], expr#127=[+($t1, $t126)], expr#128=[64:BIGINT], expr#129=[+($t1, $t128)], expr#130=[65:BIGINT], expr#131=[+($t1, $t130)], expr#132=[66:BIGINT], expr#133=[+($t1, $t132)], expr#134=[67:BIGINT], expr#135=[+($t1, $t134)], expr#136=[68:BIGINT], expr#137=[+($t1, $t136)], expr#138=[69:BIGINT], expr#139=[+($t1, $t138)], expr#140=[70:BIGINT], expr#141=[+($t1, $t140)], expr#142=[71:BIGINT], expr#143=[+($t1, $t142)], expr#144=[72:BIGINT], expr#145=[+($t1, $t144)], expr#146=[73:BIGINT], expr#147=[+($t1, $t146)], expr#148=[74:BIGINT], expr#149=[+($t1, $t148)], expr#150=[75:BIGINT], expr#151=[+($t1, $t150)], expr#152=[76:BIGINT], expr#153=[+($t1, $t152)], expr#154=[77:BIGINT], expr#155=[+($t1, $t154)], expr#156=[78:BIGINT], expr#157=[+($t1, $t156)], expr#158=[79:BIGINT], expr#159=[+($t1, $t158)], expr#160=[80:BIGINT], expr#161=[+($t1, $t160)], expr#162=[81:BIGINT], expr#163=[+($t1, $t162)], expr#164=[82:BIGINT], expr#165=[+($t1, $t164)], expr#166=[83:BIGINT], expr#167=[+($t1, $t166)], expr#168=[84:BIGINT], expr#169=[+($t1, $t168)], expr#170=[85:BIGINT], expr#171=[+($t1, $t170)], expr#172=[86:BIGINT], expr#173=[+($t1, $t172)], expr#174=[87:BIGINT], expr#175=[+($t1, $t174)], expr#176=[88:BIGINT], expr#177=[+($t1, $t176)], expr#178=[89:BIGINT], expr#179=[+($t1, $t178)], ResolutionWidth=[$t0], $f90=[$t3], $f91=[$t5], $f92=[$t7], $f93=[$t9], $f94=[$t11], $f95=[$t13], $f96=[$t15], $f97=[$t17], $f98=[$t19], $f99=[$t21], $f100=[$t23], $f101=[$t25], $f102=[$t27], $f103=[$t29], $f104=[$t31], $f105=[$t33], $f106=[$t35], $f107=[$t37], $f108=[$t39], $f109=[$t41], $f110=[$t43], $f111=[$t45], $f112=[$t47], $f113=[$t49], $f114=[$t51], $f115=[$t53], $f116=[$t55], $f117=[$t57], $f118=[$t59], $f119=[$t61], $f120=[$t63], $f121=[$t65], $f122=[$t67], $f123=[$t69], $f124=[$t71], $f125=[$t73], $f126=[$t75], $f127=[$t77], $f128=[$t79], $f129=[$t81], $f130=[$t83], $f131=[$t85], $f132=[$t87], $f133=[$t89], $f134=[$t91], $f135=[$t93], $f136=[$t95], $f137=[$t97], $f138=[$t99], $f139=[$t101], $f140=[$t103], $f141=[$t105], $f142=[$t107], $f143=[$t109], $f144=[$t111], $f145=[$t113], $f146=[$t115], $f147=[$t117], $f148=[$t119], $f149=[$t121], $f150=[$t123], $f151=[$t125], $f152=[$t127], $f153=[$t129], $f154=[$t131], $f155=[$t133], $f156=[$t135], $f157=[$t137], $f158=[$t139], $f159=[$t141], $f160=[$t143], $f161=[$t145], $f162=[$t147], $f163=[$t149], $f164=[$t151], $f165=[$t153], $f166=[$t155], $f167=[$t157], $f168=[$t159], $f169=[$t161], $f170=[$t163], $f171=[$t165], $f172=[$t167], $f173=[$t169], $f174=[$t171], $f175=[$t173], $f176=[$t175], $f177=[$t177], $f178=[$t179]) CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[ResolutionWidth]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["ResolutionWidth"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml index bf40fe857ed..12fc0646da6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], SearchEngineID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(SearchEngineID=[$65], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($65), IS NOT NULL($76))]) LogicalFilter(condition=[<>($63, '')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchEngineID|ClientIP":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchEngineID|ClientIP":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml index 81236b33d51..9cdd38482bc 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], WatchID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(WatchID=[$41], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) LogicalFilter(condition=[<>($63, '')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml index ccda84ba38a..a64c682196a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], WatchID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(WatchID=[$41], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=SUM($1),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($1),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml index 9c41efa9139..48debf64773 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) LogicalProject(sum=[$1], state=[$0]) - LogicalAggregate(group=[{0}], sum=[SUM($1)]) + LogicalAggregate(group=[{0}], sum=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=SUM($0)), PROJECT->[sum, state], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=CHECKED_LONG_SUM($0)), PROJECT->[sum, state], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml index f2105ce0d3c..5dad5e945b4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$1], span(age,5)=[$0]) - LogicalAggregate(group=[{1}], sum(balance)=[SUM($0)]) + LogicalAggregate(group=[{1}], sum(balance)=[CHECKED_LONG_SUM($0)]) LogicalProject(balance=[$7], span(age,5)=[SPAN($10, 5, null:NULL)]) LogicalFilter(condition=[IS NOT NULL($10)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=SUM($0)), PROJECT->[sum(balance), span(age,5)], SORT_AGG_METRICS->[0 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":1},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=CHECKED_LONG_SUM($0)), PROJECT->[sum(balance), span(age,5)], SORT_AGG_METRICS->[0 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":1},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml index cd0355241fe..4e87192da1d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$1], c=[$2], dc(employer)=[$3], state=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], c=[COUNT()], dc(employer)=[COUNT(DISTINCT $2)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], c=[COUNT()], dc(employer)=[COUNT(DISTINCT $2)]) LogicalProject(state=[$7], balance=[$3], employer=[$6]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), PROJECT->[sum(balance), c, dc(employer), state], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=CHECKED_LONG_SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), PROJECT->[sum(balance), c, dc(employer), state], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml index 59cd137ca59..3e3a45b6386 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$2], count()=[$3], d=[$4], gender=[$0], new_state=[$1]) - LogicalAggregate(group=[{0, 1}], sum(balance)=[SUM($2)], count()=[COUNT()], d=[COUNT(DISTINCT $3)]) + LogicalAggregate(group=[{0, 1}], sum(balance)=[CHECKED_LONG_SUM($2)], count()=[COUNT()], d=[COUNT(DISTINCT $3)]) LogicalProject(gender=[$4], new_state=[$17], balance=[$3], employer=[$6]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($17))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), PROJECT->[sum(balance), count(), d, gender, new_state], SORT_AGG_METRICS->[2 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=CHECKED_LONG_SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), PROJECT->[sum(balance), count(), d, gender, new_state], SORT_AGG_METRICS->[2 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml index 68cb12a49dd..c4c572ce9e5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) LogicalProject(c=[$2], s=[$3], span(age,5)=[$1], state=[$0]) - LogicalAggregate(group=[{0, 2}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0, 2}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3], span(age,5)=[SPAN($8, 5, null:NULL)]) LogicalFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=CHECKED_LONG_SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index bc65d5c4c29..a1d7c439d9e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum=[$2], len=[$0], gender=[$1]) - LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) + LogicalAggregate(group=[{0, 1}], sum=[CHECKED_LONG_SUM($2)]) LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=CHECKED_LONG_SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index 1d664d5cd43..f6402d44d63 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)=[CHECKED_LONG_SUM($2)], sum(balance - 100)=[CHECKED_LONG_SUM($3)], sum(balance * 100)=[CHECKED_LONG_SUM($4)], sum(balance / 100)=[CHECKED_LONG_SUM($5)]) LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=CHECKED_LONG_SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=CHECKED_LONG_SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json index 064aa294a2d..f265a37f292 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml deleted file mode 100644 index e7f26a14a96..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml +++ /dev/null @@ -1,13 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($19, 1)]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $11)]) - LogicalFilter(condition=[IS NOT NULL($11)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - physical: | - EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..13=[{inputs}], expr#14=[1], expr#15=[<=($t13, $t14)], proj#0..12=[{exprs}], $condition=[$t15]) - EnumerableWindow(window#0=[window(partition {11} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->IS NOT NULL($11)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"email","boost":1.0}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_push.yaml new file mode 100644 index 00000000000..11aa4be7da2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[<=($19, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $11)]) + LogicalFilter(condition=[IS NOT NULL($11)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"email":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["email"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email"},{"field":"account_number"},{"field":"firstname"},{"field":"address"},{"field":"birthdate"},{"field":"gender"},{"field":"city"},{"field":"lastname"},{"field":"balance"},{"field":"employer"},{"field":"state"},{"field":"age"},{"field":"male"}]}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml index 3767a38b3a9..2fb92e7c652 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) LogicalProject(c=[$1], s=[$2], state=[$0]) - LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=CHECKED_LONG_SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml index 520f729b7f9..b5192ed01ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) LogicalProject(c=[$1], s=[$2], state=[$0]) - LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=CHECKED_LONG_SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml index 0478b24369c..9955ef801c2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml @@ -10,9 +10,9 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml index a1cf6ae00e9..902af30e1d7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml @@ -10,9 +10,9 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableSort(sort0=[$4], dir0=[ASC]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml index 324960f28dd..d0f762c426d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml @@ -22,17 +22,16 @@ calcite: EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableAggregate(group=[{0, 1, 2, 3}], avg_age=[AVG($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml index 42b50e7eb5f..213eef91aa2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml @@ -24,17 +24,16 @@ calcite: EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], sort1=[$1], sort2=[$2], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) - EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableAggregate(group=[{0, 1, 2}], avg_age=[AVG($4)]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml index 0818c18eabb..bbb9c602016 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml @@ -19,21 +19,14 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | - CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) + CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], avg(cpu_usage)=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t0, $t3, $t4)], host=[$t1], cpu_usage=[$t2], @timestamp0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp","host","cpu_usage"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(cpu_usage)=AVG($1)), PROJECT->[@timestamp0, host, avg(cpu_usage)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"host":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["host"]}},"missing_bucket":true,"missing_order":"last","order":"asc"}}},{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], avg(cpu_usage)=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], @timestamp0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), PROJECT->[host, cpu_usage, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","cpu_usage","@timestamp"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(cpu_usage)=AVG($1)), PROJECT->[host, avg(cpu_usage)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"host":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["host"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml index f26cb9e5822..a8f49b59ee4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml @@ -23,13 +23,8 @@ calcite: EnumerableAggregate(group=[{0, 1}], count()=[$SUM0($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], count()=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], count()=[$t2]) - EnumerableAggregate(group=[{0, 1}], count()=[COUNT()]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t0, $t2, $t3)], host=[$t1], @timestamp0=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"_source":{"includes":["@timestamp","host"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[@timestamp0, host, count()], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"host":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["host"]}},"missing_bucket":true,"missing_order":"last","order":"asc"}}},{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{0}], __grand_total__=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0), PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->IS NOT NULL($0), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"host":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["host"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml new file mode 100644 index 00000000000..610d9aa1410 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], F=[$1], M=[$2]) + LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml new file mode 100644 index 00000000000..a8419925e99 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml @@ -0,0 +1,16 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], avg_balance: F=[$1], avg_balance: M=[$3], cnt: F=[$2], cnt: M=[$4]) + LogicalAggregate(group=[{2}], F_avg_balance=[MAX($0) FILTER $3], F_cnt=[MAX($1) FILTER $3], M_avg_balance=[MAX($0) FILTER $4], M_cnt=[MAX($1) FILTER $4]) + LogicalProject(avg_balance=[$2], cnt=[$3], state=[$1], $f4=[IS TRUE(=($0, 'F'))], $f5=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)], cnt=[COUNT()]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableCalc(expr#0..4=[{inputs}], proj#0..1=[{exprs}], avg_balance: M=[$t3], cnt: F=[$t2], cnt: M=[$t4]) + EnumerableAggregate(group=[{2}], F_avg_balance=[MAX($0) FILTER $3], F_cnt=[MAX($1) FILTER $3], M_avg_balance=[MAX($0) FILTER $4], M_cnt=[MAX($1) FILTER $4]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=['F'], expr#5=[=($t0, $t4)], expr#6=[IS TRUE($t5)], expr#7=['M'], expr#8=[=($t0, $t7)], expr#9=[IS TRUE($t8)], avg_balance=[$t2], cnt=[$t3], state=[$t1], $f4=[$t6], $f5=[$t9]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml new file mode 100644 index 00000000000..eaf89c53c2b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], F_avg_balance=[$1], M_avg_balance=[$2]) + LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml index 059caa2e2d2..79252ca2148 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], state=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_balance=[$t2], age_range=[$t0], state=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=[CASE($t20, $t21, $t11)], age_range=[$t22], state=[$t9], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml index 43e27cd2d5d..b3ec99de2cf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t4, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t3)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t4)], avg(balance)=[$t11], count()=[$t5], age_range=[$t0], state=[$t1], gender=[$t2]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)], count()=[COUNT()]) + EnumerableCalc(expr#0..4=[{inputs}], avg(balance)=[$t3], count()=[$t4], age_range=[$t0], state=[$t1], gender=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}], avg(balance)=[AVG($3)], count()=[COUNT()]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], age_range=[$t23], state=[$t9], gender=[$t4], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml index 6dfa7cd65a3..e49dee2d350 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], avg_balance=[$t10], age_range=[$t0], balance_range=[$t1], state=[$t2]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) + EnumerableCalc(expr#0..3=[{inputs}], avg_balance=[$t3], age_range=[$t0], balance_range=[$t1], state=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}], avg_balance=[AVG($3)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=['a35':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], expr#24=[20000], expr#25=[<($t7, $t24)], expr#26=['medium':VARCHAR], expr#27=['high':VARCHAR], expr#28=[CASE($t25, $t26, $t27)], age_range=[$t23], balance_range=[$t28], state=[$t9], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml index 41ed8ba61fc..1dd00811008 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(balance)=[$t9], state=[$t0], age_range=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg(balance)=[$t2], state=[$t0], age_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], state=[$t9], age_range=[$t23], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml index 67ad0f0fd07..1ae9205fa10 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(age)=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[30..40)]], expr#23=[SEARCH($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml index 10ead7ad449..14664cb5df6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], age_range=[$t0]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..1=[{inputs}], avg(balance)=[$t1], age_range=[$t0]) + EnumerableAggregate(group=[{0}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[35..40), [80..+∞)]], expr#23=[SEARCH($t10, $t22)], expr#24=['30-40 or >=80':VARCHAR], expr#25=[null:NULL], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml index a81e208bdbf..6a33abfd5df 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg_age=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml index 404726f6083..1aee0d0ced4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], balance_range=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_balance=[$t2], age_range=[$t0], balance_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[20000], expr#28=[<($t7, $t27)], expr#29=['medium':VARCHAR], expr#30=['high':VARCHAR], expr#31=[CASE($t28, $t29, $t30)], age_range=[$t26], balance_range=[$t31], balance=[$t7]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml index fe925e0a80a..1d1b9bfec33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml @@ -26,15 +26,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 10}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 10}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[SAFE_CAST($t10)], expr#22=[IS NOT NULL($t21)], expr#23=[AND($t19, $t20, $t22)], proj#0..18=[{exprs}], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml index beb3275a6c6..1876916cb25 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml @@ -26,15 +26,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[SAFE_CAST($t15)], expr#19=[IS NOT NULL($t18)], expr#20=[AND($t16, $t17, $t19)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml index 8224f075819..d6fd5118aa5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml @@ -9,7 +9,6 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) - EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableAggregate(group=[{4}], avg(balance)=[AVG($7)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml index 16aa3871687..52fb3848a8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml @@ -9,7 +9,7 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) - EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], state=[$t1], gender=[$t0], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 9}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t9)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml index 1db12fc013f..a5d14f85dcf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml @@ -2,11 +2,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum=[$2], len=[$0], gender=[$1]) - LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) + LogicalAggregate(group=[{0, 1}], sum=[CHECKED_LONG_SUM($2)]) LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100:BIGINT], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) - EnumerableAggregate(group=[{4}], sum_SUM=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) + EnumerableAggregate(group=[{4}], sum_SUM=[CHECKED_LONG_SUM($7)], sum_COUNT=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index 655e16839ed..0a06b733276 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -2,12 +2,12 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)=[CHECKED_LONG_SUM($2)], sum(balance - 100)=[CHECKED_LONG_SUM($3)], sum(balance * 100)=[CHECKED_LONG_SUM($4)], sum(balance / 100)=[CHECKED_LONG_SUM($5)]) LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) + EnumerableAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[CHECKED_LONG_SUM($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json index a31d2acfc61..0b9e873d52c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml index ac3728eacb9..a283b195883 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml @@ -8,7 +8,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0]) - EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_age=[$t2], state=[$t1], city=[$t0]) + EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml index f781995261c..5acfa39c392 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -19,7 +19,7 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t11)], state=[$t1], age2=[$t11], $condition=[$t12]) - EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[+($t2, $t3)], expr#5=[IS NOT NULL($t2)], state=[$t1], age2=[$t4], $condition=[$t5]) + EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json index 028a56cb020..126f559f4d0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0])\n EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..2=[{inputs}], avg_age=[$t2], state=[$t1], city=[$t0])\n EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json index 9d64b554b18..1be834a39ba 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], state=[$t0])\n EnumerableAggregate(group=[{7}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], avg(balance)=[$t1], state=[$t0])\n EnumerableAggregate(group=[{7}], avg(balance)=[AVG($3)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml index 0bf9a2c50ce..f901475ef8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml @@ -10,10 +10,10 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml index d72bf7b429f..2816bf68f20 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml @@ -10,10 +10,10 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableSort(sort0=[$4], dir0=[ASC]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml index 3ec98ba9382..b131d24ba2c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml @@ -23,17 +23,16 @@ calcite: EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $14=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableAggregate(group=[{0, 1, 2, 3}], avg_age=[AVG($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml index 40fb4087001..e0ceaba3192 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml @@ -23,17 +23,16 @@ calcite: EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) - EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableAggregate(group=[{0, 1, 2}], avg_age=[AVG($4)]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/flat_object.json b/integ-test/src/test/resources/flat_object.json new file mode 100644 index 00000000000..03ed0d15d67 --- /dev/null +++ b/integ-test/src/test/resources/flat_object.json @@ -0,0 +1,6 @@ +{"index":{"_id":"1"}} +{"name":"alpha","status":200,"attributes":{"service":"checkout","region":"us-east-1"}} +{"index":{"_id":"2"}} +{"name":"beta","status":500,"attributes":{"service":"search","region":"us-west-2"}} +{"index":{"_id":"3"}} +{"name":"gamma","status":200,"attributes":{"service":"checkout","region":"eu-west-1"}} diff --git a/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json new file mode 100644 index 00000000000..5721d2c3773 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json @@ -0,0 +1,15 @@ +{ + "mappings": { + "properties": { + "name": { + "type": "keyword" + }, + "status": { + "type": "integer" + }, + "attributes": { + "type": "flat_object" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/ppl_lint_disabled_object_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/ppl_lint_disabled_object_index_mapping.json new file mode 100644 index 00000000000..6893eceb163 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/ppl_lint_disabled_object_index_mapping.json @@ -0,0 +1,13 @@ +{ + "mappings": { + "properties": { + "session": { + "type": "object", + "enabled": false + }, + "status": { + "type": "keyword" + } + } + } +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json new file mode 100644 index 00000000000..5b9ece1480b --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json @@ -0,0 +1,136 @@ +{ + "schemaVersion": 4, + "ruleId": "agg-on-text", + "channel": "lint", + "note": "The standard engine accepts both text aggregations: avg(text) returns null while sum(text) returns a non-empty numeric result (observed as 0.0). The warning catches this misleading coercion rather than predicting backend rejection.", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "agg-on-text", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + } + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "firstname": "text", + "balance": "long" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "avg-text-field": { + "role": "trigger", + "query": "source={{index}} | stats avg(firstname) as avg_firstname" + }, + "sum-text-field": { + "role": "trigger", + "query": "source={{index}} | stats sum(firstname) as sum_firstname" + }, + "avg-numeric-control": { + "role": "control", + "query": "source={{index}} | stats avg(balance) as avg_balance" + } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "avg-text-field": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "Numeric aggregation on a text field may return no value (null), because text is not stored as a number.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "avg_firstname" + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "avg_firstname" + } + } + } + }, + "sum-text-field": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "Numeric aggregation on a text field may return no value (null), because text is not stored as a number.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + }, + "avg-numeric-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json new file mode 100644 index 00000000000..b1dbcda115a --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -0,0 +1,87 @@ +{ + "schemaVersion": 3, + "ruleId": "dedup-consecutive-unsupported", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "dedup-consecutive-unsupported", + "enabled": false, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "sourceScoped": false, + "appliesTo": { + "minVersion": "3.3.0", + "engine": "calcite" + } + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": true + } + }, + "frontendContext": { + "isCalcite": true, + "forceEnable": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "dedup-consecutive-true": { + "role": "trigger", + "query": "source={{index}} | dedup firstname consecutive=true" + }, + "dedup-consecutive-true-multi-field": { + "role": "trigger", + "query": "source={{index}} | dedup firstname, lastname consecutive=true" + }, + "dedup-plain-control": { + "role": "control", + "query": "source={{index}} | dedup firstname" + } + }, + "expectations": [ + { + "version": ">=3.3.0", + "engine": "calcite", + "queries": { + "dedup-consecutive-true": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + }, + "dedup-consecutive-true-multi-field": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + }, + "dedup-plain-control": { + "detectorCount": 0, + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json new file mode 100644 index 00000000000..ebfe44389ac --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -0,0 +1,92 @@ +{ + "schemaVersion": 3, + "ruleId": "disabled-join-type", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "disabled-join-type", + "enabled": false, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "sourceScoped": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false, + "allJoinTypesAllowed": false + } + }, + "frontendContext": { + "isCalcite": true, + "forceEnable": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "right-join-disabled": { + "role": "trigger", + "query": "source={{index}} | right join left=l right=r on l.account_number=r.account_number {{index}}" + }, + "cross-join-disabled": { + "role": "trigger", + "query": "source={{index}} | cross join left=l right=r on l.account_number=r.account_number {{index}}" + }, + "inner-join-control": { + "role": "control", + "query": "source={{index}} | join left=l right=r on l.account_number=r.account_number {{index}} | head 1" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "right-join-disabled": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SemanticCheckException", + "reason": "Invalid Query" + } + } + } + }, + "cross-join-disabled": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SemanticCheckException", + "reason": "Invalid Query" + } + } + } + }, + "inner-join-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json new file mode 100644 index 00000000000..5bb88408554 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -0,0 +1,157 @@ +{ + "schemaVersion": 4, + "ruleId": "division-by-zero", + "note": "The detector flags both division and modulo by a literal zero because both operations return null silently. The backend result-shape oracle verifies that behavior independently for each operator.", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "division-by-zero", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {}, + "sourceScoped": false + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "divide-by-zero-literal": { + "role": "trigger", + "query": "source={{index}} | eval ratio = balance / 0 | fields ratio | head 1" + }, + "divide-by-decimal-zero-literal": { + "role": "trigger", + "query": "source={{index}} | eval ratio = balance / 0.0 | fields ratio | head 1" + }, + "divide-by-nonzero-control": { + "role": "control", + "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" + }, + "modulo-by-zero-literal": { + "role": "trigger", + "query": "source={{index}} | eval m = balance % 0 | fields m | head 1" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "divide-by-zero-literal": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "Dividing by zero returns no value (null) instead of an error.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + } + } + }, + "divide-by-decimal-zero-literal": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "Dividing by zero returns no value (null) instead of an error.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + } + } + }, + "divide-by-nonzero-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + }, + "modulo-by-zero-literal": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "Dividing by zero returns no value (null) instead of an error.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "m" + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "m" + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json new file mode 100644 index 00000000000..366a7f85bb1 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json @@ -0,0 +1,138 @@ +{ + "schemaVersion": 4, + "ruleId": "enabled-false-object", + "channel": "lint", + "note": "The standard Calcite route can still project and filter enabled:false object values from _source. These result-shape oracles pin that observed acceptance; the warning communicates that the object is not indexed/searchable through ordinary OpenSearch field semantics.", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "enabled-false-object", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + } + }, + "backendFixture": { + "indices": [ + "PPL_LINT_DISABLED_OBJECT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "status": "keyword" + }, + "disabledObjectFields": [ + "session" + ] + }, + "index": "opensearch-sql_test_index_ppl_lint_disabled_object", + "queries": { + "disabled-object-field": { + "role": "trigger", + "query": "source={{index}} | fields session.id | head 1" + }, + "disabled-object-filter": { + "role": "trigger", + "query": "source={{index}} | where session.id = 'abc' | fields status" + }, + "indexed-field-control": { + "role": "control", + "query": "source={{index}} | where status = 'ok' | fields status | head 1" + } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "disabled-object-field": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "This field is stored but not searchable, so PPL returns null for it.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + }, + "disabled-object-filter": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "This field is stored but not searchable, so PPL returns null for it.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 1 + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 1 + } + } + } + }, + "indexed-field-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json new file mode 100644 index 00000000000..f9c68833a02 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -0,0 +1,364 @@ +{ + "schemaVersion": 4, + "ruleId": "field-validation", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "field-validation", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {}, + "sourceScoped": true + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "visibleIndices": [ + "{{index}}" + ], + "deriveFromMapping": { + "account_number": "long", + "balance": "long", + "age": "long", + "firstname": "text", + "lastname": "text", + "gender": "text", + "address": "text", + "employer": "text", + "email": "text", + "city": "text", + "state": "text" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "unknown-field-existence": { + "role": "trigger", + "query": "source={{index}} | where nonexistent_field > 3" + }, + "grok-field-slot-shape-typo": { + "role": "trigger", + "query": "source={{index}} | grok field=firstname \"%{WORD:w}\"" + }, + "known-field-control": { + "role": "control", + "query": "source={{index}} | where age > 30 | head 1" + } + }, + "expectations": [ + { + "version": "<3.4.0", + "note": "This rule has an empty appliesTo, so it ships to users on EVERY engine, including pre-3.4. Detector behavior is live-verified identical from 2.19 up (1/1/0 on the compiled surface at 2.19.0, 3.0.0, 3.5.0, 3.7.0). The backend oracle deliberately omits error.type/reason: this engine's wording for an unknown field has not been observed live, and inventing one would either fail spuriously or get 'fixed' by pinning whatever CI first happened to see. A compiled-surface leg records the real wording; pin it then.", + "queries": { + "unknown-field-existence": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Unknown field \"nonexistent_field\".", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + } + }, + "grok-field-slot-shape-typo": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "grok expects a field name here, not an expression.", + "deterministicFix": { + "offered": true, + "title": "Remove \"field=\" (use \"firstname\")", + "text": "firstname", + "range": { + "startLine": 1, + "startColumn": 48, + "endLine": 1, + "endColumn": 63 + }, + "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + } + }, + "known-field-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + }, + { + "version": ">=3.4.0 <3.7.0", + "queries": { + "unknown-field-existence": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Unknown field \"nonexistent_field\".", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + } + }, + "grok-field-slot-shape-typo": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "grok expects a field name here, not an expression.", + "deterministicFix": { + "offered": true, + "title": "Remove \"field=\" (use \"firstname\")", + "text": "firstname", + "range": { + "startLine": 1, + "startColumn": 48, + "endLine": 1, + "endColumn": 63 + }, + "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + } + }, + "known-field-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + }, + { + "version": ">=3.7.0", + "queries": { + "unknown-field-existence": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Unknown field \"nonexistent_field\".", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [nonexistent_field] not found." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [nonexistent_field] not found." + } + } + } + } + }, + "grok-field-slot-shape-typo": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "grok expects a field name here, not an expression.", + "deterministicFix": { + "offered": true, + "title": "Remove \"field=\" (use \"firstname\")", + "text": "firstname", + "range": { + "startLine": 1, + "startColumn": 48, + "endLine": 1, + "endColumn": 63 + }, + "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [field] not found." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [field] not found." + } + } + } + } + }, + "known-field-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json new file mode 100644 index 00000000000..70006fafbdf --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json @@ -0,0 +1,249 @@ +{ + "schemaVersion": 4, + "ruleId": "flat-object-subfield", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": [ + "qualifiedName", + "wcQualifiedName" + ], + "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true). The analytics feature build cannot create flat_object in composite/Parquet storage, so every analytics backend oracle is explicitly non-applicable while the frontend detector assertions still run.", + "wiring": { + "detector": "flat-object-subfield", + "enabled": false, + "severity": "error", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": { + "minVersion": "3.8.0", + "engine": "calcite" + } + }, + "backendFixture": { + "indices": [ + "FLAT_OBJECT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "forceEnable": true, + "deriveFromMapping": { + "name": "keyword", + "status": "integer", + "attributes": "flat_object" + } + }, + "index": "opensearch-sql_test_index_flat_object", + "queries": { + "flat-object-dotted-subfield": { + "role": "trigger", + "query": "source={{index}} | fields attributes.service" + }, + "flat-object-bare-root": { + "role": "trigger", + "query": "source={{index}} | fields attributes" + }, + "flat-object-in-where": { + "role": "trigger", + "query": "source={{index}} | where attributes.service = 'checkout'" + }, + "non-flat-field-control": { + "role": "control", + "query": "source={{index}} | fields name, status | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "flat-object-dotted-subfield": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" + } + } + }, + "flat-object-bare-root": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" + } + } + }, + "flat-object-in-where": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" + } + } + }, + "non-flat-field-control": { + "detectorCount": 0, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" + } + } + } + } + }, + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "flat-object-dotted-subfield": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" + } + } + }, + "flat-object-bare-root": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes] not found." + } + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" + } + } + }, + "flat-object-in-where": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" + } + } + }, + "non-flat-field-control": { + "detectorCount": 0, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json new file mode 100644 index 00000000000..cb1d65825d0 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -0,0 +1,83 @@ +{ + "schemaVersion": 3, + "ruleId": "head-without-sort", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "head-without-sort", + "enabled": false, + "severity": "info", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "sourceScoped": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "forceEnable": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "head-without-sort": { + "role": "trigger", + "query": "source={{index}} | head 5" + }, + "head-without-sort-after-where": { + "role": "trigger", + "query": "source={{index}} | where age > 20 | head 5" + }, + "head-with-sort-control": { + "role": "control", + "query": "source={{index}} | sort age | head 5" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "head-without-sort": { + "detectorCount": 1, + "severity": "info", + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + }, + "head-without-sort-after-where": { + "detectorCount": 1, + "severity": "info", + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + }, + "head-with-sort-control": { + "detectorCount": 0, + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json new file mode 100644 index 00000000000..b6a5779ff08 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -0,0 +1,250 @@ +{ + "schemaVersion": 4, + "ruleId": "invalid-capture-group-name", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": [ + "rexCommand", + "stringLiteral" + ], + "wiring": { + "detector": "invalid-capture-group-name", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": { + "minVersion": "3.4.0" + }, + "sourceScoped": false + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "email": "text" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "rex-capture-name-underscore": { + "role": "trigger", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email" + }, + "rex-capture-name-hyphen": { + "role": "trigger", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email" + }, + "rex-capture-name-alphanumeric-control": { + "role": "control", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email, username, domain | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "rex-capture-name-underscore": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Capture group name \"user_name\" is invalid. Start with a letter and use only letters and numbers.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + } + }, + "rex-capture-name-hyphen": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Capture group name \"user-name\" is invalid. Start with a letter and use only letters and numbers.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + } + }, + "rex-capture-name-alphanumeric-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + }, + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "rex-capture-name-underscore": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Capture group name \"user_name\" is invalid. Start with a letter and use only letters and numbers.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user_name'." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user_name'." + } + } + } + } + }, + "rex-capture-name-hyphen": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Capture group name \"user-name\" is invalid. Start with a letter and use only letters and numbers.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user-name'." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user-name'." + } + } + } + } + }, + "rex-capture-name-alphanumeric-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json new file mode 100644 index 00000000000..fe720056ba0 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -0,0 +1,56 @@ +{ + "schemaVersion": 4, + "description": "PPL frontend/backend compatibility corpus for the approved 12 detector rules.", + "contracts": [ + "agg-on-text.spec.json", + "division-by-zero.spec.json", + "enabled-false-object.spec.json", + "field-validation.spec.json", + "invalid-capture-group-name.spec.json", + "multisearch-min-subsearch.spec.json", + "replace-wildcard-asymmetry.spec.json", + "rex-scan-cost.spec.json", + "type-mismatch-numeric.spec.json", + "union-min-datasets.spec.json", + "unsupported-window-function-in-eventstats.spec.json", + "wildcard-source-zero-match.spec.json" + ], + "dormantContracts": [ + "dedup-consecutive-unsupported.spec.json", + "disabled-join-type.spec.json", + "flat-object-subfield.spec.json", + "head-without-sort.spec.json" + ], + "enforced": [ + "field-validation.spec.json", + "invalid-capture-group-name.spec.json", + "multisearch-min-subsearch.spec.json", + "replace-wildcard-asymmetry.spec.json", + "union-min-datasets.spec.json", + "unsupported-window-function-in-eventstats.spec.json" + ], + "defaultError": [ + "field-validation.spec.json", + "invalid-capture-group-name.spec.json", + "multisearch-min-subsearch.spec.json", + "replace-wildcard-asymmetry.spec.json", + "union-min-datasets.spec.json", + "unsupported-window-function-in-eventstats.spec.json" + ], + "requiredSyntaxFeatures": [], + "nonEnforcing": [ + "agg-on-text.spec.json", + "division-by-zero.spec.json", + "enabled-false-object.spec.json", + "rex-scan-cost.spec.json", + "type-mismatch-numeric.spec.json", + "wildcard-source-zero-match.spec.json" + ], + "notes": { + "enforced": "Reviewed lint error contracts with deterministic backend behavior.", + "defaultError": "Exact approved six-rule detector error census.", + "requiredSyntaxFeatures": "Reserved for future syntax-channel compatibility contracts; currently empty.", + "nonEnforcing": "Oracle-quality classification for warning, info, advisory, and result-shape contracts; scheduling determines execution, not this list.", + "dormantContracts": "Preserved default-off detector regression contracts. They do not count toward active shipping coverage and must force-enable their detector when run." + } +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json new file mode 100644 index 00000000000..f13d9d4408c --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -0,0 +1,149 @@ +{ + "schemaVersion": 4, + "ruleId": "multisearch-min-subsearch", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": [ + "multisearchCommand", + "subSearch" + ], + "notes": "Query-initial (no leading pipe) on purpose — see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", + "wiring": { + "detector": "multisearch-min-subsearch", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { + "minVersion": "3.4.0" + }, + "sourceScoped": false + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "multisearch-single-subsearch": { + "role": "trigger", + "query": "multisearch [ search source={{index}} ]" + }, + "multisearch-single-subsearch-with-where": { + "role": "trigger", + "query": "multisearch [ search source={{index}} | where age > 30 ]" + }, + "multisearch-two-subsearches-control": { + "role": "control", + "query": "multisearch [ search source={{index}} ] [ search source={{index}} ]" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "queries": { + "multisearch-single-subsearch": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The multisearch command requires at least two subsearches.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } + } + } + }, + "multisearch-single-subsearch-with-where": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The multisearch command requires at least two subsearches.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } + } + } + }, + "multisearch-two-subsearches-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json new file mode 100644 index 00000000000..0ff54ce9fc3 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -0,0 +1,248 @@ +{ + "schemaVersion": 4, + "ruleId": "replace-wildcard-asymmetry", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": [ + "replacePair", + "stringLiteral" + ], + "wiring": { + "detector": "replace-wildcard-asymmetry", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { + "minVersion": "3.4.0", + "engine": "calcite" + }, + "sourceScoped": false + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "replace-wildcard-count-mismatch": { + "role": "trigger", + "query": "source={{index}} | replace \"*_a\" with \"b_*_*\" in firstname" + }, + "replace-wildcard-count-mismatch-reverse": { + "role": "trigger", + "query": "source={{index}} | replace \"*_a_*\" with \"b_*\" in firstname" + }, + "replace-symmetric-control": { + "role": "control", + "query": "source={{index}} | replace \"*_a\" with \"b_*\" in firstname | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "replace-wildcard-count-mismatch": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + } + }, + "replace-wildcard-count-mismatch-reverse": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + } + }, + "replace-symmetric-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + }, + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "replace-wildcard-count-mismatch": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + } + } + } + } + }, + "replace-wildcard-count-mismatch-reverse": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 2 wildcard(s), replacement has 1. Replacement must have same number of wildcards or none." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 2 wildcard(s), replacement has 1. Replacement must have same number of wildcards or none." + } + } + } + } + }, + "replace-symmetric-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json new file mode 100644 index 00000000000..43d02469f67 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json @@ -0,0 +1,113 @@ +{ + "schemaVersion": 4, + "ruleId": "rex-scan-cost", + "channel": "lint", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "rex-scan-cost", + "enabled": false, + "severity": "info", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": {} + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "forceEnable": true, + "deriveFromMapping": { + "email": "text" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "parse-text-field": { + "role": "trigger", + "query": "source={{index}} | parse email '.+@(?.+)' | fields email, host | head 1" + }, + "grok-text-field": { + "role": "trigger", + "query": "source={{index}} | grok email '.+@%{HOSTNAME:grok_host}' | fields email, grok_host | head 1" + }, + "plain-field-control": { + "role": "control", + "query": "source={{index}} | fields email | head 1" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "parse-text-field": { + "frontend": { + "count": 1, + "severity": "info", + "messageEquals": "parse runs the pattern against every input row from text field \"email\", even when it finds no match.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "advisory", + "httpStatus": 200 + }, + "analytics": { + "kind": "advisory", + "httpStatus": 200 + } + } + }, + "grok-text-field": { + "frontend": { + "count": 1, + "severity": "info", + "messageEquals": "grok runs the pattern against every input row from text field \"email\", even when it finds no match.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "advisory", + "httpStatus": 200 + }, + "analytics": { + "kind": "advisory", + "httpStatus": 200 + } + } + }, + "plain-field-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "advisory", + "httpStatus": 200 + }, + "analytics": { + "kind": "advisory", + "httpStatus": 200 + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json new file mode 100644 index 00000000000..635f862a9d4 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json @@ -0,0 +1,134 @@ +{ + "schemaVersion": 4, + "ruleId": "type-mismatch-numeric", + "channel": "lint", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "type-mismatch-numeric", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + } + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "age": "long" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "numeric-field-string-value": { + "role": "trigger", + "query": "source={{index}} | where age = \"thirty\" | fields age" + }, + "string-value-numeric-field": { + "role": "trigger", + "query": "source={{index}} | where \"thirty\" = age | fields age" + }, + "numeric-string-control": { + "role": "control", + "query": "source={{index}} | where age = \"32\" | fields age | head 1" + } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "numeric-field-string-value": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "This field is numeric, but the compared value is not a number, so the comparison returns no rows.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 0 + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 0 + } + } + } + }, + "string-value-numeric-field": { + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "This field is numeric, but the compared value is not a number, so the comparison returns no rows.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 0 + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 0 + } + } + } + }, + "numeric-string-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json new file mode 100644 index 00000000000..60551229d9b --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -0,0 +1,152 @@ +{ + "schemaVersion": 4, + "ruleId": "union-min-datasets", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": [ + "unionCommand", + "unionDataset", + "pplCommands" + ], + "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' — a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", + "wiring": { + "detector": "union-min-datasets", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + }, + "sourceScoped": false + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { + "role": "trigger", + "query": "union [ source={{index}} ]" + }, + "union-single-dataset-with-fields": { + "role": "trigger", + "query": "union [ source={{index}} | fields firstname ]" + }, + "union-two-datasets-control": { + "role": "control", + "query": "union [ source={{index}} ] [ source={{index}} ]" + } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The union command requires at least two datasets.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + } + } + }, + "union-single-dataset-with-fields": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The union command requires at least two datasets.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + } + } + }, + "union-two-datasets-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json new file mode 100644 index 00000000000..478ed49c0be --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -0,0 +1,340 @@ +{ + "schemaVersion": 4, + "ruleId": "unsupported-window-function-in-eventstats", + "detectorPath": "packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "unsupported-window-function-in-eventstats", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": { + "minVersion": "3.4.0" + }, + "sourceScoped": false + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "eventstats-rank": { + "role": "trigger", + "query": "source={{index}} | eventstats rank() as rank_value" + }, + "eventstats-dense-rank": { + "role": "trigger", + "query": "source={{index}} | eventstats dense_rank() as rank_value" + }, + "eventstats-avg-control": { + "role": "control", + "query": "source={{index}} | eventstats avg(age) as avg_age" + } + }, + "notes": "The DETECTOR verdict is identical on 3.6/3.7/3.8 (1 diagnostic on the trigger, 0 on the control, live-observed). Only the engine's rejection shape moves, in three epochs: 3.6 masks the cause entirely (HTTP 500 UnsupportedOperationException / 'There was internal problem at backend'); 3.7 keeps the 500 but names the function; 3.8 turned it into a proper HTTP 400 CalciteUnsupportedException. Each epoch is pinned separately so the multi-version check proves the rule still fires on older engines instead of reporting the wording difference as drift. A 500 is a poor rejection oracle (it cannot distinguish this failure from an unrelated crash), which is why the <3.8 entries additionally rely on the detector count for discrimination.", + "expectations": [ + { + "version": ">=3.4.0 <3.7.0", + "queries": { + "eventstats-rank": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + } + } + }, + "eventstats-dense-rank": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + } + } + }, + "eventstats-avg-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + }, + { + "version": ">=3.7.0 <3.8.0", + "queries": { + "eventstats-rank": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: rank" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: rank" + } + } + } + } + }, + "eventstats-dense-rank": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: dense_rank" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: dense_rank" + } + } + } + } + }, + "eventstats-avg-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + }, + { + "version": ">=3.8.0", + "queries": { + "eventstats-rank": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } + } + } + } + }, + "eventstats-dense-rank": { + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: dense_rank" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: dense_rank" + } + } + } + } + }, + "eventstats-avg-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json new file mode 100644 index 00000000000..43e94902d56 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json @@ -0,0 +1,100 @@ +{ + "schemaVersion": 4, + "ruleId": "wildcard-source-zero-match", + "channel": "lint", + "grammarSurface": "both", + "schedule": "pr", + "wiring": { + "detector": "wildcard-source-zero-match", + "enabled": true, + "severity": "info", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "visibleIndices": [ + "{{index}}" + ] + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "missing-wildcard-source": { + "role": "trigger", + "query": "source={{index}}-definitely-missing-* | head 1" + }, + "matching-wildcard-control": { + "role": "control", + "query": "source={{index}}* | head 1" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "missing-wildcard-source": { + "frontend": { + "count": 1, + "severity": "info", + "messageEquals": "Wildcard source pattern matches no known index.", + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 404, + "body": { + "status": 404 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 404, + "body": { + "status": 404 + } + } + } + }, + "matching-wildcard-control": { + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + } + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl_lint_disabled_object.json b/integ-test/src/test/resources/ppl_lint_disabled_object.json new file mode 100644 index 00000000000..30759d7a585 --- /dev/null +++ b/integ-test/src/test/resources/ppl_lint_disabled_object.json @@ -0,0 +1,4 @@ +{"index":{"_id":"1"}} +{"session":{"id":"abc","raw":"not-indexed"},"status":"ok"} +{"index":{"_id":"2"}} +{"session":{"id":"def","raw":"not-indexed"},"status":"error"} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.analyze.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.analyze.yml new file mode 100644 index 00000000000..a1087dbe6da --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.analyze.yml @@ -0,0 +1,129 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + - do: + indices.create: + index: ppl_analyze + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + message: + type: keyword + age: + type: integer + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "ppl_analyze", "_id": 1}}' + - '{"message": "hello", "age": 25}' + - '{"index": {"_index": "ppl_analyze", "_id": 2}}' + - '{"message": "world", "age": 35}' + +--- +teardown: + - do: + indices.delete: + index: ppl_analyze + ignore_unavailable: true + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Analyze returns full response for ppl query": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=ppl_analyze | fields message' + analyze: true + - match: {query: 'source=ppl_analyze | fields message'} + - is_true: querySegments + - is_true: logicalPlan + - is_true: physicalPlan + - is_true: operator_tree + - is_true: recommendations + - is_true: profile + - gt: {profile.summary.total_time_ms: 0.0} + - gt: {profile.phases.execute.time_ms: 0.0} + - is_true: schema + - is_true: datarows + - match: {total: 2} + - match: {size: 2} + +--- +"Analyze returns operator tree with pushdown info": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=ppl_analyze | where age > 30 | fields message' + analyze: true + - is_true: operator_tree + - match: {total: 1} + - match: {size: 1} + +--- +"Analyze returns recommendations field": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=ppl_analyze | where age > 30 | fields message' + analyze: true + - is_true: recommendations + +--- +"Analyze ignored for explain api": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: 'source=ppl_analyze | fields message' + analyze: true + - match: {query: null} + - match: {operator_tree: null} + +--- +"Analyze with non-jdbc format still returns response": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=ppl_analyze | fields message' + analyze: true + - is_true: query diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.profile.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.profile.yml index 882757fd8b7..d8bbb3d5724 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.profile.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.profile.yml @@ -53,9 +53,18 @@ teardown: - gt: {profile.phases.analyze.time_ms: 0.0} - gt: {profile.phases.optimize.time_ms: 0.0} - gt: {profile.phases.execute.time_ms: 0.0} - - gt: {profile.phases.format.time_ms: 0.0} + - gte: {profile.phases.format.time_ms: 0.0} - gt: {profile.plan.time_ms: 0.0} - match: {profile.plan.rows: 2} + - match: {query: 'source=ppl_profile | fields message'} + - is_true: logicalPlan + - is_true: physicalPlan + - is_true: operator_tree + - is_true: recommendations + - is_true: schema + - is_true: datarows + - match: {total: 2} + - match: {size: 2} --- "Profile ignored for explain api": diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml new file mode 100644 index 00000000000..d638ed4495f --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml @@ -0,0 +1,230 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5164 +# SUM/AVG over a BIGINT (long) column near 2^63 must not silently wrap to a negative value. +# +# The enumerable SUM accumulator for a long argument is a plain long, so a running sum past 2^63 +# wraps to a negative result (e.g. SUM(long) returned -5594372458244005145). AVG reduces to +# SUM(field)/COUNT(field), so its intermediate long SUM wraps the same way (AVG returned a negative +# average). SUM(long) now uses checked BIGINT accumulation with Math.addExact in the Calcite +# fallback. The pushdown path uses OpenSearch's native double-based sum and checks its final value +# before narrowing to BIGINT. AVG(long) is averaged in DOUBLE, which holds the true average without +# wrapping. + +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + - do: + indices.create: + index: test_agg_overflow + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_overflow + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - do: + indices.create: + index: test_agg_in_range + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_in_range + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 1000000000000}' + - '{"index": {}}' + - '{"long_field": 2000000000000}' + - '{"index": {}}' + - '{"long_field": 3000000000000}' + - do: + indices.create: + index: test_agg_exact + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_exact + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 4611686018427387904}' + - '{"index": {}}' + - '{"long_field": 1}' + - do: + indices.create: + index: test_agg_reduce_left + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + index: + index: test_agg_reduce_left + refresh: true + body: + long_field: 9223372036854775807 + - do: + indices.create: + index: test_agg_reduce_right + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + index: + index: test_agg_reduce_right + refresh: true + body: + long_field: 4096 + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + - do: + indices.delete: + index: test_agg_overflow + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_in_range + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_exact + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_reduce_* + ignore_unavailable: true + +--- +"SUM of large longs overflowing BIGINT throws error": + - skip: + features: + - headers + # 3 * (2^63 - 1) far exceeds the BIGINT range; historically this wrapped to a negative value. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_overflow | stats sum(long_field) + - match: { "$body": "/[Oo]verflow/" } + +--- +"Pushed SUM whose final value exceeds BIGINT throws error": + - skip: + features: + - headers + # Each one-shard index has an in-range partial. The final result is far enough beyond 2^63 to be + # distinguishable from Long.MAX_VALUE after native double accumulation. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_reduce_* | stats sum(long_field) + - match: { "$body": "/[Oo]verflow/" } + +--- +"AVG of large longs does not overflow or error": + - skip: + features: + - headers + # The average of three identical values is that value; averaging in DOUBLE avoids the wrap that + # a long intermediate SUM would cause. 9223372036854775807 is returned as a double (9.223...E18). + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_overflow | stats avg(long_field) + - match: { total: 1 } + - match: { datarows: [[9.223372036854776e18]] } + +--- +"SUM of longs within BIGINT range does not error": + - skip: + features: + - headers + # 1e12 + 2e12 + 3e12 = 6e12, well within long range; must return the exact sum, no error. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_in_range | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[6000000000000]] } + +--- +"Pushed SUM of large in-range longs uses native double precision": + - skip: + features: + - headers + # 2^62 + 1 fits in BIGINT but cannot be represented exactly by a double accumulator. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_exact | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[4611686018427387904]] } + +--- +"Fallback SUM of large in-range longs retains low-order precision": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_exact | head 2 | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[4611686018427387905]] } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/text_agg_pushdown.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/text_agg_pushdown.yml new file mode 100644 index 00000000000..f13798a7729 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/text_agg_pushdown.yml @@ -0,0 +1,282 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + + # Primary index — appId is text-only, no keyword sub-field. Before the fix this + # scenario silently degraded to a full _source scan with client-side aggregation. + - do: + indices.create: + index: logs_text_only + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + appId: + type: text + level: + type: keyword + + - do: + bulk: + index: logs_text_only + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"appId": "app-1", "level": "info"}' + - '{"index": {"_id": "2"}}' + - '{"appId": "app-1", "level": "warn"}' + - '{"index": {"_id": "3"}}' + - '{"appId": "app-2", "level": "info"}' + - '{"index": {"_id": "4"}}' + - '{"appId": "app-3", "level": "info"}' + - '{"index": {"_id": "5"}}' + - '{"appId": "app-1", "level": "info"}' + + # Companion index — appId is text with a keyword sub-field. Sits under the + # same logs_text* pattern so multi-index queries also exercise the + # conflicting-mapping case where the merged type loses its keyword sub-field. + - do: + indices.create: + index: logs_text_with_keyword + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + appId: + type: text + fields: + keyword: + type: keyword + ignore_above: 256 + + - do: + bulk: + index: logs_text_with_keyword + refresh: true + body: + - '{"index": {}}' + - '{"appId": "app-1"}' + - '{"index": {}}' + - '{"appId": "app-1"}' + - '{"index": {}}' + - '{"appId": "app-2"}' + +--- +teardown: + - do: + indices.delete: + index: logs_text_only,logs_text_with_keyword + ignore_unavailable: true + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + +--- +"top command on a text field returns correct counts": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text_only | top 10 appId + - match: { total: 3 } + - match: + schema: + - { name: appId, type: string } + - { name: count, type: bigint } + - match: + datarows: + - [ "app-1", 3 ] + - [ "app-2", 1 ] + - [ "app-3", 1 ] + +--- +"stats by text field returns correct counts": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text_only | stats count() by appId + - match: { total: 3 } + - match: + schema: + - { name: "count()", type: bigint } + - { name: appId, type: string } + - match: + datarows: + - [ 3, "app-1" ] + - [ 1, "app-2" ] + - [ 1, "app-3" ] + +--- +"top on text field pushes down as composite terms with _source script": + # AGGREGATION pushdown means the physical plan carries a composite_buckets + # aggregation whose terms source uses a script (not a field), because appId + # is a text field with no .keyword sub-field. Presence of both AGGREGATION + # in PushDownContext and the composite/script bucket confirms the DSL is + # pushed to OpenSearch instead of falling back to an unbounded scan. + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text_only | top 10 appId + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/composite_buckets/" } + - match: { "calcite.physical": "/\"appId\":\\{\"terms\":\\{\"script\"/" } + +--- +"stats by text field pushes down as composite terms with _source script": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text_only | stats count() by appId + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/composite_buckets/" } + - match: { "calcite.physical": "/\"appId\":\\{\"terms\":\\{\"script\"/" } + +--- +"count(text_field) on text-only pushes down as scripted value_count": + # count(FIELD) reaches the metric path with a bare RexInputRef (Calcite does not + # insert a numeric cast around COUNT). For a text field without .keyword, the + # metric aggregation must still push down by using a script value source that + # reads the field from _source, not fall back to an unbounded scan. + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text_only | stats count(appId) + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/\"count\\(appId\\)\":\\{\"value_count\":\\{\"script\"/" } + +--- +"count(text_field) on text-only returns correct count": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text_only | stats count(appId) + - match: { total: 1 } + - match: + schema: + - { name: "count(appId)", type: bigint } + - match: + datarows: + - [ 5 ] + +--- +"text field with .keyword sub-field pushes down using the sub-field": + # A text field that carries `.keyword` must be rewritten to `.keyword` + # in the DSL, not routed through the script path. Baseline that guarantees the + # fix did NOT alter behavior for keyword-backed fields. + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text_with_keyword | stats count() by appId + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/composite_buckets/" } + - match: { "calcite.physical": "/\"appId\":\\{\"terms\":\\{\"field\":\"appId\\.keyword\"/" } + +--- +"multi-index stats across text-only + text+keyword pushes down via _source script": + # `source=logs_text*` spans a text-only and a text+keyword index. The + # merged `appId` field loses its .keyword sub-field (because one member has + # none). Before the fix this fell back to a full _source scan and client-side + # aggregation; after the fix the composite terms bucket uses the _source- + # reading script for the merged field type. + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text* | stats count() by appId + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/composite_buckets/" } + - match: { "calcite.physical": "/\"appId\":\\{\"terms\":\\{\"script\"/" } + +--- +"multi-index stats returns correct merged counts": + # logs_text_only: app-1=3, app-2=1, app-3=1 + # logs_text_with_keyword: app-1=2, app-2=1 + # merged: app-1=5, app-2=2, app-3=1 + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text* | stats count() by appId + - match: { total: 3 } + - match: + schema: + - { name: "count()", type: bigint } + - { name: appId, type: string } + - match: + datarows: + - [ 5, "app-1" ] + - [ 2, "app-2" ] + - [ 1, "app-3" ] + +--- +"multi-index top returns correct merged counts": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text* | top 10 appId + - match: { total: 3 } + - match: + schema: + - { name: appId, type: string } + - { name: count, type: bigint } + - match: + datarows: + - [ "app-1", 5 ] + - [ "app-2", 2 ] + - [ "app-3", 1 ] diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 5ca10c7091c..03d16cad083 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -32,6 +32,7 @@ plugins { dependencies { api project(':core') + api project(':ppl-rest-spi') api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:${resilience4j_version}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 3b9c3619521..68350c5a0fd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,105 +114,4 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); - - /** - * Read-only cluster health snapshot for the {@code rest} command (backs {@code - * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's - * security thread-context; performs no privilege escalation and mutates nothing. - * - * @param params endpoint query args (already allow-list-validated) - * @return a single map of health field name to value - */ - default Map clusterHealth(Map params) { - throw new UnsupportedOperationException("clusterHealth is not supported by this client"); - } - - /** - * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One - * map per index. Runs under the caller's security thread-context; read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per index - */ - default List> catIndices(Map params) { - throw new UnsupportedOperationException("catIndices is not supported by this client"); - } - - /** - * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map - * per node with resource state. Runs under the caller's security thread-context; read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per node - */ - default List> catNodes(Map params) { - throw new UnsupportedOperationException("catNodes is not supported by this client"); - } - - /** - * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code - * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map describing the cluster manager node - */ - default List> catClusterManager(Map params) { - throw new UnsupportedOperationException("catClusterManager is not supported by this client"); - } - - /** - * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One - * map per installed plugin per node. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per plugin - */ - default List> catPlugins(Map params) { - throw new UnsupportedOperationException("catPlugins is not supported by this client"); - } - - /** - * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map - * per shard. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per shard - */ - default List> catShards(Map params) { - throw new UnsupportedOperationException("catShards is not supported by this client"); - } - - /** - * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code - * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, - * cluster_manager_node). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return a single map of cluster-state field name to value - */ - default Map clusterState(Map params) { - throw new UnsupportedOperationException("clusterState is not supported by this client"); - } - - /** - * Read-only cluster-settings listing for the {@code rest} command (backs {@code - * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per setting - */ - default List> clusterSettings(Map params) { - throw new UnsupportedOperationException("clusterSettings is not supported by this client"); - } - - /** - * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). - * One map per resolved index, alias, or data stream (name, type). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per resolved name - */ - default List> resolveIndex(Map params) { - throw new UnsupportedOperationException("resolveIndex is not supported by this client"); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 080a8627894..d9681898f73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,8 +18,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -27,18 +25,20 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; -import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; +import org.opensearch.core.tasks.TaskId; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; import org.opensearch.sql.common.error.ErrorCode; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.opensearch.executor.OpenSearchQueryManager; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; +import org.opensearch.tasks.CancellableTask; import org.opensearch.transport.client.node.NodeClient; /** OpenSearch connection by node client. */ @@ -164,7 +164,18 @@ public Map getIndexMaxResultWindows(String... indexExpression) @Override public OpenSearchResponse search(OpenSearchRequest request) { return request.search( - req -> client.search(req).actionGet(), req -> client.searchScroll(req).actionGet()); + req -> { + applyParentTask(req); + return client.search(req).actionGet(); + }, + req -> client.searchScroll(req).actionGet()); + } + + private void applyParentTask(SearchRequest req) { + CancellableTask task = OpenSearchQueryManager.getCancellableTask(); + if (task != null) { + req.setParentTask(new TaskId(client.getLocalNodeId(), task.getId())); + } } /** @@ -288,274 +299,4 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } - - @Override - public Map clusterHealth(Map params) { - ClusterHealthRequest request = new ClusterHealthRequest(); - if (params != null && Boolean.parseBoolean(params.get("local"))) { - request.local(true); - } - ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); - return flattenHealth(response); - } - - @Override - public List> catIndices(Map params) { - ClusterHealthResponse response = - client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (Map.Entry entry : response.getIndices().entrySet()) { - ClusterIndexHealth health = entry.getValue(); - Map row = new java.util.LinkedHashMap<>(); - row.put("index", entry.getKey()); - row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); - row.put("pri", health.getNumberOfShards()); - row.put("rep", health.getNumberOfReplicas()); - row.put("active_shards", health.getActiveShards()); - rows.add(row); - } - String healthFilter = params == null ? null : params.get("health"); - if (healthFilter != null) { - rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); - } - return rows; - } - - @Override - public List> catNodes(Map params) { - org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = - new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); - statsRequest.all(); - org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = - client.admin().cluster().nodesStats(statsRequest).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { - org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); - Map row = new java.util.LinkedHashMap<>(); - row.put("name", node.getName()); - row.put("ip", node.getHostAddress()); - row.put( - "node_role", - node.getRoles().stream() - .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) - .sorted() - .collect(java.util.stream.Collectors.joining(","))); - row.put( - "heap_percent", - ns.getJvm() == null || ns.getJvm().getMem() == null - ? null - : (int) ns.getJvm().getMem().getHeapUsedPercent()); - row.put( - "ram_percent", - ns.getOs() == null || ns.getOs().getMem() == null - ? null - : (int) ns.getOs().getMem().getUsedPercent()); - row.put( - "cpu", - ns.getProcess() == null || ns.getProcess().getCpu() == null - ? null - : (int) ns.getProcess().getCpu().getPercent()); - rows.add(row); - } - return rows; - } - - @Override - public List> catClusterManager(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - org.opensearch.cluster.node.DiscoveryNode cm = - response.getState().nodes().getClusterManagerNode(); - List> rows = new java.util.ArrayList<>(); - if (cm != null) { - Map row = new java.util.LinkedHashMap<>(); - row.put("id", cm.getId()); - row.put("host", cm.getHostName()); - row.put("ip", cm.getHostAddress()); - row.put("node", cm.getName()); - rows.add(row); - } - return rows; - } - - @Override - public List> catPlugins(Map params) { - org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = - new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); - infoRequest.all(); - org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = - client.admin().cluster().nodesInfo(infoRequest).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { - org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = - info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); - if (plugins == null) { - continue; - } - for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("name", info.getNode().getName()); - row.put("component", pi.getName()); - row.put("version", pi.getVersion()); - rows.add(row); - } - } - return rows; - } - - @Override - public List> catShards(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.cluster.routing.ShardRouting sr : - response.getState().getRoutingTable().allShards()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("index", sr.getIndexName()); - row.put("shard", sr.id()); - row.put("prirep", sr.primary() ? "p" : "r"); - row.put("state", sr.state().name()); - org.opensearch.cluster.node.DiscoveryNode n = - sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); - row.put("node", n == null ? null : n.getName()); - rows.add(row); - } - return rows; - } - - @Override - public Map clusterState(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - Map row = new java.util.LinkedHashMap<>(); - row.put("cluster_name", response.getClusterName().value()); - row.put("state_uuid", response.getState().stateUUID()); - row.put("version", response.getState().version()); - org.opensearch.cluster.node.DiscoveryNode cm = - response.getState().nodes().getClusterManagerNode(); - row.put("cluster_manager_node", cm == null ? null : cm.getName()); - return row; - } - - @Override - public List> clusterSettings(Map params) { - // The transport path has no SettingsFilter of its own; it is published from - // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings - // into memory when we cannot redact them, matching native GET /_cluster/settings. - org.opensearch.common.settings.SettingsFilter filter = - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); - if (filter == null) { - throw new IllegalStateException( - "cluster settings redaction filter is not initialized; refusing to return unredacted" - + " settings"); - } - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - List> rows = new java.util.ArrayList<>(); - org.opensearch.common.settings.Settings persistent = - filter.filter(response.getState().metadata().persistentSettings()); - org.opensearch.common.settings.Settings transientSettings = - filter.filter(response.getState().metadata().transientSettings()); - collectSettings(persistent, "persistent", rows); - collectSettings(transientSettings, "transient", rows); - return rows; - } - - private void collectSettings( - org.opensearch.common.settings.Settings settings, - String tier, - List> rows) { - if (settings == null) { - return; - } - for (String key : settings.keySet()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("setting", key); - String value = settings.get(key); - if (value == null) { - // List-valued settings return null from get(); fall back to the joined list form. - java.util.List list = settings.getAsList(key); - value = list.isEmpty() ? null : String.join(",", list); - } - row.put("value", value); - row.put("tier", tier); - rows.add(row); - } - } - - @Override - public List> resolveIndex(Map params) { - String expandWildcards = params == null ? null : params.get("expand_wildcards"); - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = - expandWildcards == null - ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( - new String[] {"*"}) - : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( - new String[] {"*"}, - org.opensearch.action.support.IndicesOptions.fromParameters( - expandWildcards, - null, - null, - null, - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request - .DEFAULT_INDICES_OPTIONS)); - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = - client - .execute( - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) - .actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : - response.getIndices()) { - rows.add(resolveRow(idx.getName(), "index")); - } - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : - response.getAliases()) { - rows.add(resolveRow(alias.getName(), "alias")); - } - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : - response.getDataStreams()) { - rows.add(resolveRow(ds.getName(), "data_stream")); - } - return rows; - } - - private Map resolveRow(String name, String type) { - Map row = new java.util.LinkedHashMap<>(); - row.put("name", name); - row.put("type", type); - return row; - } - - private Map flattenHealth(ClusterHealthResponse response) { - Map row = new java.util.LinkedHashMap<>(); - row.put("cluster_name", response.getClusterName()); - row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); - row.put("number_of_nodes", response.getNumberOfNodes()); - row.put("number_of_data_nodes", response.getNumberOfDataNodes()); - row.put("active_primary_shards", response.getActivePrimaryShards()); - row.put("active_shards", response.getActiveShards()); - row.put("relocating_shards", response.getRelocatingShards()); - row.put("initializing_shards", response.getInitializingShards()); - row.put("unassigned_shards", response.getUnassignedShards()); - row.put("timed_out", response.isTimedOut()); - return row; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index e98c5bf95f4..f369c0003b8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,19 +8,15 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -32,7 +28,6 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; -import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -277,267 +272,4 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } - - @Override - public Map clusterHealth(Map params) { - try { - ClusterHealthRequest request = new ClusterHealthRequest(); - if (params != null && Boolean.parseBoolean(params.get("local"))) { - request.local(true); - } - ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); - return flattenHealth(response); - } catch (IOException e) { - throw new IllegalStateException("Failed to get cluster health", e); - } - } - - @Override - public List> catIndices(Map params) { - try { - ClusterHealthResponse response = - client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); - List> rows = new ArrayList<>(); - for (Map.Entry entry : response.getIndices().entrySet()) { - ClusterIndexHealth health = entry.getValue(); - Map row = new HashMap<>(); - row.put("index", entry.getKey()); - row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); - row.put("pri", health.getNumberOfShards()); - row.put("rep", health.getNumberOfReplicas()); - row.put("active_shards", health.getActiveShards()); - rows.add(row); - } - String healthFilter = params == null ? null : params.get("health"); - if (healthFilter != null) { - rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); - } - return rows; - } catch (IOException e) { - throw new IllegalStateException("Failed to get cat indices", e); - } - } - - @Override - public List> catNodes(Map params) { - List> raw = - catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("name", r.get("name")); - row.put("ip", r.get("ip")); - row.put("node_role", r.get("node.role")); - row.put("heap_percent", asInt(r.get("heap.percent"))); - row.put("ram_percent", asInt(r.get("ram.percent"))); - row.put("cpu", asInt(r.get("cpu"))); - rows.add(row); - } - return rows; - } - - @Override - public List> catClusterManager(Map params) { - List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("id", r.get("id")); - row.put("host", r.get("host")); - row.put("ip", r.get("ip")); - row.put("node", r.get("node")); - rows.add(row); - } - return rows; - } - - @Override - public List> catPlugins(Map params) { - List> raw = catJson("/_cat/plugins", "name,component,version"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("name", r.get("name")); - row.put("component", r.get("component")); - row.put("version", r.get("version")); - rows.add(row); - } - return rows; - } - - @Override - public List> catShards(Map params) { - List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("index", r.get("index")); - row.put("shard", asInt(r.get("shard"))); - row.put("prirep", r.get("prirep")); - row.put("state", r.get("state")); - row.put("node", r.get("node")); - rows.add(row); - } - return rows; - } - - /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ - @SuppressWarnings("unchecked") - private List> catJson(String path, String columns) { - try { - org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); - request.addParameter("format", "json"); - request.addParameter("h", columns); - org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); - try (org.opensearch.core.xcontent.XContentParser parser = - org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( - org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, - org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, - response.getEntity().getContent())) { - List list = parser.list(); - List> rows = new ArrayList<>(); - for (Object o : list) { - rows.add((Map) o); - } - return rows; - } - } catch (IOException e) { - throw new IllegalStateException("Failed GET " + path, e); - } - } - - private static Integer asInt(Object value) { - if (value == null) { - return null; - } - try { - return (int) Double.parseDouble(value.toString().trim()); - } catch (NumberFormatException e) { - return null; - } - } - - @Override - @SuppressWarnings("unchecked") - public Map clusterState(Map params) { - Map state = - getJsonMap( - "/_cluster/state/master_node,version,metadata,nodes", - // nodes.*.name resolves the manager id to a name without over-fetching node IPs. - Map.of( - "filter_path", - "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); - Map row = new HashMap<>(); - row.put("cluster_name", state.get("cluster_name")); - row.put("state_uuid", state.get("state_uuid")); - row.put("version", asLong(state.get("version"))); - Object cmId = state.get("cluster_manager_node"); - String cmName = null; - Object nodes = state.get("nodes"); - if (cmId != null && nodes instanceof Map) { - Object n = ((Map) nodes).get(cmId.toString()); - if (n instanceof Map) { - Object name = ((Map) n).get("name"); - cmName = name == null ? null : name.toString(); - } - } - row.put("cluster_manager_node", cmName); - return row; - } - - @Override - @SuppressWarnings("unchecked") - public List> clusterSettings(Map params) { - Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); - List> rows = new ArrayList<>(); - for (String tier : new String[] {"persistent", "transient"}) { - Object section = body.get(tier); - if (section instanceof Map) { - for (Map.Entry e : ((Map) section).entrySet()) { - Map row = new HashMap<>(); - row.put("setting", e.getKey()); - row.put("value", e.getValue() == null ? null : e.getValue().toString()); - row.put("tier", tier); - rows.add(row); - } - } - } - return rows; - } - - /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ - @SuppressWarnings("unchecked") - private Map getJsonMap(String path, Map params) { - try { - org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); - if (params != null) { - params.forEach(request::addParameter); - } - org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); - try (org.opensearch.core.xcontent.XContentParser parser = - org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( - org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, - org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, - response.getEntity().getContent())) { - return parser.map(); - } - } catch (IOException e) { - throw new IllegalStateException("Failed GET " + path, e); - } - } - - private static Long asLong(Object value) { - if (value == null) { - return null; - } - try { - return (long) Double.parseDouble(value.toString().trim()); - } catch (NumberFormatException e) { - return null; - } - } - - @Override - @SuppressWarnings("unchecked") - public List> resolveIndex(Map params) { - String expandWildcards = params == null ? null : params.get("expand_wildcards"); - Map body = - getJsonMap( - "/_resolve/index/*", - expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); - List> rows = new ArrayList<>(); - addResolved(body.get("indices"), "index", rows); - addResolved(body.get("aliases"), "alias", rows); - addResolved(body.get("data_streams"), "data_stream", rows); - return rows; - } - - @SuppressWarnings("unchecked") - private void addResolved(Object section, String type, List> rows) { - if (section instanceof List) { - for (Object o : (List) section) { - if (o instanceof Map) { - Map row = new HashMap<>(); - row.put("name", ((Map) o).get("name")); - row.put("type", type); - rows.add(row); - } - } - } - } - - private Map flattenHealth(ClusterHealthResponse response) { - Map row = new HashMap<>(); - row.put("cluster_name", response.getClusterName()); - row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); - row.put("number_of_nodes", response.getNumberOfNodes()); - row.put("number_of_data_nodes", response.getNumberOfDataNodes()); - row.put("active_primary_shards", response.getActivePrimaryShards()); - row.put("active_shards", response.getActiveShards()); - row.put("relocating_shards", response.getRelocatingShards()); - row.put("initializing_shards", response.getInitializingShards()); - row.put("unassigned_shards", response.getUnassignedShards()); - row.put("timed_out", response.isTimedOut()); - return row; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 76de0c30a08..2a70502f392 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -31,6 +31,7 @@ public enum MappingType { Text("text", ExprCoreType.UNKNOWN), MatchOnlyText("match_only_text", ExprCoreType.UNKNOWN), Keyword("keyword", ExprCoreType.STRING), + ConstantKeyword("constant_keyword", ExprCoreType.STRING), Ip("ip", ExprCoreType.IP), GeoPoint("geo_point", ExprCoreType.UNKNOWN), Binary("binary", ExprCoreType.UNKNOWN), diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index e8c7cfc7c68..483f2684d61 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -42,10 +42,14 @@ import org.locationtech.jts.geom.Point; import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.TimewrapPivot; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.ResourceLimitExceededException; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; @@ -259,7 +263,7 @@ public void explain( } })) { // triggers the hook - OpenSearchRelRunners.run(context, rel); + OpenSearchRelRunners.run(context, CalciteToolsHelper.optimize(rel, context)); } if (physicalError.get() != null) { @@ -306,7 +310,7 @@ public void explain( CalcitePlanContext.skipEncoding.set(true); } // triggers the hook - OpenSearchRelRunners.run(context, rel); + OpenSearchRelRunners.run(context, CalciteToolsHelper.optimize(rel, context)); } listener.onResponse( new ExplainResponse( @@ -336,11 +340,47 @@ public void execute( listener.onResponse(response); } catch (SQLException e) { + if (isPitContextLimitReached(e)) { + // reason (title) comes from the wrapped cause's message; keep it short and put the + // explanation and remedy in details. + ResourceLimitExceededException pitException = + new ResourceLimitExceededException( + "Too many open Point-In-Time (PIT) contexts on this node.", e); + throw ErrorReport.wrap(pitException) + .code(ErrorCode.RESOURCE_LIMIT_EXCEEDED) + .details( + "This query opened a Point-In-Time (PIT) context on each shard and reached" + + " the limit set by [search.max_open_pit_context]. Increase that" + + " setting.") + .build(); + } throw new RuntimeException(e); } }); } + /** + * Substring of the error OpenSearch's {@code SearchService} raises when a node has no free PIT + * contexts. The engine opens a PIT (one reader context per shard) to page over a query it cannot + * push down -- e.g. a {@code stats} that groups by a text field with no {@code keyword} sub-field + * -- and a busy node exhausts its per-node budget. The raw failure is an opaque internal message, + * so it is replaced with an actionable one when this marker appears anywhere in the cause chain. + */ + private static final String PIT_CONTEXT_LIMIT_MARKER = "too many Point In Time contexts"; + + /** Package-private for testing. Walks the cause chain guarding against self-referential loops. */ + static boolean isPitContextLimitReached(Throwable t) { + for (Throwable cause = t; + cause != null && cause != cause.getCause(); + cause = cause.getCause()) { + String message = cause.getMessage(); + if (message != null && message.contains(PIT_CONTEXT_LIMIT_MARKER)) { + return true; + } + } + return false; + } + /** * Process values recursively, handling geo points, nested maps, structs and arrays. When a {@link * RelDataType} is provided, struct values (StructImpl) are converted to Maps keyed by field diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java index 7aaaaa6655e..c391153fca6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java @@ -32,6 +32,7 @@ public class OpenSearchQueryManager implements QueryManager { private final Settings settings; public static final String SQL_WORKER_THREAD_POOL_NAME = "sql-worker"; + public static final String SQL_COMPLEX_WORKER_THREAD_POOL_NAME = "sql-complex-worker"; public static final String SQL_BACKGROUND_THREAD_POOL_NAME = "sql_background_io"; private static final ThreadLocal cancellableTask = new ThreadLocal<>(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ScriptDetector.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ScriptDetector.java new file mode 100644 index 00000000000..aafb2f307b3 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ScriptDetector.java @@ -0,0 +1,116 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import java.util.Set; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Window; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexVisitorImpl; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.context.AggSpec; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; + +/** + * Inspects a Calcite plan tree to determine whether execution will be expensive. Detects: (1) + * user-defined functions (REX_EXTRACT, PARSE, etc.) that become per-document scripts, (2) join + * nodes requiring in-memory merge, and (3) window functions requiring in-memory evaluation. + * + *

Works on both logical plans (before optimization, where PushDownContext is empty) and physical + * plans (after optimization, where PushDownContext tracks scripts). + */ +public final class ScriptDetector { + + private static final Set EXPENSIVE_UDFS = + Set.of("REX_EXTRACT", "REX_EXTRACT_MULTI", "PARSE", "PATTERN_PARSER"); + + private ScriptDetector() {} + + /** + * Returns true if the plan contains patterns indicating expensive execution: UDF calls that + * produce scripts, join nodes, or window functions. + */ + public static boolean hasScripts(RelNode plan) { + boolean[] found = {false}; + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (found[0]) { + return; + } + // Physical plan: check PushDownContext for scripts already detected by optimizer + if (node instanceof AbstractCalciteIndexScan scan) { + found[0] = scanHasScripts(scan); + } + // Logical plan: detect join nodes (always require in-memory processing) + if (!found[0] && node instanceof Join) { + found[0] = true; + } + // Logical plan: detect window rel nodes (eventstats, dedup patterns) + if (!found[0] && node instanceof Window) { + found[0] = true; + } + // Logical plan: check projections for UDFs or window expressions + if (!found[0] && node instanceof Project project) { + found[0] = projectHasExpensiveExpressions(project); + } + if (!found[0]) { + super.visit(node, ordinal, parent); + } + } + }.go(plan); + return found[0]; + } + + private static boolean scanHasScripts(AbstractCalciteIndexScan scan) { + PushDownContext ctx = scan.getPushDownContext(); + if (ctx.isScriptPushed()) { + return true; + } + if (ctx.isSortExprPushed()) { + return true; + } + AggSpec aggSpec = ctx.getAggSpec(); + return aggSpec != null && aggSpec.getScriptCount() > 0; + } + + private static boolean projectHasExpensiveExpressions(Project project) { + for (RexNode expr : project.getProjects()) { + if (hasExpensiveRex(expr)) { + return true; + } + } + return false; + } + + private static boolean hasExpensiveRex(RexNode expr) { + boolean[] found = {false}; + expr.accept( + new RexVisitorImpl(true) { + @Override + public Void visitOver(RexOver over) { + found[0] = true; + return null; + } + + @Override + public Void visitCall(RexCall call) { + String name = call.getOperator().getName(); + if (name != null && EXPENSIVE_UDFS.contains(name)) { + found[0] = true; + return null; + } + return super.visitCall(call); + } + }); + return found[0]; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcher.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcher.java new file mode 100644 index 00000000000..3de8bb85c2b --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcher.java @@ -0,0 +1,174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_COMPLEX_WORKER_THREAD_POOL_NAME; +import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_WORKER_THREAD_POOL_NAME; + +import java.util.Map; +import java.util.function.Consumer; +import lombok.RequiredArgsConstructor; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQueryBase; +import org.apache.calcite.runtime.Hook; +import org.apache.calcite.util.Holder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.ThreadContext; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.executor.ExecutionDispatcher; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.monitor.profile.ProfileContext; +import org.opensearch.sql.monitor.profile.QueryProfiling; +import org.opensearch.tasks.CancellableTask; +import org.opensearch.threadpool.Scheduler.Cancellable; +import org.opensearch.threadpool.ThreadPool; + +/** + * Dispatches query execution to either the fast or complex worker thread pool based on whether the + * plan contains scripts. Plans with scripts require in-memory evaluation and are routed to the + * complex pool so they don't block fast pushdown-only queries. + */ +@RequiredArgsConstructor +public class ThreadPoolExecutionDispatcher implements ExecutionDispatcher { + + private static final Logger LOG = LogManager.getLogger(ThreadPoolExecutionDispatcher.class); + + private final ThreadPool threadPool; + private final Settings settings; + + @Override + public void dispatch( + RelNode plan, + CalcitePlanContext context, + ResponseListener listener, + ExecutionEngine engine) { + dispatchInternal(plan, context, () -> engine.execute(plan, context, listener), listener); + } + + @Override + public void dispatchTask(RelNode plan, CalcitePlanContext context, Runnable task) { + dispatchInternal(plan, context, task, null); + } + + private void dispatchInternal( + RelNode optimizedPlan, + CalcitePlanContext context, + Runnable task, + @Nullable ResponseListener failureListener) { + if (isComplexPoolEnabled() && ScriptDetector.hasScripts(optimizedPlan)) { + LOG.debug("Query plan contains scripts, dispatching to complex worker pool"); + // Capture thread-local state to propagate across thread boundary + Map ctx = ThreadContext.getImmutableContext(); + CancellableTask cancellableTask = OpenSearchQueryManager.getCancellableTask(); + ProfileContext profileContext = QueryProfiling.current(); + CalcitePlanContext.ThreadLocalSnapshot snapshot = CalcitePlanContext.snapshotThreadLocals(); + @Nullable JaninoRelMetadataProvider metadataProvider = + RelMetadataQueryBase.THREAD_PROVIDERS.get(); + long currentTime = Hook.CURRENT_TIME.get(-1L); + TimeValue timeout = settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT); + + threadPool.schedule( + () -> { + final Thread executionThread = Thread.currentThread(); + Cancellable timeoutHandle = + threadPool.schedule( + () -> { + LOG.warn( + "Query execution timed out after {}. Interrupting execution thread.", + timeout); + executionThread.interrupt(); + }, + timeout, + ThreadPool.Names.GENERIC); + Cancellable cancelPoller = scheduleCancellationPoller(cancellableTask, executionThread); + Hook.Closeable hookHandle = null; + try { + // Restore state from caller thread + ThreadContext.putAll(ctx); + OpenSearchQueryManager.setCancellableTask(cancellableTask); + QueryProfiling.set(profileContext); + CalcitePlanContext.restoreThreadLocals(snapshot); + // Override execution pool to indicate complex pool + CalcitePlanContext.executionPool.set(SQL_COMPLEX_WORKER_THREAD_POOL_NAME); + if (metadataProvider != null) { + RelMetadataQueryBase.THREAD_PROVIDERS.set(metadataProvider); + } + if (currentTime >= 0) { + hookHandle = + Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(currentTime)); + } + task.run(); + } catch (Exception e) { + LOG.error("Exception during task execution on complex pool", e); + if (failureListener != null) { + failureListener.onFailure(e); + } + } finally { + timeoutHandle.cancel(); + cancelPoller.cancel(); + Thread.interrupted(); + if (hookHandle != null) { + hookHandle.close(); + } + OpenSearchQueryManager.clearCancellableTask(); + RelMetadataQueryBase.THREAD_PROVIDERS.remove(); + CalcitePlanContext.clearTimewrapSignals(); + QueryProfiling.clear(); + } + }, + new TimeValue(0), + SQL_COMPLEX_WORKER_THREAD_POOL_NAME); + } else { + CalcitePlanContext.executionPool.set(SQL_WORKER_THREAD_POOL_NAME); + task.run(); + } + } + + private static final TimeValue CANCEL_POLL_INTERVAL = new TimeValue(500); + private static final Cancellable NOOP_CANCELLABLE = + new Cancellable() { + @Override + public boolean cancel() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + }; + + /** + * Polls the cancellable task and interrupts the execution thread when cancelled. This bridges the + * gap where OpenSearchQueryManager's timeout interrupt targets the sql-worker thread but + * execution has moved to the complex-worker thread. + */ + private Cancellable scheduleCancellationPoller( + @Nullable CancellableTask cancellableTask, Thread executionThread) { + if (cancellableTask == null) { + return NOOP_CANCELLABLE; + } + return threadPool.scheduleWithFixedDelay( + () -> { + if (cancellableTask.isCancelled()) { + LOG.debug("Task cancelled, interrupting complex pool execution thread"); + executionThread.interrupt(); + } + }, + CANCEL_POLL_INTERVAL, + ThreadPool.Names.GENERIC); + } + + private boolean isComplexPoolEnabled() { + return settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java index 83b1915f6b5..d2e3c52bb6b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java @@ -19,12 +19,12 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.geospatial.action.IpEnrichmentActionClient; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.transport.client.node.NodeClient; @@ -60,8 +60,8 @@ public SqlReturnTypeInference getReturnTypeInference() { public UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrapUDT( List.of( - List.of(ExprCoreType.STRING, ExprCoreType.IP), - List.of(ExprCoreType.STRING, ExprCoreType.IP, ExprCoreType.STRING))); + List.of(PPLOperandTypes.STRING_T, PPLOperandTypes.IP_UDT), + List.of(PPLOperandTypes.STRING_T, PPLOperandTypes.IP_UDT, PPLOperandTypes.STRING_T))); } public static class GeoIPImplementor implements NotNullImplementor { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 775b0278683..6aba75cbcde 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -89,11 +89,13 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.NamedFieldExpression; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.ScriptQueryExpression; import org.opensearch.sql.opensearch.response.agg.ArgMaxMinParser; import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CheckedLongSumParser; import org.opensearch.sql.opensearch.response.agg.CountAsTotalHitsParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -158,7 +160,13 @@ > T build(RexNode node, T sourceBuilde T build(RexNode node, Function fieldBuilder, Function scriptBuilder) { if (node == null) return fieldBuilder.apply(METADATA_FIELD); else if (node instanceof RexInputRef ref) { - return fieldBuilder.apply(inferNamedField(node).getReferenceForTermQuery()); + String fieldRef = inferNamedField(node).getReferenceForTermQuery(); + // Text field with no .keyword sub-field is not aggregatable directly. Fall back to a + // Calcite script that reads the value from _source. + if (fieldRef == null) { + return scriptBuilder.apply(inferScript(node).getScript()); + } + return fieldBuilder.apply(fieldRef); } else if (node instanceof RexCall || node instanceof RexLiteral) { return scriptBuilder.apply(inferScript(node).getScript()); } @@ -175,7 +183,7 @@ NamedFieldExpression inferNamedField(RexNode node) { } ScriptQueryExpression inferScript(RexNode node) { - if (node instanceof RexCall || node instanceof RexLiteral) { + if (node instanceof RexCall || node instanceof RexLiteral || node instanceof RexInputRef) { return new ScriptQueryExpression( node, rowType, fieldTypes, cluster, Collections.emptyMap()); } @@ -477,6 +485,10 @@ private static Pair createRegularAggregation( AggregateBuilderHelper helper, List dedupSortKeys) { + if (aggCall.getAggregation() == PPLBuiltinOperators.CHECKED_LONG_SUM) { + return createCheckedLongSumAggregation(args, aggName, helper); + } + return switch (aggCall.getAggregation().kind) { case AVG -> Pair.of( @@ -651,6 +663,17 @@ yield switch (functionName) { }; } + private static Pair createCheckedLongSumAggregation( + List> args, String aggName, AggregateBuilderHelper helper) { + if (args.size() != 1) { + throw new AggregateAnalyzerException("CHECKED_LONG_SUM requires exactly one argument"); + } + + return Pair.of( + helper.build(args.getFirst().getKey(), AggregationBuilders.sum(aggName)), + new CheckedLongSumParser(aggName)); + } + private static boolean supportsMaxMinAggregation(ExprType fieldType) { ExprType coreType = (fieldType instanceof OpenSearchDataType) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java new file mode 100644 index 00000000000..7ea18298b13 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.search.aggregations.Aggregation; +import org.opensearch.search.aggregations.metrics.NumericMetricsAggregation; + +/** + * Narrows OpenSearch's double-based native sum to the BIGINT type declared by CHECKED_LONG_SUM. + * + *

OpenSearch may already have lost low-order precision before this parser receives the result. A + * double at the positive BIGINT boundary is also ambiguous because {@code Long.MAX_VALUE} rounds to + * {@code 2^63}; Java's narrowing conversion saturates that value to {@code Long.MAX_VALUE}. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class CheckedLongSumParser implements MetricParser { + + private static final double TWO_POW_63 = 0x1p63; + + @Getter private final String name; + + @Override + public List> parse(Aggregation aggregation) { + double value = ((NumericMetricsAggregation.SingleValue) aggregation).value(); + Long narrowed = Double.isNaN(value) ? null : narrow(value); + return Collections.singletonList( + new HashMap<>(Collections.singletonMap(aggregation.getName(), narrowed))); + } + + static long narrow(double value) { + if (!Double.isFinite(value) || value > TWO_POW_63 || value < -TWO_POW_63) { + throw new ArithmeticException("BIGINT overflow in SUM"); + } + return (long) value; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index ce2bdd4960b..0a7cf512210 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -71,14 +71,10 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); - public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = - Setting.boolSetting( - Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); - public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = Setting.listSetting( Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), - List.of(), + List.of("/_cluster/health"), Function.identity(), Setting.Property.NodeScope); @@ -363,6 +359,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting SQL_COMPLEX_WORKER_POOL_ENABLED_SETTING = + Setting.boolSetting( + Key.SQL_COMPLEX_WORKER_POOL_ENABLED.getKeyValue(), + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + /** Construct OpenSearchSetting. The OpenSearchSetting must be singleton. */ @SuppressWarnings("unchecked") public OpenSearchSettings(ClusterSettings clusterSettings) { @@ -391,11 +394,6 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); - registerNonDynamicSettings( - settingBuilder, - clusterSettings, - Key.PPL_REST_REDACTION_ENABLED, - PPL_REST_REDACTION_ENABLED_SETTING); registerNonDynamicSettings( settingBuilder, clusterSettings, @@ -631,6 +629,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.FIELD_TYPE_TOLERANCE, FIELD_TYPE_TOLERANCE_SETTING, new Updater(Key.FIELD_TYPE_TOLERANCE)); + register( + settingBuilder, + clusterSettings, + Key.SQL_COMPLEX_WORKER_POOL_ENABLED, + SQL_COMPLEX_WORKER_POOL_ENABLED_SETTING, + new Updater(Key.SQL_COMPLEX_WORKER_POOL_ENABLED)); defaultSettings = settingBuilder.build(); } @@ -726,6 +730,7 @@ public static List> pluginSettings() { .add(SESSION_INACTIVITY_TIMEOUT_MILLIS_SETTING) .add(STREAMING_JOB_HOUSEKEEPER_INTERVAL_SETTING) .add(FIELD_TYPE_TOLERANCE_SETTING) + .add(SQL_COMPLEX_WORKER_POOL_ENABLED_SETTING) .build(); } @@ -734,7 +739,6 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) - .add(PPL_REST_REDACTION_ENABLED_SETTING) .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 7b911471242..62ab089c1a8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -19,6 +19,7 @@ import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; @@ -52,9 +53,10 @@ public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { private Table restTable(String name) { RestSpec spec = decodeRestSpec(name); - RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry registry = RestEndpointRegistryHolder.get(); + registry.resolve(spec.getEndpoint()); List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); - if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { + if (allowed == null || !allowed.contains(spec.getEndpoint())) { throw new IllegalArgumentException( allowed == null || allowed.isEmpty() ? "the rest command is disabled on this cluster" @@ -63,7 +65,6 @@ private Table restTable(String name) { + "] is not enabled on this cluster. Enabled endpoints: " + allowed); } - boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); - return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); + return new OpenSearchCatalogTable(new RestCatalogSource(registry, spec, client), settings); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java new file mode 100644 index 00000000000..f746beb54a9 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.transport.client.node.NodeClient; + +/** + * The built-in {@link RestEndpointProvider}. It ships a single read-only, in-cluster endpoint, + * {@code /_cluster/health}, expressed as a {@link RestEndpointDefinition}. It is a uniform client + * of the same SPI an external plugin uses. Additional endpoints are left to follow-up changes. + * + *

Like any external provider, it fetches at execution time through the transport node client the + * context carries ({@link RestEndpointContext#client()}), so it holds no reference to the sql + * storage client and runs under the caller's security thread-context. + * + *

It returns the full health response in a single JSON {@code response} column; a query extracts + * the fields it needs with the {@code spath} command or the {@code json_extract} function. + */ +public final class CoreEndpointsProvider implements RestEndpointProvider { + + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .argSpec(ArgSpec.builder().arg("local", Set.of("true", "false")).build()) + .handler(CoreEndpointsProvider::clusterHealth) + .build()); + } + + private static List clusterHealth(RestEndpointContext ctx) { + NodeClient client = ctx.client(); + if (client == null) { + throw new IllegalStateException( + "the /_cluster/health rest endpoint requires an in-cluster node client"); + } + ClusterHealthRequest request = new ClusterHealthRequest(); + if (Boolean.parseBoolean(ctx.args().get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + return List.of(builder.toString()); + } catch (IOException e) { + throw new IllegalStateException("failed to serialize the /_cluster/health response", e); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java index 96779726a7e..e0c5c77423d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -17,8 +17,8 @@ /** * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management - * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. - * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. + * endpoint resolved against a {@link RestEndpointRegistry} instance, exposing the fixed endpoint + * schema. Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. */ @Getter public class RestCatalogSource implements CatalogSource { @@ -26,19 +26,13 @@ public class RestCatalogSource implements CatalogSource { private final OpenSearchClient client; private final RestSpec spec; private final RestEndpointRegistry.Endpoint endpoint; - private final boolean redact; - public RestCatalogSource(OpenSearchClient client, RestSpec spec) { - this(client, spec, false); - } - - public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { + public RestCatalogSource(RestEndpointRegistry registry, RestSpec spec, OpenSearchClient client) { this.client = client; this.spec = spec; - this.redact = redact; // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. - this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); - RestEndpointRegistry.validate(spec); + this.endpoint = registry.resolve(spec.getEndpoint()); + registry.validate(spec); } @Override @@ -48,7 +42,7 @@ public Map getFieldTypes() { @Override public OpenSearchSystemRequest createRequest() { - return new RestRequest(client, endpoint, spec, redact); + return new RestRequest(client, endpoint, spec); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index e64a91ab54a..ac4a7c72df6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -5,277 +5,141 @@ package org.opensearch.sql.opensearch.storage.rest; -import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; -import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; -import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; -import static org.opensearch.sql.data.model.ExprValueUtils.longValue; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; -import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import lombok.Getter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointHandler; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** - * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps - * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so - * the Calcite plan can fix its row type at plan time), and the query args it accepts. + * The read-only endpoint allow-list, built by merging every {@link RestEndpointProvider} (the + * built-in {@link CoreEndpointsProvider} plus any externally contributed providers) into one map of + * endpoint name to an internal {@link Endpoint}. A built-in and an externally contributed endpoint + * are uniform entries here, except that a built-in name cannot be shadowed by an external provider. * - *

This is the single place the read-only allow-list is enforced. Endpoints outside the registry, - * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. - * Adding an endpoint is a reviewed change here, never arbitrary pass-through. + *

This is the single place the read-only allow-list is enforced. An endpoint that no provider + * registered, including every mutating endpoint, is rejected by {@link #resolve} with a clear + * exception, and an arg the endpoint's {@link ArgSpec} does not accept is rejected by {@link + * #validate}. Adding an endpoint is a reviewed change to a provider, never arbitrary pass-through. */ public final class RestEndpointRegistry { - private RestEndpointRegistry() {} + private static final Logger LOG = LogManager.getLogger(RestEndpointRegistry.class); - /** Produces the raw rows for an endpoint via a read-only client call. */ - @FunctionalInterface - public interface RowFetcher { - List> fetch(OpenSearchClient client, RestSpec spec); + private final Map registry; + + public RestEndpointRegistry(List providers) { + Map m = new LinkedHashMap<>(); + Set disabled = new HashSet<>(); + for (RestEndpointProvider provider : providers) { + boolean builtIn = provider instanceof CoreEndpointsProvider; + for (RestEndpointDefinition definition : provider.getEndpoints()) { + String name = definition.name(); + if (disabled.contains(name)) { + continue; + } + Endpoint existing = m.get(name); + if (existing == null) { + m.put(name, new Endpoint(definition, builtIn)); + continue; + } + if (existing.isBuiltIn() || builtIn) { + LOG.warn( + "rest endpoint [{}] collides with a built-in endpoint; ignoring the duplicate from" + + " provider [{}]", + name, + provider.getClass().getName()); + continue; + } + LOG.warn( + "rest endpoint [{}] is registered by multiple external providers; disabling it." + + " Conflicting provider [{}]", + name, + provider.getClass().getName()); + m.remove(name); + disabled.add(name); + } + } + this.registry = m; } - /** A single allow-listed endpoint description. */ + /** The single column every rest endpoint surfaces: one JSON {@code response} string. */ + static final String RESPONSE_COLUMN = "response"; + + /** A single allow-listed endpoint, adapted from a provider's {@link RestEndpointDefinition}. */ @Getter public static final class Endpoint { private final String path; private final LinkedHashMap schema; - private final Set allowedArgs; - private final RowFetcher fetcher; + private final ArgSpec argSpec; + private final RestEndpointHandler handler; + private final boolean builtIn; - Endpoint( - String path, - LinkedHashMap schema, - Set allowedArgs, - RowFetcher fetcher) { - this.path = path; - this.schema = schema; - this.allowedArgs = allowedArgs; - this.fetcher = fetcher; - } - - /** Dispatch the read-only call and shape the response into fixed-schema rows. */ - public List toRows(OpenSearchClient client, RestSpec spec) { - return toRows(client, spec, false); + Endpoint(RestEndpointDefinition definition, boolean builtIn) { + this.path = definition.name(); + this.schema = new LinkedHashMap<>(); + this.schema.put(RESPONSE_COLUMN, STRING); + this.argSpec = definition.argSpec(); + this.handler = definition.handler(); + this.builtIn = builtIn; } /** - * Shape the response into fixed-schema rows, masking network identifiers when redaction is - * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value - * column is zone-masked. Off by default. + * Invoke the provider's handler and wrap each returned string into the single {@code response} + * column. Runs at execution time (scan open). A provider that masks sensitive values does so + * before serializing the response it returns, so the values surfaced here are already redacted. */ - public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { - boolean redactCat = redact && path.startsWith("/_cat"); - boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); + public List toRows(RestEndpointContext ctx) { List out = new ArrayList<>(); - for (Map raw : fetcher.fetch(client, spec)) { + for (String response : handler.fetch(ctx)) { LinkedHashMap tuple = new LinkedHashMap<>(); - for (Map.Entry col : schema.entrySet()) { - ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); - tuple.put( - col.getKey(), - maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); - } + tuple.put(RESPONSE_COLUMN, response == null ? ExprNullValue.of() : stringValue(response)); out.add(new ExprTupleValue(tuple)); } return out; } - - private static ExprValue maskCell( - String column, - ExprType type, - ExprValue value, - boolean redactCat, - boolean redactSettingsValue) { - if (type != STRING || value.isNull()) { - return value; - } - if (redactCat) { - return stringValue(RestResponseRedactor.redact(value.stringValue())); - } - if (redactSettingsValue && "value".equals(column)) { - return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); - } - return value; - } - } - - private static final Map REGISTRY = buildRegistry(); - - private static Map buildRegistry() { - Map m = new LinkedHashMap<>(); - - // /_cluster/health — single-row cluster health snapshot (read-only monitor action). - LinkedHashMap healthSchema = new LinkedHashMap<>(); - healthSchema.put("cluster_name", STRING); - healthSchema.put("status", STRING); - healthSchema.put("number_of_nodes", INTEGER); - healthSchema.put("number_of_data_nodes", INTEGER); - healthSchema.put("active_primary_shards", INTEGER); - healthSchema.put("active_shards", INTEGER); - healthSchema.put("relocating_shards", INTEGER); - healthSchema.put("initializing_shards", INTEGER); - healthSchema.put("unassigned_shards", INTEGER); - healthSchema.put("timed_out", BOOLEAN); - m.put( - "/_cluster/health", - new Endpoint( - "/_cluster/health", - healthSchema, - Set.of("local"), - (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); - - // /_cat/indices — one row per index (read-only monitor action). - LinkedHashMap catSchema = new LinkedHashMap<>(); - catSchema.put("index", STRING); - catSchema.put("health", STRING); - catSchema.put("pri", INTEGER); - catSchema.put("rep", INTEGER); - catSchema.put("active_shards", INTEGER); - m.put( - "/_cat/indices", - new Endpoint( - "/_cat/indices", - catSchema, - Set.of("health"), - (client, spec) -> client.catIndices(spec.getArgs()))); - - // /_cat/nodes — one row per node with resource state (read-only monitor action). - LinkedHashMap nodesSchema = new LinkedHashMap<>(); - nodesSchema.put("name", STRING); - nodesSchema.put("ip", STRING); - nodesSchema.put("node_role", STRING); - nodesSchema.put("heap_percent", INTEGER); - nodesSchema.put("ram_percent", INTEGER); - nodesSchema.put("cpu", INTEGER); - m.put( - "/_cat/nodes", - new Endpoint( - "/_cat/nodes", - nodesSchema, - Set.of(), - (client, spec) -> client.catNodes(spec.getArgs()))); - - // /_cat/cluster_manager — single row identifying the elected cluster manager node. - LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); - clusterManagerSchema.put("id", STRING); - clusterManagerSchema.put("host", STRING); - clusterManagerSchema.put("ip", STRING); - clusterManagerSchema.put("node", STRING); - m.put( - "/_cat/cluster_manager", - new Endpoint( - "/_cat/cluster_manager", - clusterManagerSchema, - Set.of(), - (client, spec) -> client.catClusterManager(spec.getArgs()))); - - // /_cat/plugins — one row per installed plugin per node (read-only monitor action). - LinkedHashMap pluginsSchema = new LinkedHashMap<>(); - pluginsSchema.put("name", STRING); - pluginsSchema.put("component", STRING); - pluginsSchema.put("version", STRING); - m.put( - "/_cat/plugins", - new Endpoint( - "/_cat/plugins", - pluginsSchema, - Set.of(), - (client, spec) -> client.catPlugins(spec.getArgs()))); - - // /_cat/shards — one row per shard (read-only monitor action). - LinkedHashMap shardsSchema = new LinkedHashMap<>(); - shardsSchema.put("index", STRING); - shardsSchema.put("shard", INTEGER); - shardsSchema.put("prirep", STRING); - shardsSchema.put("state", STRING); - shardsSchema.put("node", STRING); - m.put( - "/_cat/shards", - new Endpoint( - "/_cat/shards", - shardsSchema, - Set.of(), - (client, spec) -> client.catShards(spec.getArgs()))); - - // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). - LinkedHashMap stateSchema = new LinkedHashMap<>(); - stateSchema.put("cluster_name", STRING); - stateSchema.put("state_uuid", STRING); - stateSchema.put("version", LONG); - stateSchema.put("cluster_manager_node", STRING); - m.put( - "/_cluster/state", - new Endpoint( - "/_cluster/state", - stateSchema, - Set.of(), - (client, spec) -> List.of(client.clusterState(spec.getArgs())))); - - // /_cluster/settings — one row per configured setting (persistent/transient tier). - LinkedHashMap settingsSchema = new LinkedHashMap<>(); - settingsSchema.put("setting", STRING); - settingsSchema.put("value", STRING); - settingsSchema.put("tier", STRING); - m.put( - "/_cluster/settings", - new Endpoint( - "/_cluster/settings", - settingsSchema, - Set.of(), - (client, spec) -> client.clusterSettings(spec.getArgs()))); - - // /_resolve/index — one row per resolved index/alias/data_stream name. - LinkedHashMap resolveSchema = new LinkedHashMap<>(); - resolveSchema.put("name", STRING); - resolveSchema.put("type", STRING); - m.put( - "/_resolve/index", - new Endpoint( - "/_resolve/index", - resolveSchema, - Set.of("expand_wildcards"), - (client, spec) -> client.resolveIndex(spec.getArgs()))); - - return m; } /** - * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, + * Resolve an allow-listed endpoint. Anything no provider registered (unknown path, mutating verb, * {@code /services/*}, plugin admin endpoints) is refused here. */ - public static Endpoint resolve(String path) { + public Endpoint resolve(String path) { if (path == null || path.isBlank()) { throw new IllegalArgumentException( - "rest endpoint must be a non-empty path. Supported read-only endpoints: " - + REGISTRY.keySet()); + "rest endpoint must be a non-empty path. Only the following endpoints are supported: " + + registry.keySet()); } - Endpoint endpoint = REGISTRY.get(path); + Endpoint endpoint = registry.get(path); if (endpoint == null) { throw new IllegalArgumentException( "rest endpoint [" + path - + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " - + REGISTRY.keySet()); + + "] is not allow-listed. Only the following endpoints are supported: " + + registry.keySet()); } return endpoint; } - /** Validate that every supplied query arg is accepted by the endpoint. */ - public static void validate(RestSpec spec) { + /** Validate the count, the reserved timeout token, and every supplied query arg. */ + public void validate(RestSpec spec) { Endpoint endpoint = resolve(spec.getEndpoint()); if (spec.getCount() != null && spec.getCount() < 0) { throw new IllegalArgumentException( @@ -294,135 +158,19 @@ public static void validate(RestSpec spec) { "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); } if (spec.getArgs() != null) { + ArgSpec argSpec = endpoint.getArgSpec(); for (String arg : spec.getArgs().keySet()) { - if (!endpoint.getAllowedArgs().contains(arg)) { + if (!argSpec.allows(arg)) { throw new IllegalArgumentException( "rest endpoint [" + spec.getEndpoint() + "] does not accept arg [" + arg + "]. Allowed args: " - + endpoint.getAllowedArgs()); - } - validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); - } - } - } - - // Allowed value domains for the get-args that are applied server-side. Keys are validated against - // the per-endpoint allow-list above; values are validated here so a user-supplied value is never - // passed unchecked into an admin transport request. - private static final Map> ARG_VALUE_DOMAINS = - Map.of( - "local", Set.of("true", "false"), - "health", Set.of("green", "yellow", "red")); - - private static final Set EXPAND_WILDCARDS_VALUES = - Set.of("open", "closed", "hidden", "none", "all"); - - /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ - private static void validateArgValue(String endpoint, String arg, String value) { - Set domain = ARG_VALUE_DOMAINS.get(arg); - if (domain != null) { - if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [" - + arg - + "] has an unsupported value [" - + value - + "]. Allowed values: " - + domain); - } - } else if ("expand_wildcards".equals(arg)) { - if (value == null || value.isBlank()) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [expand_wildcards] has an unsupported value [" - + value - + "]. Allowed values: " - + EXPAND_WILDCARDS_VALUES); - } - for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { - if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [expand_wildcards] has an unsupported value [" - + value - + "]. Allowed values: " - + EXPAND_WILDCARDS_VALUES); + + argSpec.allowedArgs()); } + argSpec.validateValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); } } } - - private static ExprValue coerce(String column, ExprType type, Object value) { - if (value == null) { - return ExprNullValue.of(); - } - try { - if (type == INTEGER) { - return integerValue(toNumber(value).intValue()); - } - if (type == LONG) { - return longValue(toNumber(value).longValue()); - } - if (type == DOUBLE) { - return doubleValue(toNumber(value).doubleValue()); - } - if (type == BOOLEAN) { - return booleanValue(toBoolean(value)); - } - } catch (IllegalArgumentException | ClassCastException e) { - // Surface a clear client error (HTTP 400) instead of a raw HTTP 500 when an endpoint - // returns an unexpected value shape. NumberFormatException extends IllegalArgumentException, - // so toNumber parse failures and toBoolean's "not a boolean" are both caught here; genuinely - // unexpected faults (NPE, etc.) are left to propagate. - throw new IllegalArgumentException( - "rest endpoint value for column [" - + column - + "] could not be coerced to " - + type - + ": [" - + value - + "]"); - } - return stringValue(String.valueOf(value)); - } - - /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ - private static Number toNumber(Object value) { - if (value instanceof Number n) { - return n; - } - String s = String.valueOf(value).trim(); - if (s.isEmpty()) { - throw new NumberFormatException("empty string"); - } - if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { - return Double.parseDouble(s); - } - return Long.parseLong(s); - } - - /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ - private static boolean toBoolean(Object value) { - if (value instanceof Boolean b) { - return b; - } - String s = String.valueOf(value).trim(); - if (s.isEmpty()) { - throw new IllegalArgumentException("empty string is not a boolean"); - } - if (s.equalsIgnoreCase("true")) { - return true; - } - if (s.equalsIgnoreCase("false")) { - return false; - } - throw new IllegalArgumentException("not a boolean: " + value); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java new file mode 100644 index 00000000000..18669c05187 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +/** + * Bridge for sharing the merged {@link RestEndpointRegistry} between plugin bootstrap (where the + * SQL plugin builds it from the built-in provider plus every provider discovered via {@code + * ExtensiblePlugin.loadExtensions}) and {@code OpenSearchStorageEngine.getTable} (which resolves a + * {@code rest} endpoint at query time). + * + *

Why a static holder: {@code loadExtensions} runs during node bootstrap, before the Node-level + * Guice injector exists, so the merged registry cannot be injected into the storage engine. + * Publishing it here once at bootstrap lets the storage engine read the same instance without going + * through the injector. Mirrors {@code AnalyticsExecutorHolder}. + */ +public final class RestEndpointRegistryHolder { + + private static volatile RestEndpointRegistry registry; + + private RestEndpointRegistryHolder() {} + + public static void set(RestEndpointRegistry instance) { + registry = instance; + } + + public static RestEndpointRegistry get() { + return registry; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java index 868dabdd64e..22f73c63005 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -9,41 +9,39 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.spi.rest.RestEndpointContext; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; +import org.opensearch.transport.client.node.NodeClient; /** - * Dispatches an allow-listed, read-only management endpoint through the transport node client under + * Dispatches an allow-listed, read-only management endpoint through the endpoint's handler under * the caller's security thread-context and returns the response shaped to the endpoint's fixed * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to - * the system-index scan family. + * the system-index scan family. This is the lazy scan: the handler runs here at {@link #search} + * (execution), never at planning time. */ public class RestRequest implements OpenSearchSystemRequest { private final OpenSearchClient client; private final RestEndpointRegistry.Endpoint endpoint; private final RestSpec spec; - private final boolean redact; public RestRequest( OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { - this(client, endpoint, spec, false); - } - - public RestRequest( - OpenSearchClient client, - RestEndpointRegistry.Endpoint endpoint, - RestSpec spec, - boolean redact) { this.client = client; this.endpoint = endpoint; this.spec = spec; - this.redact = redact; } @Override public List search() { - List rows = endpoint.toRows(client, spec, redact); + // The node transport client every provider handler fetches through, sourced from the same + // OpenSearchClient the storage engine uses so it runs under the caller's security + // thread-context. + NodeClient nodeClient = client == null ? null : client.getNodeClient().orElse(null); + RestEndpointContext ctx = RestEndpointContext.of(spec.getArgs(), nodeClient); + List rows = endpoint.toRows(ctx); if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { return rows.subList(0, spec.getCount()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java deleted file mode 100644 index fd674dccde4..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.List; -import java.util.regex.Pattern; - -/** - * Masks network identifiers in rest command cell values. Enabled per deployment via {@code - * plugins.ppl.rest.redaction.enabled}; off by default. - */ -public final class RestResponseRedactor { - - private RestResponseRedactor() {} - - private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; - private static final Pattern IPV4 = - Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); - private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); - private static final Pattern EC2_HOST = - Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); - private static final Pattern IPV6 = - Pattern.compile( - "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" - + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", - Pattern.CASE_INSENSITIVE); - private static final Pattern AZ_NAME = - Pattern.compile( - "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", - Pattern.CASE_INSENSITIVE); - - private record Mask(Pattern pattern, String replacement) {} - - private static final List MASKS = - List.of( - new Mask(IPV4, "x.x.x.x"), - new Mask(INET, "inet[/x.x.x.x:y]"), - new Mask(EC2_HOST, ""), - new Mask(IPV6, "x.x.x.x"), - new Mask(AZ_NAME, "xx-xxxxx-xx")); - - /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ - public static String redact(String text) { - if (text == null || text.isEmpty()) { - return text; - } - String out = text; - for (Mask mask : MASKS) { - out = mask.pattern().matcher(out).replaceAll(mask.replacement()); - } - return out; - } - - /** Mask availability-zone names only. */ - public static String maskAvailabilityZone(String text) { - if (text == null || text.isEmpty()) { - return text; - } - return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java deleted file mode 100644 index d425c361e6a..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import org.opensearch.common.settings.SettingsFilter; - -/** - * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' - * /_cluster/settings'} fetcher. - * - *

The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with - * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the - * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads - * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the - * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction - * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. - * - *

Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in - * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link - * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code - * getRestHandlers} fires lets the fetcher read the same instance without going back through the - * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. - */ -public final class RestSettingsFilterHolder { - - private static volatile SettingsFilter settingsFilter; - - private RestSettingsFilterHolder() {} - - public static void set(SettingsFilter instance) { - settingsFilter = instance; - } - - public static SettingsFilter get() { - return settingsFilter; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java index 65ba189b0d9..3aa347b70fa 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java @@ -15,6 +15,7 @@ import javax.annotation.Nullable; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchSecurityException; +import org.opensearch.core.tasks.TaskCancelledException; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.NonFallbackCalciteException; import org.opensearch.sql.monitor.profile.ProfileContext; @@ -118,7 +119,10 @@ private OpenSearchResponse getCurrentResponse(OpenSearchRequest request) { return nextBatchFuture.get(); } catch (OpenSearchSecurityException e) { throw e; - } catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TaskCancelledException("The task is cancelled."); + } catch (ExecutionException e) { if (e.getCause() instanceof OpenSearchSecurityException) { throw (OpenSearchSecurityException) e.getCause(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java index 05e0907d934..dd88cf7a6c6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java @@ -43,8 +43,14 @@ private static String convert(String text, boolean escapeStarQuestion) { for (char currentChar : text.toCharArray()) { switch (currentChar) { case DEFAULT_ESCAPE: - escaped = true; - convertedString.append(currentChar); + if (escaped) { + convertedString.deleteCharAt(convertedString.length() - 1); + convertedString.append(currentChar); + escaped = false; + } else { + escaped = true; + convertedString.append(currentChar); + } break; case '%': if (escaped) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java index dff9b47265a..a0e75ce9459 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java @@ -53,9 +53,13 @@ public OpenSearchCatalogEnumerator( @Override public Object current() { - return fields.stream() - .map(k -> current.tupleValue().getOrDefault(k, ExprNullValue.of()).valueForCalcite()) - .toArray(); + Object[] row = + fields.stream() + .map(k -> current.tupleValue().getOrDefault(k, ExprNullValue.of()).valueForCalcite()) + .toArray(); + // Calcite represents a single-column row as the bare scalar (the ARRAY row format optimizes to + // SCALAR for a one-field row type), so return the value directly instead of a length-one array. + return row.length == 1 ? row[0] : row; } @Override diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java deleted file mode 100644 index 33be6cc8482..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.client; - -import static java.util.stream.Collectors.toSet; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Answers.RETURNS_DEEP_STUBS; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.opensearch.action.admin.cluster.state.ClusterStateRequest; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.settings.SettingsFilter; -import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; -import org.opensearch.transport.client.node.NodeClient; - -/** - * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using - * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. - */ -class OpenSearchNodeClientClusterSettingsFilterTest { - - @AfterEach - void clearHolder() { - RestSettingsFilterHolder.set(null); - } - - private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { - NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); - ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); - when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) - .thenReturn(stateResp); - when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); - when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); - return new OpenSearchNodeClient(nodeClient); - } - - @Test - void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { - Settings persistent = - Settings.builder() - .put("cluster.routing.allocation.enable", "all") - .put("plugins.secret.token", "supersecret") - .build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - // Publish a filter that redacts the secret key, exactly as the native endpoint would. - RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); - - List> rows = client.clusterSettings(Map.of()); - Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); - - assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); - assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); - } - - @Test - void clusterSettingsRedactsByGlobPattern() { - Settings persistent = - Settings.builder() - .put("cluster.routing.allocation.enable", "all") - .put("s3.client.default.secret_key", "AKIAEXAMPLE") - .build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); - - Set keys = - client.clusterSettings(Map.of()).stream() - .map(r -> (String) r.get("setting")) - .collect(toSet()); - - assertTrue(keys.contains("cluster.routing.allocation.enable")); - assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); - } - - @Test - void clusterSettingsFailsClosedWhenNoFilterPublished() { - Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw - // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter - // before any query, so this path is unreachable in cluster. - assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 1479ccfb615..247b7a754e0 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -91,6 +91,7 @@ public void typeName() { assertEquals("DOUBLE", OpenSearchDataType.of(MappingType.Double).typeName()); assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).typeName()); assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).typeName()); + assertEquals("CONSTANT_KEYWORD", OpenSearchDataType.of(MappingType.ConstantKeyword).typeName()); } @Test @@ -115,6 +116,7 @@ private static Stream getTestDataWithType() { return Stream.of( Arguments.of(MappingType.Text, "text", OpenSearchTextType.of()), Arguments.of(MappingType.Keyword, "keyword", STRING), + Arguments.of(MappingType.ConstantKeyword, "constant_keyword", STRING), Arguments.of(MappingType.Byte, "byte", BYTE), Arguments.of(MappingType.Short, "short", SHORT), Arguments.of(MappingType.Integer, "integer", INTEGER), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java index 01d61288173..3da901b567a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java @@ -12,7 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -22,6 +22,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -71,7 +72,9 @@ class OpenSearchExecutionEngineTest { @BeforeEach void setUp() { - doAnswer( + // lenient: the static PIT-detection tests below exercise no mock, so this stub is unused there. + lenient() + .doAnswer( invocation -> { // Run task immediately Runnable task = invocation.getArgument(0); @@ -262,6 +265,45 @@ public void onFailure(Exception e) { assertTrue(plan.hasClosed); } + @Test + void detects_pit_context_limit_in_nested_cause() { + // The create-PIT rejection surfaces wrapped several layers deep, mirroring the real chain: + // SQLException -> RuntimeException -> ExecutionException(all shards failed) -> rejection. + Throwable rejection = + new IllegalStateException( + "Trying to create too many Point In Time contexts. Must be less than or equal to: [0]." + + " This limit can be set by changing the [search.max_open_pit_context] setting."); + Throwable chain = + new SQLException( + "exception while executing query: Error occurred while creating PIT", + new RuntimeException("all shards failed", rejection)); + + assertTrue(OpenSearchExecutionEngine.isPitContextLimitReached(chain)); + } + + @Test + void does_not_flag_unrelated_failures_as_pit_context_limit() { + Throwable chain = + new SQLException( + "exception while executing query", new RuntimeException("all shards failed")); + + assertFalse(OpenSearchExecutionEngine.isPitContextLimitReached(chain)); + } + + @Test + void pit_context_limit_check_survives_self_referential_cause() { + // A throwable whose cause is itself must not loop forever (getCause() == this). + RuntimeException selfReferential = + new RuntimeException("boom") { + @Override + public synchronized Throwable getCause() { + return this; + } + }; + + assertFalse(OpenSearchExecutionEngine.isPitContextLimitReached(selfReferential)); + } + @RequiredArgsConstructor private static class FakePhysicalPlan extends TableScanOperator implements SerializablePlan { private final Iterator it; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ScriptDetectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ScriptDetectorTest.java new file mode 100644 index 00000000000..e44809e40a9 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ScriptDetectorTest.java @@ -0,0 +1,212 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.logical.LogicalWindow; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.sql.SqlOperator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.context.AggSpec; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class ScriptDetectorTest { + + @Test + void returnsFalseForNonScanNode() { + RelNode mockNode = createMockNode(); + assertFalse(ScriptDetector.hasScripts(mockNode)); + } + + @Test + void returnsTrueWhenFilterScriptPushed() { + AbstractCalciteIndexScan scan = createMockScan(true, false, 0); + assertTrue(ScriptDetector.hasScripts(scan)); + } + + @Test + void returnsTrueWhenAggScriptPresent() { + AbstractCalciteIndexScan scan = createMockScan(false, false, 1); + assertTrue(ScriptDetector.hasScripts(scan)); + } + + @Test + void returnsTrueWhenSortExprPushed() { + AbstractCalciteIndexScan scan = createMockScan(false, true, 0); + assertTrue(ScriptDetector.hasScripts(scan)); + } + + @Test + void returnsFalseWhenNoScripts() { + AbstractCalciteIndexScan scan = createMockScan(false, false, 0); + assertFalse(ScriptDetector.hasScripts(scan)); + } + + @Test + void detectsScriptsInNestedPlan() { + AbstractCalciteIndexScan scan = createMockScan(false, false, 3); + RelNode parent = createMockNode(scan); + assertTrue(ScriptDetector.hasScripts(parent)); + } + + @Test + void detectsJoinNode() { + LogicalJoin join = mock(LogicalJoin.class); + when(join.getJoinType()).thenReturn(JoinRelType.LEFT); + when(join.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(join).childrenAccept(any(RelVisitor.class)); + assertTrue(ScriptDetector.hasScripts(join)); + } + + @Test + void detectsWindowNode() { + LogicalWindow window = mock(LogicalWindow.class); + when(window.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(window).childrenAccept(any(RelVisitor.class)); + assertTrue(ScriptDetector.hasScripts(window)); + } + + @Test + void detectsExpensiveUdfInProject() { + SqlOperator rexExtractOp = mock(SqlOperator.class); + when(rexExtractOp.getName()).thenReturn("REX_EXTRACT"); + RexCall udfCall = mock(RexCall.class); + when(udfCall.getOperator()).thenReturn(rexExtractOp); + when(udfCall.getOperands()).thenReturn(List.of()); + RelDataType type = mock(RelDataType.class); + when(udfCall.getType()).thenReturn(type); + doAnswer(inv -> inv.>getArgument(0).visitCall(udfCall)) + .when(udfCall) + .accept(any()); + + LogicalProject project = mock(LogicalProject.class); + when(project.getProjects()).thenReturn(List.of(udfCall)); + when(project.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(project).childrenAccept(any(RelVisitor.class)); + + assertTrue(ScriptDetector.hasScripts(project)); + } + + @Test + void ignoresCheapUdfInProject() throws Exception { + SqlOperator cheapOp = mock(SqlOperator.class); + when(cheapOp.getName()).thenReturn("NOW"); + RexCall cheapCall = mock(RexCall.class); + when(cheapCall.getOperator()).thenReturn(cheapOp); + RelDataType type = mock(RelDataType.class); + when(cheapCall.getType()).thenReturn(type); + // RexVisitorImpl.visitCall accesses call.operands field directly, set it via reflection + java.lang.reflect.Field operandsField = RexCall.class.getDeclaredField("operands"); + operandsField.setAccessible(true); + operandsField.set(cheapCall, com.google.common.collect.ImmutableList.of()); + doAnswer(inv -> inv.>getArgument(0).visitCall(cheapCall)) + .when(cheapCall) + .accept(any()); + + LogicalProject project = mock(LogicalProject.class); + when(project.getProjects()).thenReturn(List.of((RexNode) cheapCall)); + when(project.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(project).childrenAccept(any(RelVisitor.class)); + + assertFalse(ScriptDetector.hasScripts(project)); + } + + @Test + void detectsRexOverInProject() { + RexOver rexOver = mock(RexOver.class); + SqlOperator op = mock(SqlOperator.class); + when(rexOver.getOperator()).thenReturn(op); + when(rexOver.getOperands()).thenReturn(List.of()); + RelDataType type = mock(RelDataType.class); + when(rexOver.getType()).thenReturn(type); + doAnswer(inv -> inv.>getArgument(0).visitOver(rexOver)) + .when(rexOver) + .accept(any(org.apache.calcite.rex.RexVisitor.class)); + + LogicalProject project = mock(LogicalProject.class); + when(project.getProjects()).thenReturn(List.of(rexOver)); + when(project.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(project).childrenAccept(any(RelVisitor.class)); + + assertTrue(ScriptDetector.hasScripts(project)); + } + + @Test + void returnsFalseForSimpleFieldProject() { + RexInputRef fieldRef = mock(RexInputRef.class); + RelDataType type = mock(RelDataType.class); + when(fieldRef.getType()).thenReturn(type); + + LogicalProject project = mock(LogicalProject.class); + when(project.getProjects()).thenReturn(List.of((RexNode) fieldRef)); + when(project.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(project).childrenAccept(any(RelVisitor.class)); + + assertFalse(ScriptDetector.hasScripts(project)); + } + + private static RelNode createMockNode(RelNode... children) { + RelNode node = mock(RelNode.class); + List childList = List.of(children); + when(node.getInputs()).thenReturn(childList); + doAnswer( + invocation -> { + RelVisitor visitor = invocation.getArgument(0); + for (int i = 0; i < childList.size(); i++) { + visitor.visit(childList.get(i), i, node); + } + return null; + }) + .when(node) + .childrenAccept(any(RelVisitor.class)); + return node; + } + + private static AbstractCalciteIndexScan createMockScan( + boolean scriptPushed, boolean sortExprPushed, long aggScriptCount) { + AbstractCalciteIndexScan scan = mock(AbstractCalciteIndexScan.class); + + PushDownContext ctx = mock(PushDownContext.class); + when(ctx.isScriptPushed()).thenReturn(scriptPushed); + when(ctx.isSortExprPushed()).thenReturn(sortExprPushed); + + if (aggScriptCount > 0) { + AggSpec aggSpec = mock(AggSpec.class); + when(aggSpec.getScriptCount()).thenReturn(aggScriptCount); + when(ctx.getAggSpec()).thenReturn(aggSpec); + } else { + when(ctx.getAggSpec()).thenReturn(null); + } + + when(scan.getPushDownContext()).thenReturn(ctx); + when(scan.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(scan).childrenAccept(any(RelVisitor.class)); + return scan; + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcherTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcherTest.java new file mode 100644 index 00000000000..bf6adbf985e --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcherTest.java @@ -0,0 +1,399 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_COMPLEX_WORKER_THREAD_POOL_NAME; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQueryBase; +import org.apache.logging.log4j.ThreadContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; +import org.opensearch.tasks.CancellableTask; +import org.opensearch.threadpool.Scheduler.Cancellable; +import org.opensearch.threadpool.Scheduler.ScheduledCancellable; +import org.opensearch.threadpool.ThreadPool; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class ThreadPoolExecutionDispatcherTest { + + @Mock private ThreadPool threadPool; + @Mock private Settings settings; + @Mock private CalcitePlanContext context; + @Mock private ResponseListener listener; + @Mock private ExecutionEngine engine; + + private ThreadPoolExecutionDispatcher dispatcher; + + @BeforeEach + void setUp() { + dispatcher = new ThreadPoolExecutionDispatcher(threadPool, settings); + // Mock schedule calls to return non-null cancellables (for both outer dispatch and inner + // timeout) + when(threadPool.schedule(any(Runnable.class), any(TimeValue.class), any())) + .thenReturn(mock(ScheduledCancellable.class)); + when(threadPool.scheduleWithFixedDelay(any(Runnable.class), any(TimeValue.class), any())) + .thenReturn(mock(Cancellable.class)); + } + + @AfterEach + void tearDown() { + ThreadContext.clearAll(); + OpenSearchQueryManager.clearCancellableTask(); + RelMetadataQueryBase.THREAD_PROVIDERS.remove(); + CalcitePlanContext.clearTimewrapSignals(); + } + + @Test + void executesInlineWhenNoScripts() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + RelNode plan = createMockNode(); + + dispatcher.dispatch(plan, context, listener, engine); + + verify(engine).execute(plan, context, listener); + verify(threadPool, never()).schedule(any(), any(TimeValue.class), any()); + } + + @Test + void dispatchesToSlowPoolWhenScriptsDetected() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + + dispatcher.dispatch(scan, context, listener, engine); + + verify(threadPool) + .schedule( + any(Runnable.class), eq(new TimeValue(0)), eq(SQL_COMPLEX_WORKER_THREAD_POOL_NAME)); + verify(engine, never()).execute(any(RelNode.class), any(), any(ResponseListener.class)); + } + + @Test + void executesInlineWhenSlowPoolDisabled() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(false); + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + + dispatcher.dispatch(scan, context, listener, engine); + + verify(engine).execute(scan, context, listener); + verify(threadPool, never()).schedule(any(), any(TimeValue.class), any()); + } + + @Test + void scheduledRunnableCallsEngine() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + verify(engine).execute(scan, context, listener); + } + + @Test + void propagatesCancellableTaskToSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + CancellableTask mockTask = mock(CancellableTask.class); + OpenSearchQueryManager.setCancellableTask(mockTask); + + AtomicReference taskOnSlowPool = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + // Simulate running on a different thread — clear the ThreadLocal first + OpenSearchQueryManager.clearCancellableTask(); + task.run(); + taskOnSlowPool.set(OpenSearchQueryManager.getCancellableTask()); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + // During execution, the task should have been available + // (we check via a side-channel since the finally block clears it) + verify(engine).execute(scan, context, listener); + // After execution, it should be cleaned up + assertNull( + OpenSearchQueryManager.getCancellableTask(), + "CancellableTask should be cleared after execution"); + } + + @Test + void propagatesLog4jThreadContextToSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + ThreadContext.put("request.id", "test-123"); + ThreadContext.put("user", "admin"); + + AtomicReference requestIdOnSlowPool = new AtomicReference<>(); + AtomicReference userOnSlowPool = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + // Simulate a different thread — clear MDC + ThreadContext.clearAll(); + task.run(); + requestIdOnSlowPool.set(ThreadContext.get("request.id")); + userOnSlowPool.set(ThreadContext.get("user")); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + assertEquals("test-123", requestIdOnSlowPool.get()); + assertEquals("admin", userOnSlowPool.get()); + } + + @Test + void propagatesMetadataProviderToSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + JaninoRelMetadataProvider provider = mock(JaninoRelMetadataProvider.class); + RelMetadataQueryBase.THREAD_PROVIDERS.set(provider); + + AtomicReference providerOnSlowPool = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + RelMetadataQueryBase.THREAD_PROVIDERS.remove(); + task.run(); + providerOnSlowPool.set(RelMetadataQueryBase.THREAD_PROVIDERS.get()); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + // After finally block, metadata provider should be cleaned up + assertNull( + RelMetadataQueryBase.THREAD_PROVIDERS.get(), + "Metadata provider should be cleaned up after execution"); + } + + @Test + void propagatesTimewrapSignalsToSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + CalcitePlanContext.stripNullColumns.set(true); + CalcitePlanContext.timewrapUnitName.set("HOUR"); + CalcitePlanContext.timewrapSeries.set("timestamp"); + + AtomicReference stripOnSlowPool = new AtomicReference<>(); + AtomicReference unitOnSlowPool = new AtomicReference<>(); + AtomicReference seriesOnSlowPool = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + // Clear thread-locals to simulate new thread + CalcitePlanContext.clearTimewrapSignals(); + CalcitePlanContext.stripNullColumns.set(false); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + // Capture the values during engine.execute + doAnswer( + invocation -> { + stripOnSlowPool.set(CalcitePlanContext.stripNullColumns.get()); + unitOnSlowPool.set(CalcitePlanContext.timewrapUnitName.get()); + seriesOnSlowPool.set(CalcitePlanContext.timewrapSeries.get()); + return null; + }) + .when(engine) + .execute(any(RelNode.class), any(), any(ResponseListener.class)); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + assertEquals(true, stripOnSlowPool.get()); + assertEquals("HOUR", unitOnSlowPool.get()); + assertEquals("timestamp", seriesOnSlowPool.get()); + } + + @Test + void forwardsExceptionToListenerOnSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + RuntimeException error = new RuntimeException("execution failed"); + doThrow(error).when(engine).execute(any(RelNode.class), any(), any(ResponseListener.class)); + + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + verify(listener).onFailure(error); + } + + @Test + void cleansUpThreadLocalsAfterException() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + CancellableTask mockTask = mock(CancellableTask.class); + OpenSearchQueryManager.setCancellableTask(mockTask); + CalcitePlanContext.timewrapUnitName.set("DAY"); + RelMetadataQueryBase.THREAD_PROVIDERS.set(mock(JaninoRelMetadataProvider.class)); + + doThrow(new RuntimeException("boom")) + .when(engine) + .execute(any(RelNode.class), any(), any(ResponseListener.class)); + + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + assertNull(OpenSearchQueryManager.getCancellableTask()); + assertNull(RelMetadataQueryBase.THREAD_PROVIDERS.get()); + // timewrapSignals cleared via clearTimewrapSignals() + assertNull(CalcitePlanContext.timewrapUnitName.get()); + } + + @Test + void cancellableTaskAvailableDuringExecution() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + CancellableTask mockTask = mock(CancellableTask.class); + OpenSearchQueryManager.setCancellableTask(mockTask); + + AtomicReference taskDuringExecution = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + // Simulate running on a different thread + OpenSearchQueryManager.clearCancellableTask(); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + doAnswer( + invocation -> { + taskDuringExecution.set(OpenSearchQueryManager.getCancellableTask()); + return null; + }) + .when(engine) + .execute(any(RelNode.class), any(), any(ResponseListener.class)); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + assertNotNull( + taskDuringExecution.get(), "CancellableTask should be available during execution"); + assertEquals(mockTask, taskDuringExecution.get()); + } + + private static RelNode createMockNode(RelNode... children) { + RelNode node = mock(RelNode.class); + List childList = List.of(children); + when(node.getInputs()).thenReturn(childList); + doAnswer( + invocation -> { + RelVisitor visitor = invocation.getArgument(0); + for (int i = 0; i < childList.size(); i++) { + visitor.visit(childList.get(i), i, node); + } + return null; + }) + .when(node) + .childrenAccept(any(RelVisitor.class)); + return node; + } + + private static AbstractCalciteIndexScan createMockScanWithScripts() { + AbstractCalciteIndexScan scan = mock(AbstractCalciteIndexScan.class); + PushDownContext ctx = mock(PushDownContext.class); + when(ctx.isScriptPushed()).thenReturn(true); + when(ctx.isSortExprPushed()).thenReturn(false); + when(ctx.getAggSpec()).thenReturn(null); + when(scan.getPushDownContext()).thenReturn(ctx); + when(scan.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(scan).childrenAccept(any(RelVisitor.class)); + return scan; + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index 4779332abac..521ac7f109f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -41,12 +40,14 @@ import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import org.opensearch.sql.opensearch.request.AggregateAnalyzer.ExpressionNotAnalyzableException; import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CheckedLongSumParser; import org.opensearch.sql.opensearch.response.agg.FilterParser; import org.opensearch.sql.opensearch.response.agg.MetricParserHelper; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -176,6 +177,64 @@ void analyze_aggCall_simple() throws ExpressionNotAnalyzableException { }); } + @Test + void analyze_checkedLongSum() throws ExpressionNotAnalyzableException { + AggregateCall checkedLongSumCall = + AggregateCall.create( + PPLBuiltinOperators.CHECKED_LONG_SUM, + false, + false, + false, + ImmutableList.of(), + ImmutableList.of(0), + -1, + null, + RelCollations.EMPTY, + typeFactory.createSqlType(SqlTypeName.BIGINT), + "checked_sum"); + Aggregate aggregate = createMockAggregate(List.of(checkedLongSumCall), ImmutableBitSet.of()); + Project project = createMockProject(List.of(0)); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); + + Pair, OpenSearchAggregationResponseParser> result = + AggregateAnalyzer.analyze(aggregate, project, List.of("checked_sum"), helper); + + SumAggregationBuilder builder = + assertInstanceOf(SumAggregationBuilder.class, result.getLeft().getFirst()); + assertEquals("a", builder.field()); + NoBucketAggregationParser parser = + assertInstanceOf(NoBucketAggregationParser.class, result.getRight()); + assertInstanceOf( + CheckedLongSumParser.class, + parser.getMetricsParser().getMetricParserMap().get("checked_sum")); + } + + @Test + void analyze_checkedLongSumExpressionUsesNativeScriptedSum() + throws ExpressionNotAnalyzableException { + buildAggregation("checked_sum") + .withAggCall( + b -> + b.aggregateCall( + PPLBuiltinOperators.CHECKED_LONG_SUM, + b.call(SqlStdOperatorTable.PLUS, b.field("a"), b.literal(1))) + .as("checked_sum")) + .expectDslTemplate("[{\"checked_sum\":{\"sum\":{\"script\":*}}}]") + .expectResponseParser( + new MetricParserHelper(List.of(new CheckedLongSumParser("checked_sum")))) + .verify(); + } + + @Test + void analyze_bigintAvgUsesNativeField() throws ExpressionNotAnalyzableException { + buildAggregation("avg") + .withAggCall(b -> b.aggregateCall(PPLBuiltinOperators.BIGINT_AVG, b.field("a")).as("avg")) + .expectDslQuery("[{\"avg\":{\"avg\":{\"field\":\"a\"}}}]") + .expectResponseParser(new MetricParserHelper(List.of(new SingleValueParser("avg")))) + .verify(); + } + @Test void analyze_aggCall_extended() throws ExpressionNotAnalyzableException { AggregateCall varSampCall = @@ -301,33 +360,81 @@ void analyze_groupBy() throws ExpressionNotAnalyzableException { } @Test - void analyze_aggCall_TextWithoutKeyword() { + void analyze_aggCall_TextWithoutKeyword_countPushesDownAsScript() + throws ExpressionNotAnalyzableException { + // count(FIELD) on a text field with no .keyword sub-field must not fall back to an + // unbounded client-side scan. The metric aggregation is built with a script value source + // that reads the value from _source, matching the pattern used by TermQuery/LikeQuery. + Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); + + SchemaPlus root = Frameworks.createRootSchema(true); + root.add( + "test", + new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory tf) { + return rowType; + } + }); + RelBuilder rb = + RelBuilder.create(Frameworks.newConfigBuilder().defaultSchema(root).build()).scan("test"); + AggregateCall aggCall = AggregateCall.create( - SqlStdOperatorTable.SUM, + SqlStdOperatorTable.COUNT, false, false, false, ImmutableList.of(), - ImmutableList.of(0), + // arg #2 in the row type is `c` — text without .keyword sub-field + ImmutableList.of(2), -1, null, RelCollations.EMPTY, - typeFactory.createSqlType(SqlTypeName.INTEGER), - "sum"); - Aggregate aggregate = createMockAggregate(List.of(aggCall), ImmutableBitSet.of()); - Project project = createMockProject(List.of(2)); + typeFactory.createSqlType(SqlTypeName.BIGINT), + "cnt"); + + RelNode rel = rb.aggregate(rb.groupKey(), List.of(aggCall)).build(); + Aggregate aggregate = (Aggregate) rel; + AggregateAnalyzer.AggregateBuilderHelper helper = - new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); - ExpressionNotAnalyzableException exception = - assertThrows( - ExpressionNotAnalyzableException.class, - () -> AggregateAnalyzer.analyze(aggregate, project, List.of("sum"), helper)); - assertEquals("[field] must not be null: [sum]", exception.getCause().getMessage()); + new AggregateAnalyzer.AggregateBuilderHelper( + rowType, fieldTypes, aggregate.getCluster(), true, BUCKET_SIZE); + Pair, OpenSearchAggregationResponseParser> result = + AggregateAnalyzer.analyze(aggregate, null, List.of("cnt"), helper); + + String dsl = result.getLeft().toString(); + // The value_count metric must use a script value source (not a raw field) on `c`. + assertTrue( + dsl.contains("\"cnt\":{\"value_count\":{\"script\":{"), + "expected value_count metric on 'c' to use a script, but got: " + dsl); + assertTrue( + dsl.contains("\"lang\":\"opensearch_compounded_script\""), + "expected compounded script lang, but got: " + dsl); + assertTrue( + dsl.contains("\"DIGESTS\":[\"c\"]"), + "expected script to reference field 'c' via DIGESTS, but got: " + dsl); } @Test - void analyze_groupBy_TextWithoutKeyword() { + void analyze_groupBy_TextWithoutKeyword() throws ExpressionNotAnalyzableException { + // Grouping by a text field with no .keyword sub-field must not fall back to an unbounded + // client-side scan. Instead, the composite terms bucket is built with a script value source + // that reads the field from _source, matching the pattern used by TermQuery/LikeQuery. + Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); + + SchemaPlus root = Frameworks.createRootSchema(true); + root.add( + "test", + new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory tf) { + return rowType; + } + }); + RelBuilder rb = + RelBuilder.create(Frameworks.newConfigBuilder().defaultSchema(root).build()).scan("test"); + AggregateCall aggCall = AggregateCall.create( SqlStdOperatorTable.COUNT, @@ -341,16 +448,32 @@ void analyze_groupBy_TextWithoutKeyword() { RelCollations.EMPTY, typeFactory.createSqlType(SqlTypeName.INTEGER), "cnt"); - List outputFields = List.of("c", "cnt"); - Aggregate aggregate = createMockAggregate(List.of(aggCall), ImmutableBitSet.of(0)); - Project project = createMockProject(List.of(2)); + + RelNode rel = rb.aggregate(rb.groupKey(ImmutableBitSet.of(2)), List.of(aggCall)).build(); + Aggregate aggregate = (Aggregate) rel; + Project project = null; + AggregateAnalyzer.AggregateBuilderHelper helper = - new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); - ExpressionNotAnalyzableException exception = - assertThrows( - ExpressionNotAnalyzableException.class, - () -> AggregateAnalyzer.analyze(aggregate, project, outputFields, helper)); - assertEquals("[field] must not be null", exception.getCause().getMessage()); + new AggregateAnalyzer.AggregateBuilderHelper( + rowType, fieldTypes, aggregate.getCluster(), true, BUCKET_SIZE); + Pair, OpenSearchAggregationResponseParser> result = + AggregateAnalyzer.analyze(aggregate, project, List.of("c", "cnt"), helper); + + String dsl = result.getLeft().toString(); + // Composite bucket for `c` must use a script value source (not a raw field). + assertTrue( + dsl.contains( + "\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[" + + "{\"c\":{\"terms\":{\"script\":{"), + "expected composite terms bucket on 'c' to use a script, but got: " + dsl); + // The script must be tagged as a Calcite compounded script and reference the `c` field. + assertTrue( + dsl.contains("\"lang\":\"opensearch_compounded_script\""), + "expected compounded script lang, but got: " + dsl); + assertTrue( + dsl.contains("\"DIGESTS\":[\"c\"]"), + "expected script to reference field 'c' via DIGESTS, but got: " + dsl); + assertInstanceOf(BucketAggregationParser.class, result.getRight()); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java new file mode 100644 index 00000000000..8a5c940ada6 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.search.aggregations.metrics.NumericMetricsAggregation; + +class CheckedLongSumParserTest { + + private final CheckedLongSumParser parser = new CheckedLongSumParser("sum"); + + @Test + void narrowsNativeSumToLong() { + assertEquals(42L, value(parser.parse(aggregation(42d)))); + } + + @Test + void preservesNativeDoublePrecisionBehavior() { + double rounded = (double) ((1L << 62) + 1L); + assertEquals(1L << 62, value(parser.parse(aggregation(rounded)))); + } + + @Test + void saturatesAmbiguousPositiveBoundary() { + assertEquals(Long.MAX_VALUE, value(parser.parse(aggregation((double) Long.MAX_VALUE)))); + } + + @Test + void rejectsValueClearlyOutsideLongRange() { + assertThrows(ArithmeticException.class, () -> parser.parse(aggregation(Math.nextUp(0x1p63)))); + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Math.nextDown(-0x1p63)))); + } + + @Test + void rejectsInfiniteValue() { + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Double.POSITIVE_INFINITY))); + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Double.NEGATIVE_INFINITY))); + } + + @Test + void convertsNanToNull() { + assertNull(value(parser.parse(aggregation(Double.NaN)))); + } + + private static NumericMetricsAggregation.SingleValue aggregation(double value) { + NumericMetricsAggregation.SingleValue aggregation = + mock(NumericMetricsAggregation.SingleValue.class); + when(aggregation.getName()).thenReturn("sum"); + when(aggregation.value()).thenReturn(value); + return aggregation; + } + + private static Object value(List> rows) { + return rows.getFirst().get("sum"); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 0c570098924..4d77df2c992 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -17,7 +17,6 @@ import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; -import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -79,10 +78,8 @@ void pluginNonDynamicSettings() { @Test void restSettingsAreNonDynamic() { - assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); - assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index 102ec4da8f7..e883ee7b548 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -16,6 +16,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -24,6 +25,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.storage.rest.CoreEndpointsProvider; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.SystemIndexUtils; @@ -35,6 +39,13 @@ class OpenSearchStorageEngineTest { @Mock private Settings settings; + @BeforeEach + void publishRestRegistry() { + // restTable() reads the merged registry from the holder (published by SQLPlugin in production); + // publish a built-in-only registry here so the rest endpoints resolve in this unit test. + RestEndpointRegistryHolder.set(new RestEndpointRegistry(List.of(new CoreEndpointsProvider()))); + } + @Test public void getTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); @@ -62,28 +73,30 @@ public void getSystemTable() { } @Test - public void getRestTableAllowedByWildcard() { + public void wildcardNoLongerEnablesEndpoints() { when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) .thenReturn(List.of("*")); - when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); String name = SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); - Table table = - engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); - assertTrue(table instanceof OpenSearchCatalogTable); + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("is not enabled on this cluster")); } @Test public void getRestTableAllowedBySubset() { when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("/_cat/nodes")); - when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + .thenReturn(List.of("/_cluster/health")); OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); String name = SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); assertTrue( engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) instanceof OpenSearchCatalogTable); @@ -91,12 +104,13 @@ public void getRestTableAllowedBySubset() { @Test public void getRestTableRejectedWhenEndpointNotInSubset() { + // The endpoint resolves in the registry but is absent from the (non-empty) enabled subset. when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("/_cat/nodes")); + .thenReturn(List.of("/_some/other")); OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); String name = SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); IllegalArgumentException e = assertThrows( IllegalArgumentException.class, @@ -112,7 +126,7 @@ public void getRestTableDisabledWhenListEmpty() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); String name = SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); IllegalArgumentException e = assertThrows( IllegalArgumentException.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java new file mode 100644 index 00000000000..2fad35408b1 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Proves the built-in {@code /_cluster/health} provider fetches through the transport node client + * the context carries (the same seam an external provider uses) and returns the response as a + * single JSON {@code response} column, holding no reference to the sql storage client. + */ +class CoreEndpointsProviderTest { + + @Test + void clusterHealthFetchesViaContextNodeClient() throws IOException { + ClusterHealthResponse response = mock(ClusterHealthResponse.class); + when(response.toXContent(any(XContentBuilder.class), any())) + .thenAnswer( + invocation -> { + XContentBuilder builder = invocation.getArgument(0); + return builder + .startObject() + .field("status", "green") + .field("number_of_nodes", 3) + .endObject(); + }); + + NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); + when(nodeClient.admin().cluster().health(any(ClusterHealthRequest.class)).actionGet()) + .thenReturn(response); + + RestEndpointRegistry registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); + List rows = + registry.resolve("/_cluster/health").toRows(RestEndpointContext.of(Map.of(), nodeClient)); + + assertEquals(1, rows.size()); + String json = rows.get(0).tupleValue().get("response").stringValue(); + assertTrue(json.contains("\"status\":\"green\"")); + assertTrue(json.contains("\"number_of_nodes\":3")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java index 8676373d2a6..e3de1b19a74 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java @@ -10,14 +10,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -25,38 +24,59 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** - * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, - * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) - * path. + * Covers the {@code rest} {@link RestCatalogSource} against the PR1 endpoint set (only {@code + * /_cluster/health}): fixed endpoint schema, allow-list enforcement, response row shaping and + * truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) path. Schema and gating + * resolve against a registry built from the built-in {@link CoreEndpointsProvider}; row shaping and + * truncation use a fake provider returning canned rows, independent of the health transport fetch. */ @ExtendWith(MockitoExtension.class) class RestCatalogSourceTest { @Mock private OpenSearchClient client; + private RestEndpointRegistry registry; + + @BeforeEach + void buildRegistry() { + registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); + } + private RestSpec healthSpec() { return new RestSpec("/_cluster/health", Map.of(), null, null); } + private static RestEndpointRegistry fakeHealthRegistry(List responses) { + RestEndpointProvider provider = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .handler(ctx -> responses) + .build()); + return new RestEndpointRegistry(List.of(provider)); + } + @Test void getFieldTypesReturnsFixedEndpointSchema() { - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + RestCatalogSource source = new RestCatalogSource(registry, healthSpec(), client); Map fieldTypes = source.getFieldTypes(); - assertThat(fieldTypes, hasEntry("status", STRING)); - assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); + assertThat(fieldTypes, hasEntry("response", STRING)); } @Test void isScannable() { - assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); + assertTrue(new RestCatalogSource(registry, healthSpec(), client).isScannable()); } @Test void implementV2IsUnsupported() { - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + RestCatalogSource source = new RestCatalogSource(registry, healthSpec(), client); assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); } @@ -65,7 +85,8 @@ void constructorRejectsNonAllowListedEndpoint() { assertThrows( IllegalArgumentException.class, () -> - new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + new RestCatalogSource( + registry, new RestSpec("/_cluster/reroute", Map.of(), null, null), client)); } @Test @@ -74,14 +95,18 @@ void constructorRejectsDisallowedArg() { IllegalArgumentException.class, () -> new RestCatalogSource( - client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + registry, + new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null), + client)); } @Test void constructorRejectsNegativeCount() { assertThrows( IllegalArgumentException.class, - () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); + () -> + new RestCatalogSource( + registry, new RestSpec("/_cluster/health", Map.of(), -1, null), client)); } @Test @@ -89,35 +114,30 @@ void constructorRejectsTimeoutArg() { assertThrows( IllegalArgumentException.class, () -> - new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + new RestCatalogSource( + registry, new RestSpec("/_cluster/health", Map.of(), null, "5s"), client)); } @Test void restRequestShapesResponseRows() { - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", 1); - when(client.clusterHealth(any())).thenReturn(health); - - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + when(client.getNodeClient()).thenReturn(Optional.empty()); + String response = "{\"status\":\"green\",\"number_of_nodes\":1}"; + RestCatalogSource source = + new RestCatalogSource(fakeHealthRegistry(List.of(response)), healthSpec(), client); List rows = source.createRequest().search(); assertEquals(1, rows.size()); - assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); - assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + assertEquals(response, rows.get(0).tupleValue().get("response").stringValue()); } @Test void countTruncatesRows() { - Map idx1 = new LinkedHashMap<>(); - idx1.put("index", "a"); - Map idx2 = new LinkedHashMap<>(); - idx2.put("index", "b"); - when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); - + // count=0 exercises the truncation path (subList to empty) over a single-row response. + when(client.getNodeClient()).thenReturn(Optional.empty()); RestCatalogSource source = - new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); - List rows = source.createRequest().search(); - assertEquals(1, rows.size()); - assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); + new RestCatalogSource( + fakeHealthRegistry(List.of("{\"status\":\"green\"}")), + new RestSpec("/_cluster/health", Map.of(), 0, null), + client); + assertTrue(source.createRequest().search().isEmpty()); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java new file mode 100644 index 00000000000..2a950c76f52 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Proves the {@code rest} framework treats the built-in {@link CoreEndpointsProvider} and an + * externally contributed {@link RestEndpointProvider} as uniform clients of one registry: endpoints + * from BOTH resolve, validate against the same allow-list, and fetch rows the same way. The + * built-in provider holds no privileged position. + */ +class RestEndpointExtensibilityTest { + + /** A stand-in external plugin provider: contributes one endpoint that echoes a query arg. */ + private static final class FakeEchoProvider implements RestEndpointProvider { + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_plugin/echo") + .argSpec(ArgSpec.builder().arg("text").build()) + .handler(ctx -> List.of(ctx.args().getOrDefault("text", "default"))) + .build()); + } + } + + private RestEndpointRegistry mergedRegistry() { + return new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), new FakeEchoProvider())); + } + + @Test + void bothBuiltInAndExternalEndpointsResolve() { + RestEndpointRegistry registry = mergedRegistry(); + + assertEquals("/_cluster/health", registry.resolve("/_cluster/health").getPath()); + assertEquals("/_plugin/echo", registry.resolve("/_plugin/echo").getPath()); + } + + @Test + void externalEndpointFetchesThroughTheSamePath() { + RestEndpointRegistry.Endpoint echo = mergedRegistry().resolve("/_plugin/echo"); + List rows = echo.toRows(RestEndpointContext.of(Map.of("text", "hi"), null)); + assertEquals(1, rows.size()); + assertEquals("hi", rows.get(0).tupleValue().get("response").stringValue()); + } + + @Test + void externalEndpointArgsValidatedByTheSameAllowList() { + RestEndpointRegistry registry = mergedRegistry(); + // Declared arg is accepted. + registry.validate(new RestSpec("/_plugin/echo", Map.of("text", "x"), null, null)); + // Undeclared arg is rejected by the same validation path that guards built-in endpoints. + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> + registry.validate( + new RestSpec("/_plugin/echo", Map.of("not_allowed", "x"), null, null))); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void duplicateOfBuiltInNameIsDroppedSoBuiltInWins() { + // An external provider that re-declares a built-in name (/_cluster/health) is dropped rather + // than failing the build; a built-in name cannot be shadowed, so the built-in wins. + RestEndpointProvider shadowsCore = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .handler(ctx -> List.of()) + .build()); + RestEndpointRegistry registry = + new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), shadowsCore)); + + assertEquals("/_cluster/health", registry.resolve("/_cluster/health").getPath()); + assertTrue(registry.resolve("/_cluster/health").isBuiltIn()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java index c5a978bb495..c89c8160b16 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -8,131 +8,121 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import java.util.LinkedHashMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; -@ExtendWith(MockitoExtension.class) +/** + * Covers the {@code rest} {@link RestEndpointRegistry} against the PR1 endpoint set (only {@code + * /_cluster/health}): allow-list resolution, arg validation, count/timeout gating, and single + * response-column row shaping. Shaping is exercised through a fake provider that returns canned + * response strings, so it stays independent of how any one endpoint fetches. A provider that masks + * sensitive values does so inside its own handler, so there is no framework redaction step here. + */ class RestEndpointRegistryTest { - @Mock private OpenSearchClient client; + private RestEndpointRegistry registry; - @Test - void resolveAllowListedEndpoint() { - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - assertEquals("/_cluster/health", endpoint.getPath()); - assertEquals(STRING, endpoint.getSchema().get("status")); - assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); + @BeforeEach + void buildRegistry() { + registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); } - @Test - void resolveRejectsNonAllowListedEndpoint() { - // A mutating endpoint is simply absent from the registry and is refused here. - assertThrows( - IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); - assertThrows( - IllegalArgumentException.class, - () -> RestEndpointRegistry.resolve("/services/server/info")); + private static RestEndpointContext ctx(Map args) { + return RestEndpointContext.of(args, null); } - @Test - void validateRejectsUnknownArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + private static RestEndpointRegistry.Endpoint fakeEndpoint(List responses) { + RestEndpointProvider provider = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_test/probe") + .handler(context -> responses) + .build()); + return new RestEndpointRegistry(List.of(provider)).resolve("/_test/probe"); } @Test - void validateAcceptsAllowedArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); - RestEndpointRegistry.validate(spec); // no throw + void resolveAllowListedEndpoint() { + RestEndpointRegistry.Endpoint endpoint = registry.resolve("/_cluster/health"); + assertEquals("/_cluster/health", endpoint.getPath()); + assertEquals(STRING, endpoint.getSchema().get("response")); + assertEquals(1, endpoint.getSchema().size()); } @Test - void catEndpointsRedactAddressesWhenRedactionEnabled() { - Map node = new LinkedHashMap<>(); - node.put("name", "ip-10-0-0-7"); - node.put("ip", "10.0.0.7"); - node.put("node_role", "dir"); - node.put("heap_percent", 44); - node.put("ram_percent", 95); - node.put("cpu", 2); - when(client.catNodes(any())).thenReturn(List.of(node)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); - RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); - - Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("x.x.x.x", redacted.get("ip").stringValue()); - assertEquals("", redacted.get("name").stringValue()); - assertEquals(44, redacted.get("heap_percent").integerValue()); - - Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); - assertEquals("10.0.0.7", plain.get("ip").stringValue()); - assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); + void twoExternalProvidersSameName_disableTheName() { + RestEndpointProvider a = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_plugin/dup") + .handler(c -> List.of("{\"a\":\"x\"}")) + .build()); + RestEndpointProvider b = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_plugin/dup") + .handler(c -> List.of("{\"b\":\"y\"}")) + .build()); + RestEndpointRegistry reg = new RestEndpointRegistry(List.of(a, b)); + assertThrows(IllegalArgumentException.class, () -> reg.resolve("/_plugin/dup")); } @Test - void catClusterManagerRedactsHostAndIp() { - Map row = new LinkedHashMap<>(); - row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); - row.put("host", "10.0.0.7"); - row.put("ip", "10.0.0.7"); - row.put("node", "71d03b567bb755839a73d437b2b066d4"); - when(client.catClusterManager(any())).thenReturn(List.of(row)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); - RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); - - Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("x.x.x.x", redacted.get("host").stringValue()); - assertEquals("x.x.x.x", redacted.get("ip").stringValue()); - assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); - assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); + void externalProviderCannotShadowBuiltIn() { + RestEndpointProvider shadow = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .handler(c -> List.of("{\"hijacked\":\"yes\"}")) + .build()); + RestEndpointRegistry reg = + new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), shadow)); + RestEndpointRegistry.Endpoint health = reg.resolve("/_cluster/health"); + assertTrue(health.isBuiltIn()); + assertEquals(STRING, health.getSchema().get("response")); } @Test - void nonCatEndpointNotRedactedEvenWhenEnabled() { - Map health = new LinkedHashMap<>(); - health.put("cluster_name", "10.0.0.7"); - health.put("status", "green"); - health.put("number_of_nodes", 3); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); - - Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); + void resolveRejectsNonAllowListedEndpoint() { + // A mutating endpoint, and any endpoint deferred out of PR1, is simply absent and refused here. + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/_cluster/reroute")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/_cat/nodes")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/services/server/info")); } @Test - void clusterSettingsMasksAvailabilityZoneInValue() { - Map setting = new LinkedHashMap<>(); - setting.put("setting", "cluster.routing.allocation.awareness.attributes"); - setting.put("value", "zone:us-east-1a"); - setting.put("tier", "persistent"); - when(client.clusterSettings(any())).thenReturn(List.of(setting)); + void resolveRejectsBlankEndpoint() { + IllegalArgumentException emptyEx = + assertThrows(IllegalArgumentException.class, () -> registry.resolve("")); + assertTrue(emptyEx.getMessage().contains("non-empty path")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve(" ")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve(null)); + } - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); - RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); + @Test + void validateRejectsUnknownArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); + } - Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); - assertEquals( - "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); - assertEquals("persistent", row.get("tier").stringValue()); + @Test + void validateAcceptsAllowedArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); + registry.validate(spec); // no throw } @Test @@ -140,151 +130,59 @@ void validateRejectsDroppedLevelArg() { // level was dropped (no-op against the fixed cluster-level health schema); now unknown. RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); assertTrue(ex.getMessage().contains("does not accept arg")); } - @Test - void validateRejectsDroppedFlatSettingsArg() { - // flat_settings was dropped (redundant: settings are already flattened to dotted keys). - RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - } - - @Test - void validateAcceptsValidArgValues() { - RestEndpointRegistry.validate( - new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); - RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); - RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); - } - @Test void validateRejectsBadArgValue() { - IllegalArgumentException health = - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); - assertTrue(health.getMessage().contains("unsupported value")); - IllegalArgumentException local = assertThrows( IllegalArgumentException.class, () -> - RestEndpointRegistry.validate( + registry.validate( new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); assertTrue(local.getMessage().contains("unsupported value")); - - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec( - "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); - } - - @Test - void resolveRejectsBlankEndpoint() { - IllegalArgumentException emptyEx = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); - assertTrue(emptyEx.getMessage().contains("non-empty path")); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); } @Test void validateRejectsNegativeCount() { - RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), -1, null); IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); assertTrue(ex.getMessage().contains("non-negative")); } @Test void validateAcceptsZeroCount() { - RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); - RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), 0, null); + registry.validate(spec); // no throw: 0 is a valid limit } @Test void validateRejectsTimeoutArg() { RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); assertTrue(ex.getMessage().contains("timeout")); } @Test - void coerceParsesNumericStringValues() { - // The cat JSON API returns numeric columns as strings; coerce must parse them. - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", "3"); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - List rows = - endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); - - assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - } - - @Test - void coerceThrowsClearErrorOnUncoercibleValue() { - // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), - // not a raw ClassCastException / NumberFormatException (HTTP 500). - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", "not-a-number"); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - IllegalArgumentException ex = - assertThrows( - IllegalArgumentException.class, - () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); - assertTrue(ex.getMessage().contains("number_of_nodes")); - assertTrue(ex.getMessage().contains("not-a-number")); - } - - @Test - void clusterHealthRowsAreShapedToFixedSchema() { - Map health = new LinkedHashMap<>(); - health.put("cluster_name", "test-cluster"); - health.put("status", "green"); - health.put("number_of_nodes", 1); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - List rows = - endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); - + void rowsAreWrappedIntoResponseColumn() { + String response = "{\"status\":\"green\",\"number_of_nodes\":1}"; + RestEndpointRegistry.Endpoint endpoint = fakeEndpoint(List.of(response)); + List rows = endpoint.toRows(ctx(Map.of())); assertEquals(1, rows.size()); - assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); - assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - // a declared column the action did not return becomes null, never absent. - assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); + assertEquals(STRING, endpoint.getSchema().get("response")); + assertEquals(response, rows.get(0).tupleValue().get("response").stringValue()); } @Test - void catIndicesRowsAreShapedToFixedSchema() { - Map idx = new LinkedHashMap<>(); - idx.put("index", "books"); - idx.put("health", "yellow"); - idx.put("pri", 1); - idx.put("rep", 1); - when(client.catIndices(any())).thenReturn(List.of(idx)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); - List rows = - endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); - + void nullResponseBecomesNull() { + List withNull = new ArrayList<>(); + withNull.add(null); + List rows = fakeEndpoint(withNull).toRows(ctx(Map.of())); assertEquals(1, rows.size()); - assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); - assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); + assertTrue(rows.get(0).tupleValue().get("response").isNull()); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java deleted file mode 100644 index 3b038306da5..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -class RestResponseRedactorTest { - - @Test - void masksIpv4() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); - assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); - assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); - } - - @Test - void doesNotMaskInvalidOrPartialIpv4() { - assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); - assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); - assertEquals("44", RestResponseRedactor.redact("44")); - } - - @Test - void masksEc2HostName() { - assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); - assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); - assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); - assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); - } - - @Test - void masksFullIpv6() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); - } - - @Test - void masksCompressedIpv6() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); - } - - @Test - void masksInetAddress() { - assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); - } - - @Test - void masksAvailabilityZones() { - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); - // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); - assertEquals( - "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); - } - - @Test - void maskAvailabilityZoneMasksOnlyZones() { - assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); - assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); - assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); - } - - @Test - void leavesNonAddressesIntact() { - assertEquals( - "e4e136ea81e27370ff73cf753ba22d39", - RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); - assertEquals( - "data,ingest,remote_cluster_client", - RestResponseRedactor.redact("data,ingest,remote_cluster_client")); - assertEquals( - "x.x.x.x 44 95 imr - e4e136ea", - RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); - } - - @Test - void handlesNullAndEmpty() { - assertEquals(null, RestResponseRedactor.redact(null)); - assertEquals("", RestResponseRedactor.redact("")); - assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); - assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java index 24ee9b12907..50fb6a5ea8f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java @@ -47,4 +47,22 @@ public void test_escape_sql_wildcards_safe() { assertEquals("foo\\*bar", StringUtils.convertSqlWildcardToLuceneSafe("foo*bar")); assertEquals("foo\\?bar", StringUtils.convertSqlWildcardToLuceneSafe("foo?bar")); } + + @Test + public void test_escaping_backslash_itself() { + assertEquals("\\", StringUtils.convertSqlWildcardToLucene("\\\\")); + assertEquals("\\", StringUtils.convertSqlWildcardToLuceneSafe("\\\\")); + assertEquals("*\\*", StringUtils.convertSqlWildcardToLucene("%\\\\%")); + assertEquals("*\\*", StringUtils.convertSqlWildcardToLuceneSafe("%\\\\%")); + assertEquals("\\*", StringUtils.convertSqlWildcardToLucene("\\\\%")); + assertEquals("\\*", StringUtils.convertSqlWildcardToLuceneSafe("\\\\%")); + assertEquals("*\\", StringUtils.convertSqlWildcardToLucene("%\\\\")); + assertEquals("*\\", StringUtils.convertSqlWildcardToLuceneSafe("%\\\\")); + assertEquals("\\\\", StringUtils.convertSqlWildcardToLucene("\\\\\\\\")); + assertEquals("\\\\", StringUtils.convertSqlWildcardToLuceneSafe("\\\\\\\\")); + assertEquals("\\\\*", StringUtils.convertSqlWildcardToLucene("\\\\\\\\%")); + assertEquals("\\\\*", StringUtils.convertSqlWildcardToLuceneSafe("\\\\\\\\%")); + assertEquals("\\%", StringUtils.convertSqlWildcardToLucene("\\\\\\%")); + assertEquals("\\%", StringUtils.convertSqlWildcardToLuceneSafe("\\\\\\%")); + } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index d9437fded5b..31f14e3411b 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.datasource.model.DataSourceMetadata.defaultOpenSearchDataSourceMetadata; import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_BACKGROUND_THREAD_POOL_NAME; +import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_COMPLEX_WORKER_THREAD_POOL_NAME; import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_WORKER_THREAD_POOL_NAME; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_REQUEST_BUFFER_INDEX_NAME; @@ -104,6 +105,9 @@ import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.opensearch.storage.OpenSearchDataSourceFactory; +import org.opensearch.sql.opensearch.storage.rest.CoreEndpointsProvider; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine; import org.opensearch.sql.plugin.config.EngineExtensionsHolder; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; @@ -135,6 +139,7 @@ import org.opensearch.sql.spark.transport.model.CancelAsyncQueryActionResponse; import org.opensearch.sql.spark.transport.model.CreateAsyncQueryActionResponse; import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionResponse; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.sql.domain.SQLQueryRequest; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.threadpool.ExecutorBuilder; @@ -154,6 +159,7 @@ public class SQLPlugin extends Plugin private static final Logger LOGGER = LogManager.getLogger(SQLPlugin.class); private List executionEngineExtensions = List.of(); + private List restEndpointProviders = List.of(); private ClusterService clusterService; /** Settings should be inited when bootstrap the plugin. */ @@ -182,6 +188,16 @@ public void loadExtensions(ExtensionLoader loader) { executionEngineExtensions.size(), executionEngineExtensions.stream().map(e -> e.getClass().getSimpleName()).toList()); } + + List restProviders = loader.loadExtensions(RestEndpointProvider.class); + this.restEndpointProviders = restProviders != null ? List.copyOf(restProviders) : List.of(); + } + + private void publishRestCommandRegistries() { + List providers = new ArrayList<>(); + providers.add(new CoreEndpointsProvider()); + providers.addAll(this.restEndpointProviders); + RestEndpointRegistryHolder.set(new RestEndpointRegistry(providers)); } @Override @@ -198,10 +214,6 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); - // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can - // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); - return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), @@ -237,7 +249,13 @@ private BiFunction createSqlAnalyticsRout } cached[0] = new RestUnifiedQueryAction( - client, clusterService, executor, contextProvider, pluginSettings); + client, + clusterService, + executor, + contextProvider, + pluginSettings, + new org.opensearch.sql.opensearch.executor.ThreadPoolExecutionDispatcher( + client.threadPool(), pluginSettings)); } return cached[0]; }; @@ -376,6 +394,9 @@ public Collection createComponents( this.clusterService = clusterService; this.pluginSettings = new OpenSearchSettings(clusterService.getClusterSettings()); this.client = (NodeClient) client; + + publishRestCommandRegistries(); + this.dataSourceService = createDataSourceService(); dataSourceService.createDataSource(defaultOpenSearchDataSourceMetadata()); LocalClusterState.state().setClusterService(clusterService); @@ -450,10 +471,12 @@ public ScheduledJobParser getJobParser() { @Override public List> getExecutorBuilders(Settings settings) { - // The worker pool is the primary pool where most of the work is done. The background thread - // pool is a separate queue for asynchronous requests to other nodes. We keep them separate to - // prevent deadlocks during async fetches on small node counts. Tasks in the background pool - // should do no work except I/O to other services. + // The worker pool is the primary pool where most of the work is done. The complex-worker pool + // handles queries that require scripts (table scans that can't be pushed to Lucene) so they + // don't starve fast queries. The background thread pool is a separate queue for asynchronous + // requests to other nodes. We keep them separate to prevent deadlocks during async fetches on + // small node counts. Tasks in the background pool should do no work except I/O to other + // services. return List.of( new FixedExecutorBuilder( settings, @@ -461,11 +484,17 @@ public List> getExecutorBuilders(Settings settings) { OpenSearchExecutors.allocatedProcessors(settings), 1000, "thread_pool." + SQL_WORKER_THREAD_POOL_NAME), + new FixedExecutorBuilder( + settings, + SQL_COMPLEX_WORKER_THREAD_POOL_NAME, + OpenSearchExecutors.allocatedProcessors(settings), + 1000, + "thread_pool." + SQL_COMPLEX_WORKER_THREAD_POOL_NAME), new FixedExecutorBuilder( settings, SQL_BACKGROUND_THREAD_POOL_NAME, settings.getAsInt( - "thread_pool.search.size", OpenSearchExecutors.allocatedProcessors(settings)), + "thread_pool.search.size", 2 * OpenSearchExecutors.allocatedProcessors(settings)), 1000, "thread_pool." + SQL_BACKGROUND_THREAD_POOL_NAME)); } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java index 057c88c9a02..816f1071310 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java @@ -15,6 +15,7 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.executor.DelegatingExecutionEngine; +import org.opensearch.sql.executor.ExecutionDispatcher; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryManager; import org.opensearch.sql.executor.QueryService; @@ -26,6 +27,7 @@ import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; import org.opensearch.sql.opensearch.executor.OpenSearchQueryManager; +import org.opensearch.sql.opensearch.executor.ThreadPoolExecutionDispatcher; import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; import org.opensearch.sql.opensearch.executor.protector.OpenSearchExecutionProtector; import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; @@ -114,13 +116,19 @@ public SQLService sqlService( /** {@link QueryPlanFactory}. */ @Provides public QueryPlanFactory queryPlanFactory( - DataSourceService dataSourceService, ExecutionEngine executionEngine, Settings settings) { + DataSourceService dataSourceService, + ExecutionEngine executionEngine, + Settings settings, + NodeClient nodeClient) { Analyzer analyzer = new Analyzer( new ExpressionAnalyzer(functionRepository), dataSourceService, functionRepository); Planner planner = new Planner(LogicalPlanOptimizer.create()); + ExecutionDispatcher executionDispatcher = + new ThreadPoolExecutionDispatcher(nodeClient.threadPool(), settings); QueryService queryService = - new QueryService(analyzer, executionEngine, planner, dataSourceService, settings); + new QueryService( + analyzer, executionEngine, planner, dataSourceService, settings, executionDispatcher); return new QueryPlanFactory(queryService); } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java index bb87bf7fa91..800fcbe1692 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java @@ -30,6 +30,7 @@ public class PPLQueryRequestFactory { private static final String DEFAULT_EXPLAIN_MODE = "standard"; private static final String QUERY_PARAMS_PRETTY = "pretty"; private static final String QUERY_PARAMS_PROFILE = "profile"; + private static final String QUERY_PARAMS_ANALYZE = "analyze"; private static final String QUERY_PARAMS_FETCH_SIZE = "fetch_size"; /** @@ -82,9 +83,12 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques try { jsonContent = new JSONObject(content); boolean profileRequested = jsonContent.optBoolean(QUERY_PARAMS_PROFILE, false); + boolean analyzeRequested = jsonContent.optBoolean(QUERY_PARAMS_ANALYZE, false); String queryString = jsonContent.optString(PPL_FIELD_NAME, ""); - boolean enableProfile = - profileRequested && isProfileSupported(restRequest.path(), format, queryString); + // if both profile and analyze are requested, profile overrides analyze + boolean profileSupported = isProfileSupported(restRequest.path(), format, queryString); + boolean enableProfile = profileRequested && profileSupported; + boolean enableAnalyze = analyzeRequested && !profileRequested && profileSupported; // Support fetch_size as a URL parameter if not already in the JSON body if (!jsonContent.has(QUERY_PARAMS_FETCH_SIZE) && restRequest.params().containsKey(QUERY_PARAMS_FETCH_SIZE)) { @@ -104,7 +108,8 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques restRequest.path(), format.getFormatName(), explainMode, - enableProfile); + enableProfile, + enableAnalyze); // set sanitize option if csv format if (format.equals(Format.CSV)) { pplRequest.sanitize(getSanitizeOption(restRequest.params())); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 5685d539541..609b4a71099 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -69,18 +69,21 @@ public class RestUnifiedQueryAction { private final ClusterService clusterService; private final org.opensearch.analytics.EngineContextProvider contextProvider; private final org.opensearch.sql.common.setting.Settings pluginSettings; + private final org.opensearch.sql.executor.ExecutionDispatcher executionDispatcher; public RestUnifiedQueryAction( NodeClient client, ClusterService clusterService, QueryPlanExecutor> planExecutor, org.opensearch.analytics.EngineContextProvider contextProvider, - org.opensearch.sql.common.setting.Settings pluginSettings) { + org.opensearch.sql.common.setting.Settings pluginSettings, + org.opensearch.sql.executor.ExecutionDispatcher executionDispatcher) { this.client = client; this.clusterService = clusterService; this.analyticsEngine = new AnalyticsExecutionEngine(planExecutor); this.contextProvider = contextProvider; this.pluginSettings = pluginSettings; + this.executionDispatcher = executionDispatcher; } /** @@ -230,19 +233,25 @@ private void doExecute( // string, so apply the equivalent top-level limit here before the system cap. plan = addFetchSizeLimit(plan, planContext, fetchSize); plan = addQuerySizeLimit(plan, planContext); - if (profiling) { - analyticsEngine.executeWithProfile( - plan, - planContext, - queryCtx, - createQueryListener(queryType, profileCtx, closingListener)); - } else { - analyticsEngine.execute( - plan, - planContext, - queryCtx, - createQueryListener(queryType, profileCtx, closingListener)); - } + plan = + org.opensearch.sql.calcite.utils.CalciteToolsHelper.optimize( + plan, planContext); + RelNode finalPlan = plan; + Runnable executeTask = + profiling + ? () -> + analyticsEngine.executeWithProfile( + finalPlan, + planContext, + queryCtx, + createQueryListener(queryType, profileCtx, closingListener)) + : () -> + analyticsEngine.execute( + finalPlan, + planContext, + queryCtx, + createQueryListener(queryType, profileCtx, closingListener)); + executionDispatcher.dispatchTask(finalPlan, planContext, executeTask); } catch (Exception e) { closingListener.onFailure(e); } finally { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index 678ed58f37f..772f1ec123f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -33,6 +33,7 @@ import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.sql.executor.AnalyzeResponse; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.legacy.metrics.MetricName; @@ -135,7 +136,13 @@ private void buildUnifiedQueryHandlerIfReady() { if (executor != null && contextProvider != null) { this.unifiedQueryHandler = new RestUnifiedQueryAction( - clientRef, clusterServiceRef, executor, contextProvider, pluginSettingsRef); + clientRef, + clusterServiceRef, + executor, + contextProvider, + pluginSettingsRef, + new org.opensearch.sql.opensearch.executor.ThreadPoolExecutionDispatcher( + clientRef.threadPool(), pluginSettingsRef)); } } @@ -211,6 +218,13 @@ protected void doExecute( if (transformedRequest.isExplainRequest()) { pplService.explain( transformedRequest, createExplainResponseListener(transformedRequest, clearingListener)); + /** + * Removing `|| transformedRequest.profile()` from line 200 will separate the `profile` and + * `analyze` endpoints. See PR #5568. + */ + } else if (transformedRequest.analyze() || transformedRequest.profile()) { + pplService.analyze( + transformedRequest, createAnalyzeResponseListener(transformedRequest, clearingListener)); } else { pplService.execute( transformedRequest, @@ -219,6 +233,29 @@ protected void doExecute( } } + private ResponseListener createAnalyzeResponseListener( + PPLQueryRequest request, ActionListener listener) { + return new ResponseListener() { + @Override + public void onResponse(AnalyzeResponse response) { + JsonResponseFormatter formatter = + new JsonResponseFormatter<>(PRETTY) { + @Override + protected Object buildJsonObject(AnalyzeResponse response) { + return response; + } + }; + listener.onResponse( + new TransportPPLQueryResponse(formatter.format(response), formatter.contentType())); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }; + } + /** * TODO: need to extract an interface for both SQL and PPL action handler and move these common * methods to the interface. This is not easy to do now because SQL action handler is still in diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java index 4ba1a53d872..68a96a4f924 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java @@ -53,6 +53,11 @@ public class TransportPPLQueryRequest extends ActionRequest { @Accessors(fluent = true) private boolean profile = false; + @Setter + @Getter + @Accessors(fluent = true) + private boolean analyze = false; + @Setter @Getter @Accessors(fluent = true) @@ -67,6 +72,7 @@ public TransportPPLQueryRequest(PPLQueryRequest pplQueryRequest) { sanitize = pplQueryRequest.sanitize(); style = pplQueryRequest.style(); profile = pplQueryRequest.profile(); + analyze = pplQueryRequest.analyze(); explainMode = pplQueryRequest.mode().getModeName(); queryId = pplQueryRequest.queryId(); } @@ -83,6 +89,7 @@ public TransportPPLQueryRequest(StreamInput in) throws IOException { sanitize = in.readBoolean(); style = in.readEnum(JsonResponseFormatter.Style.class); profile = in.readBoolean(); + analyze = in.readBoolean(); queryId = in.readOptionalString(); } @@ -116,6 +123,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(sanitize); out.writeEnum(style); out.writeBoolean(profile); + out.writeBoolean(analyze); out.writeOptionalString(queryId); } @@ -172,7 +180,7 @@ public String getDescription() { /** Convert to PPLQueryRequest. */ public PPLQueryRequest toPPLQueryRequest() { PPLQueryRequest pplQueryRequest = - new PPLQueryRequest(pplQuery, jsonContent, path, format, explainMode, profile); + new PPLQueryRequest(pplQuery, jsonContent, path, format, explainMode, profile, analyze); pplQueryRequest.sanitize(sanitize); pplQueryRequest.style(style); pplQueryRequest.queryId(queryId); diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 0cf87f0604e..516c31940c3 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -54,7 +54,8 @@ public void setUp() { clusterService, executor, mock(EngineContextProvider.class), - mock(org.opensearch.sql.common.setting.Settings.class)); + mock(org.opensearch.sql.common.setting.Settings.class), + new org.opensearch.sql.executor.DirectExecutionDispatcher()); } @Test diff --git a/ppl-rest-spi/README.md b/ppl-rest-spi/README.md new file mode 100644 index 00000000000..2b7caacaaaa --- /dev/null +++ b/ppl-rest-spi/README.md @@ -0,0 +1,171 @@ +# ppl-rest-spi + +A generic way for any OpenSearch plugin to integrate with PPL: contribute your own read-only data +as a table queryable via `rest ''`, without adding a new PPL command or grammar keyword and +without a compile dependency on the sql plugin's internals. Each contributed endpoint composes with +ordinary PPL (`| where`, `| stats`, `| head`), so a plugin extends the query surface without any +change to the language itself. + +This module is intentionally thin: it depends only on OpenSearch core, and the row values it +exchanges are plain `java.lang` types (String / Number / Boolean / nested Map), so there is no +cross-classloader type-identity problem. Whatever type a handler returns, PPL surfaces it as a +string column, and a query casts the fields it needs. + +## Example: `/_cluster/health` + +The built-in `/_cluster/health` endpoint is implemented against this SPI exactly like an external +provider would. It declares a single `response` column and, at execution time, calls the +cluster-health transport action and serializes the whole response into that column: + +```java +RestEndpointDefinition.builder() + .name("/_cluster/health") + .argSpec(ArgSpec.builder().arg("local", Set.of("true", "false")).build()) + .handler(ctx -> { + ClusterHealthResponse health = + ctx.client().admin().cluster().health(new ClusterHealthRequest()).actionGet(); + XContentBuilder json = XContentFactory.jsonBuilder(); + health.toXContent(json, ToXContent.EMPTY_PARAMS); // serialize the full response as-is + return List.of(json.toString()); + }) + .build(); +``` + +Querying it returns one row whose `response` column holds the full health JSON: + +``` +> rest '/_cluster/health' + +response +-------------------------------------------------------------------------------------------- +{"cluster_name":"opensearch-cluster","status":"green","timed_out":false,"number_of_nodes":1, +"number_of_data_nodes":1,"discovered_cluster_manager":true,"active_primary_shards":0, +"active_shards":0,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":0, +"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0, +"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":100.0} +``` + +Pull individual fields downstream with `spath` (or `json_extract`), casting where a numeric type +is needed: + +``` +> rest '/_cluster/health' | spath input=response path=status output=status | fields status + +status +------ +green +``` + +``` +> rest '/_cluster/health' + | spath input=response path=number_of_nodes output=nodes + | where cast(nodes as int) >= 1 + | fields nodes + +nodes +----- +1 +``` + +## Contract + +| Type | Role | +|---|---| +| `RestEndpointProvider` | Your entry point: `List getEndpoints()`. | +| `RestEndpointDefinition` | One endpoint: `name()`, `argSpec()`, `handler()`. Build with `RestEndpointDefinition.builder()`. Every endpoint surfaces a single `response` string column. | +| `ArgSpec` | The query args the endpoint accepts and each arg's allowed value domain; an unknown or out-of-domain arg is rejected. `ArgSpec.NONE` accepts none. | +| `RestEndpointHandler` | `List fetch(RestEndpointContext)`: each string is one row's `response` cell (typically a serialized JSON document). Runs at scan execution (not planning), so the scan is lazy and `EXPLAIN` is side-effect free. | +| `RestEndpointContext` | The validated `args()` plus an optional core `NodeClient` (`client()`) for a handler that issues its own read-only transport action. | + +## Add an endpoint + +1. Depend on the published `ppl-rest-spi` artifact `compileOnly` (the installed sql plugin + provides it at runtime, so it is never bundled), and declare the sql plugin as an extended + plugin so `loadExtensions` discovers your provider: + + ```gradle + dependencies { + compileOnly "org.opensearch.query:ppl-rest-spi:${sqlPluginVersion}" + } + opensearchplugin { extendedPlugins = ['opensearch-sql'] } + ``` + +2. Implement `RestEndpointProvider`: + + ```java + public final class MyRestProvider implements RestEndpointProvider { + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_my/thing") + .argSpec(ArgSpec.builder().arg("verbose", Set.of("true", "false")).build()) + .handler(MyRestProvider::fetch) + .build()); + } + + private static List fetch(RestEndpointContext ctx) { + // ctx.args() is already validated; ctx.client() is a NodeClient for transport calls (may be null). + MyThing thing = readMyThing(ctx); // your own read-only transport call + XContentBuilder json = XContentFactory.jsonBuilder(); + thing.toXContent(json, ToXContent.EMPTY_PARAMS); // serialize the whole response + return List.of(json.toString()); + } + } + ``` + +3. Register the provider as a service in + `META-INF/services/org.opensearch.sql.spi.rest.RestEndpointProvider`, containing your class's + fully-qualified name. + +4. Add the endpoint name to the sql plugin's default allow list. Steps 1 to 3 only *register* the + provider, which does not make the endpoint queryable on its own. A name is queryable only when it + is also present in `plugins.ppl.rest.allowed_endpoints`. Enable your endpoint by submitting a + change to the sql plugin that adds its name to the default list in `OpenSearchSettings.java`: + + ```java + public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = + Setting.listSetting( + Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), + List.of("/_cluster/health", "/_my/thing"), // add your endpoint name here + Function.identity(), + Setting.Property.NodeScope); + ``` + + Once that change is merged and released, the endpoint is enabled by default on every cluster + running that sql version. The sql maintainers' review of this change is the gate that decides + which endpoint names PPL is allowed to expose. + +Then `rest '/_my/thing' verbose=true | spath input=response path=count output=count | where cast(count as int) > 0` composes like any scan; pull fields out of the `response` JSON with `spath` (or `json_extract`) and cast the ones you compute on. + +## Rules and guarantees + +- Endpoints are read-only. The handler produces rows; every value is surfaced as a string column, + and a query extracts and casts the fields it needs (for example with `json_extract` or `spath`). +- `plugins.ppl.rest.allowed_endpoints` is the enable list, whose default is maintained in the sql + plugin's `OpenSearchSettings.java`: a name is queryable only when it is both registered by a + provider and listed there; anything else is rejected before any transport call. Listing a name no + provider registered has no effect. +- Endpoint names are global across all providers, and `allowed_endpoints` does not resolve name + collisions: it only enables names, it does not make two providers that claim the same name + coexist. Collisions are handled separately, at registration time: a built-in name cannot be + shadowed by an external provider (the built-in wins and the external duplicate is dropped with a + logged warning), and if two external providers register the same name that name is disabled + entirely (logged warning, and the node still starts). So even when every name is listed in + `allowed_endpoints`, a name owned by two providers will not silently pick a winner. +- Redaction is the endpoint's own responsibility, applied to the response before it is streamed + back. The framework surfaces exactly what the handler returns and adds no masking step, so there + is no central redaction seam to implement and an endpoint with nothing sensitive does nothing. + Because an endpoint returns a single `response` JSON column, redact the sensitive fields (IPs, + hostnames, tokens) on the response object *before* serializing it into that column, so the masked + value is what is streamed out: + + ```java + .handler(ctx -> { + MyStats stats = fetchStats(ctx); // your own read-only transport call + stats.maskIp(); // redact on the object first + XContentBuilder json = XContentFactory.jsonBuilder(); + stats.toXContent(json, ToXContent.EMPTY_PARAMS); // then serialize the masked response + return List.of(json.toString()); + }) + ``` diff --git a/ppl-rest-spi/build.gradle b/ppl-rest-spi/build.gradle new file mode 100644 index 00000000000..d0c6ccd5f4f --- /dev/null +++ b/ppl-rest-spi/build.gradle @@ -0,0 +1,106 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * The `rest` command extension SPI. + * + * This module is intentionally thin: it declares ONLY the interfaces a plugin implements to + * contribute read-only `rest` endpoints, plus the JDK-typed row/column/arg value objects those + * interfaces exchange. It depends on the OpenSearch core artifact ONLY (for the transport client + * handed to a handler) and on NO sql module, so external plugins can compileOnly it without a + * dependency cycle and without a cross-classloader type-identity problem (row values are plain + * java.lang types). + */ + +plugins { + id 'java-library' + id "io.freefair.lombok" + id 'com.diffplug.spotless' +} + +dependencies { + // Core only. Deliberately no `project(':core')` / `project(':opensearch')` dependency so the + // sql plugin (which depends on this module) never forms a cycle, and so an external plugin can + // depend on this module alone. + compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + + testImplementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + testImplementation('org.junit.jupiter:junit-jupiter-api:5.9.3') + testImplementation('org.junit.jupiter:junit-jupiter-params:5.9.3') + testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine') + testRuntimeOnly('org.junit.platform:junit-platform-launcher') +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + } +} + +// Published as a standalone Maven artifact so an external plugin can depend on the SPI +// `compileOnly`. Intentionally NOT added to the root `publishedModules` list, which would prefix +// the artifactId with `unified-query-`; this is a public extension SPI, so it keeps the plain +// `ppl-rest-spi` artifactId. +apply plugin: 'maven-publish' + +publishing { + publications { + restSpi(MavenPublication) { + from components.java + groupId = "org.opensearch.query" + artifactId = "ppl-rest-spi" + + pom { + name = "ppl-rest-spi" + description = "OpenSearch PPL rest command extension SPI" + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + name = 'OpenSearch' + url = 'https://github.com/opensearch-project/sql' + } + } + } + } + } + + repositories { + maven { + name = "Snapshots" + url = "https://ci.opensearch.org/ci/dbc/snapshots/maven/" + url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO") + credentials(AwsCredentials) { + accessKey = System.getenv("AWS_ACCESS_KEY_ID") + secretKey = System.getenv("AWS_SECRET_ACCESS_KEY") + sessionToken = System.getenv("AWS_SESSION_TOKEN") + } + } + } +} + +spotless { + java { + target fileTree('.') { + include '**/*.java' + exclude '**/build/**', '**/build-*/**' + } + importOrder() + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") + removeUnusedImports() + trimTrailingWhitespace() + endWithNewline() + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java new file mode 100644 index 00000000000..003c56cca6a --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * The query args a {@code rest} endpoint accepts, with the allowed value domain of each. An + * undeclared arg is rejected; a declared arg whose value is outside its domain is rejected; an + * empty domain accepts any value. + */ +public final class ArgSpec { + + public static final ArgSpec NONE = builder().build(); + + // arg name -> allowed values (empty set means "any value allowed"). + private final Map> valueDomains; + + private ArgSpec(Map> valueDomains) { + this.valueDomains = valueDomains; + } + + public Set allowedArgs() { + return valueDomains.keySet(); + } + + public boolean allows(String arg) { + return valueDomains.containsKey(arg); + } + + /** Validate an accepted arg's value against its domain (no-op if the domain is empty). */ + public void validateValue(String endpoint, String arg, String value) { + Set domain = valueDomains.get(arg); + if (domain == null || domain.isEmpty()) { + return; + } + if (value == null || !domain.contains(value.toLowerCase(Locale.ROOT))) { + throw unsupported(endpoint, arg, value, domain); + } + } + + private static IllegalArgumentException unsupported( + String endpoint, String arg, String value, Set domain) { + return new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [" + + arg + + "] has an unsupported value [" + + value + + "]. Allowed values: " + + domain); + } + + public static Builder builder() { + return new Builder(); + } + + /** Builder for {@link ArgSpec}. Declaration order is preserved for stable error messages. */ + public static final class Builder { + private final LinkedHashMap> valueDomains = new LinkedHashMap<>(); + + public Builder arg(String name) { + valueDomains.put(name, Set.of()); + return this; + } + + public Builder arg(String name, Set domain) { + valueDomains.put(name, Set.copyOf(domain)); + return this; + } + + public ArgSpec build() { + return new ArgSpec(new LinkedHashMap<>(valueDomains)); + } + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java new file mode 100644 index 00000000000..3cf78911deb --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.Map; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Per-invocation context handed to {@link RestEndpointHandler#fetch} at execution time (scan open), + * carrying the validated query args and the node transport client a provider may use for its own + * read-only transport action. A provider that already holds a client may ignore {@link #client()}. + */ +public interface RestEndpointContext { + + /** Validated query args for this invocation (never null; empty when none supplied). */ + Map args(); + + /** Node transport client for a provider that issues its own transport action; may be null. */ + NodeClient client(); + + static RestEndpointContext of(Map args, NodeClient client) { + Map safeArgs = args == null ? Map.of() : args; + return new RestEndpointContext() { + @Override + public Map args() { + return safeArgs; + } + + @Override + public NodeClient client() { + return client; + } + }; + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java new file mode 100644 index 00000000000..ca74f768966 --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.Objects; + +/** + * One read-only {@code rest} endpoint from a {@link RestEndpointProvider}: a unique name (the token + * after {@code rest}, e.g. {@code /_cluster/health}), the {@link ArgSpec} it accepts, and the + * {@link RestEndpointHandler} that produces its rows. Every endpoint surfaces a single {@code + * response} string column; a query extracts the fields it needs with {@code spath} or {@code + * json_extract}. A provider that needs to mask sensitive values does so inside its handler before + * returning the response. Immutable; build with {@link #builder()}. + */ +public interface RestEndpointDefinition { + + String name(); + + ArgSpec argSpec(); + + RestEndpointHandler handler(); + + static Builder builder() { + return new Builder(); + } + + final class Builder { + private String name; + private ArgSpec argSpec = ArgSpec.NONE; + private RestEndpointHandler handler; + + private Builder() {} + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder argSpec(ArgSpec argSpec) { + this.argSpec = argSpec; + return this; + } + + public Builder handler(RestEndpointHandler handler) { + this.handler = handler; + return this; + } + + public RestEndpointDefinition build() { + String endpointName = Objects.requireNonNull(name, "rest endpoint name is required"); + ArgSpec spec = Objects.requireNonNull(argSpec, "argSpec is required"); + RestEndpointHandler endpointHandler = Objects.requireNonNull(handler, "handler is required"); + return new RestEndpointDefinition() { + @Override + public String name() { + return endpointName; + } + + @Override + public ArgSpec argSpec() { + return spec; + } + + @Override + public RestEndpointHandler handler() { + return endpointHandler; + } + }; + } + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java new file mode 100644 index 00000000000..ed94d02441b --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.List; + +/** + * Produces the rows of a {@code rest} endpoint. Invoked at execution time (scan open), never at + * planning, so the scan stays lazy and EXPLAIN is side-effect free; a transport-backed provider may + * block on {@code ctx.client().execute(...).actionGet()} here. Each row is one string, surfaced as + * the single {@code response} column, typically a serialized JSON document; a query extracts and + * casts the fields it needs with {@code spath} or {@code json_extract}. A provider that must mask + * sensitive values does so before serializing the response it returns here. + */ +@FunctionalInterface +public interface RestEndpointHandler { + + /** + * Fetch the rows for one invocation. + * + * @param ctx the validated query args and (optional) transport client for this invocation + * @return one response string per row (never null; empty when there are no rows) + */ + List fetch(RestEndpointContext ctx); +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java new file mode 100644 index 00000000000..db04c016f8f --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.List; + +/** + * The extension point a plugin implements to contribute read-only {@code rest} endpoints, via + * {@code ExtensiblePlugin.loadExtensions(RestEndpointProvider.class)}. The sql plugin merges every + * discovered provider with its own built-in one into a single registry, so built-in and external + * endpoints are uniform clients of the same contract. A provider declares data (name, schema, args) + * and a handler; it never touches the PPL grammar. + */ +public interface RestEndpointProvider { + + /** The endpoints this provider contributes. Called once when the registry is built. */ + List getEndpoints(); +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index fab30b66c3f..eb3f0bb7eaa 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -15,6 +15,10 @@ DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; REST: 'REST'; TIMEOUT: 'TIMEOUT'; +MAKERESULTS: 'MAKERESULTS'; +FORMAT: 'FORMAT'; +CSV: 'CSV'; +DATA: 'DATA'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; @@ -58,6 +62,8 @@ APPENDCOL: 'APPENDCOL'; ADDTOTALS: 'ADDTOTALS'; ADDCOLTOTALS: 'ADDCOLTOTALS'; GRAPHLOOKUP: 'GRAPHLOOKUP'; +XYSERIES: 'XYSERIES'; +SEP: 'SEP'; TIMEWRAP: 'TIMEWRAP'; ALIGN: 'ALIGN'; SERIES: 'SERIES'; @@ -184,6 +190,8 @@ UNION: 'UNION'; MAXOUT: 'MAXOUT'; COUNTFIELD: 'COUNTFIELD'; SHOWCOUNT: 'SHOWCOUNT'; +PERCENTFIELD: 'PERCENTFIELD'; +SHOWPERC: 'SHOWPERC'; LIMIT: 'LIMIT'; USEOTHER: 'USEOTHER'; OTHERSTR: 'OTHERSTR'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index fda8d66d135..ce949f8534b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -47,6 +47,7 @@ pplCommands : describeCommand | restCommand | showDataSourcesCommand + | makeresultsCommand | searchCommand | multisearchCommand | graphLookupCommand @@ -99,6 +100,7 @@ commands | fieldformatCommand | nomvCommand | graphLookupCommand + | xyseriesCommand | unionCommand | timewrapCommand ; @@ -154,7 +156,9 @@ commandName | NOMV | TRANSPOSE | GRAPHLOOKUP + | XYSERIES | TIMEWRAP + | MAKERESULTS ; searchCommand @@ -226,6 +230,16 @@ showDataSourcesCommand : SHOW DATASOURCES ; +makeresultsCommand + : MAKERESULTS makeresultsArg* + ; + +makeresultsArg + : COUNT EQUAL integerLiteral + | FORMAT EQUAL (CSV | JSON) + | DATA EQUAL stringLiteral + ; + whereCommand : WHERE logicalExpression ; @@ -487,6 +501,8 @@ rareTopCommand rareTopOption : COUNTFIELD EQUAL countField = stringLiteral | SHOWCOUNT EQUAL showCount = booleanLiteral + | PERCENTFIELD EQUAL percentField = stringLiteral + | SHOWPERC EQUAL showPerc = booleanLiteral | USENULL EQUAL useNull = booleanLiteral ; @@ -764,6 +780,19 @@ graphLookupArgs | (FILTER EQUAL LT_PRTHS logicalExpression RT_PRTHS) ; +xyseriesCommand + : XYSERIES xyseriesOption* xField = fieldExpression yNameField = fieldExpression IN LT_PRTHS xyseriesPivotValues RT_PRTHS yDataFields = fieldList + ; + +xyseriesOption + : SEP EQUAL sep = stringLiteral + | FORMAT EQUAL format = stringLiteral + ; + +xyseriesPivotValues + : stringLiteral (COMMA stringLiteral)* + ; + // clauses fromClause : SOURCE EQUAL tableOrSubqueryClause @@ -1761,6 +1790,9 @@ searchableKeyWord // ARGUMENT KEYWORDS | KEEPEMPTY | CONSECUTIVE + | FORMAT + | CSV + | DATA | DEDUP_SPLITVALUES | PARTITIONS | ALLNUM @@ -1788,6 +1820,8 @@ searchableKeyWord | ANOMALY_SCORE_THRESHOLD | COUNTFIELD | SHOWCOUNT + | PERCENTFIELD + | SHOWPERC | MAXOUT | PATH | INPUT @@ -1855,4 +1889,5 @@ searchableKeyWord | EDGE // rest command token, also usable as a free-text search term / identifier | TIMEOUT + | SEP ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index 6ad9032432c..7f117e7bb0b 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -8,18 +8,26 @@ import static org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import static org.opensearch.sql.executor.execution.QueryPlanFactory.NO_CONSUMER_RESPONSE_LISTENER; +import java.util.ArrayList; +import java.util.List; import lombok.extern.log4j.Log4j2; +import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.tree.ParseTree; +import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.ast.statement.Statement; +import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.QueryContext; +import org.opensearch.sql.executor.AnalyzeResponse; +import org.opensearch.sql.executor.AnalyzeResponse.QuerySegment; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.QueryManager; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.executor.execution.AbstractPlan; import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.ppl.parser.AstBuilder; import org.opensearch.sql.ppl.parser.AstStatementBuilder; @@ -85,6 +93,93 @@ public void explain(PPLQueryRequest request, ResponseListener l } } + /** + * Analyze the query: produces the AST node and logical plan RelNode. + * + * @param request {@link PPLQueryRequest} + * @param listener {@link ResponseListener} for analyze response + */ + public void analyze(PPLQueryRequest request, ResponseListener listener) { + try { + String queryText = request.getRequest(); + ParseTree cst = parser.parse(queryText); + Statement statement = + cst.accept( + new AstStatementBuilder( + new AstBuilder(queryText, settings), + AstStatementBuilder.StatementBuilderContext.builder() + .isExplain(false) + .fetchSize(request.getFetchSize()) + .highlightConfig(request.getHighlightConfig()) + .format( + request.getFormat() != null && !request.getFormat().isEmpty() + ? org.opensearch.sql.protocol.response.format.Format.ofExplain( + request.getFormat()) + .orElse(null) + : null) + .build())); + + log.info( + "[{}] Incoming request {}", + QueryContext.getRequestId(), + anonymizer.anonymizeStatement(statement)); + + List querySegments = extractQuerySegments(cst, queryText); + UnresolvedPlan unresolvedPlan = ((Query) statement).getPlan(); + queryManager.submit( + queryExecutionFactory.createAnalyzePlan( + queryText, querySegments, unresolvedPlan, PPL_QUERY, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + private List extractQuerySegments(ParseTree cst, String queryText) { + List segments = new ArrayList<>(); + OpenSearchPPLParser.QueryStatementContext queryStmt = findQueryStatement(cst); + if (queryStmt == null) { + return segments; + } + + // First segment: the search/source command (pplCommands) + OpenSearchPPLParser.PplCommandsContext pplCommands = queryStmt.pplCommands(); + if (pplCommands != null) { + segments.add(buildSegment(pplCommands, queryText)); + } + + // Remaining segments: each piped command + for (OpenSearchPPLParser.CommandsContext cmd : queryStmt.commands()) { + segments.add(buildSegment(cmd, queryText)); + } + return segments; + } + + private OpenSearchPPLParser.QueryStatementContext findQueryStatement(ParseTree tree) { + if (tree instanceof OpenSearchPPLParser.QueryStatementContext ctx) { + return ctx; + } + for (int i = 0; i < tree.getChildCount(); i++) { + OpenSearchPPLParser.QueryStatementContext result = findQueryStatement(tree.getChild(i)); + if (result != null) { + return result; + } + } + return null; + } + + private QuerySegment buildSegment(ParserRuleContext ctx, String queryText) { + int start = ctx.getStart().getStartIndex(); + int stop = ctx.getStop().getStopIndex(); + String source = queryText.substring(start, stop + 1); + // For wrapper rules like CommandsContext, drill into the specific child command + ParserRuleContext target = ctx; + if (ctx.getChildCount() == 1 && ctx.getChild(0) instanceof ParserRuleContext child) { + target = child; + } + String nodeType = target.getClass().getSimpleName().replace("Context", ""); + return QuerySegment.builder().nodeType(nodeType).source(source).build(); + } + private AbstractPlan plan( PPLQueryRequest request, ResponseListener queryListener, diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java b/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java index 2d044da58e6..c25e4655bf8 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java @@ -25,7 +25,8 @@ public class OpenSearchSparkSqlDialect extends SparkSqlDialect { ImmutableMap.of( "ARG_MIN", "MIN_BY", "ARG_MAX", "MAX_BY", - "SAFE_CAST", "TRY_CAST"); + "SAFE_CAST", "TRY_CAST", + "CHECKED_LONG_SUM", "SUM"); private static final Map CALL_SEPARATOR = ImmutableMap.of("SAFE_CAST", "AS"); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java index 06c7fe1c38e..0ef1de27fc3 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java @@ -52,6 +52,11 @@ public class PPLQueryRequest { @Accessors(fluent = true) private boolean profile = false; + @Setter + @Getter + @Accessors(fluent = true) + private boolean analyze = false; + @Setter @Getter @Accessors(fluent = true) @@ -62,10 +67,9 @@ public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path) { } public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path, String format) { - this(pplQuery, jsonContent, path, format, ExplainMode.STANDARD.getModeName(), false); + this(pplQuery, jsonContent, path, format, ExplainMode.STANDARD.getModeName(), false, false); } - /** Constructor of PPLQueryRequest. */ public PPLQueryRequest( String pplQuery, JSONObject jsonContent, @@ -73,12 +77,25 @@ public PPLQueryRequest( String format, String explainMode, boolean profile) { + this(pplQuery, jsonContent, path, format, explainMode, profile, false); + } + + /** Constructor of PPLQueryRequest. */ + public PPLQueryRequest( + String pplQuery, + JSONObject jsonContent, + String path, + String format, + String explainMode, + boolean profile, + boolean analyze) { this.pplQuery = pplQuery; this.jsonContent = jsonContent; this.path = Optional.ofNullable(path).orElse(DEFAULT_PPL_PATH); this.format = format; this.explainMode = explainMode; this.profile = profile; + this.analyze = analyze; } public String getRequest() { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index e7d42793fe8..7aca48a4666 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -97,6 +97,7 @@ import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.MinSpanBin; import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.MvCombine; @@ -129,6 +130,7 @@ import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -146,6 +148,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StatsByClauseContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; +import org.opensearch.sql.ppl.utils.MakeResultsDataParser; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; import org.opensearch.sql.utils.SystemIndexUtils; @@ -261,12 +264,9 @@ public UnresolvedPlan visitShowDataSourcesCommand( /** * Rest command.
- * Leading command that reads an allow-listed, read-only in-cluster management endpoint - * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved - * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves - * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE - * resolves to a system index. Allow-list/authorization enforcement happens at source-table - * construction in the storage engine (it owns the transport actions and per-endpoint schemas). + * Encodes the validated endpoint spec into a reserved table name (via {@link + * org.opensearch.sql.utils.SystemIndexUtils#restTable}) that resolves through the storage engine + * to a REST source table on the Calcite path, mirroring how DESCRIBE resolves to a system index. */ @Override public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { @@ -290,6 +290,46 @@ public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ct return new RestRelation(new QualifiedName(token)); } + /** makeresults command. */ + @Override + public UnresolvedPlan visitMakeresultsCommand(OpenSearchPPLParser.MakeresultsCommandContext ctx) { + int count = 1; + String format = null; + String data = null; + for (OpenSearchPPLParser.MakeresultsArgContext arg : ctx.makeresultsArg()) { + if (arg.integerLiteral() != null) { + String raw = arg.integerLiteral().getText(); + try { + count = Integer.parseInt(raw); + } catch (NumberFormatException e) { + throw new SyntaxCheckException( + "makeresults count \"" + raw + "\" is not a valid integer within the allowed range"); + } + } else if (arg.stringLiteral() != null) { + data = StringUtils.unquoteText(arg.stringLiteral().getText()); + } else if (arg.JSON() != null) { + format = "json"; + } else if (arg.CSV() != null) { + format = "csv"; + } + } + if (data != null || format != null) { + if (data == null || format == null) { + throw new SyntaxCheckException("makeresults format and data must be provided together"); + } + return MakeResultsDataParser.parse(format, data); + } + if (count < 0) { + // Negative count yields zero rows. + count = 0; + } + if (count > 5000) { + // Inline literal rows hit the JVM 64 KB per-method bytecode limit. + throw new SyntaxCheckException("makeresults count must not exceed 5000"); + } + return new MakeResults(count); + } + /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { @@ -1828,4 +1868,37 @@ public UnresolvedPlan visitGraphLookupCommand(OpenSearchPPLParser.GraphLookupCom .filter(filter) .build(); } + + /** Xyseries command. */ + @Override + public UnresolvedPlan visitXyseriesCommand(OpenSearchPPLParser.XyseriesCommandContext ctx) { + UnresolvedExpression xField = internalVisitExpression(ctx.xField); + UnresolvedExpression yNameField = internalVisitExpression(ctx.yNameField); + + // Parse pivot values from IN (...) clause + List pivotValues = + ctx.xyseriesPivotValues().stringLiteral().stream() + .map(s -> StringUtils.unquoteText(s.getText())) + .distinct() + .collect(Collectors.toList()); + + // Parse y-data fields + List yDataFields = + ctx.yDataFields.fieldExpression().stream() + .map(this::internalVisitExpression) + .collect(Collectors.toList()); + + // Parse options + String separator = ": "; + String format = null; + for (OpenSearchPPLParser.XyseriesOptionContext optCtx : ctx.xyseriesOption()) { + if (optCtx.SEP() != null) { + separator = StringUtils.unquoteText(optCtx.sep.getText()); + } else if (optCtx.FORMAT() != null) { + format = StringUtils.unquoteText(optCtx.format.getText()); + } + } + + return new Xyseries(xField, yNameField, pivotValues, yDataFields, separator, format); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index 2cdc702b785..3aba4faf99c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -307,6 +307,18 @@ public static List getArgumentList( new Argument( RareTopN.Option.showCount.name(), opt.isPresent() ? getArgumentValue(opt.get().showCount) : Literal.TRUE)); + opt = ctx.rareTopOption().stream().filter(op -> op.percentField != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.percentField.name(), + opt.isPresent() + ? getArgumentValue(opt.get().percentField) + : new Literal("percent", DataType.STRING))); + opt = ctx.rareTopOption().stream().filter(op -> op.showPerc != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.showPerc.name(), + opt.isPresent() ? getArgumentValue(opt.get().showPerc) : Literal.FALSE)); opt = ctx.rareTopOption().stream().filter(op -> op.useNull != null).findFirst(); list.add( new Argument( diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/MakeResultsDataParser.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/MakeResultsDataParser.java new file mode 100644 index 00000000000..225f3d0280f --- /dev/null +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/MakeResultsDataParser.java @@ -0,0 +1,384 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.tree.Values; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.data.type.ExprCoreType; + +/** + * Parses the inline {@code makeresults format=csv|json data="..."} literal into a shared {@link + * Values} node of typed literal rows. JSON values infer their type (integer to long, decimal to + * float, boolean, string); a CSV {@code name:type} header declares the type, a bare name is string. + * UDT types (timestamp/date/time/ip/json) are not yet supported on this path. + */ +public final class MakeResultsDataParser { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private MakeResultsDataParser() {} + + public static Values parse(String format, String data) { + String fmt = format == null ? null : format.toLowerCase(Locale.ROOT); + Values result; + if ("json".equals(fmt)) { + result = parseJson(data); + } else if ("csv".equals(fmt)) { + result = parseCsv(data); + } else { + throw new SyntaxCheckException("makeresults format must be 'csv' or 'json'"); + } + // Cap inline cells (rows x columns): the generated literal method hits the JVM 64 KB + // per-method bytecode limit above ~6900 cells, so a flat row cap is unsafe for wide data. + if (result.getValues() != null && !result.getValues().isEmpty()) { + int rows = result.getValues().size(); + int cols = result.getValues().get(0).size(); + long cells = (long) rows * cols; + if (cells > 5000) { + throw new SyntaxCheckException( + "makeresults data must not exceed 5000 cells (rows x columns); got " + + rows + + " rows x " + + cols + + " columns = " + + cells); + } + // A single string literal must fit the 65535-byte constant-pool CONSTANT_Utf8 limit. + for (List row : result.getValues()) { + for (Literal cell : row) { + Object v = cell.getValue(); + if (v instanceof String && ((String) v).length() > 60000) { + throw new SyntaxCheckException( + "makeresults data cell value must not exceed 60000 characters; got " + + ((String) v).length()); + } + } + } + } + return result; + } + + private static Values toValues( + List names, + List types, + List> rows, + boolean withImplicitTimestamp) { + List dataTypes = new ArrayList<>(); + for (ExprCoreType t : types) { + dataTypes.add(exprToDataType(t)); + } + List> literalRows = new ArrayList<>(); + for (List row : rows) { + List out = new ArrayList<>(); + for (int i = 0; i < names.size(); i++) { + Object v = row.get(i); + out.add(v == null ? new Literal(null, DataType.NULL) : new Literal(v, dataTypes.get(i))); + } + literalRows.add(out); + } + return new Values(literalRows, names, types, withImplicitTimestamp); + } + + private static DataType exprToDataType(ExprCoreType t) { + switch (t) { + case BOOLEAN: + return DataType.BOOLEAN; + case INTEGER: + return DataType.INTEGER; + case LONG: + return DataType.LONG; + case FLOAT: + return DataType.FLOAT; + case DOUBLE: + return DataType.DOUBLE; + case STRING: + default: + return DataType.STRING; + } + } + + private static Values parseJson(String data) { + JsonNode arr; + try { + arr = MAPPER.readTree(data); + } catch (Exception e) { + throw new SyntaxCheckException("makeresults data is not valid JSON: " + e.getMessage()); + } + if (arr == null || !arr.isArray()) { + throw new SyntaxCheckException("makeresults JSON data must be an array of objects"); + } + LinkedHashMap cols = new LinkedHashMap<>(); + List> raw = new ArrayList<>(); + for (JsonNode node : arr) { + if (!node.isObject()) { + throw new SyntaxCheckException("makeresults JSON data must be an array of objects"); + } + Map row = new LinkedHashMap<>(); + for (Iterator> it = node.fields(); it.hasNext(); ) { + Map.Entry f = it.next(); + String name = f.getKey(); + JsonNode v = f.getValue(); + ExprCoreType inferred = inferJsonType(v); + if (inferred == null) { + cols.putIfAbsent(name, null); + } else { + cols.merge(name, inferred, MakeResultsDataParser::widen); + } + row.put(name, jsonValue(v)); + } + raw.add(row); + } + List names = new ArrayList<>(cols.keySet()); + List types = new ArrayList<>(); + for (String n : names) { + ExprCoreType t = cols.get(n); + if (t == null) { + throw new SyntaxCheckException( + "makeresults column '" + + n + + "' has only null values; provide at least one non-null" + + " value so its type can be determined"); + } + types.add(t); + } + List> rows = new ArrayList<>(); + for (Map r : raw) { + List out = new ArrayList<>(); + for (int i = 0; i < names.size(); i++) { + out.add(coerce(r.get(names.get(i)), types.get(i))); + } + rows.add(out); + } + return toValues(names, types, rows, true); + } + + private static Values parseCsv(String data) { + String[] lines = data.split("\r?\n", -1); + if (lines.length == 0 || lines[0].trim().isEmpty()) { + throw new SyntaxCheckException("makeresults CSV data must start with a header line"); + } + String[] header = splitCsvLine(lines[0]).toArray(new String[0]); + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (String token : header) { + String t = token.trim(); + String name = t; + ExprCoreType type = ExprCoreType.STRING; + int colon = t.lastIndexOf(':'); + if (colon > 0 && colon < t.length() - 1) { + ExprCoreType declared = resolveType(t.substring(colon + 1).trim()); + if (declared != null) { + name = t.substring(0, colon).trim(); + type = declared; + } + } + if (name.isEmpty()) { + throw new SyntaxCheckException( + "makeresults CSV header has a blank column name: " + lines[0]); + } + names.add(name); + types.add(type); + } + names = uniquify(names); + List> rows = new ArrayList<>(); + for (int li = 1; li < lines.length; li++) { + if (lines[li].trim().isEmpty()) { + continue; + } + List cells = splitCsvLine(lines[li]); + if (cells.size() > names.size()) { + throw new SyntaxCheckException( + "makeresults CSV row has more columns than the header: " + lines[li]); + } + List out = new ArrayList<>(); + for (int i = 0; i < names.size(); i++) { + String cell = i < cells.size() ? cells.get(i).trim() : ""; + out.add(coerce(cell.isEmpty() ? null : cell, types.get(i))); + } + rows.add(out); + } + return toValues(names, types, rows, false); + } + + private static List splitCsvLine(String line) { + List out = new ArrayList<>(); + StringBuilder cur = new StringBuilder(); + boolean inQuotes = false; + for (int i = 0; i < line.length(); i++) { + char c = line.charAt(i); + if (inQuotes) { + if (c == '"') { + if (i + 1 < line.length() && line.charAt(i + 1) == '"') { + cur.append('"'); + i++; + } else { + inQuotes = false; + } + } else { + cur.append(c); + } + } else if (c == '"') { + inQuotes = true; + } else if (c == ',') { + out.add(cur.toString()); + cur.setLength(0); + } else { + cur.append(c); + } + } + if (inQuotes) { + throw new SyntaxCheckException( + "makeresults CSV data has an unterminated quoted field: " + line); + } + out.add(cur.toString()); + return out; + } + + private static List uniquify(List names) { + List out = new ArrayList<>(); + Set seen = new HashSet<>(); + for (String n : names) { + String candidate = n; + int suffix = 0; + while (!seen.add(candidate)) { + candidate = n + suffix++; + } + out.add(candidate); + } + return out; + } + + private static ExprCoreType inferJsonType(JsonNode v) { + if (v.isNull()) { + return null; + } + if (v.isObject() || v.isArray()) { + return ExprCoreType.STRING; + } + if (v.isBoolean()) { + return ExprCoreType.BOOLEAN; + } + if (v.isIntegralNumber()) { + // A JSON integer wider than long keeps full precision as a string rather than overflowing. + return v.canConvertToLong() ? ExprCoreType.LONG : ExprCoreType.STRING; + } + if (v.isNumber()) { + return ExprCoreType.FLOAT; + } + return ExprCoreType.STRING; + } + + private static ExprCoreType widen(ExprCoreType a, ExprCoreType b) { + if (a == null) { + return b; + } + if (b == null || a == b) { + return a; + } + boolean an = a == ExprCoreType.LONG || a == ExprCoreType.FLOAT; + boolean bn = b == ExprCoreType.LONG || b == ExprCoreType.FLOAT; + if (an && bn) { + return ExprCoreType.FLOAT; + } + return ExprCoreType.STRING; + } + + private static Object jsonValue(JsonNode v) { + if (v.isNull()) { + return null; + } + if (v.isObject() || v.isArray()) { + return v.toString(); + } + if (v.isBoolean()) { + return v.booleanValue(); + } + if (v.isIntegralNumber()) { + return v.canConvertToLong() ? v.longValue() : v.asText(); + } + if (v.isNumber()) { + return v.doubleValue(); + } + return v.asText(); + } + + private static ExprCoreType resolveType(String name) { + switch (name.toLowerCase(Locale.ROOT)) { + case "string": + return ExprCoreType.STRING; + case "boolean": + return ExprCoreType.BOOLEAN; + case "int": + case "integer": + return ExprCoreType.INTEGER; + case "long": + return ExprCoreType.LONG; + case "float": + return ExprCoreType.FLOAT; + case "double": + return ExprCoreType.DOUBLE; + case "date": + case "time": + case "timestamp": + case "ip": + case "json": + throw new SyntaxCheckException( + "makeresults inline type '" + name + "' is not yet supported; use string and cast"); + default: + return null; + } + } + + private static Object coerce(Object value, ExprCoreType type) { + if (value == null) { + return null; + } + String s = String.valueOf(value); + try { + switch (type) { + case STRING: + return s; + case BOOLEAN: + return value instanceof Boolean ? value : parseBooleanStrict(s.trim()); + case INTEGER: + return Integer.parseInt(s.trim()); + case LONG: + return value instanceof Long ? value : Long.parseLong(s.trim()); + case FLOAT: + case DOUBLE: + return value instanceof Double ? value : Double.parseDouble(s.trim()); + default: + return s; + } + } catch (NumberFormatException e) { + throw new SyntaxCheckException( + "makeresults cannot parse \"" + s + "\" as " + type.typeName()); + } + } + + private static Boolean parseBooleanStrict(String s) { + if ("true".equalsIgnoreCase(s)) { + return Boolean.TRUE; + } + if ("false".equalsIgnoreCase(s)) { + return Boolean.FALSE; + } + throw new SyntaxCheckException( + "makeresults cannot parse \"" + s + "\" as boolean; expected true or false"); + } +} diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 540869d1642..1ccd9561e25 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -114,6 +114,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; @@ -478,13 +479,16 @@ public String visitRareTopN(RareTopN node, String context) { Integer noOfResults = node.getNoOfResults(); String countField = (String) arguments.get(RareTopN.Option.countField.name()).getValue(); Boolean showCount = (Boolean) arguments.get(RareTopN.Option.showCount.name()).getValue(); + String percentField = (String) arguments.get(RareTopN.Option.percentField.name()).getValue(); + Boolean showPerc = (Boolean) arguments.get(RareTopN.Option.showPerc.name()).getValue(); Boolean useNull = (Boolean) arguments.get(RareTopN.Option.useNull.name()).getValue(); String fields = visitFieldList(node.getFields()); String group = visitExpressionList(node.getGroupExprList()); String options = UnresolvedPlanHelper.isCalciteEnabled(settings) ? StringUtils.format( - "countield='%s' showcount=%s usenull=%s ", countField, showCount, useNull) + "countfield='%s' showcount=%s percentfield='%s' showperc=%s usenull=%s ", + countField, showCount, percentField, showPerc, useNull) : ""; return StringUtils.format( "%s | %s %d %s%s", @@ -843,6 +847,26 @@ public String visitTranspose(Transpose node, String context) { return anonymized.toString(); } + @Override + public String visitXyseries(Xyseries node, String context) { + String child = node.getChild().get(0).accept(this, context); + StringBuilder command = new StringBuilder(); + command.append(" | xyseries"); + if (node.getSeparator() != null && !": ".equals(node.getSeparator())) { + command.append(" sep=").append(MASK_LITERAL); + } + if (node.getFormat() != null) { + command.append(" format=").append(MASK_LITERAL); + } + command.append(" ").append(visitExpression(node.getXField())); + command.append(" ").append(visitExpression(node.getYNameField())); + command.append(" in (").append(MASK_LITERAL).append(")"); + String dataFields = + node.getYDataFields().stream().map(this::visitExpression).collect(Collectors.joining(",")); + command.append(" ").append(dataFields); + return StringUtils.format("%s%s", child, command.toString()); + } + @Override public String visitAppendCol(AppendCol node, String context) { String child = node.getChild().get(0).accept(this, context); @@ -903,6 +927,11 @@ public String visitValues(Values node, String context) { return ""; } + @Override + public String visitMakeResults(org.opensearch.sql.ast.tree.MakeResults node, String context) { + return "makeresults"; + } + private String visitFieldList(List fieldList) { return fieldList.stream().map(this::visitExpression).collect(Collectors.joining(",")); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 0c5ad9dd4fe..42abefc8dc6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -398,6 +398,25 @@ public void testRareCommandWithGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testRareCommandWithShowPercShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t a=1 | rare showperc=true a"); + assertNotEquals(null, tree); + } + + @Test + public void testRareCommandWithShowPercAndGroupByShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | rare showperc=true a by b"); + assertNotEquals(null, tree); + } + + @Test + public void testRareCommandWithPercentFieldShouldPass() { + ParseTree tree = + new PPLSyntaxParser().parse("source=t | rare showperc=true percentfield='pct' a"); + assertNotEquals(null, tree); + } + @Test public void testTopCommandWithoutNShouldPass() { ParseTree tree = new PPLSyntaxParser().parse("source=t a=1 | top a"); @@ -422,6 +441,25 @@ public void testTopCommandWithoutNAndGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testTopCommandWithShowPercShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | top showperc=true a"); + assertNotEquals(null, tree); + } + + @Test + public void testTopCommandWithShowPercAndGroupByShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | top showperc=true a by b"); + assertNotEquals(null, tree); + } + + @Test + public void testTopCommandWithPercentFieldShouldPass() { + ParseTree tree = + new PPLSyntaxParser().parse("source=t | top showperc=true percentfield='pct' a"); + assertNotEquals(null, tree); + } + @Test public void testCanParseMultiMatchRelevanceFunction() { assertNotEquals( diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java index 415acc5558b..850cefd7308 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java @@ -482,7 +482,7 @@ public void testJoinWithRelationSubquery() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalProject(sum=[$1], JOB=[$0])\n" - + " LogicalAggregate(group=[{0}], sum=[SUM($1)])\n" + + " LogicalAggregate(group=[{0}], sum=[CHECKED_LONG_SUM($1)])\n" + " LogicalProject(JOB=[$2], MGR=[$3])\n" + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMakeResultsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMakeResultsTest.java new file mode 100644 index 00000000000..8f2b61685f0 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMakeResultsTest.java @@ -0,0 +1,254 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; + +/** Logical-plan tests for the makeresults leading command (count path + format/data path). */ +public class CalcitePPLMakeResultsTest extends CalcitePPLAbstractTest { + public CalcitePPLMakeResultsTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + private void expectError(String ppl, String messageFragment) { + try { + getRelNode(ppl); + fail("expected an error for: " + ppl); + } catch (Exception e) { + String msg = String.valueOf(e.getMessage()); + assertTrue( + "expected message containing '" + messageFragment + "' but got: " + msg, + msg.contains(messageFragment)); + } + } + + @Test + public void testMakeResultsBare() { + RelNode root = getRelNode("makeresults"); + verifyLogical(root, "LogicalProject(@timestamp=[NOW()])\n LogicalValues(tuples=[[{ 0 }]])\n"); + } + + @Test + public void testMakeResultsCount() { + RelNode root = getRelNode("makeresults count=3"); + verifyLogical( + root, + "LogicalProject(@timestamp=[NOW()])\n" + + " LogicalValues(tuples=[[{ 0 }, { 1 }, { 2 }]])\n"); + } + + @Test + public void testMakeResultsCountZero() { + RelNode root = getRelNode("makeresults count=0"); + verifyLogical(root, "LogicalValues(tuples=[[]])\n"); + } + + @Test + public void testMakeResultsJson() { + RelNode root = + getRelNode( + "makeresults format=json data='[{\"name\":\"John\",\"age\":35,\"score\":3.5}," + + "{\"name\":\"Sarah\",\"age\":39,\"score\":4.0}]'"); + verifyLogical( + root, + "LogicalProject(@timestamp=[NOW()], name=[CAST($0):VARCHAR NOT NULL]," + + " age=[CAST($1):BIGINT NOT NULL], score=[CAST($2):REAL NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', 35, 3.5E0 }, { 'Sarah', 39, 4.0E0 }]])\n"); + } + + @Test + public void testMakeResultsTypedCsv() { + RelNode root = + getRelNode("makeresults format=csv data='name:string,age:int\nJohn,35\nSarah,39'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], age=[$1])\n" + + " LogicalValues(tuples=[[{ 'John', 35 }, { 'Sarah', 39 }]])\n"); + } + + @Test + public void testMakeResultsBareCsv() { + RelNode root = getRelNode("makeresults format=csv data='name,age\nJohn,35\nSarah,39'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], age=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', '35' }, { 'Sarah', '39' }]])\n"); + } + + @Test + public void testMakeResultsHeaderOnlyCsv() { + RelNode root = getRelNode("makeresults format=csv data='name,age'"); + verifyLogical(root, "LogicalValues(tuples=[[]])\n"); + } + + @Test + public void testMakeResultsCsvQuotedComma() { + RelNode root = getRelNode("makeresults format=csv data='name,note\nJohn,\"a,b\"'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], note=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', 'a,b' }]])\n"); + } + + @Test + public void testMakeResultsJsonBigIntKeepsPrecision() { + RelNode root = getRelNode("makeresults format=json data='[{\"n\":99999999999999999999}]'"); + verifyLogical( + root, + "LogicalProject(@timestamp=[NOW()], n=[CAST($0):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ '99999999999999999999' }]])\n"); + } + + @Test + public void testMakeResultsSerializesNestedJson() { + RelNode root = getRelNode("makeresults format=json data='[{\"a\":{\"x\":1},\"b\":[1,2]}]'"); + verifyLogical( + root, + "LogicalProject(@timestamp=[NOW()], a=[CAST($0):VARCHAR NOT NULL]," + + " b=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ '{\"x\":1}', '[1,2]' }]])\n"); + } + + @Test + public void testMakeResultsRejectsAllNullColumn() { + expectError("makeresults format=json data='[{\"a\":null},{\"a\":null}]'", "only null values"); + } + + @Test + public void testMakeResultsRejectsDataWithoutFormat() { + expectError("makeresults data='[{\"a\":1}]'", "format and data must be provided together"); + } + + @Test + public void testMakeResultsNegativeCountYieldsZeroRows() { + // A negative count silently yields zero rows rather than an error. + RelNode root = getRelNode("makeresults count=-1"); + verifyLogical(root, "LogicalValues(tuples=[[]])\n"); + } + + @Test + public void testMakeResultsRejectsCountOverCap() { + // count > 5000 is rejected cleanly, not with a Janino 64 KB codegen failure. + expectError("makeresults count=6000", "must not exceed 5000"); + } + + @Test + public void testMakeResultsRejectsCellBudget() { + // 50 columns x 120 rows = 6000 cells > 5000; a flat row cap would miss this. + expectError(csvData(50, 120), "cells (rows x columns)"); + } + + @Test + public void testMakeResultsAllowsAtCellBudget() { + // 50 columns x 100 rows = 5000 cells is exactly at budget and must be accepted. + assertNotNull(getRelNode(csvData(50, 100))); + } + + @Test + public void testMakeResultsRejectsOversizedCellValue() { + // A single value over the 60000 per-value guard is rejected, not a codegen error. + String wide = "x".repeat(60001); + expectError( + "makeresults format=csv data='c0\n" + wide + "'", "cell value must not exceed 60000"); + } + + @Test + public void testMakeResultsRejectsCsvRowWithMoreColumns() { + expectError( + "makeresults format=csv data='name,age\nJohn,35,extra'", "more columns than the header"); + } + + @Test + public void testMakeResultsCsvRowWithFewerColumnsPadsNull() { + RelNode root = getRelNode("makeresults format=csv data='name,age\nJohn,35\nSarah'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], age=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', '35' }, { 'Sarah', null }]])\n"); + } + + @Test + public void testMakeResultsCountAtCap() { + assertNotNull(getRelNode("makeresults count=5000")); + } + + @Test + public void testMakeResultsAllowsCellValueAtLimit() { + assertNotNull(getRelNode("makeresults format=csv data='c0\n" + "x".repeat(60000) + "'")); + } + + private static String csvData(int cols, int rows) { + StringBuilder sb = new StringBuilder("makeresults format=csv data='"); + for (int j = 0; j < cols; j++) sb.append(j == 0 ? "" : ",").append("c").append(j); + for (int r = 0; r < rows; r++) { + sb.append("\n"); + for (int j = 0; j < cols; j++) sb.append(j == 0 ? "" : ",").append("1"); + } + return sb.append("'").toString(); + } + + @Test + public void testMakeResultsRejectsCountOverflow() { + // T3: a count outside int range yields a clean validation error, not a raw + // NumberFormatException. + expectError("makeresults count=99999999999999", "not a valid integer"); + } + + @Test + public void testMakeResultsJsonUserTimestampWins() { + RelNode root = getRelNode("makeresults format=json data='[{\"@timestamp\":\"2020\",\"x\":1}]'"); + verifyLogical( + root, + "LogicalProject(@timestamp=[CAST($0):VARCHAR NOT NULL], x=[CAST($1):BIGINT NOT NULL])\n" + + " LogicalValues(tuples=[[{ '2020', 1 }]])\n"); + } + + @Test + public void testMakeResultsRejectsInvalidBoolean() { + expectError( + "makeresults format=csv data='active:boolean\nnot true'", "cannot parse \"not true\""); + } + + @Test + public void testMakeResultsAcceptsBooleanCaseInsensitive() { + RelNode root = getRelNode("makeresults format=csv data='active:boolean\nTRUE\nFalse'"); + verifyLogical(root, "LogicalValues(tuples=[[{ true }, { false }]])\n"); + } + + @Test + public void testMakeResultsRejectsUnterminatedQuote() { + expectError("makeresults format=csv data='name\n\"unterminated'", "unterminated quoted field"); + } + + @Test + public void testMakeResultsUniquifiesDuplicateCsvHeaders() { + RelNode root = getRelNode("makeresults format=csv data='name,name\nJohn,Doe'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], name0=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', 'Doe' }]])\n"); + } + + @Test + public void testMakeResultsRejectsBlankCsvHeader() { + expectError("makeresults format=csv data=',field\n1,2'", "blank column name"); + } + + @Test + public void testMakeResultsSkipsWhitespaceOnlyCsvLines() { + RelNode root = getRelNode("makeresults format=csv data='name,age\nJohn,35\n \nSarah,39'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], age=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', '35' }, { 'Sarah', '39' }]])\n"); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java index 21aa15c2e64..356e8a533af 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java @@ -222,6 +222,206 @@ public void failWithDuplicatedName() { } } + @Test + public void testRareShowPerc() { + String ppl = "source=EMP | rare showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; count=1; percent=7.142857\n" + + "JOB=ANALYST; count=2; percent=14.285714\n" + + "JOB=MANAGER; count=3; percent=21.428571\n" + + "JOB=CLERK; count=4; percent=28.571429\n" + + "JOB=SALESMAN; count=4; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithGroupBy() { + String ppl = "source=EMP | rare showperc=true JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3])\n" + + " LogicalFilter(condition=[<=($4, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2, $1)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($2):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($2) OVER (PARTITION BY $0)):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=MANAGER; count=1; percent=20.0\n" + + "DEPTNO=20; JOB=ANALYST; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=CLERK; count=2; percent=40.0\n" + + "DEPTNO=10; JOB=CLERK; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=MANAGER; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1; percent=33.333333\n" + + "DEPTNO=30; JOB=CLERK; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=MANAGER; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=SALESMAN; count=4; percent=66.666667\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`, `percent`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, `count`, `percent`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY `count` NULLS LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS" + + " DOUBLE) / CAST(SUM(COUNT(*)) OVER (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithoutShowCount() { + String ppl = "source=EMP | rare showcount=false showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; percent=7.142857\n" + + "JOB=ANALYST; percent=14.285714\n" + + "JOB=MANAGER; percent=21.428571\n" + + "JOB=CLERK; percent=28.571429\n" + + "JOB=SALESMAN; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercCustomField() { + String ppl = "source=EMP | rare percentfield='pct' showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], pct=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], pct=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " pct=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; count=1; pct=7.142857\n" + + "JOB=ANALYST; count=2; pct=14.285714\n" + + "JOB=MANAGER; count=3; pct=21.428571\n" + + "JOB=CLERK; count=4; pct=28.571429\n" + + "JOB=SALESMAN; count=4; pct=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `pct`\n" + + "FROM (SELECT `JOB`, `count`, `pct`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `pct`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithLimit() { + String ppl = "source=EMP | rare 1 showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Should show percentage relative to full dataset, not 100% + // PRESIDENT has 1 out of 14 total employees = 7.142857% + String expectedResult = "JOB=PRESIDENT; count=1; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testTop() { String ppl = "source=EMP | top JOB"; @@ -408,4 +608,205 @@ public void testTopUseNullFalse() { + "WHERE `_row_number_rare_top_` <= 10"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testTopShowPerc() { + String ppl = "source=EMP | top showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; count=4; percent=28.571429\n" + + "JOB=SALESMAN; count=4; percent=28.571429\n" + + "JOB=MANAGER; count=3; percent=21.428571\n" + + "JOB=ANALYST; count=2; percent=14.285714\n" + + "JOB=PRESIDENT; count=1; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithGroupBy() { + String ppl = "source=EMP | top showperc=true JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3])\n" + + " LogicalFilter(condition=[<=($4, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC, $1)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($2):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($2) OVER (PARTITION BY $0)):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=ANALYST; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=CLERK; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=MANAGER; count=1; percent=20.0\n" + + "DEPTNO=10; JOB=CLERK; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=MANAGER; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1; percent=33.333333\n" + + "DEPTNO=30; JOB=SALESMAN; count=4; percent=66.666667\n" + + "DEPTNO=30; JOB=CLERK; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=MANAGER; count=1; percent=16.666667\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`, `percent`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, `count`, `percent`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY `count` DESC NULLS FIRST, `JOB` NULLS LAST)" + + " `_row_number_rare_top_`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS" + + " DOUBLE) / CAST(SUM(COUNT(*)) OVER (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithoutShowCount() { + String ppl = "source=EMP | top showcount=false showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; percent=28.571429\n" + + "JOB=SALESMAN; percent=28.571429\n" + + "JOB=MANAGER; percent=21.428571\n" + + "JOB=ANALYST; percent=14.285714\n" + + "JOB=PRESIDENT; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercCustomField() { + String ppl = "source=EMP | top percentfield='pct' showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], pct=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], pct=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " pct=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; count=4; pct=28.571429\n" + + "JOB=SALESMAN; count=4; pct=28.571429\n" + + "JOB=MANAGER; count=3; pct=21.428571\n" + + "JOB=ANALYST; count=2; pct=14.285714\n" + + "JOB=PRESIDENT; count=1; pct=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `pct`\n" + + "FROM (SELECT `JOB`, `count`, `pct`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `pct`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithLimit() { + String ppl = "source=EMP | top 1 showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Should show percentage relative to full dataset, not 100% + // CLERK has 4 out of 14 total employees = 28.571429% + String expectedResult = "JOB=CLERK; count=4; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java index da9424c9c8f..60a81d90b23 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -42,24 +42,24 @@ private Node parse(String ppl) { @Test public void restHealthProjectsDeclaredColumns() { - Project project = - (Project) parse("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + Project project = (Project) parse("| rest \"/_cluster/health\" | fields response"); RestRelation rest = (RestRelation) project.getChild().get(0); SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); assertEquals("/_cluster/health", spec.getEndpoint()); // downstream fields compose on top of the rest row source. - assertEquals(2, project.getProjectList().size()); + assertEquals(1, project.getProjectList().size()); } @Test public void restReservedNameRoundTrips() { RestRelation rest = - (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); + (RestRelation) + parse("| rest \"/_cluster/health\" count=10 timeout=\"5s\" health=\"green\""); String reserved = rest.getTableQualifiedName().toString(); assertTrue(SystemIndexUtils.isRestSource(reserved)); SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); - assertEquals("/_cat/indices", spec.getEndpoint()); + assertEquals("/_cluster/health", spec.getEndpoint()); assertEquals(Integer.valueOf(10), spec.getCount()); assertEquals("5s", spec.getTimeout()); assertEquals("green", spec.getArgs().get("health")); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java index 66027839f8e..63cc0572453 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java @@ -86,7 +86,7 @@ public void testTimewrapDayProducesUnpivotedPlan() { + ":BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)])\n" + " LogicalSort(sort0=[$0], dir0=[ASC])\n" + " LogicalProject(@timestamp=[$0], sum(value)=[$1])\n" - + " LogicalAggregate(group=[{1}], sum(value)=[SUM($0)])\n" + + " LogicalAggregate(group=[{1}], sum(value)=[CHECKED_LONG_SUM($0)])\n" + " LogicalProject(value=[$1], @timestamp0=[SPAN($0, 6, 'h')])\n" + " LogicalFilter(condition=[AND(>=($0, TIMESTAMP('2024-07-01" + " 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2024-07-03 18:00:00':VARCHAR)), IS NOT" @@ -109,7 +109,8 @@ public void testTimewrapDaySparkSql() { + " `__base_offset__`, ((MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE" + " BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + " CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) / 86400 + 1 `__period__`\n" - + "FROM (SELECT SPAN(`@timestamp`, 6, 'h') `@timestamp`, SUM(`value`) `sum(value)`\n" + + "FROM (SELECT SPAN(`@timestamp`, 6, 'h') `@timestamp`," + + " SUM(`value`) `sum(value)`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` >= TIMESTAMP('2024-07-01 00:00:00') AND `@timestamp` <=" + " TIMESTAMP('2024-07-03 18:00:00') AND `value` IS NOT NULL\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrackingTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrackingTest.java new file mode 100644 index 00000000000..8469348f399 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrackingTest.java @@ -0,0 +1,266 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.opensearch.sql.executor.QueryType.PPL; + +import java.util.List; +import org.apache.calcite.plan.Contexts; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; +import org.junit.Before; +import org.junit.Test; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.CalcitePlanContext.NodeIdMapping; +import org.opensearch.sql.calcite.CalciteRelNodeVisitor; +import org.opensearch.sql.calcite.SysLimit; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.ppl.parser.AstStatementBuilder; + +public class CalcitePPLTrackingTest { + + private final Frameworks.ConfigBuilder config; + private final CalciteRelNodeVisitor planTransformer; + private final Settings settings; + private final DataSourceService dataSourceService; + private final PPLSyntaxParser pplParser = new PPLSyntaxParser(); + + public CalcitePPLTrackingTest() { + this.dataSourceService = mock(DataSourceService.class); + this.planTransformer = new CalciteRelNodeVisitor(dataSourceService); + this.settings = mock(Settings.class); + this.config = + Frameworks.newConfigBuilder() + .defaultSchema( + CalciteAssert.addSchema( + Frameworks.createRootSchema(true), + CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL)) + .programs(); + } + + @Before + public void init() { + doReturn(true).when(settings).getSettingValue(Settings.Key.CALCITE_ENGINE_ENABLED); + doReturn(true).when(settings).getSettingValue(Settings.Key.CALCITE_SUPPORT_ALL_JOIN_TYPES); + doReturn(true).when(settings).getSettingValue(Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED); + doReturn(-1).when(settings).getSettingValue(Settings.Key.PPL_JOIN_SUBSEARCH_MAXOUT); + doReturn(-1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + doReturn(false).when(dataSourceService).dataSourceExists(any()); + } + + private CalcitePlanContext createContext() { + config.context(Contexts.of(RelBuilder.Config.DEFAULT)); + return CalcitePlanContext.create(config.build(), SysLimit.fromSettings(settings), PPL); + } + + private Node plan(String query) { + final AstStatementBuilder builder = + new AstStatementBuilder( + new AstBuilder(query, settings), + AstStatementBuilder.StatementBuilderContext.builder().build()); + return builder.visit(pplParser.parse(query)); + } + + private RelNode getRelNode(String ppl, CalcitePlanContext context) { + Query query = (Query) plan(ppl); + planTransformer.analyze(query.getPlan(), context); + return context.relBuilder.build(); + } + + @Test + public void testTrackingProducesSameLogicalPlanAsNonTracking() { + String ppl = "source=EMP | eval a = 1 | fields EMPNO, a"; + + CalcitePlanContext withoutTracking = createContext(); + RelNode expected = getRelNode(ppl, withoutTracking); + + CalcitePlanContext withTracking = createContext(); + withTracking.setTrackingEnabled(true); + RelNode actual = getRelNode(ppl, withTracking); + + assertEquals(expected.explain().replace("\r\n", "\n"), actual.explain().replace("\r\n", "\n")); + } + + @Test + public void testTrackingDisabledProducesNoMappings() { + String ppl = "source=EMP | eval a = 1"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(false); + getRelNode(ppl, context); + + assertTrue(context.getNodeIdMappings().isEmpty()); + } + + @Test + public void testTrackingEvalRecordsMappings() { + String ppl = "source=EMP | eval a = 1"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + assertFalse(mappings.isEmpty()); + + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Eval mapping", astTypes.contains("Eval")); + } + + @Test + public void testTrackingFilterRecordsMappings() { + String ppl = "source=EMP | where SAL > 1000"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Filter mapping", astTypes.contains("Filter")); + } + + @Test + public void testTrackingSortRecordsMappings() { + String ppl = "source=EMP | sort SAL"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Sort mapping", astTypes.contains("Sort")); + } + + @Test + public void testTrackingMultipleCommandsRecordsMappings() { + String ppl = "source=EMP | where SAL > 1000 | eval bonus = SAL * 0.1 | sort SAL"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Filter mapping", astTypes.contains("Filter")); + assertTrue("Should contain Eval mapping", astTypes.contains("Eval")); + assertTrue("Should contain Sort mapping", astTypes.contains("Sort")); + } + + @Test + public void testTrackingMappingsHaveNonEmptyRelNodeIds() { + String ppl = "source=EMP | eval a = 1"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + for (NodeIdMapping mapping : context.getNodeIdMappings()) { + assertFalse( + "Mapping for " + mapping.astNodeType() + " should have non-empty RelNode IDs", + mapping.relNodeIds().isEmpty()); + } + } + + @Test + public void testTrackingProjectRecordsMappings() { + String ppl = "source=EMP | fields EMPNO, ENAME"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Project mapping", astTypes.contains("Project")); + } + + @Test + public void testTrackingAggregationRecordsMappings() { + String ppl = "source=EMP | stats count() by DEPTNO"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Aggregation mapping", astTypes.contains("Aggregation")); + } + + @Test + public void testTrackingMultipleCommandsProducesSameLogicalPlan() { + String ppl = "source=EMP | where SAL > 1000 | eval bonus = SAL * 0.1 | sort SAL | head 10"; + + CalcitePlanContext withoutTracking = createContext(); + RelNode expected = getRelNode(ppl, withoutTracking); + + CalcitePlanContext withTracking = createContext(); + withTracking.setTrackingEnabled(true); + RelNode actual = getRelNode(ppl, withTracking); + + assertEquals(expected.explain().replace("\r\n", "\n"), actual.explain().replace("\r\n", "\n")); + } + + @Test + public void testVisitChildrenCapturesSubtreeContribution() { + // visitChildren records a child's SUBTREE contribution (all RelNodes produced + // by that child and its descendants). For a multi-command pipeline, the mapping + // for Filter should include RelNodes from its own subtree (Relation + Filter itself). + String ppl = "source=EMP | where SAL > 1000 | eval bonus = SAL * 0.1"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + + // Relation is a leaf — should produce exactly 1 RelNode (the scan) + NodeIdMapping relationMapping = + mappings.stream().filter(m -> m.astNodeType().equals("Relation")).findFirst().orElseThrow(); + assertFalse( + "Relation (leaf) should produce at least one RelNode", + relationMapping.relNodeIds().isEmpty()); + + // Filter's subtree includes Relation beneath it, so visitChildren should + // capture more RelNode IDs for Filter than for Relation alone. + NodeIdMapping filterMapping = + mappings.stream().filter(m -> m.astNodeType().equals("Filter")).findFirst().orElseThrow(); + assertTrue( + "Filter subtree should produce more RelNodes than Relation alone", + filterMapping.relNodeIds().size() > relationMapping.relNodeIds().size()); + } + + @Test + public void testVisitChildrenRecordsAllChildrenSeparately() { + // visitChildren iterates over node.getChild() and records each one. + // For a pipeline with multiple commands, each command gets its own mapping entry. + String ppl = "source=EMP | where SAL > 1000 | sort SAL | head 5"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + + // Each command in the pipeline should have a separate mapping entry + assertTrue("Should record Relation", astTypes.contains("Relation")); + assertTrue("Should record Filter", astTypes.contains("Filter")); + assertTrue("Should record Sort", astTypes.contains("Sort")); + assertTrue("Should record Head", astTypes.contains("Head")); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d57ca8a69bb..9f874423712 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -81,7 +81,9 @@ import org.opensearch.sql.ast.tree.Join; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.exception.SemanticCheckException; @@ -739,6 +741,8 @@ public void testRareCommand() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -755,6 +759,8 @@ public void testRareCommandWithGroupBy() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); @@ -771,12 +777,50 @@ public void testRareCommandWithMultipleFields() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), field("b"))); } + @Test + public void testRareCommandWithShowPerc() { + assertEqual( + "source=t | rare showperc=true a", + rareTopN( + relation("t"), + CommandType.RARE, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + emptyList(), + field("a"))); + } + + @Test + public void testRareCommandWithShowPercAndGroupBy() { + assertEqual( + "source=t | rare showperc=true a by b", + rareTopN( + relation("t"), + CommandType.RARE, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + exprList(field("b")), + field("a"))); + } + @Test public void testTopCommandWithN() { assertEqual( @@ -788,6 +832,8 @@ public void testTopCommandWithN() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -804,6 +850,8 @@ public void testTopCommandWithoutNAndGroupBy() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -820,6 +868,8 @@ public void testTopCommandWithNAndGroupBy() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); @@ -836,6 +886,8 @@ public void testTopCommandWithMultipleFields() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), @@ -853,11 +905,49 @@ public void testTopCommandWithUseNullFalse() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(false))), exprList(field("b")), field("a"))); } + @Test + public void testTopCommandWithShowPerc() { + assertEqual( + "source=t | top showperc=true a", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + emptyList(), + field("a"))); + } + + @Test + public void testTopCommandWithShowPercAndGroupBy() { + assertEqual( + "source=t | top showperc=true a by b", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + exprList(field("b")), + field("a"))); + } + @Test public void testTopCommandWithLegacyFalse() { when(settings.getSettingValue(Key.PPL_SYNTAX_LEGACY_PREFERRED)).thenReturn(false); @@ -870,6 +960,8 @@ public void testTopCommandWithLegacyFalse() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(false))), exprList(field("b")), field("a"))); @@ -1107,6 +1199,12 @@ public void testDescribeCommand() { assertEqual("describe t", describe(mappingTable("t"))); } + @Test + public void testMakeResultsCommand() { + assertEqual("makeresults", new MakeResults(1)); + assertEqual("makeresults count=5", new MakeResults(5)); + } + @Test public void testDescribeMatchAllCrossClusterSearchCommand() { assertEqual("describe *:t", describe(mappingTable("*:t"))); @@ -1820,6 +1918,78 @@ public void testMalformedPipeProducesSyntaxError() { plan("source=t | invalidCmd |"); } + // Xyseries tests + + @Test + public void testXyseriesCommandBasic() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt")), + ": ", + null); + expected.attach(relation("t")); + assertEqual("source=t | xyseries url response in (\"200\", \"404\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandMultipleDataFields() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt"), field("method_cnt")), + ": ", + null); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries url response in (\"200\", \"404\") host_cnt, method_cnt", expected); + } + + @Test + public void testXyseriesCommandWithSep() { + Xyseries expected = + new Xyseries( + field("url"), field("response"), List.of("200"), List.of(field("host_cnt")), "-", null); + expected.attach(relation("t")); + assertEqual("source=t | xyseries sep=\"-\" url response in (\"200\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandWithFormat() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200"), + List.of(field("host_cnt")), + ": ", + "$VAL$+$AGG$"); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries format=\"$VAL$+$AGG$\" url response in (\"200\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandWithSepAndFormat() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt")), + "-", + "$AGG$_$VAL$"); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries sep=\"-\" format=\"$AGG$_$VAL$\" url response in (\"200\", \"404\")" + + " host_cnt", + expected); + } + @Test public void testUnionWithSubsearches() { plan("| union [search source=t1 | where age > 30] " + "[search source=t2 | where age < 20]"); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 1822b690feb..00a3693594e 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -36,6 +36,11 @@ public void testSearchCommand() { assertEquals("source=table identifier = ***", anonymize("search source=t a=1")); } + @Test + public void testMakeResultsCommand() { + assertEquals("makeresults", anonymize("makeresults count=5")); + } + @Test public void testTableFunctionCommand() { assertEquals( @@ -334,6 +339,29 @@ public void testChartCommandOverBy() { anonymize("source=t | chart sum(amount) over gender by age")); } + @Test + public void testXyseriesCommand() { + assertEquals( + "source=table | stats avg(identifier) by identifier,identifier" + + " | xyseries identifier identifier in (***) identifier", + anonymize( + "source=t | stats avg(balance) by gender, state" + + " | xyseries state gender in (\"F\",\"M\") avg_balance")); + } + + @Test + public void testXyseriesCommandWithOptions() { + assertEquals( + "source=table | stats avg(identifier),max(identifier) by identifier,identifier" + + " | xyseries sep=*** format=*** identifier identifier in (***)" + + " identifier,identifier", + anonymize( + "source=t | stats avg(balance) as avg_balance, max(balance) as max_balance" + + " by gender, state" + + " | xyseries sep=\"_\" format=\"$AGG$_$VAL$\" state gender" + + " in (\"F\",\"M\") avg_balance, max_balance")); + } + // todo, sort order is ignored, it doesn't impact the log analysis. @Test public void testSortCommandWithOptions() { @@ -413,20 +441,38 @@ public void testTopCommandWithNAndGroupBy() { public void testRareCommandWithGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | rare 10 countield='count' showcount=true usenull=true identifier by" - + " identifier", + "source=table | rare 10 countfield='count' showcount=true percentfield='percent'" + + " showperc=false usenull=true identifier by identifier", anonymize("source=t | rare a by b")); } + @Test + public void testRareCommandWithShowPercWithCalCite() { + when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); + assertEquals( + "source=table | rare 10 countfield='count' showcount=true percentfield='percent'" + + " showperc=true usenull=true identifier", + anonymize("source=t | rare showperc=true a ")); + } + @Test public void testTopCommandWithNAndGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | top 1 countield='count' showcount=true usenull=true identifier by" - + " identifier", + "source=table | top 1 countfield='count' showcount=true percentfield='percent'" + + " showperc=false usenull=true identifier by identifier", anonymize("source=t | top 1 a by b")); } + @Test + public void testTopCommandWithShowPercWithCalcite() { + when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); + assertEquals( + "source=table | top 1 countfield='count' showcount=true percentfield='percent'" + + " showperc=true usenull=true identifier by identifier", + anonymize("source=t | top 1 showperc=true a by b")); + } + @Test public void testAndExpression() { assertEquals( diff --git a/release-notes/opensearch-sql.release-notes-3.8.0.0.md b/release-notes/opensearch-sql.release-notes-3.8.0.0.md new file mode 100644 index 00000000000..8dbff4fd2c0 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-3.8.0.0.md @@ -0,0 +1,79 @@ +## Version 3.8.0 Release Notes + +Compatible with OpenSearch and OpenSearch Dashboards version 3.8.0 + +### Features + +* Add PPL `xyseries` command for pivoting row-oriented grouped results into wide tables ([#5343](https://github.com/opensearch-project/sql/pull/5343)) +* Add PPL `timewrap` command for time-period comparison over timechart output ([#5241](https://github.com/opensearch-project/sql/pull/5241)) +* Add PPL `foreach` command for iterating over field lists, multivalue fields, and JSON arrays ([#5613](https://github.com/opensearch-project/sql/pull/5613)) +* Add PPL `makeresults` command for generating in-memory rows without an index scan ([#5622](https://github.com/opensearch-project/sql/pull/5622)) + +### Enhancements + +* Anonymize `xyseries` command and mark it as experimental in documentation ([#5643](https://github.com/opensearch-project/sql/pull/5643)) +* Suggest similar field names in 'field not found' error messages ([#5402](https://github.com/opensearch-project/sql/pull/5402)) +* Support `constant_keyword` field type in PPL, treating it as a string ([#5639](https://github.com/opensearch-project/sql/pull/5639)) +* Decouple Calcite PPL planning from ExprType, operating on RelDataType directly ([#5633](https://github.com/opensearch-project/sql/pull/5633)) +* Support bare-field join criteria shorthand (`join on `) in PPL ([#5517](https://github.com/opensearch-project/sql/pull/5517)) +* Classify unsupported-feature errors as client errors (4xx) on the SQL path ([#5569](https://github.com/opensearch-project/sql/pull/5569)) +* Reject unsupported output formats on the analytics-engine route with a 4xx error ([#5570](https://github.com/opensearch-project/sql/pull/5570)) +* Widen narrow integer operands in PPL arithmetic to prevent silent overflow ([#5603](https://github.com/opensearch-project/sql/pull/5603)) +* Add configurable expression depth limit during AST building to prevent stack overflow ([#5602](https://github.com/opensearch-project/sql/pull/5602)) +* Add `json_tree` machine-readable explain format accessible via `_explain?format=json_tree` ([#5576](https://github.com/opensearch-project/sql/pull/5576)) +* Onboard new backport-pr reusable GitHub workflow ([#5586](https://github.com/opensearch-project/sql/pull/5586)) +* Return all columns including struct and nested fields when using `head` command ([#5518](https://github.com/opensearch-project/sql/pull/5518)) +* Bring `CalcitePPLBasicIT` to parity on the analytics-engine route ([#5542](https://github.com/opensearch-project/sql/pull/5542)) +* Bring `CalciteWhereCommandIT` to parity on the analytics-engine route ([#5546](https://github.com/opensearch-project/sql/pull/5546)) +* Stabilize order-dependent PPL ITs with explicit sort for multi-shard analytics runs ([#5537](https://github.com/opensearch-project/sql/pull/5537)) +* Align `DateTimeComparisonIT` today's date computation to UTC for analytics-engine compatibility ([#5543](https://github.com/opensearch-project/sql/pull/5543)) +* Fix NPE on `case()` with incompatible branch types, returning a clean 400 error ([#5575](https://github.com/opensearch-project/sql/pull/5575)) +* Fix NPE when `rex` sits inside `appendcol` subsearch for the analytics engine ([#5574](https://github.com/opensearch-project/sql/pull/5574)) + +### Bug Fixes + +* Fix `ClassCastException` in PPL multisearch on indexes with `@timestamp` alias field ([#5577](https://github.com/opensearch-project/sql/pull/5577)) +* Fix PPL `foreach` JSON array type coercion to handle non-numeric elements gracefully ([#5637](https://github.com/opensearch-project/sql/pull/5637)) +* Detect long (BIGINT) arithmetic overflow instead of silently wrapping ([#5604](https://github.com/opensearch-project/sql/pull/5604)) +* Preserve SQL-layer profiling alongside the analytics-engine profile ([#5571](https://github.com/opensearch-project/sql/pull/5571)) +* Propagate request-task cancellation into the analytics PPL route ([#5563](https://github.com/opensearch-project/sql/pull/5563)) +* Return 4xx instead of 500 for unsupported window functions ([#5587](https://github.com/opensearch-project/sql/pull/5587)) +* Fix `SHOW`/`DESCRIBE` statement routing under `cluster.pluggable.dataformat` setting ([#5528](https://github.com/opensearch-project/sql/pull/5528)) +* Handle opaque `NullPointerException` for unresolvable alias-type field path with a clear error ([#5536](https://github.com/opensearch-project/sql/pull/5536)) +* Fix invalid field or index error misclassified as internal 500 failures ([#5532](https://github.com/opensearch-project/sql/pull/5532)) +* Fix `GROUP BY` expression resolution in `SELECT`/`HAVING`/`ORDER BY` ([#5548](https://github.com/opensearch-project/sql/pull/5548)) +* Fix SQL window functions with `ORDER BY`/`LIMIT` on unified query path ([#5592](https://github.com/opensearch-project/sql/pull/5592)) +* Fix dedup field name mapping to handle alias collision when rename and eval resolve to the same source field ([#5593](https://github.com/opensearch-project/sql/pull/5593)) +* Allow partial pushdown for semi-scripted predicates so pushable filters are not blocked by unsupported ones ([#5565](https://github.com/opensearch-project/sql/pull/5565)) +* Gracefully handle malformed documents in result scanning instead of crashing ([#5618](https://github.com/opensearch-project/sql/pull/5618)) +* Honor PPL `fetch_size` on the analytics-engine route ([#5567](https://github.com/opensearch-project/sql/pull/5567)) +* Strip analytics-engine-unsupported fields from test data and exclude affected ITs ([#5541](https://github.com/opensearch-project/sql/pull/5541)) +* Repair two pre-existing IT failures on main (error type assertion and explain flake) ([#5545](https://github.com/opensearch-project/sql/pull/5545)) + +### Infrastructure + +* Bring `CalciteBinCommandIT` and `CalciteMultisearchCommandIT` to parity on the analytics-engine route ([#5551](https://github.com/opensearch-project/sql/pull/5551)) +* Bring `CalcitePPLEnhancedCoalesceIT` to parity on the analytics-engine route ([#5552](https://github.com/opensearch-project/sql/pull/5552)) +* Bring `CalcitePPLJoinIT` to parity on the analytics-engine route ([#5554](https://github.com/opensearch-project/sql/pull/5554)) +* Stabilize `CalcitePPLConditionBuiltinFunctionIT` on the analytics-engine route ([#5556](https://github.com/opensearch-project/sql/pull/5556)) +* Stabilize `CalciteStreamstatsCommandIT` on the analytics-engine route ([#5582](https://github.com/opensearch-project/sql/pull/5582)) +* Stabilize PPL ITs on the analytics-engine route (array/map-path/datatype/basic) ([#5562](https://github.com/opensearch-project/sql/pull/5562)) +* Stabilize PPL ITs on the analytics-engine route (case/string/full-text/like/appendpipe) ([#5561](https://github.com/opensearch-project/sql/pull/5561)) +* Stabilize PPL ITs on the analytics-engine route (percentile/float/datetime/json/dedup/union/rename/chart) ([#5564](https://github.com/opensearch-project/sql/pull/5564)) +* Stabilize PPL ITs on the analytics-engine route (sort/streamstats/IP-UDT/metadata/strip-verifier) ([#5566](https://github.com/opensearch-project/sql/pull/5566)) +* Stabilize subquery PPL ITs on the analytics-engine route ([#5555](https://github.com/opensearch-project/sql/pull/5555)) +* Recover concrete schema type for ANY-typed columns on the analytics route (fixes eval max/min) ([#5557](https://github.com/opensearch-project/sql/pull/5557)) +* Fix SQL IT test queries, assertions, and data for engine-agnostic compatibility ([#5584](https://github.com/opensearch-project/sql/pull/5584)) +* Gate analytics-engine incompatible IT tests with capability matrix annotations ([#5585](https://github.com/opensearch-project/sql/pull/5585)) +* Decouple IT from execution backend with capability-based gating ([#5560](https://github.com/opensearch-project/sql/pull/5560)) +* Fix doctest job-scheduler dependency resolution for 3.8.0 ([#5540](https://github.com/opensearch-project/sql/pull/5540)) +* Bump Apache Calcite 1.41.0 → 1.42.0 (CVE-2026-46718) ([#5619](https://github.com/opensearch-project/sql/pull/5619)) +* Bump `get-ci-image-tag.yml` ref to SHA-pinned opensearch-build commit to unblock CI ([#5583](https://github.com/opensearch-project/sql/pull/5583)) +* Case test patches for missed optimizations ([#5531](https://github.com/opensearch-project/sql/pull/5531)) +* Use engine-zone today in `DateTimeFunctionIT` now()-based assertions ([#5553](https://github.com/opensearch-project/sql/pull/5553)) +* Update datetime tests to stay within analytics-engine epoch bounds ([#5534](https://github.com/opensearch-project/sql/pull/5534)) + +### Maintenance + +* Fix flaky TPC-H Q15 floating-point assertion ([#5629](https://github.com/opensearch-project/sql/pull/5629)) +* Fix lychee link checker ([#5451](https://github.com/opensearch-project/sql/pull/5451)) diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh new file mode 100755 index 00000000000..11e77f1e4ba --- /dev/null +++ b/scripts/ppl-lint-rule-validation.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Local developer entry point for the PPL lint rule validation contract. +# +# Runs both halves of the cross-repository check from a SQL checkout, in the same +# order as CI (design §3.1): +# 1. Backend: runs the Gradle integration test against a live /_plugins/_ppl +# endpoint on the SQL plugin built from this checkout, and — while the +# cluster is alive — exports the candidate runtime grammar bundle +# (ppl-grammar-bundle.json), a target manifest (target.json), and the +# observed backend report (backend-report.json). +# 2. Detector: bootstraps an OpenSearch-Dashboards (OSD) checkout, deserializes +# the candidate bundle through OSD's headless lint API, runs the real +# detectors against the same queries, and asserts the detector-vs-backend +# differential. +# +# The backend half must run first: the detector half lints against the bundle it +# exports. Use SKIP_BACKEND=1 only if you already have the three artifacts. +# +# Usage: +# # OSD main detector check plus SQL backend IT (fetches OSD into .ci/) +# ./scripts/ppl-lint-rule-validation.sh +# +# # Reuse an existing OSD checkout (skips clone + bootstrap if node_modules present) +# OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh +# +# # Reproduce a CI run against a specific OSD revision +# OSD_REF= ./scripts/ppl-lint-rule-validation.sh +# +# # Skip one half (detector needs the backend artifacts to exist already) +# SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh +# SKIP_DETECTOR=1 ./scripts/ppl-lint-rule-validation.sh +# +# # Run the full nightly corpus (all rules + coverage assertion) +# PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh +# +# # Run the same corpus through composite/Parquet + DataFusion +# RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh +# +# # Pass local analytics plugin ZIP overrides through to Gradle +# RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh \ +# -PanalyticsEngineZip=/path/to/analytics-engine.zip + +set -euo pipefail + +SQL_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$SQL_ROOT" + +OSD_REPO_URL="${OSD_REPO_URL:-https://github.com/opensearch-project/OpenSearch-Dashboards.git}" +OSD_REF="${OSD_REF:-main}" +DEFAULT_OSD_CHECKOUT="$SQL_ROOT/.ci/OpenSearch-Dashboards" +CONTRACT_DIR="$SQL_ROOT/integ-test/src/test/resources/ppl-lint/contracts" +DETECTOR_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" +IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" +# pr (fast, blocking subset) or nightly (full corpus + coverage assertion). +PPL_LINT_SCHEDULE="${PPL_LINT_SCHEDULE:-pr}" +RUN_ANALYTICS="${RUN_ANALYTICS:-0}" + +# Candidate artifacts the backend half exports and the detector half consumes. +GRAMMAR_BUNDLE="$SQL_ROOT/ppl-grammar-bundle.json" +TARGET_MANIFEST="$SQL_ROOT/target.json" +BACKEND_REPORT="$SQL_ROOT/backend-report.json" +DETECTOR_REPORT="$SQL_ROOT/detector-report.json" + +log() { echo "[ppl-lint-rule-validation] $*"; } + +run_backend() { + local backend="standard" + local gradle_args=( + :integ-test:integTest + --tests "$IT_CLASS" + ) + if [[ "$RUN_ANALYTICS" == "1" ]]; then + backend="analytics" + gradle_args=(:integ-test:analyticsEnginePplLintIT) + # The checked-in schema-v3 contracts intentionally have no analytics + # oracles yet. Execute them once and retain their raw observations without + # borrowing the standard route's oracle. + gradle_args+=(-Dppl.lint.observe.only=true) + fi + + log "Running $backend backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" + gradle_args+=( + -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" + -Dppl.lint.execution_backend="$backend" + -Dppl.lint.sql_sha="$(git rev-parse HEAD)" + -Dppl.lint.report="$BACKEND_REPORT" + -Dppl.lint.grammar.bundle="$GRAMMAR_BUNDLE" + -Dppl.lint.target="$TARGET_MANIFEST" + ) + ./gradlew "${gradle_args[@]}" "$@" + log "Backend integration test passed. Exported: $(basename "$GRAMMAR_BUNDLE"), $(basename "$TARGET_MANIFEST")." +} + +run_detector() { + local osd_checkout="$1" + + if [[ ! -f "$GRAMMAR_BUNDLE" ]]; then + log "ERROR: $GRAMMAR_BUNDLE not found. Run the backend half first (do not set SKIP_BACKEND=1)." + exit 2 + fi + + if [[ ! -d "$osd_checkout/node_modules" ]]; then + log "Bootstrapping OSD at $osd_checkout (this can take a while)..." + (cd "$osd_checkout" && yarn osd bootstrap) + else + log "Reusing bootstrapped OSD at $osd_checkout (node_modules present)." + fi + + log "Running detector validation against the candidate bundle (schedule=$PPL_LINT_SCHEDULE)..." + ( + cd "$osd_checkout" + PPL_LINT_CONTRACT_DIR="$CONTRACT_DIR" \ + PPL_LINT_SCHEDULE="$PPL_LINT_SCHEDULE" \ + PPL_LINT_GRAMMAR_BUNDLE="$GRAMMAR_BUNDLE" \ + PPL_LINT_TARGET_MANIFEST="$TARGET_MANIFEST" \ + PPL_LINT_BACKEND_REPORT="$BACKEND_REPORT" \ + PPL_LINT_REPORT="$DETECTOR_REPORT" \ + PPL_LINT_OBSERVE_ONLY="$RUN_ANALYTICS" \ + PPL_LINT_OBSERVE_ANALYTICS="$RUN_ANALYTICS" \ + node -r ./src/setup_node_env "$DETECTOR_SCRIPT" + ) + log "Detector validation passed." +} + +if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then + run_backend "$@" +else + log "SKIP_BACKEND=1 — skipping the SQL backend integration test (using existing artifacts)." +fi + +if [[ "${SKIP_DETECTOR:-0}" != "1" ]]; then + if [[ -n "${OSD_SOURCE_PATH:-}" ]]; then + OSD_CHECKOUT="$(cd "$OSD_SOURCE_PATH" && pwd)" + log "Using existing OSD checkout: $OSD_CHECKOUT" + else + OSD_CHECKOUT="$DEFAULT_OSD_CHECKOUT" + if [[ ! -d "$OSD_CHECKOUT/.git" ]]; then + log "Cloning OSD ($OSD_REF) into $OSD_CHECKOUT ..." + mkdir -p "$(dirname "$OSD_CHECKOUT")" + git clone --depth 1 --branch "$OSD_REF" "$OSD_REPO_URL" "$OSD_CHECKOUT" 2>/dev/null || + git clone "$OSD_REPO_URL" "$OSD_CHECKOUT" + fi + log "Checking out OSD ref: $OSD_REF" + git -C "$OSD_CHECKOUT" fetch --depth 1 origin "$OSD_REF" 2>/dev/null || true + git -C "$OSD_CHECKOUT" checkout "$OSD_REF" 2>/dev/null || + git -C "$OSD_CHECKOUT" checkout FETCH_HEAD + fi + + OSD_SHA="$(git -C "$OSD_CHECKOUT" rev-parse HEAD)" + log "OSD revision under test: $OSD_SHA" + + run_detector "$OSD_CHECKOUT" +else + log "SKIP_DETECTOR=1 — skipping the OSD detector contract." +fi + +log "Done." diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md new file mode 100644 index 00000000000..769435da88b --- /dev/null +++ b/scripts/ppl-lint/README.md @@ -0,0 +1,518 @@ +# PPL lint rule validation + +A cross-repository GitHub Actions check that proves the OpenSearch Dashboards +(OSD) PPL lint detectors still agree with the SQL backend on the **same +candidate runtime grammar** built by a SQL pull request. + +PPL language behavior lives in SQL; PPL lint detectors live in OSD. A SQL change +can silently invalidate an OSD rule (a parser refactor stops a detector matching, +or a semantic change makes a flagged query valid) without touching OSD. Neither +repository's own unit tests catch that. This check does. + +- **Multi-version design:** [`docs/dev/ppl-lint-runtime-compatibility-ci-design.md`](../../docs/dev/ppl-lint-runtime-compatibility-ci-design.md) +- **Deferred analytics design:** [`docs/dev/ppl-lint-analytics-engine-ci-validation.md`](../../docs/dev/ppl-lint-analytics-engine-ci-validation.md) +- **Workflow:** [`.github/workflows/ppl-lint-rule-validation.yml`](../../.github/workflows/ppl-lint-rule-validation.yml) +- **Contracts:** [`integ-test/src/test/resources/ppl-lint/contracts/`](../../integ-test/src/test/resources/ppl-lint/contracts) + +## The pipeline + +Three jobs run in a line; artifacts are the only bridge between them. + +``` +backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.json)──▶ + detector-validation ──▶ validation-result (the single required check) +``` + +1. **backend-validation** (OpenSearch CI container). Builds the SQL PR, starts + the Gradle test cluster, runs each contract's trigger/control queries against + `POST /_plugins/_ppl`, and — while the cluster is alive — exports: + - `ppl-grammar-bundle.json` — the candidate runtime grammar (`GET /_plugins/_ppl/_grammar`); + - `target.json` — schema-v2 engine, grammar, execution-backend, storage, and route identity; + - `backend-report.json` — the observed HTTP behavior and execution backend per query. +2. **detector-validation** (`ubuntu-latest`). Checks out and bootstraps OSD as a + Node code dependency (no OSD server, no Monaco, no browser), then runs + [`run-frontend-contract.mjs`](run-frontend-contract.mjs). That runner + deserializes the candidate bundle through OSD's production headless APIs and + runs each query with either the real lint detectors or the shared runtime + syntax listener on the **candidate** grammar. It then asserts the + frontend-vs-backend differential. +3. **validation-result**. `if: always()`, `needs: [backend-validation, + detector-validation]`. Fails unless both succeeded — so a skipped detector + (because the backend failed first) still reds the check instead of looking + green. It writes the per-rule PR summary and uploads `run-manifest.json`. This + is the **only** job repo admins pin to branch protection. + +## Workflow inputs and modes + +| Trigger | Mode | OSD ref | Enforcing? | +| --- | --- | --- | --- | +| `pull_request` | SQL PR validation | `main` | **Yes** — the required check | +| `workflow_dispatch` (`osd_ref`) | OSD-branch evidence | the given commit/branch | No — pre-merge evidence only | +| `schedule` (nightly) | full corpus + coverage | `main` | No | + +The required PR corpus contains 12 detector contracts. The nightly mode runs +the same active corpus plus four dormant report-only contracts across the +supported version and execution-backend matrix. + +`workflow_dispatch` inputs: + +- `osd_repo` — the OSD repository to check out, for validating an unmerged change + that lives on a fork. Defaults to `opensearch-project/OpenSearch-Dashboards`. + The `osd_ref` must exist in this repo (a purely local commit cannot be fetched). +- `osd_ref` — an OSD commit or branch to validate instead of `main`. Resolved to + an immutable commit SHA and recorded in the run manifest. A manual run **cannot** + satisfy branch protection; merge the OSD change first, then rerun the required + `pull_request` check against OSD `main`. +- `schedule` — `pr` (reviewed blocking contracts) or `nightly` (all active contracts). + +To validate an OSD change that is not yet merged, push it to a branch on your OSD +fork and dispatch with `osd_repo=/OpenSearch-Dashboards` and +`osd_ref=`. + +## Local reproduction + +From the SQL checkout: + +```bash +# Backend IT (exports the bundle) then detector check against OSD main. +./scripts/ppl-lint-rule-validation.sh + +# Reuse an already-bootstrapped OSD checkout. +OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh + +# Reproduce a specific CI run's OSD revision (from run-manifest.json). +OSD_REF= ./scripts/ppl-lint-rule-validation.sh + +# Full nightly corpus + coverage assertion. +PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh + +# Run the corpus through the full composite/Parquet + DataFusion stack. +# The published default stack is Linux/x64; other platforms need compatible +# local plugin artifacts and -PnativeLibPath. +RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh + +# Use locally built analytics plugins (all trailing arguments pass to Gradle). +RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh \ + -PanalyticsEngineZip=/path/to/analytics-engine.zip \ + -PnativeLibPath=/path/to/native/release + +# Re-run only one half (detector needs the backend artifacts to exist). +SKIP_DETECTOR=1 ./scripts/ppl-lint-rule-validation.sh +SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh +``` + +The backend half writes `ppl-grammar-bundle.json`, `target.json`, and +`backend-report.json` to the SQL repo root; the detector half consumes them and +writes `detector-report.json`. + +### Runner environment contract + +`run-frontend-contract.mjs` is run from inside the OSD checkout with +`node -r ./src/setup_node_env` and reads: + +| Env var | Meaning | +| --- | --- | +| `PPL_LINT_CONTRACT_DIR` | directory of `*.spec.json` + `manifest.json` | +| `PPL_LINT_SCHEDULE` | `pr` or `nightly` | +| `PPL_LINT_SURFACE` | `runtime-bundle` (default) or explicit `compiled-simplified` | +| `PPL_LINT_ENGINE_MODE` | optional `calcite` or `legacy` identity for compatibility filtering/context | +| `PPL_LINT_APPLICABLE_ONLY` | `1` omits contracts excluded by surface, version, or engine mode | +| `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required on the runtime surface) | +| `PPL_LINT_TARGET_MANIFEST` | schema-v2 `target.json` (engine, grammar, execution backend, and storage identity) | +| `PPL_LINT_BACKEND_REPORT` | `backend-report.json` (enables the differential) | +| `PPL_LINT_REPORT` | where to write `detector-report.json` | +| `PPL_LINT_CONTRACT_FILE` | (optional) run a single spec instead of the dir | + +## Contract format (schema v3 and v4) + +One JSON file per rule or syntax feature under `contracts/`, listed in +`manifest.json`. Each file has `channel: "lint"|"syntax"` (missing defaults to +`lint`), a top-level `queries` map, and version-scoped `expectations[]`. +`suppression-control` is syntax-only: the frontend must retain a raw syntax error +without producing the contracted friendly rewrite. + +```jsonc +{ + "schemaVersion": 4, + "ruleId": "union-min-datasets", + "channel": "lint", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "wiring": { "detector": "union-min-datasets", "enabled": true, "severity": "error", ... }, + "backendFixture": { "indices": ["ACCOUNT"], "clusterSettings": { "calcite": true, "calciteFallback": false } }, + "frontendContext": { "isCalcite": true }, + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { "role": "trigger", "query": "| union [ source={{index}} ]" }, + "union-two-datasets-control": { "role": "control", "query": "| union [ source={{index}} ] [ source={{index}} ]" } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "frontend": { "count": 1, "severity": "error" }, + "backends": { + "standard": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } }, + "analytics": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } } + } + }, + "union-two-datasets-control": { + "frontend": { "count": 0 }, + "backends": { + "standard": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } }, + "analytics": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + } + ] +} +``` + +Legacy lint expectations using `detectorCount`, `severity`, and `matchMessage` +normalize to the same internal frontend oracle. Syntax expectations use +`frontend.code`, `fixText`, `rawMessage`, and `totalErrors`. + +Schema v3's `backend` is read only as `backends.standard`; it is never an +implicit analytics oracle. Schema v4's `backends` selects the configured +`standard` or `analytics` execution backend. Missing analytics oracles are +recorded as unscored coverage during observation, including the raw backend +response needed to review a schema-v4 oracle, and are fatal before promotion to +the required check. The selected expectation must contain exactly the same query +names as the top-level `queries` map. + +Each backend `kind` is one of `rejection` (contracted 4xx + error type/reason), +`result-shape` (200 with datarow expectations), or `advisory` (soft 200-only +oracle). `not-applicable` requires a reason, owner, and tracking issue. When a behavior +changes in a new version, keep **both** version-scoped +expectations so the nightly matrix proves the rule still fires on the old version +while the candidate check proves the fix on the new one. + +### Pitfall: do not write pipe-first (`| command …`) trigger queries + +The detector half and the backend half must run the **byte-identical** query +(design's "Same queries" requirement). OSD's runtime lint path prepends a +synthetic `source=t ` prefix to any query that starts with a pipe, so linting +`| union [ source=idx ]` actually parses `source=t | union [ source=idx ]` — a +valid *mid-pipeline* union whose implicit upstream dataset makes the detector +stay silent. The backend, receiving the raw pipe-first query, still rejects it. +The two halves then disagree even though nothing is wrong. Write triggers in a +**query-initial** form (`union [ source=idx ]`, `multisearch [ search source=idx ]`) +that both sides accept verbatim. Until SQL emits `pipeStartRuleIndex` in the +grammar bundle (design §6, D-pipe), a pipe-first trigger with a distinct start +rule cannot be validated end to end. + +### The enforced set + +`manifest.json` partitions the corpus: + +- `enforced` — the six reviewed detector error contracts with deterministic + backend behavior. +- `defaultError` — every rule that ships **enabled at error severity** in OSD's + `rules_catalog.json`; it contains exactly six detector rules. +- `requiredSyntaxFeatures` — reserved for future syntax compatibility contracts; + currently empty. +- `nonEnforcing` — oracle-quality classification for warning, info, advisory, + and result-shape contracts. Scheduling determines whether a contract runs. +- `dormantContracts` — four preserved default-off detector contracts. They do + not count as active shipping coverage and explicitly force-enable their rule. + +The `enforced` / `nonEnforcing` split describes **oracle quality and review +status, not blocking behavior**. + +## Multi-version validation + +The check above validates **one** engine: the build from the PR. But a lint rule +ships to every user, and each user's cluster is on whatever version they run. A +rule that is correct on `main` can be a false positive or false negative on a +released cluster, and the single-version check cannot see it. + +[`ppl-lint-multiversion-validation.yml`](../../.github/workflows/ppl-lint-multiversion-validation.yml) +validates every active shipping detector against three planned configurations +and reports **what to change** when one disagrees: + +- the OSD compiled-simplified fallback grammar against OpenSearch 2.19.6; +- the runtime grammar exported by the highest official GA release at or below + the normalized SQL PR target; +- the runtime grammar exported by the SQL PR build. + +``` +plan configurations + ├── observe 2.19.6 backend (compiled comparison) + ├── observe latest eligible GA + export runtime grammar + └── observe PR build + export runtime grammar +aggregate rule compatibility (one OSD bootstrap, three detector passes) + └── aggregate-compatibility.mjs → 36-cell schema-v3 report +``` + +Released legs run official `opensearchproject/opensearch:` images, +which bundle the matching `opensearch-sql` plugin, so no old branch is built. +The plan reads the default `opensearch.version` from `build.gradle`, removes its +prerelease/build suffix, and selects the highest exact-semver OpenSearch tag at +or below it. The `pr-build` leg uses a Gradle test cluster. All legs run the +**same** contract oracle (`PplLintRuleValidationIT`) with +`-Dppl.lint.observe.only=true`, which records real behavior instead of asserting +against expectations — on an older engine a mismatch is the signal being +collected, not a broken run. + +The 2.19.6 leg does not request a runtime bundle. Its backend observations are +joined to a detector pass over OSD's checked-in simplified grammar. Rules with +`grammarSurface: runtime-bundle` are `n/a (surface)` there; rules outside their +`wiring.appliesTo` version range are `n/a (version)`. Surface takes precedence +when both exclusions apply. Analytics, syntax-channel, dormant-rule, discovery, +and AI action tests are not part of this workflow. + +Observation jobs do not fail on compatibility differences. A failed or missing +observation remains a planned column and becomes `inconclusive`. The final +`Aggregate rule compatibility` job writes the complete expected-versus-actual +table and `drift-report.json`, uploads both report and evidence, and only then +fails for drift or inconclusive in-scope cells. + +### What a drift report tells you + +Every finding names a drift class, the evidence, and one remediation action: + +| Action | When | What you change | +| --- | --- | --- | +| `scope-rule-version` | every contracted trigger is now accepted and controls prove support | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` | +| `narrow-detector` | only some contracted triggers are now accepted | keep the version in scope and narrow the detector to invalid forms | +| `update-detector` | the detector regressed, went too broad, or its grammar anchor was renamed | the rule's detector `.ts` (named in the finding) | +| `update-contract` | the linter is right and only the pinned expectation is stale | the `expectations[]` entry for that version | +| `fix-test-leg` | a detector/backend row or target identity is missing or errored | repair or rerun the test leg before changing product behavior | + +The schema-v3 report consolidates query symptoms into four rule/configuration +classifications: `detector-regression`, `full-engine-relaxation`, +`partial-engine-relaxation`, and `contract-drift`. Missing or errored evidence +is `inconclusive`, not a compatibility classification. + +#### Full vs partial relaxation: scope the rule, or narrow the detector? + +When an engine starts accepting a query a rule flags, the fix depends on a question +a single query cannot answer: is the behavior **fully** gone on that version, or +only **partially**? + +- **Every trigger relaxed** → `full-engine-relaxation`, action + `scope-rule-version`. Nothing + the rule claims is still true on that engine, so bound it with `maxVersion`. +- **Some triggers relaxed, others still rejected** → + `partial-engine-relaxation`, action `narrow-detector`. The engine fixed *part* + of the condition. Scoping the + rule away here would drop the diagnostics that are still correct, converting a + partial engine fix into a shipped **false negative**. Narrow the detector so it + stops matching the now-valid shapes while still flagging the rest. + +This is decided per rule, not per query: the aggregator collects every trigger's +engine verdict for a rule on a leg, then emits **one** rule-level finding that +supersedes the per-query ones. A trigger with no verdict is counted as neither — +treating it as "still rejects" would let a timed-out leg masquerade as a partial fix +and send someone to narrow a healthy detector. + +The evidence always states the contracted, accepted, rejected, and missing +trigger tally. A one-trigger rule can be fully relaxed when that trigger and its +controls produce complete evidence. + +Four hard guards keep the check from passing vacuously: + +- The active manifest must contain exactly the approved 12 rule IDs. +- A leg whose artifacts are missing remains in the matrix as a complete + `inconclusive` column. A dead observe job cannot shrink the matrix into a green + result. +- A case with no engine verdict (a transport failure, recorded by the IT as + `outcome: "error"`) is **not** read as acceptance. Coercing it would report a + timeout as an engine that now accepts the query — and advise disabling a + perfectly good rule. Likewise, a contract whose fixture index failed to seed is + reported as unusable rather than as a stream of `IndexNotFoundException` + verdicts. +- A rule whose every case was uncomparable is reported `inconclusive` and **fails** + — it proved nothing. Inconclusive findings say "check that leg's logs and re-run", + never "edit the rule", because the linter is not what went wrong. + +A rule that is out of scope for a surface, version, or engine mode is not +executed or compared. Its cell is `n/a` with the corresponding reason and never +blocks the job. + +### Where a failure shows up in the GitHub UI + +The `Aggregate rule compatibility` step summary is the primary interface. It +always contains all 12 rows and all three columns, followed by blocking findings +and remediation. `ppl-lint-multiversion-drift/drift-report.json` carries the +complete schema-v3 matrix and query cases; `ppl-lint-multiversion-evidence` +carries target identities, detector/backend reports, logs, and reproduction +commands. The final step fails only after both uploads have run. + +### Running the multi-version check locally + +Use the planner with an exact-semver tag list, then point the aggregator at the +three artifact directories produced by backend and detector runs: + +```bash +git ls-remote --tags --refs https://github.com/opensearch-project/OpenSearch.git \ + > /tmp/opensearch-release-tags.txt +node scripts/ppl-lint/plan-compatibility.mjs \ + --build-file build.gradle \ + --release-tags /tmp/opensearch-release-tags.txt \ + --compiled-version 2.19.6 \ + --sql-sha "$(git rev-parse HEAD)" \ + --osd-repository opensearch-project/OpenSearch-Dashboards \ + --osd-ref main \ + --out compatibility-plan.json + +node scripts/ppl-lint/aggregate-compatibility.mjs \ + --plan compatibility-plan.json \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --artifacts legs \ + --osd-sha "" \ + --out drift-report.json +``` + +The step summary has one row per active detector. It prints the compatibility +declared by `wiring.appliesTo` and `grammarSurface` next to each actual result. + +The classifier is pure and has no cluster or OSD dependency, so its tests run +anywhere: + +```bash +node --test "scripts/ppl-lint/__tests__/*.test.mjs" +``` + +## Discovery corpus (harvested, never enforced) + +The discovery scripts remain available as standalone investigation tooling. +They are not invoked by the multi-surface compatibility workflow. + +``` +harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-frontend-contract.mjs ──▶ detector report + + discovery-specs/ └──▶ probe-discovery-backend.mjs ─▶ backend report + │ + label-discovery.mjs ──▶ findings + trigger coverage +``` + +1. **Harvest.** `harvest-queries.mjs` extracts PPL literals from OSD's own lint test + suite and attributes each to the rule whose `describe(...)` block encloses it + (matched as a prefix, so `describe('rex-scan-cost (compiled surface)')` counts). + A query with no rule-owning ancestor is recorded unattributed and dropped rather + than guessed at. Indices are rewritten onto the fixture index; JS string escapes + are unescaped so the query matches what the test actually linted. The harvested + corpus is substantially larger than the curated 12-contract required corpus. + + Each file's **lint context** is harvested alongside its queries. Seven of the + nineteen rules are `needsContext: true` and self-suppress without a `typeMap`, so + harvesting queries alone produced 26 `rex-scan-cost` queries and zero triggers — + the detector never ran, which in the report is indistinguishable from a rule that + fired on nothing. The context is taken from the test file (its `typeMap`, + `disabledObjectFields`) because its author wrote it to make exactly those queries + fire; a hand-written substitute would be a guess about which field types each + query depends on, and a wrong guess silently suppresses the detector again. A rule + tested under two different contexts gets two spec files rather than a merged one. + + A **wildcard** source is deliberately not remapped: `wildcard-source-zero-match` + exists to flag a pattern matching no visible index, so rewriting `source=\`nope-*\`` + to a concrete index destroys the only thing it detects. +2. **Observe both halves.** `--specs-out` writes the corpus as ordinary spec files so + the **existing** detector runner produces real diagnostic counts with no changes to + it; a non-zero exit is expected there and ignored, because the generated + expectations are placeholders. `probe-discovery-backend.mjs` sends each query to + `POST /_plugins/_ppl` directly — no Gradle, no test cluster, since there is + nothing to assert. +3. **Label and report.** `label-discovery.mjs` derives each role from real detector + output (fired → trigger, silent → control) and reports disagreements. + +Roles are derived; **expectations never are**. An auto-derived expectation could only +confirm current behavior, locking in whatever the detector does today including its +bugs. Promotion into the enforced corpus stays a human writing a spec entry. + +### What it reports, and how much to trust it + +| Detector | Engine | Reported as | +| --- | --- | --- | +| fires (error/warning) | accepts | `possible-false-positive` — nearly conclusive | +| fires (**info** only) | accepts | nothing — advisory rules are never contradicted by acceptance | +| silent | rejects | `possible-false-negative` — weak, verify first | +| either | no verdict | nothing; the query is labelled but claims nothing | + +The asymmetry is deliberate. A query the engine *ran* successfully but the linter +called broken is unambiguous. A rejection may be for a reason unrelated to the rule, +so rejections matching an unknown field, a missing index, an unsupported command, or +a syntax error are **suppressed** rather than reported — without that filter every +harvested query naming an invented field becomes a finding and buries the real ones. +Suppression never applies to the false-positive side. + +Advisory (`info`) rules are the other exclusion, and it was found by running this +against a live 3.8 engine: `head-without-sort` and `rex-scan-cost` flag +non-determinism and scan cost, which the engine executes happily and will never +reject. For those, "accepted + flagged" is the rule working as designed. Severity is +the discriminator because it already encodes the claim — only an error/warning rule +asserts the engine will refuse the query, and only such a claim can be contradicted +by acceptance. The judgement uses the severities the detector actually emitted, so a +mixed-severity diagnostic is still reported. + +The report also prints per-rule trigger counts and whether each rule has enough +(≥2) to support a scope decision. That table is the direct input to the full-vs-partial +question above: a rule showing **1 trigger** cannot distinguish the two, and a rule +showing **0** was not observed at all. + +Measured on the compiled surface against OSD `main`, this yields **41 triggers with +9 of 12 rules at ≥2**. The three that remain at one trigger are at the ceiling of +what OSD's tests contain — `agg-on-text`, `wildcard-source-zero-match` and +`unsupported-window-function-in-eventstats` each have exactly one trigger written +there, and their other queries are genuine controls (`stats avg(balance)` on a +numeric field is valid; `row_number` is the one window function eventstats +supports). Raising those needs queries nobody has written yet — the point where +generation, rather than harvesting, is what adds coverage. + +The job prefers the **runtime-bundle** surface, exporting the engine's grammar via +`GET /_plugins/_ppl/_grammar` and falling back to the compiled surface (with a +warning) if that fails. The runtime surface matters because `lint_runner` SKIPS the +four `runtimeOnly` rules on the compiled grammar — the productions they walk do not +exist there — and three of those ship at error severity. + +Those four are nonetheless still at **zero** harvested queries, and no surface fixes +that: OSD's lint tests contain no trigger for `union-min-datasets`, +`multisearch-min-subsearch` or `replace-wildcard-asymmetry` at all. The only place +they appear is a negative assertion that they no-op on the compiled surface +(`analyzer_lint.test.ts`, "runtime-only rules no-op"). Harvesting cannot invent what +was never written, so these are generation's job, not the harvester's. + +This job is `continue-on-error: true` and the labeler always exits zero. A finding +here is a lead, not a proven defect; failing unrelated PRs on an auto-generated +guess would destroy the check's credibility. It runs against one engine (the newest +in the matrix) because it generates leads rather than checking version drift. + +```bash +# Locally, against a running cluster and an OSD checkout: +node -e "const c=require('/packages/osd-monaco/src/ppl/lint/rules_catalog.json'); + process.stdout.write(JSON.stringify(c.map(r=>r.id)))" > /tmp/rules.json +node scripts/ppl-lint/harvest-queries.mjs --osd --catalog-rules @/tmp/rules.json \ + --index opensearch-sql_test_index_account --out /tmp/corpus.json --specs-out /tmp/specs +( cd && PPL_LINT_SURFACE=compiled-simplified PPL_LINT_CONTRACT_DIR=/tmp/specs \ + PPL_LINT_SCHEDULE=nightly PPL_LINT_REPORT=/tmp/detector.json \ + node -r ./src/setup_node_env "$PWD/../sql/scripts/ppl-lint/run-frontend-contract.mjs" || true ) +node scripts/ppl-lint/probe-discovery-backend.mjs --corpus /tmp/corpus.json --out /tmp/backend.json +node scripts/ppl-lint/label-discovery.mjs --corpus /tmp/corpus.json \ + --detector /tmp/detector.json --backend /tmp/backend.json --out /tmp/findings.json +``` + +## Interpreting a failure + +| Failure | Meaning | +| --- | --- | +| Grammar bundle export fails | The candidate SQL build does not provide a usable runtime grammar. | +| Trigger no longer parses | The grammar changed ownership of the error or regressed. | +| Detector emits no diagnostic | The detector is incompatible with the candidate parse tree. | +| Detector flags the control | The detector became too broad. | +| Backend accepts the trigger | The lint rule's premise may be fixed or stale. | +| Backend rejects the control | Query, fixture, settings, or SQL behavior regressed. | +| No version expectation matches | The rule test does not cover the candidate version. | + +CI never rewrites expected results. A behavior change is an intentional, reviewed +edit to a versioned expectation **and** the corresponding OSD rule. If a SQL +change depends on an OSD rule update, merge the OSD change first, then rerun the +required SQL check against OSD `main`. + +## Artifacts + +Every run uploads: `run-manifest.json` (exact SQL SHA, OSD SHA, mode, backend +version, grammar hash, selected validation set), the candidate grammar bundle, +the backend and detector reports, the committed contracts used, and the job logs. diff --git a/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs b/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs new file mode 100644 index 00000000000..13df6796325 --- /dev/null +++ b/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs @@ -0,0 +1,500 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { resolveBackendOracle } from '../contract-schema.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'aggregate-compatibility.mjs'); +const REPOSITORY = path.resolve(HERE, '..', '..', '..'); +const CONTRACTS = path.join( + REPOSITORY, + 'integ-test', + 'src', + 'test', + 'resources', + 'ppl-lint', + 'contracts' +); +const WORKFLOW = path.join( + REPOSITORY, + '.github', + 'workflows', + 'ppl-lint-multiversion-validation.yml' +); +const RULE_IDS = JSON.parse( + fs.readFileSync(path.join(CONTRACTS, 'manifest.json'), 'utf8') +).contracts.map((file) => file.replace(/\.spec\.json$/, '')); +const CONFIGURATIONS = [ + { + id: '2.19.6-compiled', + label: '2.19.6 compiled', + engineVersion: '2.19.6', + surface: 'compiled-simplified', + executionBackend: 'standard', + engineMode: 'legacy', + artifactName: 'ppl-lint-observation-2.19.6-compiled', + exportRuntimeBundle: false, + }, + { + id: 'latest-release-runtime', + label: 'Latest release (3.8.0) runtime', + engineVersion: '3.8.0', + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-latest-release-runtime', + exportRuntimeBundle: true, + }, + { + id: 'pr-build-runtime', + label: 'PR runtime', + engineVersion: '3.8.0-SNAPSHOT', + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-pr-build-runtime', + exportRuntimeBundle: true, + }, +]; +const temporaryDirectories = []; + +function temporaryDirectory(prefix) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +after(() => { + for (const directory of temporaryDirectories) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +function version(value) { + const match = /^(\d+)\.(\d+)\.(\d+)/.exec(value); + return match.slice(1, 4).map(Number); +} + +function compare(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] - right[index]; + } + return 0; +} + +function applicable(spec, configuration) { + const appliesTo = spec.wiring.appliesTo || {}; + const surfaces = + spec.grammarSurface === 'both' + ? ['compiled-simplified', 'runtime-bundle'] + : [spec.grammarSurface || 'runtime-bundle']; + if (!surfaces.includes(configuration.surface)) return false; + const actual = version(configuration.engineVersion); + if (appliesTo.minVersion && compare(actual, version(appliesTo.minVersion)) < 0) return false; + if (appliesTo.maxVersion && compare(actual, version(appliesTo.maxVersion)) > 0) return false; + return !appliesTo.engine || appliesTo.engine === configuration.engineMode; +} + +function rangeMatches(range, engineVersion) { + const actual = version(engineVersion); + return String(range || '') + .trim() + .split(/\s+/) + .filter(Boolean) + .every((token) => { + const match = /^(>=|<=|>|<|=)?(.+)$/.exec(token); + const comparison = compare(actual, version(match[2])); + return ( + (!match[1] && comparison === 0) || + (match[1] === '=' && comparison === 0) || + (match[1] === '>=' && comparison >= 0) || + (match[1] === '<=' && comparison <= 0) || + (match[1] === '>' && comparison > 0) || + (match[1] === '<' && comparison < 0) + ); + }); +} + +function loadSpecs() { + return new Map( + JSON.parse(fs.readFileSync(path.join(CONTRACTS, 'manifest.json'), 'utf8')).contracts.map( + (file) => { + const spec = JSON.parse(fs.readFileSync(path.join(CONTRACTS, file), 'utf8')); + return [spec.ruleId, spec]; + } + ) + ); +} + +function selectedExpectation(spec, configuration) { + return spec.expectations.find( + (expectation) => + rangeMatches(expectation.version, configuration.engineVersion) && + (!expectation.engine || expectation.engine === configuration.engineMode) + ); +} + +function target(configuration, grammarHash) { + return { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: configuration.engineVersion, + grammarHash, + grammarBundle: + configuration.surface === 'runtime-bundle' ? 'ppl-grammar-bundle.json' : '', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }; +} + +function writeHealthyArtifacts(root) { + const specs = loadSpecs(); + for (const configuration of CONFIGURATIONS) { + const directory = path.join(root, configuration.artifactName); + fs.mkdirSync(directory, { recursive: true }); + const grammarHash = + configuration.surface === 'runtime-bundle' + ? `sha256:${configuration.id}` + : 'sha256:compiled-grammar'; + const backendTarget = target( + configuration, + configuration.surface === 'runtime-bundle' ? grammarHash : '' + ); + const detectorTarget = target(configuration, grammarHash); + const detectorResults = []; + const backendResults = []; + + for (const spec of specs.values()) { + if (!applicable(spec, configuration)) continue; + const expectation = selectedExpectation(spec, configuration); + assert.ok(expectation, `fixture expectation for ${spec.ruleId} on ${configuration.id}`); + for (const [queryName, queryDefinition] of Object.entries(spec.queries)) { + const resolved = resolveBackendOracle( + spec, + expectation.queries[queryName], + 'standard' + ); + assert.equal(resolved.status, 'applicable'); + const rejected = resolved.oracle.kind === 'rejection'; + const error = resolved.oracle.body && resolved.oracle.body.error; + detectorResults.push({ + ruleId: spec.ruleId, + queryName, + role: queryDefinition.role || 'trigger', + expected: resolved.detector.count, + actual: resolved.detector.count, + severities: + resolved.detector.count > 0 && resolved.detector.severity + ? [resolved.detector.severity] + : [], + severityMatched: true, + messageMatched: true, + executionBackend: 'standard', + }); + backendResults.push({ + ruleId: spec.ruleId, + queryName, + role: queryDefinition.role || 'trigger', + rejected, + executionBackend: 'standard', + observed: { + rejected, + httpStatus: resolved.oracle.httpStatus, + ...(error && error.type ? { type: error.type } : {}), + ...(error && error.reason ? { reason: error.reason } : {}), + }, + }); + } + } + + fs.writeFileSync(path.join(directory, 'target.json'), JSON.stringify(backendTarget)); + fs.writeFileSync( + path.join(directory, 'detector-target.json'), + JSON.stringify(detectorTarget) + ); + fs.writeFileSync( + path.join(directory, 'backend-report.json'), + JSON.stringify(backendResults) + ); + fs.writeFileSync( + path.join(directory, 'detector-report.json'), + JSON.stringify({ + schemaVersion: 2, + engineVersion: configuration.engineVersion, + grammarHash, + executionBackend: 'standard', + surface: configuration.surface, + results: detectorResults, + }) + ); + if (configuration.surface === 'runtime-bundle') { + fs.writeFileSync( + path.join(directory, 'ppl-grammar-bundle.json'), + JSON.stringify({ grammarHash }) + ); + } + fs.writeFileSync(path.join(directory, 'backend-command.txt'), 'backend command\n'); + fs.writeFileSync(path.join(directory, 'detector-command.txt'), 'detector command\n'); + } +} + +function createFixture() { + const directory = temporaryDirectory('ppl-lint-compatibility-'); + const artifacts = path.join(directory, 'legs'); + fs.mkdirSync(artifacts); + writeHealthyArtifacts(artifacts); + const plan = { + schemaVersion: 1, + sqlSha: 'candidate-sql-sha', + prTargetVersion: '3.8.0-SNAPSHOT', + normalizedPrTarget: '3.8.0', + latestEligibleGa: '3.8.0', + osd: { repository: 'example/osd', ref: 'main' }, + configurations: CONFIGURATIONS, + }; + const planFile = path.join(directory, 'compatibility-plan.json'); + fs.writeFileSync(planFile, JSON.stringify(plan)); + return { directory, artifacts, planFile }; +} + +function run(fixture) { + const reportFile = path.join(fixture.directory, 'drift-report.json'); + const summaryFile = path.join(fixture.directory, 'summary.md'); + const result = spawnSync( + process.execPath, + [ + SCRIPT, + '--plan', + fixture.planFile, + '--contracts', + CONTRACTS, + '--artifacts', + fixture.artifacts, + '--osd-sha', + 'osd-sha', + '--out', + reportFile, + '--summary', + summaryFile, + ], + { encoding: 'utf8' } + ); + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + report: fs.existsSync(reportFile) + ? JSON.parse(fs.readFileSync(reportFile, 'utf8')) + : undefined, + summary: fs.existsSync(summaryFile) ? fs.readFileSync(summaryFile, 'utf8') : '', + }; +} + +function editBackend(fixture, configurationId, ruleId, queryName, patch) { + const configuration = CONFIGURATIONS.find((entry) => entry.id === configurationId); + const file = path.join( + fixture.artifacts, + configuration.artifactName, + 'backend-report.json' + ); + const report = JSON.parse(fs.readFileSync(file, 'utf8')); + const row = report.find( + (entry) => entry.ruleId === ruleId && entry.queryName === queryName + ); + Object.assign(row, patch); + Object.assign(row.observed, patch.observed || {}); + fs.writeFileSync(file, JSON.stringify(report)); +} + +test('emits exactly 12 rules, 3 configurations, and 36 complete cells', () => { + const result = run(createFixture()); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.report.schemaVersion, 3); + assert.equal(result.report.inventory.ruleCount, 12); + assert.deepEqual(result.report.inventory.ruleIds, [...RULE_IDS].sort()); + assert.equal(result.report.configurations.length, 3); + assert.equal(result.report.matrix.length, 36); + assert.deepEqual(result.report.result, { + status: 'pass', + cellCount: 36, + compatible: 28, + notApplicable: 8, + drift: 0, + inconclusive: 0, + exitCode: 0, + }); + assert.equal( + result.summary.split('\n').filter((line) => /^\| `[a-z0-9-]+` \|/.test(line)) + .length, + 12 + ); +}); + +test('uses surface before version when a compiled pair has multiple exclusions', () => { + const { report } = run(createFixture()); + assert.equal( + report.matrix.find( + (entry) => + entry.ruleId === 'invalid-capture-group-name' && + entry.configurationId === '2.19.6-compiled' + ).expected.reason, + 'surface' + ); + assert.equal( + report.matrix.find( + (entry) => + entry.ruleId === 'agg-on-text' && + entry.configurationId === '2.19.6-compiled' + ).expected.reason, + 'version' + ); +}); + +test('classifies a one-trigger full engine relaxation and keeps the complete table', () => { + const fixture = createFixture(); + editBackend( + fixture, + 'latest-release-runtime', + 'wildcard-source-zero-match', + 'missing-wildcard-source', + { + rejected: false, + observed: { rejected: false, httpStatus: 200, type: undefined, reason: undefined }, + } + ); + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + const cell = result.report.matrix.find( + (entry) => + entry.ruleId === 'wildcard-source-zero-match' && + entry.configurationId === 'latest-release-runtime' + ); + assert.equal(cell.classification, 'full-engine-relaxation'); + assert.deepEqual(cell.triggerSummary, { + contracted: 1, + acceptedByBackend: 1, + rejectedByBackend: 0, + missing: 0, + }); + assert.equal( + result.report.findings.find( + (entry) => + entry.ruleId === 'wildcard-source-zero-match' && + entry.configurationId === 'latest-release-runtime' + ).remediation.action, + 'scope-rule-version' + ); +}); + +test('classifies partial relaxation separately and never recommends version scoping', () => { + const fixture = createFixture(); + editBackend( + fixture, + 'latest-release-runtime', + 'union-min-datasets', + 'union-single-dataset', + { + rejected: false, + observed: { rejected: false, httpStatus: 200, type: undefined, reason: undefined }, + } + ); + const result = run(fixture); + const finding = result.report.findings.find( + (entry) => + entry.ruleId === 'union-min-datasets' && + entry.configurationId === 'latest-release-runtime' + ); + assert.equal(finding.classification, 'partial-engine-relaxation'); + assert.equal(finding.remediation.action, 'narrow-detector'); + assert.ok( + !result.report.findings.some( + (entry) => + entry.ruleId === 'union-min-datasets' && + entry.remediation.action === 'scope-rule-version' + ) + ); +}); + +test('a detector regression writes JSON and every summary row before exiting nonzero', () => { + const fixture = createFixture(); + const configuration = CONFIGURATIONS[1]; + const file = path.join( + fixture.artifacts, + configuration.artifactName, + 'detector-report.json' + ); + const report = JSON.parse(fs.readFileSync(file, 'utf8')); + report.results.find( + (entry) => + entry.ruleId === 'rex-scan-cost' && entry.queryName === 'parse-text-field' + ).actual = 0; + fs.writeFileSync(file, JSON.stringify(report)); + + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + assert.equal( + result.report.matrix.find( + (entry) => + entry.ruleId === 'rex-scan-cost' && + entry.configurationId === 'latest-release-runtime' + ).classification, + 'detector-regression' + ); + assert.equal( + result.summary + .split('### Blocking findings')[0] + .split('\n') + .filter((line) => /^\| `[a-z0-9-]+` \|/.test(line)).length, + 12 + ); + assert.match(result.stderr, /after writing the complete report/); +}); + +test('a missing observation preserves the full column as inconclusive', () => { + const fixture = createFixture(); + fs.rmSync( + path.join( + fixture.artifacts, + CONFIGURATIONS[1].artifactName + ), + { recursive: true } + ); + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + const column = result.report.matrix.filter( + (entry) => entry.configurationId === 'latest-release-runtime' + ); + assert.equal(column.length, 12); + assert.ok(column.every((entry) => entry.status === 'inconclusive')); + assert.equal(result.report.result.inconclusive, 12); +}); + +test('workflow uploads the mandatory report before the only enforcement step', () => { + const workflow = fs.readFileSync(WORKFLOW, 'utf8'); + const upload = workflow.indexOf('- name: Upload drift report'); + const evidence = workflow.indexOf('- name: Upload compatibility evidence'); + const enforce = workflow.indexOf('- name: Fail after publishing compatibility results'); + assert.ok(upload > 0 && evidence > upload && enforce > evidence); + const reportStep = workflow.slice(upload, evidence); + assert.match(reportStep, /path: drift-report\.json/); + assert.match(reportStep, /if-no-files-found: error/); + assert.equal( + (workflow.match(/- name: Fail after publishing compatibility results/g) || []) + .length, + 1 + ); +}); diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs new file mode 100644 index 00000000000..669b2d6bb54 --- /dev/null +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -0,0 +1,1637 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Integration tests for the multi-version aggregator. + * + * node --test scripts/ppl-lint/__tests__/aggregate-versions.test.mjs + * + * These drive the real script as a child process over synthetic leg directories + * (the four artifact files each engine leg produces), so they cover the parts the + * pure classifier tests cannot: argument handling, artifact loading, the + * in-scope/out-of-scope split, coverage holes, once-per-rule grammar drift, and + * the process exit code that makes the CI check red or green. + */ + +import assert from 'node:assert/strict'; +import { after, test } from 'node:test'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'aggregate-versions.mjs'); +const REAL_CONTRACTS = path.resolve( + HERE, + '..', + '..', + '..', + 'integ-test', + 'src', + 'test', + 'resources', + 'ppl-lint', + 'contracts' +); + +/** Contract used by every case: a >=3.7 calcite-only rule with one trigger + one control. */ +const SPEC = { + schemaVersion: 3, + ruleId: 'union-min-datasets', + grammarSurface: 'runtime-bundle', + schedule: 'pr', + requiredParserRules: ['unionCommand'], + wiring: { + detector: 'union-min-datasets', + enabled: true, + severity: 'error', + runtimeOnly: true, + appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, + }, + index: 'test-index', + queries: { + trigger: { role: 'trigger', query: 'union [ source={{index}} ]' }, + control: { role: 'control', query: 'union [ source={{index}} ] [ source={{index}} ]' }, + }, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + trigger: { + detectorCount: 1, + severity: 'error', + backend: { + kind: 'rejection', + httpStatus: 400, + body: { + status: 400, + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }, + control: { detectorCount: 0, backend: { kind: 'result-shape', httpStatus: 200 } }, + }, + }, + ], +}; + +const REJECTION = { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', +}; + +const tmpDirs = []; + +function makeTmp(prefix) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tmpDirs.push(dir); + return dir; +} + +after(() => { + for (const dir of tmpDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +/** Write a contract dir holding SPEC (optionally patched) and a manifest. */ +function writeContracts(patch = {}) { + const dir = makeTmp('ppl-lint-contracts-'); + const spec = { ...SPEC, ...patch }; + fs.writeFileSync(path.join(dir, 'union.spec.json'), JSON.stringify(spec)); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ + schemaVersion: 3, + contracts: ['union.spec.json'], + defaultError: ['union.spec.json'], + }) + ); + return dir; +} + +/** + * Write one engine leg. `cases` maps query name to + * { detector: , severities, rejected, type, reason }. + */ +function writeLeg({ + version, + cases, + parserRuleNames = ['unionCommand', 'unionDataset'], + defaultErrorRules = [SPEC.ruleId], + executionBackend = 'standard', + grammarHash = `sha256:${version}`, + surface = 'runtime-bundle', + explicitIdentity = true, + censusEnforced = false, +}) { + const dir = makeTmp(`ppl-lint-leg-${version}-`); + const target = { + engineVersion: version, + grammarHash, + ...(explicitIdentity + ? { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + executionBackend, + storage: executionBackend === 'analytics' ? 'composite-parquet' : 'lucene', + shardCount: 1, + ...(executionBackend === 'analytics' + ? { + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + } + : {}), + } + : {}), + }; + fs.writeFileSync( + path.join(dir, 'target.json'), + JSON.stringify(target) + ); + fs.writeFileSync( + path.join(dir, 'ppl-grammar-bundle.json'), + JSON.stringify({ parserRuleNames }) + ); + + const results = []; + const backend = []; + for (const [queryName, c] of Object.entries(cases)) { + const role = queryName === 'control' ? 'control' : 'trigger'; + results.push({ + ruleId: SPEC.ruleId, + queryName, + role, + expected: role === 'trigger' ? 1 : 0, + actual: c.detector, + severities: c.severities || (c.detector > 0 ? ['error'] : []), + severityMatched: c.severityMatched ?? true, + messageMatched: c.messageMatched ?? true, + ...Object.fromEntries( + [ + 'deterministicFixMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ] + .filter((field) => c[field] !== undefined) + .map((field) => [field, c[field]]) + ), + ...(c.assertions ? { assertions: c.assertions } : {}), + ...(c.mismatches ? { mismatches: c.mismatches } : {}), + ...(c.detectorOutcome ? { outcome: c.detectorOutcome } : {}), + ...(c.detectorError ? { error: c.detectorError } : {}), + ...(explicitIdentity ? { executionBackend } : {}), + }); + backend.push({ + ruleId: SPEC.ruleId, + queryName, + role, + rejected: !!c.rejected, + observed: { + httpStatus: c.httpStatus || (c.rejected ? 400 : 200), + rejected: !!c.rejected, + ...(c.rejected ? { type: c.type || REJECTION.type, reason: c.reason || REJECTION.reason } : {}), + }, + ...(c.outcome ? { outcome: c.outcome } : {}), + ...(c.error ? { error: c.error } : {}), + ...(explicitIdentity ? { executionBackend } : {}), + }); + } + const detectorIdentity = explicitIdentity + ? { + schemaVersion: 2, + executionBackend, + engineVersion: version, + grammarHash, + } + : {}; + fs.writeFileSync( + path.join(dir, 'detector-report.json'), + JSON.stringify({ + ...detectorIdentity, + surface, + results, + ...(defaultErrorRules !== null ? { defaultErrorRules } : {}), + enabledRules: [SPEC.ruleId], + activeContractRules: [SPEC.ruleId], + requiredSyntaxFeatures: [], + census: { enforced: censusEnforced }, + }) + ); + fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); + return dir; +} + +/** Run the aggregator; returns the process result plus its JSON and Markdown reports. */ +function run({ contracts, legs, extraArgs = [] }) { + const outDir = makeTmp('ppl-lint-out-'); + const out = path.join(outDir, 'drift-report.json'); + const summaryFile = path.join(outDir, 'summary.md'); + const args = [SCRIPT, '--contracts', contracts, '--out', out, '--summary', summaryFile]; + const entries = Array.isArray(legs) ? legs : Object.entries(legs); + for (const [version, dir] of entries) { + args.push('--leg', `${version}=${dir}`); + } + args.push(...extraArgs); + const result = spawnSync(process.execPath, args, { encoding: 'utf8' }); + const report = fs.existsSync(out) ? JSON.parse(fs.readFileSync(out, 'utf8')) : undefined; + const summary = fs.existsSync(summaryFile) ? fs.readFileSync(summaryFile, 'utf8') : ''; + return { + status: result.status, + stdout: result.stdout || '', + stderr: result.stderr || '', + report, + summary, + }; +} + +/** The all-agree case, reused as the base for each drift scenario. */ +function healthyLegs() { + return { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + '3.8.0': writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + }; +} + +function writeSchema4Contracts({ includeAnalytics = true } = {}) { + const routeOracles = (standard, analytics) => ({ + standard, + ...(includeAnalytics ? { analytics } : {}), + }); + return writeContracts({ + schemaVersion: 4, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + trigger: { + detectorCount: 1, + severity: 'error', + backends: routeOracles( + { + kind: 'rejection', + httpStatus: 400, + body: { status: 400, error: REJECTION }, + }, + { kind: 'result-shape', httpStatus: 200 } + ), + }, + control: { + detectorCount: 0, + backends: routeOracles( + { kind: 'result-shape', httpStatus: 200 }, + { kind: 'result-shape', httpStatus: 200 } + ), + }, + }, + }, + ], + }); +} + +test('all versions agreeing exits 0 and reports expected versus actual compatibility', () => { + const { status, report, stdout, summary } = run({ + contracts: writeContracts(), + legs: healthyLegs(), + }); + assert.equal(status, 0); + assert.equal(report.result.passed, true); + assert.equal(report.drifts.length, 0); + assert.match(stdout, /agrees with all 2 engine version\(s\)/); + assert.match(summary, /\| Rule \| Expected compatibility \| `3\.7\.0` actual \| `3\.8\.0` actual \|/); + assert.match( + summary, + /\| `union-min-datasets` \| Calcite, >= 3\.7\.0 \| compatible \| compatible \|/ + ); + // Every rule/version pair is accounted for in the matrix. + assert.equal(report.matrix.length, 2); + assert.ok(report.matrix.every((m) => m.status === 'agree')); +}); + +test('the compatibility table contains exactly the 12 active shipping detectors', () => { + const leg = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, summary } = run({ + contracts: REAL_CONTRACTS, + legs: [['pr-build', leg]], + extraArgs: ['--all-rules'], + }); + + assert.equal(status, 1, 'missing synthetic observations remain inconclusive'); + const ruleRows = summary + .split('\n') + .filter((line) => /^\| `[a-z0-9-]+` \|/.test(line)); + assert.equal(ruleRows.length, 12); + assert.match(summary, /\| `rex-scan-cost` \| all versions \|/); + assert.doesNotMatch(summary, /command-suggestion/); +}); + +test('same-version standard and analytics verdicts are classified as backend divergence', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, report, stdout } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build', analytics], + ], + }); + + assert.equal(status, 1); + assert.equal(report.schemaVersion, 2); + assert.equal(report.legs.length, 2); + assert.equal(new Set(report.legs.map((leg) => leg.key)).size, 2); + assert.deepEqual( + report.legs.map((leg) => leg.executionBackend).sort(), + ['analytics', 'standard'] + ); + assert.equal(report.backendPairs.length, 1); + assert.ok(report.matrix.every((row) => row.status === 'drift')); + assert.ok(report.matrix.every((row) => row.key.includes(row.executionBackend))); + + const divergence = report.drifts.find( + (drift) => drift.driftClass === 'execution-backend-divergence' + ); + assert.ok(divergence); + assert.deepEqual(divergence.executionBackends, ['standard', 'analytics']); + assert.match(divergence.key, /standard-vs-analytics/); + assert.equal(divergence.remediation.action, 'align-execution-backends'); + assert.doesNotMatch(divergence.remediation.detail, /maxVersion|minVersion|scope/i); + assert.equal( + report.drifts.filter( + (drift) => + drift.driftClass === 'engine-relaxed' || drift.driftClass === 'engine-tightened' + ).length, + 0, + 'route differences must not be rendered as product-version drift' + ); + const tableHeader = stdout.split('\n').find((line) => line.startsWith('| Rule |')); + assert.match(tableHeader, /Expected compatibility/); + assert.match(tableHeader, /`pr-build` actual/); + assert.doesNotMatch(tableHeader, /analytics/); +}); + +test('schema-v3 analytics has explicit backend-oracle coverage holes', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report, stdout } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 2); + assert.ok(report.coverageHoles.every((hole) => hole.executionBackend === 'analytics')); + assert.ok(report.coverageHoles.every((hole) => hole.kind === 'backend-oracle')); + assert.ok(report.coverageHoles.every((hole) => /standard-only/.test(hole.reason))); + assert.equal(report.matrix[0].status, 'uncovered'); + assert.equal(report.drifts.length, 0); + assert.match(stdout, /schema-v3 backend oracles are standard-only/); +}); + +test('analytics observation mode reports schema-v3 coverage without failing', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 0); + assert.equal(report.result.enforcedCoverageHoles, 0); + assert.equal(report.result.observedAnalyticsFindings, 2); + assert.ok(report.coverageHoles.every((hole) => hole.blocking === false)); +}); + +test('schema-v3 raw analytics observations still expose backend divergence', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ...entry, + kind: 'coverage-missing', + outcome: 'coverage-missing', + coverage: 'missing', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 0); + assert.equal( + report.drifts.filter((drift) => drift.driftClass === 'execution-backend-divergence') + .length, + 1 + ); + assert.equal(report.result.observedAnalyticsFindings, 3); +}); + +test('backend divergence cannot hide a standard-route regression', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, report } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 1); + assert.ok( + report.drifts.some( + (drift) => + drift.executionBackend === 'standard' && + drift.driftClass === 'engine-relaxed' + ), + 'the blocking standard-route regression must survive paired-route classification' + ); + assert.ok( + report.drifts.some( + (drift) => drift.driftClass === 'execution-backend-divergence' + ) + ); + assert.ok(report.result.enforcedDriftCount > 0); +}); + +test('not-applicable cannot waive an enforced analytics backend oracle', () => { + const contracts = writeSchema4Contracts(); + const contractFile = path.join(contracts, 'union.spec.json'); + const contract = JSON.parse(fs.readFileSync(contractFile, 'utf8')); + for (const query of Object.values(contract.expectations[0].queries)) { + query.backends.analytics = { + kind: 'not-applicable', + reason: 'analytics fixture is not supported yet', + owner: '@analytics-team', + issue: 'https://example.test/issues/42', + }; + } + fs.writeFileSync(contractFile, JSON.stringify(contract)); + + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: entry.executionBackend, + kind: 'not-applicable', + outcome: 'not-applicable', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts, + legs: [['pr-build-analytics', analytics]], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 2); + assert.ok(report.coverageHoles.every((hole) => hole.issue.endsWith('/42'))); + assert.equal(report.matrix[0].status, 'uncovered'); +}); + +test('analytics coverage gaps cannot hide missing raw observations', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: 'analytics', + kind: 'coverage-missing', + outcome: 'error', + error: 'connect timeout', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match(report.inconclusive[0].reasons.join(' '), /no engine verdict/); +}); + +test('--all-rules makes incomplete non-default observations fail as infrastructure', () => { + const contracts = writeContracts(); + const manifestFile = path.join(contracts, 'manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8')); + manifest.defaultError = []; + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const leg = writeLeg({ + version: '3.8.0', + defaultErrorRules: [], + cases: { control: { detector: 0, rejected: false } }, + }); + + const { status, report } = run({ + contracts, + legs: [['3.8.0', leg]], + extraArgs: ['--all-rules'], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); +}); + +test('paired detector reports must be identical across execution backends', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const detectorFile = path.join(analytics, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results.find((entry) => entry.queryName === 'trigger').actual = 0; + detector.results.find((entry) => entry.queryName === 'trigger').severities = []; + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 2); + assert.match(stderr, /detector parity failed for union-min-datasets::trigger/); +}); + +test('identical deterministic-fix mismatches across versions cannot aggregate green', () => { + const badCase = { + detector: 1, + rejected: true, + deterministicFixMatched: false, + assertions: { deterministicFix: false }, + mismatches: [ + { + field: 'deterministicFix', + expected: { offered: false }, + actual: { offered: true }, + }, + ], + }; + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + trigger: badCase, + control: { detector: 0, rejected: false }, + }, + }), + '3.8.0': writeLeg({ + version: '3.8.0', + cases: { + trigger: badCase, + control: { detector: 0, rejected: false }, + }, + }), + }; + + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const actionDrifts = report.drifts.filter( + (drift) => drift.driftClass === 'frontend-contract-mismatch' + ); + assert.equal(actionDrifts.length, 2); + assert.ok( + actionDrifts.every((drift) => + drift.frontendAssertions.includes('deterministicFixMatched') + ) + ); + assert.ok(report.matrix.every((row) => row.status === 'drift')); +}); + +test('a detector execution error is infrastructure evidence, not detector drift', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + trigger: { + detector: 0, + rejected: true, + detectorOutcome: 'error', + detectorError: 'detector crashed', + assertions: { execution: false }, + mismatches: [ + { + field: 'execution', + expected: 'completed', + actual: 'detector crashed', + }, + ], + }, + control: { detector: 0, rejected: false }, + }, + }), + }; + + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.drifts.length, 0); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match( + report.inconclusive[0].reasons.join(' '), + /trigger \(frontend execution failed: detector crashed\)/ + ); + assert.doesNotMatch(stdout, /update-detector/); +}); + +test('target and detector execution identities must match', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.executionBackend = 'standard'; + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [['pr-build-analytics', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /detector report executionBackend "standard" does not match target "analytics"/); +}); + +test('unknown target execution backends are rejected', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + cases: { trigger: { detector: 1, rejected: true } }, + }); + const targetFile = path.join(dir, 'target.json'); + const target = JSON.parse(fs.readFileSync(targetFile, 'utf8')); + target.executionBackend = 'experimental'; + fs.writeFileSync(targetFile, JSON.stringify(target)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['pr-build', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /must be "standard" or "analytics"/); +}); + +test('every backend row must match its target execution identity', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')); + backend[0].executionBackend = 'standard'; + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [['pr-build-analytics', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /backend report row .* does not match target "analytics"/); +}); + +test('duplicate backend row keys are rejected instead of overwritten', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')); + backend.push({ ...backend[0] }); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['3.8.0', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate backend report key "union-min-datasets::trigger"/); +}); + +test('duplicate detector row keys are rejected instead of selecting the first', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results.push({ ...detector.results[0] }); + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['3.8.0', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate detector report key "union-min-datasets::trigger"/); +}); + +test('duplicate backend-qualified leg identities are rejected', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [ + ['3.8.0', dir], + ['3.8.0', dir], + ], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate leg identity/); +}); + +test('paired standard and analytics legs require the same runtime grammar hash', () => { + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash: 'sha256:standard', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash: 'sha256:analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build', analytics], + ], + }); + assert.equal(status, 2); + assert.match(stderr, /different grammar hashes/); +}); + +test('a version where only one engine relaxed is red, and names just that version', () => { + const legs = healthyLegs(); + // 3.8 now accepts what 3.7 still rejects, while the detector keeps flagging. + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.result.enforcedDriftCount, 1); + const drift = report.drifts[0]; + assert.equal(drift.driftClass, 'engine-relaxed'); + assert.equal(drift.version, '3.8.0'); + assert.equal(drift.remediation.action, 'version-scope-rule'); + // The healthy version is still reported as agreeing. + assert.equal(report.matrix.find((m) => m.version === '3.7.0').status, 'agree'); +}); + +test('drift exits nonzero only after writing the JSON report and full Markdown table', () => { + const legs = healthyLegs(); + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 0, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, report, summary } = run({ contracts: writeContracts(), legs }); + + assert.equal(status, 1); + assert.equal(report.result.passed, false); + assert.equal(report.result.enforcedDriftCount, 1); + assert.match(summary, /## PPL lint multi-version validation/); + assert.match(summary, /\| Rule \| Expected compatibility \|/); + assert.match( + summary, + /\| `union-min-datasets` \| Calcite, >= 3\.7\.0 \| compatible \| \*\*drift\*\* \|/ + ); + assert.match(summary, /### Remediation/); +}); + +test('a changed rejection HTTP status is semantic drift, not agreement', () => { + const leg = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 1, rejected: true, httpStatus: 500 }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['3.8.0', leg]], + }); + + assert.equal(status, 1); + const drift = report.drifts.find( + (entry) => entry.driftClass === 'backend-oracle-mismatch' + ); + assert.ok(drift); + assert.match(drift.evidence, /HTTP status changed from 400 to 500/); + assert.equal(drift.remediation.action, 'review-backend-oracle'); +}); + +test('a same-verdict result-shape mismatch cannot pass aggregation', () => { + const leg = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { + detector: 0, + rejected: false, + outcome: 'observed-mismatch', + error: 'expected non-empty datarows', + }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['3.8.0', leg]], + }); + + assert.equal(status, 1); + const drift = report.drifts.find( + (entry) => entry.driftClass === 'backend-oracle-mismatch' + ); + assert.ok(drift); + assert.match(drift.evidence, /expected non-empty datarows/); +}); + +// --- partial vs full relaxation, end to end --------------------------------- +// +// Driven through the real script because the bug this guards is in the AGGREGATION: +// `classifyDrift` is per-query and cannot see the other triggers, so the rollup has +// to happen here or the advice is wrong whenever a rule has more than one trigger. + +/** A two-trigger contract: the shape that makes partial-vs-full decidable. */ +function writeTwoTriggerContracts() { + const dir = makeTmp('ppl-lint-contracts-multi-'); + const spec = { + ...SPEC, + queries: { + triggerA: { role: 'trigger', query: 'union [ source={{index}} ]' }, + triggerB: { role: 'trigger', query: 'union [ source={{index}} ] extra' }, + control: { role: 'control', query: 'union [ source={{index}} ] [ source={{index}} ]' }, + }, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + triggerA: { + detectorCount: 1, + severity: 'error', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400, error: REJECTION } }, + }, + triggerB: { + detectorCount: 1, + severity: 'error', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400, error: REJECTION } }, + }, + control: { detectorCount: 0, backend: { kind: 'result-shape', httpStatus: 200 } }, + }, + }, + ], + }; + fs.writeFileSync(path.join(dir, 'union.spec.json'), JSON.stringify(spec)); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ + schemaVersion: 3, + contracts: ['union.spec.json'], + defaultError: ['union.spec.json'], + }) + ); + return dir; +} + +test('ALL triggers relaxing is a full fix: one rule-level version-scope finding', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { status, report } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal(status, 1); + // Exactly ONE finding, not one per trigger: the decision is per rule. + assert.equal(report.drifts.length, 1); + const drift = report.drifts[0]; + assert.equal(drift.driftClass, 'engine-relaxed'); + assert.equal(drift.remediation.action, 'version-scope-rule'); + assert.deepEqual(drift.scope.relaxed.sort(), ['triggerA', 'triggerB']); + assert.deepEqual(drift.scope.holding, []); +}); + +test('SOME triggers relaxing is a partial fix: narrow the detector, do NOT scope', () => { + // The regression this pins: acting on the per-query view would advise + // maxVersion < 3.7, dropping the diagnostic for triggerB which the engine STILL + // rejects — converting a partial engine fix into a shipped false negative. + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { status, report, stdout } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal(status, 1); + const partial = report.drifts.find((d) => d.driftClass === 'engine-partially-relaxed'); + assert.ok(partial, 'a partial relaxation must be classified as such'); + assert.equal(partial.remediation.action, 'update-detector'); + assert.deepEqual(partial.scope.relaxed, ['triggerA']); + assert.deepEqual(partial.scope.holding, ['triggerB']); + // No finding may survive that tells the engineer to version-scope this rule. + assert.equal( + report.drifts.filter((d) => d.remediation.action === 'version-scope-rule').length, + 0, + 'a partial fix must never advise version-scoping' + ); + assert.match(stdout, /Do NOT scope/); +}); + +test('an unobserved trigger does not fake a partial fix', () => { + // If the unobserved trigger were counted as "still rejects", this would classify + // as partial and send someone to narrow a detector on the strength of a leg that + // never answered. + const legs = { + '3.7.0': writeLegWithTransportError({ + version: '3.7.0', + erroredQuery: 'triggerB', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { report } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal( + report.drifts.filter((d) => d.driftClass === 'engine-partially-relaxed').length, + 0, + 'an unobserved trigger must not be counted as holding' + ); + const relaxed = report.drifts.find((d) => d.driftClass === 'engine-relaxed'); + assert.ok(relaxed); + assert.deepEqual(relaxed.scope.unobserved, ['triggerB']); + assert.match(relaxed.remediation.detail, /produced no verdict/); +}); + +test('a rule out of scope on an older engine that accepts is not drift', () => { + const legs = healthyLegs(); + // 3.6 predates the rule's minVersion and accepts the query: intended silence. + legs['3.6.0'] = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report, summary } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.matrix.find((m) => m.version === '3.6.0').status, 'out-of-scope'); + assert.equal(report.coverageHoles.length, 0); + const row = summary + .split('\n') + .find((line) => line.startsWith('| `union-min-datasets` |')); + assert.match(row, /Calcite, >= 3\.7\.0/); + assert.match(row, /expected n\/a/); + assert.equal((row.match(/compatible/g) || []).length, 2); +}); + +test('an engine below minVersion is expected n/a even when its query rejects', () => { + const legs = healthyLegs(); + legs['3.6.0'] = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report, summary } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.drifts.filter((drift) => drift.version === '3.6.0').length, 0); + assert.equal(report.matrix.find((row) => row.version === '3.6.0').status, 'out-of-scope'); + assert.match(summary, /expected n\/a/); +}); + +test('an in-scope version with no expectation is a coverage hole, not silent success', () => { + // The rule applies from 3.7 up, but the contract only pins <3.8 — so a 3.8 + // engine runs a shipped default-error rule with nothing pinning it. + const contracts = writeContracts({ + expectations: [{ ...SPEC.expectations[0], version: '>=3.7.0 <3.8.0' }], + }); + const { status, report } = run({ contracts, legs: healthyLegs() }); + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 1); + const hole = report.coverageHoles[0]; + assert.equal(hole.version, '3.8.0'); + assert.equal(hole.enforced, true); + assert.match(report.matrix.find((m) => m.version === '3.8.0').status, /uncovered/); +}); + +test('a renamed parser rule is reported once per version, not once per query', () => { + const legs = healthyLegs(); + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + parserRuleNames: ['unionStatement', 'unionDataset'], + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const grammarDrifts = report.drifts.filter((d) => d.driftClass === 'grammar-rule-missing'); + assert.equal(grammarDrifts.length, 1, 'one grammar finding per rule/version'); + assert.equal(grammarDrifts[0].remediation.action, 'update-detector'); + assert.match(grammarDrifts[0].remediation.detail, /unionStatement/); +}); + +test('a silent detector on an unchanged engine is update-detector, never a re-pin', () => { + const legs = healthyLegs(); + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const drift = report.drifts.find((d) => d.version === '3.8.0'); + assert.equal(drift.driftClass, 'detector-silent'); + assert.equal(drift.remediation.action, 'update-detector'); +}); + +test('the leg label is corrected to the engine self-reported version', () => { + // Ask for 3.7.0 but hand over an engine that says 3.8.0: results must be + // attributed to what actually ran. + const legs = { '3.7.0': writeLeg({ version: '3.8.0', cases: { trigger: { detector: 1, rejected: true } } }) }; + const { report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(report.legs[0].label, '3.7.0'); + assert.equal(report.legs[0].engineVersion, '3.8.0'); + assert.match(stdout, /reported engineVersion "3\.8\.0"/); +}); + +test('a missing leg artifact fails loudly instead of dropping the version', () => { + const emptyLeg = makeTmp('ppl-lint-empty-leg-'); + const { status, stderr } = run({ contracts: writeContracts(), legs: { '3.8.0': emptyLeg } }); + assert.equal(status, 2, 'a broken matrix must not be able to pass'); + assert.match(stderr, /expected file not found/); +}); + +test('a default-error rule with no contract file fails the check', () => { + // OSD started shipping `brand-new-error-rule` enabled at error severity, but no + // contract pins it — so no engine version validates it. That must be red, not + // silently absent from the matrix. + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + defaultErrorRules: ['union-min-datasets', 'brand-new-error-rule'], + censusEnforced: true, + }), + }; + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.result.missingContractCount, 1); + assert.equal(report.missingContracts[0].ruleId, 'brand-new-error-rule'); + assert.match(stdout, /Unvalidated default-error rules/); + assert.match(stdout, /brand-new-error-rule.*no contract file/s); +}); + +test('a census matching the manifest keeps the check green', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + defaultErrorRules: ['union-min-datasets'], + }), + }; + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.result.missingContractCount, 0); +}); + +test('an enforced shipping census mismatch fails aggregation', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + censusEnforced: true, + }), + }; + + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.result.passed, false); + assert.ok(report.result.blockingShippingCensusProblems > 0); + assert.equal(report.shippingCensus.blocking, true); + assert.match(stdout, /shipping census problem/); + assert.match(stdout, /CENSUS ENFORCED/); + assert.doesNotMatch(stdout, /CENSUS REPORT-ONLY/); + assert.match(stdout, /### Shipping census/); +}); + +test('a report-only shipping census mismatch remains green', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }), + }; + + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.result.passed, true); + assert.equal(report.result.blockingShippingCensusProblems, 0); + assert.ok(report.shippingCensus.problems.length > 0); + assert.equal(report.shippingCensus.blocking, false); + assert.match(stdout, /CENSUS REPORT-ONLY/); + assert.doesNotMatch(stdout, /CENSUS ENFORCED/); +}); + +test('a schema-v2 detector report without a census fails closed', () => { + const dir = writeLeg({ + version: '3.8.0', + defaultErrorRules: null, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /defaultErrorRules must be a JSON array/); +}); + +test('a schema-v2 detector report with an unknown grammar surface fails closed', () => { + const dir = writeLeg({ + version: '3.8.0', + surface: 'unknown-surface', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /surface must be "runtime-bundle" or "compiled-simplified"/); +}); + +test('a schema-v2 detector census rejects duplicate rule identities', () => { + const dir = writeLeg({ + version: '3.8.0', + defaultErrorRules: [SPEC.ruleId, SPEC.ruleId], + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /defaultErrorRules contains duplicate rule/); +}); + +// --- "we don't know" must never render as "it's fine" ------------------------- + +/** Write a leg where a named query produced no engine verdict (transport failure). */ +function writeLegWithTransportError({ version, erroredQuery, cases }) { + const dir = writeLeg({ version, cases }); + const file = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(file, 'utf8')).map((entry) => + entry.queryName === erroredQuery + ? // Exactly what the IT writes on a transport failure: an `error` outcome and + // NO `rejected` field, because no verdict was ever received. + { + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: entry.executionBackend, + outcome: 'error', + error: 'connect timeout', + } + : { ...entry, outcome: 'observed' } + ); + fs.writeFileSync(file, JSON.stringify(backend)); + return dir; +} + +test('a transport error is not read as engine acceptance', () => { + // Regression: coercing a missing `rejected` to false made a timeout look like an + // engine that now ACCEPTS the trigger, and advised disabling a healthy rule. + const legs = { + '3.7.0': writeLegWithTransportError({ + version: '3.7.0', + erroredQuery: 'trigger', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + }; + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal( + report.drifts.filter((d) => d.driftClass === 'engine-relaxed').length, + 0, + 'a timeout must never be reported as the engine relaxing' + ); + assert.ok( + !/FALSE POSITIVE/.test(stdout), + 'a timeout must never advise disabling or version-scoping a rule' + ); + // This spec has a single trigger, so losing it means the rule's behavioral + // claim went unchecked: inconclusive and red, not a passing WARN. + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.match(stdout, /trigger \(no engine verdict\)/); +}); + +test('losing every trigger is inconclusive even when a control still compares', () => { + // flat-object-subfield's real shape: several triggers plus one control. The + // triggers ARE the rule's claim, so a leg that kept only the control has proven + // nothing — but `compared > 0`, so a naive count would have rendered `agree`. + const dir = writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }); + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + executionBackend: 'standard', + rejected: false, + outcome: 'observed', + observed: { httpStatus: 200, rejected: false }, + }, + ]) + ); + const { status, report } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + assert.equal(status, 1, 'a rule whose triggers all went unobserved must not read as agreement'); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.equal(report.result.enforcedInconclusive, 1); +}); + +test('a leg where nothing could be compared is inconclusive, not agreement', () => { + const dir = writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }); + // Both cases lose their verdict: the whole leg proved nothing. + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, + ]) + ); + const { status, report, stdout } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + assert.equal(status, 1, 'inconclusive must be red — "could not check" is not "passed"'); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.drifts.length, 0, 'a dead leg must not manufacture linter advice'); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match(stdout, /Inconclusive \(leg problem, not a linter problem\)/); +}); + +test('a reworded engine message does not mask a detector that went silent', () => { + // Regression: ENGINE_MESSAGE_CHANGED returned before the detector-silent check, + // so the report said "no rule change is required" while the rule had stopped + // firing. Re-pinning the string would have gone green over a dead rule. + const dir = writeLeg({ + version: '3.7.0', + cases: { + trigger: { detector: 0, rejected: true, reason: 'union needs >= 2 datasets' }, + control: { detector: 0, rejected: false }, + }, + }); + const { report } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + const drift = report.drifts.find((d) => d.queryName === 'trigger'); + assert.equal(drift.driftClass, 'detector-silent'); + assert.equal(drift.remediation.action, 'update-detector'); +}); + +test('a detector observation below minVersion remains expected n/a', () => { + const dir = writeLeg({ + version: '3.6.0', // below the rule's 3.7 minVersion + cases: { trigger: { detector: 1, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report, summary } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); + assert.equal(status, 0); + assert.equal(report.drifts.length, 0); + assert.equal(report.matrix[0].status, 'out-of-scope'); + assert.match(summary, /expected n\/a/); +}); + +test('a calcite-scoped expectation is selected rather than counted twice', () => { + // Both single-version halves drop `engine: "calcite"` entries when Calcite is + // off. Without that filter here, a per-engine pair for one range matches twice + // and is misreported as an uncovered version. + const contracts = writeContracts({ + expectations: [ + SPEC.expectations[0], + { ...SPEC.expectations[0], engine: undefined, queries: SPEC.expectations[0].queries }, + ], + }); + const { report } = run({ contracts, legs: healthyLegs() }); + // Two matching entries is genuinely ambiguous and must not silently pick one. + assert.ok( + report.coverageHoles.length > 0 || report.matrix.some((m) => m.status === 'uncovered'), + 'an ambiguous pair of expectations must be surfaced, not resolved arbitrarily' + ); +}); + +test('an errored trigger below minVersion remains expected n/a', () => { + const dir = writeLeg({ + version: '3.6.0', // below the rule's 3.7 minVersion => out of scope + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + executionBackend: 'standard', + rejected: false, + outcome: 'observed', + observed: { httpStatus: 200, rejected: false }, + }, + ]) + ); + const { status, report } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); + assert.equal(status, 0); + assert.equal(report.drifts.length, 0); + assert.equal(report.matrix[0].status, 'out-of-scope'); + assert.equal(report.result.enforcedInconclusive, 0); +}); + +test('missing rows below minVersion remain expected n/a', () => { + const dir = writeLeg({ + version: '3.6.0', + cases: { + trigger: { detector: 0, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results = detector.results.filter((entry) => entry.queryName !== 'trigger'); + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).filter( + (entry) => entry.queryName !== 'trigger' + ); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); + + assert.equal(status, 0); + assert.equal(report.matrix[0].status, 'out-of-scope'); + assert.equal(report.inconclusive.length, 0); +}); + +test('an errored control below minVersion remains expected n/a', () => { + const dir = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: true } }, + }); + const backend = JSON.parse(fs.readFileSync(path.join(dir, 'backend-report.json'), 'utf8')).map( + (e) => + e.role === 'control' + ? { + ruleId: e.ruleId, + queryName: e.queryName, + role: e.role, + executionBackend: e.executionBackend, + outcome: 'error', + error: 'timeout', + } + : { ...e, outcome: 'observed' } + ); + fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); + const { status, report, stdout, summary } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); + assert.equal(report.drifts.length, 0); + assert.ok(!/Widen "/.test(stdout)); + assert.equal(status, 0); + assert.equal(report.matrix[0].status, 'out-of-scope'); + assert.match(summary, /expected n\/a/); +}); + +test('a bad --leg argument is rejected', () => { + const result = spawnSync( + process.execPath, + [SCRIPT, '--contracts', writeContracts(), '--leg', 'no-equals-sign'], + { encoding: 'utf8' } + ); + assert.equal(result.status, 2); + assert.match(result.stderr, /--leg expects =/); +}); + +test('at least one leg is required', () => { + const result = spawnSync(process.execPath, [SCRIPT, '--contracts', writeContracts()], { + encoding: 'utf8', + }); + assert.equal(result.status, 2); + assert.match(result.stderr, /at least one --leg/); +}); diff --git a/scripts/ppl-lint/__tests__/annotate.test.mjs b/scripts/ppl-lint/__tests__/annotate.test.mjs new file mode 100644 index 00000000000..c0aca6f5d33 --- /dev/null +++ b/scripts/ppl-lint/__tests__/annotate.test.mjs @@ -0,0 +1,328 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + buildAnnotations, + buildRequiredAnnotations, + contractRepoPath, + findExpectationLine, + findRuleIdLine, + formatAnnotation, +} from '../annotate.mjs'; + +/** A contract shaped like the real ones, with two version-scoped expectations. */ +const CONTRACT = `{ + "schemaVersion": 3, + "ruleId": "invalid-capture-group-name", + "queries": { + "trigger": { "role": "trigger", "query": "source={{index}} | rex ..." } + }, + "expectations": [ + { + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": {} + }, + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": {} + } + ] +} +`; + +const readStub = (text) => () => text; +const REQUIRED_MANIFEST = `{ + "schemaVersion": 4, + "contracts": [ + "invalid-capture-group-name.spec.json" + ], + "defaultError": [ + "invalid-capture-group-name.spec.json" + ] +} +`; +const readRequired = (_dir, file) => + file === 'manifest.json' + ? REQUIRED_MANIFEST + : file === 'invalid-capture-group-name.spec.json' + ? CONTRACT + : undefined; + +test('anchors on the expectation entry that drifted, not the first one', () => { + assert.equal(findExpectationLine(CONTRACT, '>=3.7.0'), 14); + assert.equal(findExpectationLine(CONTRACT, '>=3.4.0 <3.7.0'), 9); +}); + +test('an ambiguous or absent range yields no line rather than a wrong one', () => { + // A wrong line number sends the reader to edit the wrong expectation, which is + // worse than making them find it: prefer no anchor. + const duplicated = CONTRACT.replace('">=3.4.0 <3.7.0"', '">=3.7.0"'); + assert.equal(findExpectationLine(duplicated, '>=3.7.0'), undefined); + assert.equal(findExpectationLine(CONTRACT, '>=9.9.9'), undefined); + assert.equal(findExpectationLine(undefined, '>=3.7.0'), undefined); + assert.equal(findExpectationLine(CONTRACT, undefined), undefined); +}); + +test('incidental whitespace does not defeat the anchor', () => { + const spaced = CONTRACT.replace('"version": ">=3.7.0"', '"version": ">=3.7.0"'); + assert.equal(typeof findExpectationLine(spaced, '>=3.7.0'), 'number'); +}); + +test('falls back to the ruleId line for rule-wide findings', () => { + assert.equal(findRuleIdLine(CONTRACT), 3); +}); + +test('a drift with no expectation range still anchors at the rule', () => { + // grammar-rule-missing is a fact about the rule on that engine, so it carries no + // expectationRange — it must still land on the file at a usable line. + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + driftClass: 'grammar-rule-missing', + enforced: true, + contractFile: 'invalid-capture-group-name.spec.json', + evidence: 'the candidate grammar has no parser rule(s) "rexCommand"', + remediation: { action: 'update-detector', detail: 'Re-anchor the detector.' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].line, 3); + assert.equal(annotations[0].level, 'error'); +}); + +test('inconclusive findings are warnings, never errors', () => { + // "We could not check" must not sit in the error list beside real drift, or the + // reader edits a rule because a leg timed out. + const annotations = buildAnnotations( + { + inconclusive: [ + { + ruleId: 'field-validation', + version: '3.6.0', + enforced: true, + file: 'field-validation.spec.json', + reasons: ['unknown-field-existence (no engine verdict)'], + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].level, 'warning'); + assert.match(annotations[0].message, /NOT a lint finding/); + assert.match(annotations[0].message, /Do not edit the rule/); +}); + +test('a non-enforced drift is a warning so it cannot be read as blocking', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'head-without-sort', + version: '3.8.0', + driftClass: 'detector-noisy', + enforced: false, + contractFile: 'head-without-sort.spec.json', + evidence: 'evidence', + remediation: { action: 'update-detector', detail: 'detail' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations[0].level, 'warning'); +}); + +test('backend divergence annotations name both execution routes', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + executionBackend: 'analytics', + executionBackends: ['standard', 'analytics'], + driftClass: 'execution-backend-divergence', + enforced: true, + contractFile: 'invalid-capture-group-name.spec.json', + evidence: 'standard rejected while analytics accepted', + remediation: { action: 'align-execution-backends', detail: 'Align route behavior.' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.match(annotations[0].title, /standard vs analytics/); + assert.match(annotations[0].title, /execution-backend-divergence/); +}); + +test('an unvalidated rule has no file to point at', () => { + const annotations = buildAnnotations( + { missingContracts: [{ ruleId: 'sort-on-eval-field', reason: 'has no contract file' }] }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].file, undefined); + assert.equal(annotations[0].level, 'error'); + assert.match(annotations[0].message, /manifest\.defaultError/); +}); + +test('shipping census errors point to manifest.json', () => { + const annotations = buildAnnotations( + { + shippingCensus: { + passed: false, + blocking: true, + problems: ['active lint rules do not equal enabled OSD rules'], + }, + }, + { + contractsDir: '/workspace/contracts', + workspace: '/workspace', + readFile: (_dir, file) => + file === 'manifest.json' + ? '{\n "schemaVersion": 4,\n "contracts": []\n}\n' + : undefined, + } + ); + + assert.deepEqual(annotations, [ + { + level: 'error', + file: 'contracts/manifest.json', + line: 3, + title: 'PPL lint shipping census mismatch', + message: + 'active lint rules do not equal enabled OSD rules\n' + + 'FIX: align the active SQL manifest with the approved OSD shipping catalog.', + }, + ]); +}); + +test('paths are repo-relative so GitHub can render them inline', () => { + // An absolute path still annotates the run, but never attaches to the diff. + assert.equal( + contractRepoPath('/w/integ-test/res/contracts', 'a.spec.json', '/w'), + 'integ-test/res/contracts/a.spec.json' + ); + // No workspace (a local run): absolute is the honest answer. + assert.equal(contractRepoPath('/w/c', 'a.spec.json', undefined), '/w/c/a.spec.json'); +}); + +test('workflow-command metacharacters are escaped', () => { + const line = formatAnnotation({ + level: 'error', + file: 'a,b:c.json', + line: 12, + title: 'has: comma, and colon', + message: 'first\nsecond 100% done', + }); + // Commas/colons in properties would otherwise terminate the property list. + assert.match(line, /file=a%2Cb%3Ac\.json/); + assert.match(line, /title=has%3A comma%2C and colon/); + // Newlines must survive as %0A or the annotation is truncated to one line. + assert.match(line, /first%0Asecond 100%25 done/); + assert.ok(line.startsWith('::error ')); +}); + +test('an unreadable contract still produces a file-less annotation', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'r', + version: '3.8.0', + driftClass: 'detector-silent', + enforced: true, + contractFile: 'gone.spec.json', + evidence: 'evidence', + remediation: { action: 'update-detector', detail: 'detail' }, + }, + ], + }, + { contractsDir: '/w/c', workspace: '/w', readFile: () => undefined } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].line, undefined); + assert.equal(annotations[0].file, 'c/gone.spec.json'); +}); + +test('required-lane failures anchor to their contract and census findings to the manifest', () => { + const annotations = buildRequiredAnnotations( + { + detectorFailures: [ + '[invalid-capture-group-name/trigger] expected 1 diagnostic, got 0', + ], + censusProblems: ['active lint contracts do not equal enabled catalog rules'], + censusEnforced: false, + }, + { + contractsDir: '/w/integ-test/resources/contracts', + workspace: '/w', + readFile: readRequired, + } + ); + + assert.equal(annotations.length, 2); + assert.deepEqual( + { + level: annotations[0].level, + file: annotations[0].file, + line: annotations[0].line, + }, + { + level: 'error', + file: 'integ-test/resources/contracts/invalid-capture-group-name.spec.json', + line: 3, + } + ); + assert.equal(annotations[1].level, 'warning'); + assert.equal(annotations[1].file, 'integ-test/resources/contracts/manifest.json'); + assert.match(annotations[1].message, /REPORT ONLY/); +}); + +test('required artifact row failures recover the rule identity for an inline annotation', () => { + const annotations = buildRequiredAnnotations( + { + artifactErrors: [ + 'backend row invalid-capture-group-name::trigger did not pass its oracle (outcome="fail")', + ], + }, + { + contractsDir: '/w/contracts', + workspace: '/w', + readFile: readRequired, + } + ); + + assert.equal(annotations.length, 1); + assert.equal(annotations[0].file, 'contracts/invalid-capture-group-name.spec.json'); + assert.match(annotations[0].title, /invalid-capture-group-name\/trigger/); +}); + +test('required job failures without a contract identity remain file-less', () => { + const annotations = buildRequiredAnnotations( + { backendResult: 'failure', detectorResult: 'skipped' }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readRequired } + ); + assert.equal(annotations.length, 2); + assert.ok(annotations.every((annotation) => annotation.file === undefined)); +}); + +test('a clean report emits nothing', () => { + assert.deepEqual(buildAnnotations({}, { contractsDir: '/w/c', workspace: '/w' }), []); + assert.deepEqual(buildRequiredAnnotations({}, { contractsDir: '/w/c', workspace: '/w' }), []); +}); diff --git a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs new file mode 100644 index 00000000000..cb3b1497181 --- /dev/null +++ b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs @@ -0,0 +1,296 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'assemble-run-manifest.mjs'); +const tmpDirs = []; + +function makeRun() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-manifest-')); + tmpDirs.push(dir); + fs.mkdirSync(path.join(dir, 'artifacts')); + return dir; +} + +function writeJson(dir, name, value) { + fs.writeFileSync(path.join(dir, 'artifacts', name), JSON.stringify(value)); +} + +function validArtifacts(dir) { + writeJson(dir, 'target.json', { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:test', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }); + writeJson(dir, 'backend-report.json', [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + rejected: false, + observed: { httpStatus: 200, rejected: false, response: { datarows: [] } }, + outcome: 'pass', + }, + ]); + writeJson(dir, 'detector-report.json', { + schemaVersion: 2, + executionBackend: 'standard', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:test', + surface: 'runtime-bundle', + defaultErrorRules: ['advisory-rule'], + results: [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + expected: 1, + actual: 1, + severities: ['warning'], + severityMatched: true, + messageMatched: true, + executionBackend: 'standard', + }, + ], + }); +} + +function run(dir, extraEnv = {}) { + return spawnSync(process.execPath, [SCRIPT], { + cwd: dir, + encoding: 'utf8', + env: { + ...process.env, + BACKEND_RESULT: 'success', + DETECTOR_RESULT: 'success', + SQL_SHA: 'candidate-sql-sha', + ...extraEnv, + }, + }); +} + +after(() => { + for (const dir of tmpDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('valid standard artifacts produce a passing schema-v2 manifest', () => { + const dir = makeRun(); + validArtifacts(dir); + const result = run(dir); + assert.equal(result.status, 0, result.stderr); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'run-manifest.json'), 'utf8')); + assert.equal(manifest.schemaVersion, 2); + assert.equal(manifest.executionBackend, 'standard'); + assert.equal(manifest.result.passed, true); + assert.deepEqual(manifest.result.artifactErrors, []); +}); + +test('a missing report fails closed while still writing the manifest', () => { + const dir = makeRun(); + validArtifacts(dir); + fs.rmSync(path.join(dir, 'artifacts', 'backend-report.json')); + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /required artifact is missing/); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'run-manifest.json'), 'utf8')); + assert.equal(manifest.result.passed, false); +}); + +test('detector rows must have unique identities matching the target', () => { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results.push({ ...detector.results[0] }); + fs.writeFileSync(file, JSON.stringify(detector)); + + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /duplicate row advisory-rule::trigger/); +}); + +test('a backend row without a real verdict cannot render as acceptance', () => { + const dir = makeRun(); + validArtifacts(dir); + writeJson(dir, 'backend-report.json', [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'connect timeout', + }, + ]); + + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not pass its oracle/); +}); + +test('an accepted advisory trigger is summarized from its backend outcome, not its role', () => { + const dir = makeRun(); + validArtifacts(dir); + const summary = path.join(dir, 'summary.md'); + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.equal(result.status, 0, result.stderr); + const markdown = fs.readFileSync(summary, 'utf8'); + assert.match(markdown, /advisory-rule.*accepted.*Pass/); +}); + +test('detector severity and message mismatches fail the manifest and summary', () => { + for (const field of ['severityMatched', 'messageMatched']) { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0][field] = false; + fs.writeFileSync(file, JSON.stringify(detector)); + const summary = path.join(dir, 'summary.md'); + + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, new RegExp(`did not match its ${field === 'severityMatched' ? 'severity' : 'message'}`)); + assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*accepted.*Fail/); + } +}); + +test('a detector execution error fails with its rule row instead of missing-artifact noise', () => { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0] = { + ...detector.results[0], + expected: 0, + actual: 0, + assertions: { execution: false }, + mismatches: [ + { + field: 'execution', + expected: 'completed', + actual: 'detector crashed', + }, + ], + outcome: 'error', + error: 'detector crashed', + }; + detector.failures = [ + '[advisory-rule/trigger] frontend.execution failed: detector crashed', + ]; + fs.writeFileSync(file, JSON.stringify(detector)); + const summary = path.join(dir, 'summary.md'); + + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /advisory-rule::trigger execution failed: detector crashed/); + assert.doesNotMatch(result.stderr, /count mismatch/); + assert.doesNotMatch(result.stderr, /did not match its execution assertion/); + assert.doesNotMatch(result.stderr, /required artifact is missing/); + assert.doesNotMatch(result.stderr, /has no matching detector row/); + assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*Error.*Fail/); +}); + +test('syntax-specific frontend mismatches fail artifact validation', () => { + for (const field of ['fixMatched', 'rawMessageMatched', 'totalErrorsMatched']) { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0][field] = false; + fs.writeFileSync(file, JSON.stringify(detector)); + + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /did not match its (syntax-fix|raw-parser-error|total-error) assertion/ + ); + } +}); + +test('exact deterministic fix mismatches fail artifacts and summary rows', () => { + for (const field of ['deterministicFixMatched']) { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0][field] = false; + detector.results[0].assertions = { + deterministicFix: false, + }; + detector.results[0].mismatches = [ + { + field: 'deterministicFix', + expected: { offered: false }, + actual: { offered: true }, + }, + ]; + fs.writeFileSync(file, JSON.stringify(detector)); + const summary = path.join(dir, 'summary.md'); + + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not match its deterministic-fix assertion/); + assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*accepted.*Fail/); + } +}); + +test('report-only dormant rows do not affect the required manifest result or active set', () => { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results.push({ + ruleId: 'dormant-rule', + queryName: 'trigger', + role: 'trigger', + expected: 1, + actual: 0, + severities: [], + severityMatched: false, + messageMatched: false, + assertions: { count: false }, + mismatches: [{ field: 'count', expected: 1, actual: 0 }], + executionBackend: 'standard', + reportOnly: true, + }); + fs.writeFileSync(file, JSON.stringify(detector)); + const backendFile = path.join(dir, 'artifacts', 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')); + backend.push({ + ruleId: 'dormant-rule', + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'report-only observation failed', + }); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const summary = path.join(dir, 'summary.md'); + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.equal(result.status, 0, result.stderr); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'run-manifest.json'), 'utf8')); + assert.deepEqual(manifest.validationSet, ['advisory-rule']); + assert.equal(manifest.result.passed, true); + assert.match(fs.readFileSync(summary, 'utf8'), /dormant-rule.*Report only/); +}); diff --git a/scripts/ppl-lint/__tests__/contract-schema.test.mjs b/scripts/ppl-lint/__tests__/contract-schema.test.mjs new file mode 100644 index 00000000000..0aa55eeabb6 --- /dev/null +++ b/scripts/ppl-lint/__tests__/contract-schema.test.mjs @@ -0,0 +1,881 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + assertContractSchema, + assertExactQueryCoverage, + assertShippingFrontendOracles, + classifyBackendReportRow, + contractChannel, + indexBackendReport, + normalizeFrontendOracle, + normalizeLintWiring, + normalizeTarget, + resolveBackendOracle, +} from '../contract-schema.mjs'; + +const QUERY = { + detectorCount: 1, + severity: 'error', + matchMessage: 'bad query', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, +}; + +function spec(schemaVersion, queryExpectation = QUERY) { + return { + schemaVersion, + ruleId: 'example-rule', + queries: { + trigger: { role: 'trigger', query: 'source={{index}} | bad' }, + control: { role: 'control', query: 'source={{index}} | head 1' }, + }, + expectations: [ + { + version: '>=3.7.0', + queries: { + trigger: queryExpectation, + control: { + detectorCount: 0, + ...(schemaVersion === 3 + ? { backend: { kind: 'result-shape', httpStatus: 200 } } + : { + backends: { + standard: { kind: 'result-shape', httpStatus: 200 }, + analytics: { kind: 'result-shape', httpStatus: 200 }, + }, + }), + }, + }, + }, + ], + }; +} + +test('target schema v2 requires and preserves explicit standard or analytics identity', () => { + for (const executionBackend of ['standard', 'analytics']) { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:abc', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend, + storage: executionBackend === 'analytics' ? 'composite-parquet' : 'lucene', + shardCount: 1, + ...(executionBackend === 'analytics' + ? { + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + } + : {}), + }); + assert.equal(target.executionBackend, executionBackend); + assert.equal(target.legacy, false); + } +}); + +test('unversioned targets cannot infer a standard execution identity', () => { + assert.throws( + () => + normalizeTarget({ + engineVersion: '3.7.0', + grammarHash: 'sha256:legacy', + }), + /target\.schemaVersion is required/ + ); +}); + +test('unknown target schema and execution backend are rejected', () => { + assert.throws( + () => + normalizeTarget({ + schemaVersion: 3, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'standard', + storage: 'lucene', + }), + /Unsupported target schemaVersion 3/ + ); + assert.throws( + () => + normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'experimental', + }), + /must be "standard" or "analytics"/ + ); +}); + +test('schema v3 resolves a standard oracle and never falls back for analytics', () => { + const contract = spec(3); + const standard = resolveBackendOracle(contract, QUERY, 'standard'); + const analytics = resolveBackendOracle(contract, QUERY, 'analytics'); + + assert.equal(standard.status, 'applicable'); + assert.equal(standard.oracle, QUERY.backend); + assert.deepEqual(standard.detector, analytics.detector); + assert.equal(analytics.status, 'coverage-missing'); + assert.equal(analytics.oracle, undefined); + assert.match(analytics.reason, /standard-only/); +}); + +test('analytics targets fail closed on storage and route attestation', () => { + const base = { + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }; + assert.equal(normalizeTarget(base).executionBackend, 'analytics'); + assert.throws( + () => normalizeTarget({ ...base, storage: 'lucene' }), + /storage must be "composite-parquet"/ + ); + const withoutStack = { ...base }; + delete withoutStack.analyticsStack; + assert.throws( + () => normalizeTarget(withoutStack), + /analyticsStack must be a JSON object/ + ); + assert.throws( + () => + normalizeTarget({ + ...base, + routeAttestation: { ...base.routeAttestation, explainVerified: false }, + }), + /explainVerified must be true/ + ); +}); + +test('schema-v2 standard targets require explicit storage and shard identity', () => { + const base = { + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }; + assert.equal(normalizeTarget(base).executionBackend, 'standard'); + assert.throws( + () => normalizeTarget({ ...base, storage: undefined }), + /storage must be "lucene"/ + ); + assert.throws( + () => normalizeTarget({ ...base, shardCount: undefined }), + /shardCount must be a positive integer/ + ); +}); + +test('schema v4 selects only the requested backend oracle', () => { + const queryExpectation = { + detectorCount: 1, + severity: 'warning', + backends: { + standard: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, + analytics: { kind: 'advisory', httpStatus: 200 }, + }, + }; + const contract = spec(4, queryExpectation); + + const standard = resolveBackendOracle(contract, queryExpectation, 'standard'); + const analytics = resolveBackendOracle(contract, queryExpectation, 'analytics'); + assert.equal(standard.oracle, queryExpectation.backends.standard); + assert.equal(analytics.oracle, queryExpectation.backends.analytics); + assert.deepEqual(standard.detector, analytics.detector); +}); + +test('schema v4 reports missing route coverage without using another backend oracle', () => { + const queryExpectation = { + detectorCount: 1, + severity: 'error', + backends: { + standard: { kind: 'rejection', httpStatus: 400 }, + }, + }; + const analytics = resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'); + + assert.equal(analytics.status, 'coverage-missing'); + assert.equal(analytics.oracle, undefined); + assert.match(analytics.reason, /no analytics backend oracle/); +}); + +test('not-applicable is explicit while an absent oracle is coverage-missing', () => { + const notApplicable = { + detectorCount: 1, + backends: { + analytics: { + kind: 'not-applicable', + reason: 'fixture is unsupported', + owner: '@analytics-team', + issue: 'https://example.test/issues/1', + }, + }, + }; + const missing = { + detectorCount: 1, + backends: {}, + }; + + assert.equal( + resolveBackendOracle(spec(4, notApplicable), notApplicable, 'analytics').status, + 'not-applicable' + ); + assert.equal( + resolveBackendOracle(spec(4, missing), missing, 'analytics').status, + 'coverage-missing' + ); + assert.throws( + () => + resolveBackendOracle( + spec(4, { + detectorCount: 1, + backends: { analytics: { kind: 'not-applicable' } }, + }), + { detectorCount: 1, backends: { analytics: { kind: 'not-applicable' } } }, + 'analytics' + ), + /backend oracle\.reason/ + ); + assert.throws( + () => { + const oracle = { + detectorCount: 1, + backends: { + analytics: { + kind: 'not-applicable', + reason: 'fixture is unsupported', + issue: 'https://example.test/issues/1', + }, + }, + }; + return resolveBackendOracle(spec(4, oracle), oracle, 'analytics'); + }, + /backend oracle\.owner/ + ); +}); + +test('unknown contract schema and backend oracle kind are rejected', () => { + assert.throws(() => assertContractSchema(spec(5)), /expected 3 or 4/); + const queryExpectation = { + detectorCount: 1, + backends: { analytics: { kind: 'maybe' } }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'), + /unknown analytics backend oracle.kind/ + ); + const unknownBackend = { + detectorCount: 1, + backends: { experimental: { kind: 'advisory' } }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, unknownBackend), unknownBackend, 'analytics'), + /backends key must be "standard" or "analytics"/ + ); +}); + +test('backend oracle payloads fail closed when required shapes are malformed', () => { + const cases = [ + { + oracle: { kind: 'rejection', body: { status: 400 } }, + expected: /httpStatus/, + }, + { + oracle: { kind: 'rejection', httpStatus: 400 }, + expected: /\.body must be a JSON object/, + }, + { + oracle: { + kind: 'rejection', + httpStatus: 400, + body: { status: '400' }, + }, + expected: /\.body\.status must be an integer/, + }, + { + oracle: { + kind: 'rejection', + httpStatus: 400, + body: { status: 500 }, + }, + expected: /\.httpStatus must equal .*\.body\.status/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 201, + }, + expected: /\.httpStatus must be 200/, + }, + { + oracle: { + kind: 'advisory', + httpStatus: 204, + }, + expected: /\.httpStatus must be 200/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 200, + expect: { datarowsNonEmpty: 'yes' }, + }, + expected: /datarowsNonEmpty must be a boolean/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 200, + expect: { datarowsCount: -1 }, + }, + expected: /datarowsCount must be a non-negative integer/, + }, + ]; + + for (const { oracle, expected } of cases) { + const queryExpectation = { + detectorCount: 1, + backends: { analytics: oracle }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'), + expected + ); + } +}); + +test('selected expectation query keys must exactly equal top-level query keys', () => { + const contract = spec(3); + assert.deepEqual( + assertExactQueryCoverage(contract, contract.expectations[0]), + ['control', 'trigger'] + ); + + const missing = structuredClone(contract.expectations[0]); + delete missing.queries.control; + assert.throws( + () => assertExactQueryCoverage(contract, missing), + /missing from expectation: control/ + ); + + const extra = structuredClone(contract.expectations[0]); + extra.queries.unknown = QUERY; + assert.throws( + () => assertExactQueryCoverage(contract, extra), + /not present in contract\.queries: unknown/ + ); + + assert.throws( + () => + assertExactQueryCoverage( + { ...contract, queries: {} }, + { ...contract.expectations[0], queries: {} } + ), + /contract\.queries must not be empty/ + ); +}); + +test('non-verdict backend states are never coerced to acceptance', () => { + assert.deepEqual(classifyBackendReportRow({ rejected: false }), { + status: 'observed', + rejected: false, + }); + for (const outcome of ['not-applicable', 'coverage-missing', 'error']) { + assert.equal( + classifyBackendReportRow({ outcome, rejected: false }).status, + outcome + ); + assert.equal( + classifyBackendReportRow({ outcome, rejected: false }).rejected, + undefined + ); + } + assert.equal(classifyBackendReportRow({ outcome: 'pass' }).status, 'error'); +}); + +test('backend report indexing rejects duplicate keys', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }); + const row = { + ruleId: 'example-rule', + queryName: 'trigger', + executionBackend: 'analytics', + rejected: true, + }; + assert.throws(() => indexBackendReport([row, { ...row }], target), /duplicate backend report key/); + assert.throws(() => indexBackendReport({}, target), /must be a JSON array/); +}); + +test('every schema-v2 backend report row must carry identity matching the target', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }); + const base = { ruleId: 'example-rule', queryName: 'trigger', rejected: true }; + + assert.throws(() => indexBackendReport([base], target), /missing executionBackend/); + assert.throws( + () => indexBackendReport([{ ...base, executionBackend: 'standard' }], target), + /does not match target "analytics"/ + ); + assert.equal( + indexBackendReport([{ ...base, executionBackend: 'analytics' }], target).get( + 'example-rule::trigger' + ).rejected, + true + ); +}); + +test('schema-v2 standard backend rows cannot omit identity', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.7.0', + grammarHash: 'sha256:legacy', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }); + const row = { ruleId: 'example-rule', queryName: 'trigger', rejected: true }; + + assert.throws(() => indexBackendReport([row], target), /missing executionBackend/); + assert.throws( + () => indexBackendReport([{ ...row, executionBackend: 'analytics' }], target), + /does not match target "standard"/ + ); +}); + +test('missing channel remains a backwards-compatible lint contract', () => { + const contract = spec(4, { + detectorCount: 1, + severity: 'warning', + backends: { + standard: { kind: 'advisory', httpStatus: 200 }, + }, + }); + assert.equal(contractChannel(contract), 'lint'); + assert.deepEqual( + normalizeFrontendOracle(contract, contract.expectations[0].queries.trigger), + { + channel: 'lint', + count: 1, + severity: 'warning', + messageEquals: undefined, + deterministicFix: undefined, + } + ); +}); + +test('schema-v4 lint frontend normalizes exact message and fix oracles', () => { + const contract = spec(4); + const frontend = normalizeFrontendOracle(contract, { + frontend: { + count: 1, + severity: 'warning', + messageEquals: 'Use a non-zero divisor.', + deterministicFix: { + offered: true, + title: 'Replace zero', + text: '1', + range: { + startLine: 1, + startColumn: 20, + endLine: 1, + endColumn: 21, + }, + expectedText: '0', + appliedQuery: 'source=t | eval x = 1', + }, + }, + }); + + assert.deepEqual(frontend, { + channel: 'lint', + count: 1, + severity: 'warning', + messageEquals: 'Use a non-zero divisor.', + deterministicFix: { + offered: true, + title: 'Replace zero', + text: '1', + range: { + startLine: 1, + startColumn: 20, + endLine: 1, + endColumn: 21, + }, + expectedText: '0', + appliedQuery: 'source=t | eval x = 1', + }, + }); +}); + +test('matchMessage remains available only to schema-v3 lint contracts', () => { + assert.equal( + normalizeFrontendOracle(spec(3), { + frontend: { count: 1, matchMessage: 'legacy substring' }, + }).matchMessage, + 'legacy substring' + ); + assert.throws( + () => + normalizeFrontendOracle(spec(4), { + frontend: { count: 1, matchMessage: 'not exact' }, + }), + /matchMessage is not valid/ + ); +}); + +test('schema-v4 deterministic-fix payloads fail closed on partial or extra fields', () => { + const contract = spec(4); + for (const [frontend, expected] of [ + [ + { count: 1, deterministicFix: { offered: false, title: 'unexpected' } }, + /must contain only offered/, + ], + [ + { + count: 1, + deterministicFix: { + offered: true, + title: 'Fix', + text: 'x', + range: { startLine: 0, startColumn: 0, endLine: 1, endColumn: 1 }, + appliedQuery: 'x', + }, + }, + /startLine must be a positive integer/, + ], + [ + { + count: 1, + deterministicFix: { + offered: true, + title: 'Fix', + text: 'x', + range: { startLine: 1, startColumn: 2, endLine: 1, endColumn: 1 }, + expectedText: 'y', + appliedQuery: 'x', + }, + }, + /must end at or after its start/, + ], + ]) { + assert.throws(() => normalizeFrontendOracle(contract, { frontend }), expected); + } + + assert.doesNotThrow(() => + normalizeFrontendOracle(contract, { + frontend: { + count: 1, + deterministicFix: { + offered: true, + title: 'Fix', + text: 'x', + range: { startLine: 1, startColumn: 0, endLine: 1, endColumn: 1 }, + appliedQuery: 'x', + }, + }, + }) + ); +}); + +test('active lint contracts require exact messages and deterministic-fix behavior', () => { + const contract = spec(4); + const expectation = structuredClone(contract.expectations[0]); + expectation.queries.trigger = { + frontend: { + count: 1, + severity: 'error', + messageEquals: 'Exact diagnostic.', + deterministicFix: { offered: false }, + }, + backends: { + standard: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, + analytics: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, + }, + }; + expectation.queries.control.frontend = { + count: 0, + deterministicFix: { offered: false }, + }; + delete expectation.queries.control.detectorCount; + + assert.doesNotThrow(() => assertShippingFrontendOracles(contract, expectation)); + + const missingMessage = structuredClone(expectation); + delete missingMessage.queries.trigger.frontend.messageEquals; + assert.throws( + () => assertShippingFrontendOracles(contract, missingMessage), + /messageEquals is required/ + ); + + const missingSeverity = structuredClone(expectation); + delete missingSeverity.queries.trigger.frontend.severity; + assert.throws( + () => assertShippingFrontendOracles(contract, missingSeverity), + /severity is required/ + ); + + const fixWithoutFinding = structuredClone(expectation); + fixWithoutFinding.queries.control.frontend.deterministicFix = { + offered: true, + title: 'Fix', + text: 'fixed', + range: { startLine: 1, startColumn: 0, endLine: 1, endColumn: 3 }, + expectedText: 'bad', + appliedQuery: 'fixed', + }; + assert.throws( + () => assertShippingFrontendOracles(contract, fixWithoutFinding), + /must not offer a deterministic fix/ + ); +}); + +test('syntax frontend assertions normalize stable code, fix, raw message, and error census', () => { + const contract = { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + trigger: { role: 'trigger', query: 'source=t | wherre a > 1' }, + }, + }; + const frontend = normalizeFrontendOracle(contract, { + frontend: { + count: 1, + code: 'UNKNOWN_COMMAND', + fixText: 'where', + matchMessage: 'where', + rawMessage: true, + totalErrors: 1, + }, + }); + assert.deepEqual(frontend, { + channel: 'syntax', + count: 1, + code: 'UNKNOWN_COMMAND', + fixText: 'where', + matchMessage: 'where', + rawMessage: true, + totalErrors: 1, + }); + assert.equal(assertContractSchema(contract), 4); +}); + +test('active syntax contracts require explicit fix, raw-message, and total-error assertions', () => { + const contract = { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + trigger: { role: 'trigger', query: 'source=t | wherre a > 1' }, + }, + }; + const expectation = { + queries: { + trigger: { + frontend: { + count: 1, + code: 'UNKNOWN_COMMAND', + fixText: 'where', + matchMessage: 'Unknown command "wherre". Did you mean "where"?', + rawMessage: true, + totalErrors: 1, + }, + }, + }, + }; + assert.doesNotThrow(() => assertShippingFrontendOracles(contract, expectation)); + + delete expectation.queries.trigger.frontend.fixText; + assert.throws( + () => assertShippingFrontendOracles(contract, expectation), + /fixText must explicitly assert/ + ); +}); + +test('syntax supports explicit fix absence and requires frontend code to match wiring', () => { + const contract = { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + suppressed: { role: 'suppression-control', query: 'source=t | zzzzzzzz' }, + }, + }; + assert.deepEqual( + normalizeFrontendOracle(contract, { + frontend: { + count: 0, + code: 'UNKNOWN_COMMAND', + fixText: null, + rawMessage: true, + totalErrors: 1, + }, + }), + { + channel: 'syntax', + count: 0, + code: 'UNKNOWN_COMMAND', + fixText: null, + rawMessage: true, + totalErrors: 1, + } + ); + assert.throws( + () => + normalizeFrontendOracle(contract, { + frontend: { count: 0, code: 'OTHER_ERROR' }, + }), + /does not match contract\.wiring\.code/ + ); +}); + +test('lint and syntax frontend fields cannot cross channels', () => { + assert.throws( + () => + normalizeFrontendOracle(spec(4), { + frontend: { count: 1, code: 'UNKNOWN_COMMAND' }, + }), + /code is not valid/ + ); + const syntax = { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + trigger: { role: 'trigger', query: 'source=t | wherre a > 1' }, + }, + }; + assert.throws( + () => normalizeFrontendOracle(syntax, { detectorCount: 1 }), + /must use frontend/ + ); + assert.throws( + () => + assertContractSchema({ + ...syntax, + wiring: { code: 'UNKNOWN_COMMAND', detector: 'command-suggestion' }, + }), + /must contain only/ + ); +}); + +test('suppression-control is syntax-only', () => { + assert.throws( + () => + assertContractSchema({ + ...spec(4), + queries: { + suppressed: { + role: 'suppression-control', + query: 'source=t | zzzzzzzz', + }, + }, + }), + /valid only for syntax/ + ); + assert.doesNotThrow(() => + assertContractSchema({ + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + suppressed: { + role: 'suppression-control', + query: 'source=t | zzzzzzzz', + }, + }, + }) + ); +}); + +test('normalized wiring exposes omitted version, engine, and source scope gates', () => { + const catalog = normalizeLintWiring('example-rule', { + detector: 'example-rule', + enabled: true, + severity: 'warning', + appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, + sourceScoped: true, + }); + const omittedVersion = normalizeLintWiring('example-rule', { + detector: 'example-rule', + enabled: true, + severity: 'warning', + appliesTo: { engine: 'calcite' }, + sourceScoped: true, + }); + const omittedEngine = normalizeLintWiring('example-rule', { + detector: 'example-rule', + enabled: true, + severity: 'warning', + appliesTo: { minVersion: '3.7.0' }, + sourceScoped: true, + }); + const omittedSourceScope = normalizeLintWiring('example-rule', { + detector: 'example-rule', + enabled: true, + severity: 'warning', + appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, + }); + + assert.notDeepEqual(omittedVersion, catalog); + assert.notDeepEqual(omittedEngine, catalog); + assert.notDeepEqual(omittedSourceScope, catalog); + assert.equal(omittedSourceScope.sourceScoped, false); +}); diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs new file mode 100644 index 00000000000..e8e3eecfde6 --- /dev/null +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -0,0 +1,641 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the multi-version drift classifier. + * + * Run with plain Node (no Gradle, no cluster, no OSD checkout): + * + * node --test scripts/ppl-lint/__tests__/drift.test.mjs + * + * The classifier is the part of the multi-version contract that decides what an + * engineer is told to do, so every drift class and every remediation branch is + * pinned here. Observations are hand-written rather than gathered from a + * cluster; the live-engine plumbing is exercised by the CI workflow itself. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + DRIFT_CLASSES, + REMEDIATIONS, + classifyDrift, + classifyExecutionBackendDivergence, + classifyRelaxationScope, + formatDriftReport, + parseVersion, + suggestParserRules, + versionInAppliesTo, +} from '../drift.mjs'; + +/** A trigger case that agrees on all three sides, used as the mutation base. */ +function agreeingTrigger(overrides = {}) { + return { + ruleId: 'union-min-datasets', + version: '3.7.0', + queryName: 'union-single-dataset', + role: 'trigger', + query: 'union [ source=t ]', + expected: { detectorCount: 1, severity: 'error', backendKind: 'rejection' }, + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + wiring: { appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, runtimeOnly: true }, + ...overrides, + }; +} + +/** A control case that agrees on all three sides. */ +function agreeingControl(overrides = {}) { + return { + ruleId: 'union-min-datasets', + version: '3.7.0', + queryName: 'union-two-datasets-control', + role: 'control', + query: 'union [ source=t ] [ source=t ]', + expected: { detectorCount: 0, backendKind: 'result-shape' }, + observed: { detectorCount: 0, severities: [], backendRejected: false }, + wiring: { appliesTo: { minVersion: '3.7.0', engine: 'calcite' } }, + ...overrides, + }; +} + +// --- the quiet path ----------------------------------------------------------- + +test('agreement produces no drift', () => { + assert.equal(classifyDrift(agreeingTrigger()), null); + assert.equal(classifyDrift(agreeingControl()), null); +}); + +test('a rule out of version scope on an engine that also accepts is silent', () => { + // union-min-datasets does not apply below 3.7, and a 3.6 engine that accepts + // the query is not drift — it is the reason the version window exists. + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', + observed: { detectorCount: 0, severities: [], backendRejected: false }, + }) + ); + assert.equal(drift, null); +}); + +// --- grammar moved ------------------------------------------------------------ + +test('a missing parser rule is reported as update-detector, not as a silent detector', () => { + const drift = classifyDrift( + agreeingTrigger({ + requiredParserRules: ['unionCommand', 'unionDataset'], + parserRuleNames: ['unionStatement', 'unionDataset', 'pplCommands'], + observed: { detectorCount: 0, severities: [], backendRejected: true }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.GRAMMAR_RULE_MISSING); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.target, /union_min_datasets\.ts$/); + // The likely rename is named so the engineer does not diff 259 rule names. + assert.match(drift.remediation.detail, /unionStatement/); + assert.match(drift.evidence, /no parser rule/); +}); + +test('a contract can pin a detector path that breaks the naming convention', () => { + // unsupported-window-function-in-eventstats lives in + // unsupported_window_function.ts, so the derived name would not exist. + const drift = classifyDrift( + agreeingTrigger({ + ruleId: 'unsupported-window-function-in-eventstats', + detectorPath: 'packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts', + wiring: { appliesTo: {} }, + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'CalciteUnsupportedException', + backendReason: 'Unexpected window function: rank', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal( + drift.remediation.target, + 'packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts' + ); +}); + +test('grammar-rule check is skipped when the contract declares no required rules', () => { + const drift = classifyDrift( + agreeingTrigger({ parserRuleNames: ['somethingElse'], requiredParserRules: undefined }) + ); + assert.equal(drift, null); +}); + +// --- engine behavior flips ---------------------------------------------------- + +test('same-candidate route differences use backend remediation, never version scoping', () => { + const drift = classifyExecutionBackendDivergence({ + ruleId: 'union-min-datasets', + version: '3.8.0', + queryName: 'union-single-dataset', + role: 'trigger', + query: 'union [ source=t ]', + standardObserved: { backendRejected: true, backendType: 'IllegalArgumentException' }, + analyticsObserved: { backendRejected: false }, + standardLeg: 'pr-build', + analyticsLeg: 'pr-build-analytics', + grammarHash: 'sha256:same', + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE); + assert.equal(drift.remediation.action, REMEDIATIONS.ALIGN_EXECUTION_BACKENDS); + assert.deepEqual(drift.executionBackends, ['standard', 'analytics']); + assert.doesNotMatch(drift.remediation.detail, /maxVersion|minVersion|scope/i); + assert.match(drift.evidence, /standard REJECTED.*analytics ACCEPTED/); +}); + +test('analytics oracle flips are not labeled as product-version relaxation', () => { + const drift = classifyDrift( + agreeingTrigger({ + executionBackend: 'analytics', + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.REVIEW_BACKEND_ORACLE); + assert.notEqual(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.doesNotMatch(drift.remediation.detail, /maxVersion|minVersion|scope/i); +}); + +test('engine relaxation with a still-firing detector demands version scoping', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: false, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + // Both escape hatches are spelled out: bound the version, or disable outright. + assert.match(drift.remediation.detail, /maxVersion/); + assert.match(drift.remediation.detail, /"enabled": false/); + assert.match(drift.evidence, /now ACCEPTS/); +}); + +test('engine relaxation with an already-silent detector only needs a re-pin', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { detectorCount: 0, severities: [], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); + assert.match(drift.remediation.detail, /no linter change/); +}); + +test('engine tightening on a control with a silent detector is a false negative', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command now requires matching schemas.', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_TIGHTENED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /false NEGATIVE/); + assert.match(drift.evidence, /matching schemas/); +}); + +test('engine tightening the detector already catches only needs a re-pin', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'nope', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_TIGHTENED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +// --- wording drift ------------------------------------------------------------ + +test('a reworded rejection is update-contract and points at quoted copy', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'union requires >= 2 datasets, got 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); + assert.match(drift.evidence, /error\.reason/); + assert.match(drift.remediation.detail, /quotes the old engine wording/); +}); + +test('a reworded rejection does not mask a detector that stopped firing', () => { + // Regression: this branch used to return before the detector-silent check, so a + // simultaneous rewording + detector regression reported "the detector's verdict + // is unaffected, no rule change is required". Re-pinning the string would have + // turned the check green over a rule that no longer fires at all. + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'union requires >= 2 datasets, got 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); +}); + +test('a detector firing where appliesTo excludes the version is a false positive', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', // below the rule's 3.7 minVersion + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + // The version filter's unknown-version behavior is why this reaches users. + assert.match(drift.remediation.detail, /version is unknown/); +}); + +test('an unobserved engine verdict is never treated as acceptance', () => { + // backendRejected: undefined means "we never got an answer". It must not select + // the engine-relaxed branch, which would advise disabling a healthy rule. + const drift = classifyDrift( + agreeingTrigger({ + observed: { detectorCount: 1, severities: ['error'], backendRejected: undefined }, + }) + ); + assert.equal(drift, null); +}); + +test('a changed exception type is reported even when the reason is unchanged', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED); + assert.match(drift.evidence, /error\.type/); +}); + +// --- detector-only disagreement ---------------------------------------------- + +test('a silent detector on an unchanged engine names the three silent-failure causes', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /runtimeOnly/); + assert.match(drift.remediation.detail, /typeMap/); + // Guard the anti-vacuous instruction: never silence the contract instead. + assert.match(drift.remediation.detail, /Do NOT re-pin/); +}); + +test('a noisy detector the engine disagrees with is a false positive', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { detectorCount: 2, severities: ['error', 'error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /false positive/); +}); + +test('a noisy detector the engine agrees with points at the expectation', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'bad query', + }, + // Engine tightening is the more specific story when the pinned kind is not + // a rejection, so pin the kind as rejection to isolate the noisy branch. + expected: { detectorCount: 0, backendKind: 'rejection' }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +test('a nonzero detector count mismatch is not reduced to flagged versus silent', () => { + const drift = classifyDrift( + agreeingTrigger({ + expected: { + detectorCount: 2, + severity: 'error', + backendKind: 'rejection', + }, + observed: { + ...agreeingTrigger().observed, + detectorCount: 1, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_COUNT_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /expected exactly 2.*emitted 1/); +}); + +test('a detector message mismatch is classified independently of count and severity', () => { + const drift = classifyDrift( + agreeingTrigger({ + expected: { + detectorCount: 1, + severity: 'error', + matchMessage: 'requires at least two datasets', + backendKind: 'rejection', + }, + observed: { + ...agreeingTrigger().observed, + severityMatched: true, + messageMatched: false, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_MESSAGE_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /requires at least two datasets/); +}); + +// --- severity ---------------------------------------------------------------- + +test('a downgraded severity is caught even when the count is right', () => { + const drift = classifyDrift( + agreeingTrigger({ observed: { ...agreeingTrigger().observed, severities: ['warning'] } }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.SEVERITY_MISMATCH); + assert.match(drift.remediation.detail, /Restore "union-min-datasets"\.severity/); +}); + +// --- version scoping -------------------------------------------------------- + +test('an unsupported command is not mistaken for a too-narrow version window', () => { + // Real case from CI: on 3.6 the `union` command does not exist, so BOTH the + // trigger and the control fail with SyntaxCheckException. Judging the trigger + // alone said "widen appliesTo to 3.6" — which would ship a diagnostic claiming a + // precise cause ("requires at least two datasets") for what is really + // "unsupported command". A rejected control means the version window is right. + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'Invalid Query', + }, + controlAlsoRejected: true, + }) + ); + assert.equal(drift, null); +}); + +test('an out-of-scope rule on an engine that rejects is scoped too narrowly', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', // below the rule's 3.7 minVersion + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + assert.match(drift.remediation.detail, /minVersion/); +}); + +// --- helpers ----------------------------------------------------------------- + +test('version parsing tolerates snapshot and short forms', () => { + assert.deepEqual(parseVersion('3.8.0-SNAPSHOT'), [3, 8, 0]); + assert.deepEqual(parseVersion('3.7'), [3, 7, 0]); + assert.equal(parseVersion(''), undefined); + assert.equal(parseVersion(undefined), undefined); +}); + +test('appliesTo bounds are inclusive and open-ended when absent', () => { + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, '3.7.0'), true); + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, '3.6.9'), false); + assert.equal(versionInAppliesTo({ maxVersion: '3.8.0' }, '3.8.0'), true); + assert.equal(versionInAppliesTo({ maxVersion: '3.8.0' }, '3.9.0'), false); + assert.equal(versionInAppliesTo({}, '3.9.0'), true); + // An unparseable engine version must never silently drop coverage. + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, 'weird-build'), true); +}); + +test('rename suggestions prefer containment then near spellings', () => { + assert.deepEqual(suggestParserRules('unionCommand', ['unionCommandNew', 'zzz'], 3), [ + 'unionCommandNew', + ]); + assert.deepEqual(suggestParserRules('rexCommand', ['regexCommand'], 3), ['regexCommand']); + // Nothing remotely similar: say nothing rather than guess. + assert.deepEqual(suggestParserRules('rexCommand', ['whereClause', 'sortCommand'], 3), []); +}); + +// --- partial vs full relaxation ---------------------------------------------- +// +// The distinction these tests protect: a rule whose triggers ALL relaxed should be +// scoped away from the version; a rule where only SOME relaxed must NOT be, because +// scoping it would drop the diagnostics that are still correct. Getting this +// backwards converts a partial engine fix into a shipped false negative, so each +// branch is pinned including the advice text that names the wrong action. + +const scopeBase = { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + detectorFlagged: true, +}; + +test('no relaxed trigger yields no rule-level finding', () => { + assert.equal( + classifyRelaxationScope({ ...scopeBase, relaxedTriggers: [], holdingTriggers: ['a', 'b'] }), + null + ); +}); + +test('every trigger relaxed is a FULL fix and advises version scoping', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['hyphen', 'leading-digit'], + holdingTriggers: [], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + assert.match(drift.evidence, /FULL fix, 2 of 2 observed trigger\(s\) relaxed/); +}); + +test('some triggers still rejected is a PARTIAL fix and advises the detector, NOT scoping', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['hyphen'], + holdingTriggers: ['leading-digit', 'all-digits'], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /PARTIAL fix, 1 of 3 observed trigger\(s\) relaxed/); + // The advice must say the wrong action out loud. An engineer reading only the + // action verb could still reach for maxVersion, which is the regression. + assert.match(drift.remediation.detail, /Do NOT scope .* away from 3\.8\.0/); + assert.match(drift.remediation.detail, /false NEGATIVE/); +}); + +test('a single-trigger rule warns that a FULL verdict rests on one observation', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['only-one'], + holdingTriggers: [], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.match(drift.remediation.detail, /only 1 trigger/); + assert.match(drift.remediation.detail, /confirm with more shapes/); +}); + +test('unobserved triggers are excluded from the tally and named in the advice', () => { + // The trap: counting an unobserved trigger as "holding" turns a dead leg into a + // partial fix and sends someone to narrow a healthy detector. + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['a'], + holdingTriggers: [], + unobservedTriggers: ['b'], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED, 'must not read as partial'); + assert.match(drift.evidence, /1 of 1 observed trigger\(s\) relaxed/); + assert.match(drift.evidence, /1 trigger\(s\) produced no verdict \(b\) and were NOT counted/); + assert.match(drift.remediation.detail, /re-run it before acting/); +}); + +test('a silent detector on a fully relaxed rule needs no linter change', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['a', 'b'], + holdingTriggers: [], + detectorFlagged: false, + }); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +test('the per-query relaxed finding is marked supersedable', () => { + // The aggregator drops these in favour of the rule-level verdict; without the + // marker it would report both, and the per-query one gives the wrong action. + const drift = classifyDrift( + agreeingTrigger({ observed: { detectorCount: 1, severities: ['error'], backendRejected: false } }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.supersededBy, DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED); +}); + +// --- report ------------------------------------------------------------------ + +test('the report groups by action, most urgent first', () => { + const drifts = [ + classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'X', + backendReason: 'new wording', + }, + expectedBackend: { body: { error: { type: 'X', reason: 'old wording' } } }, + }) + ), + classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ), + ]; + const report = formatDriftReport(drifts); + assert.match(report, /2 finding\(s\) across 2 engine version\(s\)/); + assert.ok( + report.indexOf(REMEDIATIONS.VERSION_SCOPE_RULE) < report.indexOf(REMEDIATIONS.UPDATE_CONTRACT), + 'version scoping (a live false positive) must be listed before a stale-string re-pin' + ); + assert.match(report, /QUERY: union \[ source=t \]/); +}); + +test('an empty drift list reports agreement', () => { + assert.match(formatDriftReport([]), /No engine\/linter drift detected/); +}); diff --git a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs new file mode 100644 index 00000000000..4d50c56155d --- /dev/null +++ b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs @@ -0,0 +1,377 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the discovery-corpus harvester. + * + * node --test scripts/ppl-lint/__tests__/harvest-queries.test.mjs + * + * The harvester's job is to attribute a query to the rule that owns it and to hand + * the labeler something the cluster can actually run. Both have a wrong-answer mode + * that is worse than dropping the query: a misattributed query produces a + * "disagreement" for a rule that never claimed anything about it, and an + * unremapped index produces a rejection that reads as engine behavior. Those two + * failure modes are what these tests pin. + */ + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; + +import { + findTestFiles, + harvestContext, + harvestFile, + referencedIdentifiers, + remapIndex, + ruleFromDescribeTitle, + toRunnerSpecs, +} from '../harvest-queries.mjs'; + +const RULES = [ + 'invalid-capture-group-name', + 'field-validation', + 'rex-scan-cost', + 'head-without-sort', + 'division-by-zero', +]; + +test('test discovery recurses through current rule locations and excludes generated data', () => { + const osd = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-harvest-')); + const lintRoot = path.join(osd, 'packages/osd-monaco/src/ppl/lint'); + const files = [ + 'rules/inline_rule.test.ts', + 'rules/__tests__/nested_rule.test.tsx', + '__tests__/catalog.test.ts', + 'rules/__fixtures__/fixture.test.ts', + 'rules/__snapshots__/snapshot.test.ts', + 'generated/generated.test.ts', + 'rules/slow.bench.test.ts', + 'rules/manual.verify.test.ts', + ]; + try { + for (const file of files) { + const absolute = path.join(lintRoot, file); + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, ''); + } + assert.deepEqual( + findTestFiles(osd).map((file) => path.relative(lintRoot, file)), + [ + '__tests__/catalog.test.ts', + 'rules/__tests__/nested_rule.test.tsx', + 'rules/inline_rule.test.ts', + ] + ); + } finally { + fs.rmSync(osd, { recursive: true, force: true }); + } +}); + +// --- attribution ------------------------------------------------------------- + +test('an exact describe title names its rule', () => { + assert.equal(ruleFromDescribeTitle('rex-scan-cost', RULES), 'rex-scan-cost'); +}); + +test('a suffixed title still names its rule', () => { + // OSD's real titles: `describe('rex-scan-cost (compiled surface)')`. Requiring an + // exact match dropped ~75% of harvestable queries. + assert.equal(ruleFromDescribeTitle('rex-scan-cost (compiled surface)', RULES), 'rex-scan-cost'); + assert.equal( + ruleFromDescribeTitle('field-validation alternate-source suppression', RULES), + 'field-validation' + ); +}); + +test('a title that merely mentions a rule mid-sentence does NOT claim it', () => { + // Prefix-only matching is the guard: this describe sits under some OTHER rule and + // must not steal attribution, or its queries get judged against the wrong rule. + assert.equal(ruleFromDescribeTitle('does not fire on rex-scan-cost candidates', RULES), null); +}); + +test('a longer rule id wins over a prefix of itself', () => { + const rules = ['field-validation', 'field-validation-shape']; + assert.equal(ruleFromDescribeTitle('field-validation-shape cases', rules), 'field-validation-shape'); +}); + +test('a rule id must end at a word boundary', () => { + assert.equal(ruleFromDescribeTitle('head-without-sorting quirks', RULES), null); +}); + +test('the innermost rule-owning describe wins', () => { + const source = ` + describe('PPL silent-failure lint rules (compiled surface)', () => { + describe('division-by-zero', () => { + it('flags it', () => { + expect(ids('source=logs | eval x = a / 0')).toContain('division-by-zero'); + }); + }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].ruleId, 'division-by-zero'); +}); + +test('a query with no rule-owning ancestor is recorded unattributed, not guessed', () => { + const source = ` + describe('some unrelated suite', () => { + it('x', () => { lint('source=logs | head 10'); }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].ruleId, null); +}); + +// --- extraction -------------------------------------------------------------- + +test('JS string escapes are unescaped to the runtime query', () => { + // A test source containing '(?\\\\d+)' is the 4 chars `\\d+` at runtime, which + // is what the detector and the engine both see. Leaving the JS layer escaped + // sends a different query than the test actually linted. + const source = String.raw` + describe('invalid-capture-group-name', () => { + it('x', () => { lint('source=logs | rex field=m "(?\\d+)"'); }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].query, 'source=logs | rex field=m "(?\\d+)"'); +}); + +test('template literals with interpolation are dropped', () => { + // `${...}` is filled at runtime; sending the placeholder to the engine tests + // nothing and would be reported as a syntax error. + const source = 'describe(\'field-validation\', () => { lint(`source=${idx} | fields a`); });'; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 0); +}); + +test('only PPL-opening strings are treated as queries', () => { + const source = ` + describe('field-validation', () => { + it('x', () => { + expect(msg).toBe('this is not a query at all'); + lint('source=logs | fields a'); + }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.deepEqual( + out.map((o) => o.query), + ['source=logs | fields a'] + ); +}); + +test('the harvest records where each query came from', () => { + const source = `describe('division-by-zero', () => {\n lint('source=logs | eval x = a / 0');\n});`; + const out = harvestFile(source, { file: 'pkg/x.test.ts', knownRules: RULES }); + assert.match(out[0].source, /^pkg\/x\.test\.ts:2$/); +}); + +// --- index remapping --------------------------------------------------------- + +test('source= is remapped onto the fixture index', () => { + assert.equal( + remapIndex('source=logs | fields a', 'acct'), + 'source=acct | fields a' + ); +}); + +test('a backticked source is remapped', () => { + assert.equal(remapIndex('source=`my-logs` | fields a', 'acct'), 'source=acct | fields a'); +}); + +test('the bare `search ` form is remapped', () => { + assert.equal( + remapIndex('search accounts | eval x = balance / 0', 'acct'), + 'search acct | eval x = balance / 0' + ); +}); + +test('remapping is a no-op without a target index', () => { + assert.equal(remapIndex('source=logs | fields a', ''), 'source=logs | fields a'); +}); + +test('a WILDCARD source is left alone', () => { + // Found by running the pipeline: rewriting `source=`nope-*`` to a concrete index + // destroyed the only thing `wildcard-source-zero-match` detects, so its single + // harvested query became a control and the rule reported zero triggers. + assert.equal(remapIndex('source=`nope-*`', 'acct'), 'source=`nope-*`'); + assert.equal(remapIndex('source=logs-* | fields a', 'acct'), 'source=logs-* | fields a'); + assert.equal(remapIndex('index=a* | head 1', 'acct'), 'index=a* | head 1'); +}); + +test('a non-wildcard source is still remapped when a wildcard appears elsewhere', () => { + // The guard must key on a wildcard in the SOURCE, not anywhere in the query — a + // regex or a field list containing `*` is unrelated to index resolution. + assert.equal( + remapIndex('source=logs | rex field=m "(?.*)"', 'acct'), + 'source=acct | rex field=m "(?.*)"' + ); +}); + +// --- context harvesting ------------------------------------------------------ + +test('a typeMap declaration is harvested', () => { + // Seven of nineteen rules are needsContext and self-suppress without this. The + // context is taken from the test file because its author wrote it to make exactly + // these queries fire; a hand-written substitute would be a guess. + const source = ` + const typeMap = new Map([ + ['age', 'long'], + ['firstname', 'text'], + ['attributes', 'flat_object'], + ]); + `; + assert.deepEqual(harvestContext(source).typeMap, { + age: 'long', + firstname: 'text', + attributes: 'flat_object', + }); +}); + +test('disabledObjectFields is harvested', () => { + const source = "const ctx = { typeMap, disabledObjectFields: new Set(['raw', 'blob']) };"; + assert.deepEqual(harvestContext(source).disabledObjectFields, ['raw', 'blob']); +}); + +test('a file with no context declaration yields an empty context', () => { + const out = harvestContext("describe('x', () => { lint('source=a | head 1'); });"); + assert.deepEqual(out.typeMap, {}); + assert.deepEqual(out.disabledObjectFields, []); +}); + +test('generated specs carry the harvested context as frontendContext', () => { + const corpus = { + index: 'acct', + queries: [ + { + ruleId: 'flat-object-subfield', + name: 'd0', + query: 'source=acct | where attributes.x = 1', + context: { typeMap: { attributes: 'flat_object' }, disabledObjectFields: ['raw'] }, + }, + ], + }; + const spec = toRunnerSpecs(corpus)[0].spec; + assert.deepEqual(spec.frontendContext.deriveFromMapping, { attributes: 'flat_object' }); + assert.deepEqual(spec.frontendContext.disabledObjectFields, ['raw']); + // Two rules ship `enabled: false` and only run when the host overrides them. + assert.equal(spec.frontendContext.forceEnable, true); +}); + +test('visibleIndices is supplied even with no typeMap', () => { + // `wildcard-source-zero-match` reads ONLY visibleIndices and self-suppresses on an + // empty list; its test file declares no typeMap, so keying this off the mapping + // left the rule permanently inert. + const spec = toRunnerSpecs({ + index: 'acct', + queries: [ + { ruleId: 'wildcard-source-zero-match', name: 'd0', query: 'source=`nope-*`', context: {} }, + ], + })[0].spec; + assert.deepEqual(spec.frontendContext.visibleIndices, ['{{index}}']); +}); + +test('one rule tested under two different contexts yields two specs', () => { + // Merging them would hand a query field types its own test never used, so the + // verdict would describe a scenario nobody wrote. + const corpus = { + index: 'acct', + queries: [ + { ruleId: 'rex-scan-cost', name: 'd0', query: 'source=acct | rex field=a ""', context: { typeMap: { a: 'text' } } }, + { ruleId: 'rex-scan-cost', name: 'd1', query: 'source=acct | rex field=b ""', context: { typeMap: { b: 'keyword' } } }, + ], + }; + const specs = toRunnerSpecs(corpus); + assert.equal(specs.length, 2); + // Suffixed only when a rule actually has more than one context. + assert.deepEqual(specs.map((s) => s.fileName).sort(), [ + 'rex-scan-cost.1.discovery.spec.json', + 'rex-scan-cost.2.discovery.spec.json', + ]); +}); + +test('the original query is kept alongside the remapped one', () => { + // Needed to explain a finding: a reader has to be able to see what the OSD test + // actually asserted before trusting a disagreement derived from the rewrite. + const source = `describe('field-validation', () => { lint('source=logs | fields a'); });`; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES, index: 'acct' }); + assert.equal(out[0].query, 'source=acct | fields a'); + assert.equal(out[0].originalQuery, 'source=logs | fields a'); +}); + +test('identifiers are collected for the labeler to intersect against the fixture', () => { + const ids = referencedIdentifiers('source=acct | eval x = durationNano / 0'); + assert.ok(ids.includes('durationNano')); + assert.ok(ids.includes('acct')); +}); + +// --- runner specs ------------------------------------------------------------ + +const CORPUS = { + index: 'acct', + queries: [ + { ruleId: 'division-by-zero', name: 'discovery-0', query: 'source=acct | eval x = a / 0' }, + { ruleId: 'division-by-zero', name: 'discovery-1', query: 'source=acct | eval x = a / 2' }, + { ruleId: 'head-without-sort', name: 'discovery-2', query: 'source=acct | head 5' }, + { ruleId: null, name: 'discovery-3', query: 'source=acct | fields a' }, + ], +}; + +test('one spec is emitted per rule, and unattributed queries are excluded', () => { + const specs = toRunnerSpecs(CORPUS); + assert.deepEqual( + specs.map((s) => s.spec.ruleId), + ['division-by-zero', 'head-without-sort'] + ); + assert.equal(Object.keys(specs[0].spec.queries).length, 2); +}); + +test('generated specs carry no wiring block', () => { + // The runner deep-equals `wiring` against the OSD catalog when present. A + // generated approximation would fail the run for a reason unrelated to discovery. + const specs = toRunnerSpecs(CORPUS); + assert.equal(specs[0].spec.wiring, undefined); +}); + +test('every generated query is declared a trigger', () => { + // Roles are derived later from real detector output. Declaring some as controls + // would make the runner apply control-specific cross-checks whose failures are + // pure noise on a corpus with no pinned verdicts. + const specs = toRunnerSpecs(CORPUS); + for (const spec of specs) { + for (const q of Object.values(spec.spec.queries)) { + assert.equal(q.role, 'trigger'); + } + } +}); + +test('exactly one expectation matches any engine version', () => { + // Two matching entries make the runner report the version as uncovered; an + // open-ended empty range is what guarantees a single match on every leg. + const specs = toRunnerSpecs(CORPUS); + for (const spec of specs) { + assert.equal(spec.spec.expectations.length, 1); + assert.equal(spec.spec.expectations[0].version, ''); + } +}); + +test('generated specs are scored on either grammar surface', () => { + const specs = toRunnerSpecs(CORPUS); + assert.ok(specs.every((s) => s.spec.grammarSurface === 'both')); +}); + +test('query names are preserved so the two halves can be joined', () => { + // The detector runner and the engine probe key on these names. If they disagreed, + // every row would lose its counterpart and the corpus would read as unobserved. + const specs = toRunnerSpecs(CORPUS); + assert.deepEqual(Object.keys(specs[0].spec.queries), ['discovery-0', 'discovery-1']); +}); diff --git a/scripts/ppl-lint/__tests__/label-discovery.test.mjs b/scripts/ppl-lint/__tests__/label-discovery.test.mjs new file mode 100644 index 00000000000..59ca138470e --- /dev/null +++ b/scripts/ppl-lint/__tests__/label-discovery.test.mjs @@ -0,0 +1,242 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for discovery-corpus labelling. + * + * node --test scripts/ppl-lint/__tests__/label-discovery.test.mjs + * + * The labeler turns two observations into a role and, sometimes, a finding. Every + * way it can produce a CONFIDENT finding from a non-observation is a way to send an + * engineer after a bug that does not exist, so the three-state read and the + * uninformative-rejection filter are pinned case by case. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + FINDINGS, + ROLES, + labelQuery, + renderMarkdown, + uninformativeRejection, +} from '../label-discovery.mjs'; + +const base = { ruleId: 'invalid-capture-group-name', query: 'source=acct | rex field=m "(?x)"' }; + +// --- role assignment --------------------------------------------------------- + +test('a query the detector fires on is a trigger', () => { + const row = labelQuery({ ...base, detectorCount: 1, backendRejected: true }); + assert.equal(row.role, ROLES.TRIGGER); +}); + +test('a query the detector ignores is a control', () => { + const row = labelQuery({ ...base, detectorCount: 0, backendRejected: false }); + assert.equal(row.role, ROLES.CONTROL); +}); + +// --- the two findings -------------------------------------------------------- + +test('detector fires + engine accepts is a possible false positive', () => { + const row = labelQuery({ + ...base, + detectorCount: 2, + severities: ['error', 'error'], + backendRejected: false, + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); + assert.match(row.finding.evidence, /ACCEPTED this query/); +}); + +test('an ADVISORY rule the engine accepts is not a false positive', () => { + // Found against a live 3.8 engine: `head-without-sort` and `rex-scan-cost` are + // `info` rules that flag non-determinism and cost. The engine runs those queries + // happily and will never reject them, so "accepted + flagged" is the rule working + // as designed. Without this, every advisory trigger becomes a finding and buries + // the real ones. + const row = labelQuery({ + ...base, + ruleId: 'head-without-sort', + detectorCount: 1, + severities: ['info'], + backendRejected: false, + }); + assert.equal(row.finding, null); + assert.equal(row.advisory, true); + // Still a trigger: it is exactly the kind of trigger the relaxation rollup counts. + assert.equal(row.role, ROLES.TRIGGER); +}); + +test('a mixed-severity diagnostic is not treated as advisory', () => { + // Only an ALL-info diagnostic is advisory. One error-severity marker means the + // rule is asserting the engine will refuse the query, which acceptance contradicts. + const row = labelQuery({ + ...base, + detectorCount: 2, + severities: ['info', 'error'], + backendRejected: false, + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); +}); + +test('an advisory rule the engine REJECTS is still evidence', () => { + // The advisory carve-out applies only to the accepted direction. A silent detector + // on a query the engine refused is unaffected by severity. + const row = labelQuery({ + ...base, + ruleId: 'head-without-sort', + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'head requires a positive integer', + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_NEGATIVE); +}); + +test('detector silent + engine rejects is a possible false negative', () => { + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SemanticCheckException', + backendReason: 'capture group name is invalid', + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_NEGATIVE); + // Must tell the reader to verify the rejection is this rule's condition; the + // engine rejecting is weaker evidence than the engine accepting. + assert.match(row.finding.evidence, /VERIFY the rejection is this rule's condition/); +}); + +test('agreement in either direction is not a finding', () => { + assert.equal(labelQuery({ ...base, detectorCount: 1, backendRejected: true }).finding, null); + assert.equal(labelQuery({ ...base, detectorCount: 0, backendRejected: false }).finding, null); +}); + +// --- the three-state read ---------------------------------------------------- + +test('no engine verdict produces no finding', () => { + // The trap this closes: coercing an absent verdict to `false` reads a timed-out + // leg as "the engine accepted this" and manufactures a false-positive finding + // against a healthy rule. + const row = labelQuery({ ...base, detectorCount: 1, backendRejected: undefined }); + assert.equal(row.finding, null); + assert.equal(row.unobserved, true); +}); + +test('a silent detector with no engine verdict is unknown, not a control', () => { + // Calling it a control would claim the rule correctly stayed quiet, which nothing + // observed. It also inflates control counts that the coverage table reports. + const row = labelQuery({ ...base, detectorCount: 0, backendRejected: undefined }); + assert.equal(row.role, ROLES.UNKNOWN); +}); + +// --- uninformative rejections ------------------------------------------------ + +test('an unknown-field rejection is suppressed, not reported', () => { + // Harvested queries name fields their original OSD test invented. Without this + // filter every such query becomes a false-negative finding and buries the real + // ones. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SemanticCheckException', + backendReason: "can't resolve Symbol(namespace=FIELD_NAME, name=durationNano)", + }); + assert.equal(row.finding, null); + assert.equal(row.suppressed, 'unknown field'); +}); + +test('a syntax error is suppressed', () => { + // After index remapping some harvested queries are genuinely malformed; a query + // the grammar cannot parse says nothing about a semantic rule. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'mismatched input ', + }); + assert.equal(row.suppressed, 'syntax error'); +}); + +test('an unsupported-command rejection is suppressed', () => { + // Same reasoning as the enforced corpus's control-also-rejected guard: the + // command not existing is not evidence about a rule's condition. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendReason: 'union is not supported in this version', + }); + assert.equal(row.suppressed, 'unsupported command'); +}); + +test('an on-topic rejection survives the filter', () => { + assert.equal( + uninformativeRejection('IllegalArgumentException', 'Union command requires at least two datasets'), + null + ); +}); + +test('a rule with no observed trigger is distinguished from one with a single trigger', () => { + // These are different problems: zero triggers means this corpus proves nothing + // about the rule at all (every harvested query was a control, or the detector is + // gated off on this surface), whereas one trigger means the rule is observable but + // a "fully relaxed" verdict would rest on a single case. Rendering both as + // "1 trigger only" hid the first, which is the more serious gap. + const markdown = renderMarkdown({ + stats: { queries: 9, triggers: 4, controls: 5, unknown: 0, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [ + { ruleId: 'none-observed', triggers: 0, controls: 4, sufficientForScopeDecision: false }, + { ruleId: 'single', triggers: 1, controls: 2, sufficientForScopeDecision: false }, + { ruleId: 'plenty', triggers: 3, controls: 2, sufficientForScopeDecision: true }, + ], + }); + assert.match(markdown, /`none-observed` \| 0 \| 4 \| \*\*none — no trigger observed\*\*/); + assert.match(markdown, /`single` \| 1 \| 2 \| \*\*no — 1 trigger only\*\*/); + assert.match(markdown, /`plenty` \| 3 \| 2 \| yes/); +}); + +test('a run with no engine half says so instead of implying agreement', () => { + // "0 finding(s)" beside 109 queries reads as "everything agrees". With no engine + // verdicts nothing was compared at all, and the report has to distinguish those. + const withoutEngine = renderMarkdown({ + differential: false, + stats: { queries: 109, triggers: 19, controls: 0, unknown: 90, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [], + }); + assert.match(withoutEngine, /No engine verdicts were supplied/); + + const withEngine = renderMarkdown({ + differential: true, + stats: { queries: 10, triggers: 4, controls: 6, unknown: 0, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [], + }); + assert.doesNotMatch(withEngine, /No engine verdicts were supplied/); +}); + +test('suppression never applies to the false-POSITIVE side', () => { + // The filter exists to protect the weak (false-negative) direction. A query the + // engine RAN successfully is conclusive regardless of what any error text says, + // so an accepted query must still report even with a suppressible-looking reason. + const row = labelQuery({ + ...base, + detectorCount: 1, + // Explicit rather than relying on the default: with an empty severities list the + // advisory check cannot fire, so the test would pass for the wrong reason and + // stop covering the suppression filter at all. + severities: ['error'], + backendRejected: false, + backendReason: "can't resolve Symbol(name=whatever)", + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); +}); diff --git a/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs b/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs new file mode 100644 index 00000000000..78e6dc45a68 --- /dev/null +++ b/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; + +import { + createPlan, + parseVersion, + releaseVersions, + selectLatestGaAtOrBelow, +} from '../plan-compatibility.mjs'; + +const temporaryDirectories = []; + +function temporaryDirectory() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-plan-')); + temporaryDirectories.push(directory); + return directory; +} + +after(() => { + for (const directory of temporaryDirectories) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +test('normalizes prerelease and build suffixes', () => { + assert.deepEqual(parseVersion('3.8.0-SNAPSHOT'), { + normalized: '3.8.0', + parts: [3, 8, 0], + }); + assert.deepEqual(parseVersion('3.8.0+build.42'), { + normalized: '3.8.0', + parts: [3, 8, 0], + }); +}); + +test('only exact semantic version tags count as official GA candidates', () => { + const tags = [ + 'a refs/tags/3.7.0', + 'b refs/tags/3.8.0-alpha1', + 'c refs/tags/3.7.1', + 'd refs/tags/v3.8.0', + 'e refs/tags/3.8.0', + ].join('\n'); + assert.deepEqual( + releaseVersions(tags).map((entry) => entry.version), + ['3.7.0', '3.7.1', '3.8.0'] + ); +}); + +test('selects the highest GA at or below the normalized PR target', () => { + const tags = ['3.6.0', '3.7.0', '3.7.2', '3.8.0', '3.9.0'].join('\n'); + assert.equal( + selectLatestGaAtOrBelow(tags, parseVersion('3.8.0-SNAPSHOT')), + '3.8.0' + ); + assert.equal( + selectLatestGaAtOrBelow(tags, parseVersion('3.7.5-SNAPSHOT')), + '3.7.2' + ); +}); + +test('plans one compiled and two runtime configurations', () => { + const directory = temporaryDirectory(); + const buildFile = path.join(directory, 'build.gradle'); + fs.writeFileSync( + buildFile, + 'opensearch_version = System.getProperty("opensearch.version", "3.8.0-SNAPSHOT")\n' + ); + const plan = createPlan({ + buildFile, + releaseTags: ['a refs/tags/3.7.0', 'b refs/tags/3.8.0'].join('\n'), + compiledVersion: '2.19.6', + sqlSha: 'sql-sha', + osdRepository: 'example/OpenSearch-Dashboards', + osdRef: 'feature', + }); + + assert.equal(plan.latestEligibleGa, '3.8.0'); + assert.deepEqual( + plan.configurations.map((configuration) => [ + configuration.id, + configuration.surface, + configuration.engineVersion, + ]), + [ + ['2.19.6-compiled', 'compiled-simplified', '2.19.6'], + ['latest-release-runtime', 'runtime-bundle', '3.8.0'], + ['pr-build-runtime', 'runtime-bundle', '3.8.0-SNAPSHOT'], + ] + ); + assert.equal(plan.releasedTargets.include.length, 2); +}); diff --git a/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs b/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs new file mode 100644 index 00000000000..5368f8afabb --- /dev/null +++ b/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the discovery engine probe's response mapping. + * + * node --test scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs + * + * The probe has one job that can go wrong quietly: turning an HTTP response into a + * verdict. Reading a non-answer as acceptance is what converts a network blip into + * "the engine now accepts this query", so the accept/reject/no-verdict split is + * pinned here. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { readResponse } from '../probe-discovery-backend.mjs'; + +test('a 200 is acceptance', () => { + const v = readResponse({ status: 200, bodyText: '{"datarows":[]}' }); + assert.equal(v.outcome, 'observed'); + assert.equal(v.rejected, false); +}); + +test('a 400 is rejection and keeps the engine type and reason', () => { + // The labeler's uninformative-rejection filter keys on these two strings; losing + // them would let an unknown-field rejection be reported as a missed diagnostic. + const v = readResponse({ + status: 400, + bodyText: JSON.stringify({ + error: { type: 'SemanticCheckException', reason: "can't resolve Symbol(name=foo)" }, + }), + }); + assert.equal(v.rejected, true); + assert.equal(v.observed.type, 'SemanticCheckException'); + assert.match(v.observed.reason, /can't resolve/); +}); + +test('a 500 is also rejection', () => { + assert.equal(readResponse({ status: 500, bodyText: '{}' }).rejected, true); +}); + +test('an unparseable body still yields a verdict from the status', () => { + // The engine answered; the body being junk does not change whether it ran the + // query. Discarding the verdict here would lose real signal. + const v = readResponse({ status: 200, bodyText: 'oops' }); + assert.equal(v.outcome, 'observed'); + assert.equal(v.rejected, false); +}); + +test('an enormous reason is truncated', () => { + const v = readResponse({ + status: 400, + bodyText: JSON.stringify({ error: { type: 'X', reason: 'y'.repeat(5000) } }), + }); + assert.equal(v.observed.reason.length, 500); +}); + +test('an accepted response carries no error fields', () => { + const v = readResponse({ status: 200, bodyText: '{"datarows":[]}' }); + assert.equal(v.observed.type, undefined); + assert.equal(v.observed.reason, undefined); +}); diff --git a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs new file mode 100644 index 00000000000..42248ce3adc --- /dev/null +++ b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs @@ -0,0 +1,457 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + assertActiveShippingContracts, + buildCensus, + buildFrontendExecutionError, + compatibilityExclusion, + evaluateFrontendAssertions, + selectManifestContractNames, +} from '../run-frontend-contract.mjs'; + +const SCRIPT = fileURLToPath(new URL('../run-frontend-contract.mjs', import.meta.url)); + +const RANGE = { + startLine: 1, + startColumn: 11, + endLine: 1, + endColumn: 14, +}; + +test('compatibility exclusion uses surface, version, then engine precedence', () => { + const runtimeCalciteRule = { + grammarSurface: 'runtime-bundle', + wiring: { + appliesTo: { minVersion: '3.4.0', engine: 'calcite' }, + }, + }; + assert.equal( + compatibilityExclusion( + runtimeCalciteRule, + '2.19.6', + 'compiled-simplified', + 'legacy' + ).reason, + 'surface' + ); + assert.equal( + compatibilityExclusion( + { ...runtimeCalciteRule, grammarSurface: 'both' }, + '2.19.6', + 'compiled-simplified', + 'legacy' + ).reason, + 'version' + ); + assert.equal( + compatibilityExclusion( + { + grammarSurface: 'both', + wiring: { appliesTo: { engine: 'calcite' } }, + }, + '3.8.0', + 'runtime-bundle', + 'legacy' + ).reason, + 'engine' + ); + assert.equal( + compatibilityExclusion( + runtimeCalciteRule, + '3.8.0-SNAPSHOT', + 'runtime-bundle', + 'calcite' + ), + undefined + ); +}); + +test('exact lint assertions materialize the effective deterministic edit', () => { + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t | bad', + matches: [ + { + ruleId: 'example-rule', + severity: 'warning', + message: 'Replace bad.', + range: RANGE, + fix: { + title: 'Replace bad', + text: 'good', + expectedText: 'bad', + }, + }, + ], + frontendOracle: { + severity: 'warning', + messageEquals: 'Replace bad.', + deterministicFix: { + offered: true, + title: 'Replace bad', + text: 'good', + range: RANGE, + expectedText: 'bad', + appliedQuery: 'source=t | good', + }, + }, + }); + + assert.deepEqual(result.mismatches, []); + assert.deepEqual(result.assertions, { + severity: true, + message: true, + deterministicFix: true, + }); + assert.deepEqual(result.deterministicFixActual, { + offered: true, + title: 'Replace bad', + text: 'good', + range: RANGE, + expectedText: 'bad', + appliedQuery: 'source=t | good', + }); +}); + +test('exact message mismatches are field-specific', () => { + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t | bad', + matches: [ + { + ruleId: 'example-rule', + severity: 'warning', + message: 'Different message.', + range: RANGE, + }, + ], + frontendOracle: { + messageEquals: 'Expected message.', + deterministicFix: { offered: false }, + }, + }); + + assert.deepEqual(result.mismatches.map(({ field }) => field), ['message']); +}); + +test('deterministic expectedText must match the source slice', () => { + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t | bad', + matches: [ + { + message: 'Replace bad.', + range: RANGE, + fix: { + title: 'Replace bad', + text: 'good', + expectedText: 'stale', + }, + }, + ], + frontendOracle: { + deterministicFix: { + offered: true, + title: 'Replace bad', + text: 'good', + range: RANGE, + expectedText: 'stale', + appliedQuery: 'source=t | good', + }, + }, + }); + + assert.equal(result.deterministicFixMatched, false); + assert.equal(result.deterministicFixActual.expectedTextMatchesSource, false); +}); + +test('a frontend execution error remains a complete report row', () => { + assert.deepEqual( + buildFrontendExecutionError({ + ruleId: 'example-rule', + channel: 'lint', + queryName: 'trigger', + role: 'trigger', + query: 'source=t | bad', + expected: 0, + surface: 'runtime-bundle', + executionBackend: 'standard', + error: new Error('detector crashed'), + }), + { + ruleId: 'example-rule', + channel: 'lint', + queryName: 'trigger', + role: 'trigger', + query: 'source=t | bad', + expected: 0, + actual: 0, + severities: [], + severityMatched: true, + messageMatched: true, + assertions: { execution: false }, + mismatches: [ + { + field: 'execution', + expected: 'completed', + actual: 'detector crashed', + }, + ], + outcome: 'error', + error: 'detector crashed', + surface: 'runtime-bundle', + executionBackend: 'standard', + backendOracleStatus: 'error', + } + ); +}); + +test('the runner writes later rule rows after one frontend execution error', (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-runner-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const osdRoot = path.join(root, 'osd'); + const contractDir = path.join(root, 'contracts'); + const reportPath = path.join(root, 'detector-report.json'); + const grammarPath = path.join(root, 'ppl-grammar-bundle.json'); + const targetPath = path.join(root, 'target.json'); + fs.mkdirSync( + path.join(osdRoot, 'src/plugins/data/public/antlr/opensearch_ppl'), + { recursive: true } + ); + fs.mkdirSync(path.join(osdRoot, 'packages/osd-monaco'), { recursive: true }); + fs.mkdirSync(contractDir, { recursive: true }); + + const wiring = (ruleId) => ({ + detector: ruleId, + enabled: true, + severity: 'info', + runtimeOnly: false, + needsContext: false, + needsExplain: false, + sourceScoped: false, + appliesTo: {}, + }); + const contract = (ruleId, query) => ({ + schemaVersion: 4, + ruleId, + grammarSurface: 'runtime-bundle', + schedule: 'pr', + wiring: wiring(ruleId), + index: 'test-index', + queries: { + trigger: { role: 'trigger', query }, + }, + expectations: [ + { + version: '>=0.0.0', + queries: { + trigger: { + frontend: { + count: 0, + deterministicFix: { offered: false }, + }, + backends: { + standard: { kind: 'advisory', httpStatus: 200 }, + analytics: { kind: 'advisory', httpStatus: 200 }, + }, + }, + }, + }, + ], + }); + const files = ['first-rule.spec.json', 'second-rule.spec.json']; + fs.writeFileSync( + path.join(contractDir, files[0]), + JSON.stringify(contract('first-rule', 'source={{index}} | fail')) + ); + fs.writeFileSync( + path.join(contractDir, files[1]), + JSON.stringify(contract('second-rule', 'source={{index}} | pass')) + ); + fs.writeFileSync( + path.join(contractDir, 'manifest.json'), + JSON.stringify({ + schemaVersion: 4, + contracts: files, + defaultError: [], + requiredSyntaxFeatures: [], + }) + ); + fs.writeFileSync( + path.join(osdRoot, 'packages/osd-monaco/ppl-lint.js'), + `const wiring = (id) => ({ + id, detector: id, enabled: true, severity: 'info', runtimeOnly: false, + needsContext: false, needsExplain: false, sourceScoped: false, appliesTo: {} + }); + exports.getBundledCatalog = () => [wiring('first-rule'), wiring('second-rule')];` + ); + fs.writeFileSync( + path.join( + osdRoot, + 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint.js' + ), + `exports.deserializeBundleOrThrow = (bundle) => bundle; + exports.lintQueryWithBundle = (query) => { + if (query.includes('| fail')) throw new Error('detector crashed'); + return { diagnostics: [] }; + };` + ); + fs.writeFileSync(grammarPath, JSON.stringify({ grammarHash: 'sha256:test' })); + fs.writeFileSync( + targetPath, + JSON.stringify({ + schemaVersion: 2, + executionBackend: 'standard', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:test', + storage: 'lucene', + shardCount: 1, + }) + ); + + const result = spawnSync(process.execPath, [SCRIPT], { + cwd: osdRoot, + encoding: 'utf8', + env: { + ...process.env, + PPL_LINT_CONTRACT_DIR: contractDir, + PPL_LINT_SCHEDULE: 'pr', + PPL_LINT_GRAMMAR_BUNDLE: grammarPath, + PPL_LINT_TARGET_MANIFEST: targetPath, + PPL_LINT_REPORT: reportPath, + }, + }); + + assert.equal(result.status, 1); + assert.match(result.stderr, /first-rule\/trigger.*frontend\.execution failed/s); + assert.match(result.stdout, /PASS second-rule\/trigger/); + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + assert.equal(report.results.length, 2); + assert.equal(report.results[0].outcome, 'error'); + assert.equal(report.results[0].error, 'detector crashed'); + assert.equal(report.results[1].ruleId, 'second-rule'); + assert.equal(report.results[1].actual, 0); +}); + +test('syntax suppression checks raw parser errors outside the suggestion code filter', () => { + const parserErrors = [ + { + code: 'PARSER_ERROR', + message: 'Unexpected command.', + rawMessage: "mismatched input 'zzzzzzzz'", + }, + ]; + const result = evaluateFrontendAssertions({ + channel: 'syntax', + query: 'source=t | zzzzzzzz', + matches: [], + allFrontendFindings: parserErrors, + frontendOracle: { + fixText: null, + rawMessage: true, + totalErrors: 1, + }, + }); + + assert.deepEqual(result.mismatches, []); + assert.deepEqual(result.assertions, { + syntaxFix: true, + rawParserError: true, + totalErrors: true, + }); + + const withUnexpectedFix = evaluateFrontendAssertions({ + channel: 'syntax', + query: 'source=t | zzzzzzzz', + matches: [], + allFrontendFindings: [ + { + ...parserErrors[0], + fix: { title: 'Rewrite', text: 'where' }, + }, + ], + frontendOracle: { fixText: null, rawMessage: true }, + }); + assert.equal(withUnexpectedFix.syntaxFixMatched, false); + assert.equal(withUnexpectedFix.mismatches[0].field, 'syntaxFix'); +}); + +test('dormant manifest contracts are opt-in and remain tagged report-only', () => { + const manifest = { + contracts: ['active.spec.json'], + dormantContracts: ['dormant.spec.json'], + }; + assert.deepEqual(selectManifestContractNames(manifest), [ + { name: 'active.spec.json', reportOnly: false }, + ]); + assert.deepEqual(selectManifestContractNames(manifest, true), [ + { name: 'active.spec.json', reportOnly: false }, + { name: 'dormant.spec.json', reportOnly: true }, + ]); + assert.throws( + () => + selectManifestContractNames( + { + contracts: ['same.spec.json'], + dormantContracts: ['same.spec.json'], + }, + true + ), + /cannot be both active and dormant/ + ); +}); + +test('discovery mode accepts legacy generated specs without shipping oracles', () => { + const contract = { + file: 'generated.discovery.spec.json', + spec: { + schemaVersion: 3, + ruleId: 'generated-rule', + queries: { trigger: { role: 'trigger', query: 'source=t' } }, + expectations: [{ version: '', queries: { trigger: { detectorCount: 0 } } }], + }, + }; + + assert.doesNotThrow(() => + assertActiveShippingContracts([contract], { discovery: true }) + ); + assert.throws( + () => assertActiveShippingContracts([contract]), + /active shipping contracts must use schemaVersion 4/ + ); +}); + +test('shipping census rejects duplicate detector IDs', () => { + const lintContract = { + file: 'example.spec.json', + spec: { + schemaVersion: 4, + ruleId: 'example', + }, + }; + const census = buildCensus( + [lintContract], + { + contracts: ['example.spec.json'], + defaultError: [], + requiredSyntaxFeatures: [], + }, + [ + { id: 'duplicate-rule', enabled: false, severity: 'info' }, + { id: 'duplicate-rule', enabled: false, severity: 'info' }, + ] + ); + + assert.ok(census.problems.some((problem) => /duplicate rule IDs/.test(problem))); +}); diff --git a/scripts/ppl-lint/aggregate-compatibility.mjs b/scripts/ppl-lint/aggregate-compatibility.mjs new file mode 100644 index 00000000000..e46a872d93f --- /dev/null +++ b/scripts/ppl-lint/aggregate-compatibility.mjs @@ -0,0 +1,961 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import { + assertContractSchema, + assertExactQueryCoverage, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; + +const ACTIVE_RULE_IDS = [ + 'agg-on-text', + 'division-by-zero', + 'enabled-false-object', + 'field-validation', + 'invalid-capture-group-name', + 'multisearch-min-subsearch', + 'replace-wildcard-asymmetry', + 'rex-scan-cost', + 'type-mismatch-numeric', + 'union-min-datasets', + 'unsupported-window-function-in-eventstats', + 'wildcard-source-zero-match', +]; + +function fatal(message) { + process.stderr.write(`[ppl-lint-compatibility] FATAL: ${message}\n`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { out: 'drift-report.json', summary: '', osdSha: '' }; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[++index]; + if (value === undefined) fatal(`${key} requires a value`); + if (key === '--plan') args.plan = value; + else if (key === '--contracts') args.contracts = value; + else if (key === '--artifacts') args.artifacts = value; + else if (key === '--osd-sha') args.osdSha = value; + else if (key === '--out') args.out = value; + else if (key === '--summary') args.summary = value; + else fatal(`unknown argument ${JSON.stringify(key)}`); + } + for (const field of ['plan', 'contracts', 'artifacts']) { + if (!args[field]) fatal(`--${field} is required`); + } + return args; +} + +function readRequiredJson(file) { + if (!fs.existsSync(file)) fatal(`required JSON file not found: ${file}`); + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + fatal(`could not parse ${file}: ${error.message}`); + } +} + +function readOptionalJson(file) { + if (!fs.existsSync(file)) { + return { value: undefined, error: `missing ${path.basename(file)}` }; + } + try { + return { value: JSON.parse(fs.readFileSync(file, 'utf8')), error: undefined }; + } catch (error) { + return { value: undefined, error: `invalid ${path.basename(file)}: ${error.message}` }; + } +} + +function readOptionalText(file) { + try { + return fs.readFileSync(file, 'utf8').trim(); + } catch { + return ''; + } +} + +function parseVersion(value) { + const match = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(value || '')); + return match ? match.slice(1, 4).map((part) => Number(part || 0)) : undefined; +} + +function compareVersion(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] < right[index] ? -1 : 1; + } + return 0; +} + +function inVersionRange(version, minVersion, maxVersion) { + const actual = parseVersion(version); + if (!actual) return false; + const min = parseVersion(minVersion); + const max = parseVersion(maxVersion); + if (min && compareVersion(actual, min) < 0) return false; + if (max && compareVersion(actual, max) > 0) return false; + return true; +} + +function matchesExpectationRange(range, version) { + if (!range || !String(range).trim()) return true; + const actual = parseVersion(version); + if (!actual) return false; + for (const token of String(range).trim().split(/\s+/)) { + const match = /^(>=|<=|>|<|=)?(\d+(?:\.\d+){0,2})$/.exec(token); + if (!match) return false; + const expected = parseVersion(match[2]); + const comparison = compareVersion(actual, expected); + const operator = match[1] || '='; + if ( + !( + (operator === '>=' && comparison >= 0) || + (operator === '<=' && comparison <= 0) || + (operator === '>' && comparison > 0) || + (operator === '<' && comparison < 0) || + (operator === '=' && comparison === 0) + ) + ) { + return false; + } + } + return true; +} + +function validatePlan(plan) { + if (!plan || plan.schemaVersion !== 1 || !Array.isArray(plan.configurations)) { + fatal('compatibility plan must be schemaVersion 1 with a configurations array'); + } + if (plan.configurations.length !== 3) { + fatal(`compatibility plan must contain exactly 3 configurations, found ${plan.configurations.length}`); + } + const ids = new Set(); + for (const configuration of plan.configurations) { + for (const field of [ + 'id', + 'label', + 'engineVersion', + 'surface', + 'executionBackend', + 'engineMode', + 'artifactName', + ]) { + if (typeof configuration[field] !== 'string' || !configuration[field]) { + fatal(`plan configuration ${JSON.stringify(configuration.id)} has invalid ${field}`); + } + } + if (!['compiled-simplified', 'runtime-bundle'].includes(configuration.surface)) { + fatal(`plan configuration ${configuration.id} has unknown surface ${configuration.surface}`); + } + if (!['calcite', 'legacy'].includes(configuration.engineMode)) { + fatal( + `plan configuration ${configuration.id} has unknown engine mode ` + + configuration.engineMode + ); + } + if (ids.has(configuration.id)) fatal(`duplicate plan configuration ${configuration.id}`); + ids.add(configuration.id); + } +} + +function loadContracts(dir) { + const manifestPath = path.join(dir, 'manifest.json'); + const manifest = readRequiredJson(manifestPath); + if (!Array.isArray(manifest.contracts)) fatal(`${manifestPath} contracts must be an array`); + const contracts = new Map(); + for (const file of manifest.contracts) { + const specPath = path.join(dir, file); + const spec = readRequiredJson(specPath); + try { + assertContractSchema(spec); + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new Error('expectations must be a non-empty array'); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + } + } catch (error) { + fatal(`invalid ${specPath}: ${error.message}`); + } + if (contracts.has(spec.ruleId)) fatal(`duplicate active rule id ${spec.ruleId}`); + contracts.set(spec.ruleId, { file, spec }); + } + const actual = [...contracts.keys()].sort(); + if (JSON.stringify(actual) !== JSON.stringify(ACTIVE_RULE_IDS)) { + fatal( + `active manifest must contain exactly the approved 12 rules; expected ` + + `${JSON.stringify(ACTIVE_RULE_IDS)}, got ${JSON.stringify(actual)}` + ); + } + return contracts; +} + +function expectedScope(spec, configuration) { + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const declaredSurface = spec.grammarSurface || 'runtime-bundle'; + const surfaces = + declaredSurface === 'both' + ? ['compiled-simplified', 'runtime-bundle'] + : [declaredSurface]; + const expected = { + applicable: true, + engine: appliesTo.engine || 'any', + minVersion: appliesTo.minVersion || null, + maxVersion: appliesTo.maxVersion || null, + surfaces, + }; + + if (!surfaces.includes(configuration.surface)) { + return { ...expected, applicable: false, reason: 'surface' }; + } + if ( + !inVersionRange( + configuration.engineVersion, + appliesTo.minVersion, + appliesTo.maxVersion + ) + ) { + return { ...expected, applicable: false, reason: 'version' }; + } + if (appliesTo.engine && appliesTo.engine !== configuration.engineMode) { + return { ...expected, applicable: false, reason: 'engine' }; + } + return expected; +} + +function selectExpectation(spec, configuration) { + const matches = spec.expectations.filter( + (expectation) => + matchesExpectationRange(expectation.version, configuration.engineVersion) && + (!expectation.engine || expectation.engine === configuration.engineMode) + ); + return matches.length === 1 ? matches[0] : undefined; +} + +function indexDetectorReport(report) { + if (!report || typeof report !== 'object' || !Array.isArray(report.results)) { + throw new Error('detector-report.json must contain a results array'); + } + const rows = new Map(); + for (const row of report.results) { + if (!row || typeof row.ruleId !== 'string' || typeof row.queryName !== 'string') { + throw new Error('detector report rows require ruleId and queryName'); + } + const key = `${row.ruleId}::${row.queryName}`; + if (rows.has(key)) throw new Error(`duplicate detector row ${key}`); + rows.set(key, row); + } + return rows; +} + +function loadEvidence(configuration, artifactsRoot, sqlSha) { + const dir = path.join(artifactsRoot, configuration.artifactName); + const errors = []; + const backendTargetRead = readOptionalJson(path.join(dir, 'target.json')); + const detectorTargetRead = readOptionalJson(path.join(dir, 'detector-target.json')); + const detectorRead = readOptionalJson(path.join(dir, 'detector-report.json')); + const backendRead = readOptionalJson(path.join(dir, 'backend-report.json')); + const bundleRead = + configuration.surface === 'runtime-bundle' + ? readOptionalJson(path.join(dir, 'ppl-grammar-bundle.json')) + : { value: undefined, error: undefined }; + + let backendTarget; + let detectorTarget; + let detectorRows = new Map(); + let backendRows = new Map(); + + for (const item of [backendTargetRead, detectorTargetRead, detectorRead, backendRead, bundleRead]) { + if (item.error) errors.push(item.error); + } + if (backendTargetRead.value) { + try { + backendTarget = normalizeTarget(backendTargetRead.value); + } catch (error) { + errors.push(`invalid target.json: ${error.message}`); + } + } + if (detectorTargetRead.value) { + try { + detectorTarget = normalizeTarget(detectorTargetRead.value); + } catch (error) { + errors.push(`invalid detector-target.json: ${error.message}`); + } + } + if (detectorRead.value) { + try { + detectorRows = indexDetectorReport(detectorRead.value); + } catch (error) { + errors.push(error.message); + } + } + if (backendRead.value && backendTarget) { + try { + backendRows = indexBackendReport(backendRead.value, backendTarget); + } catch (error) { + errors.push(`invalid backend-report.json: ${error.message}`); + } + } + + if (backendTarget) { + if ( + !backendTarget.engineVersion.startsWith( + configuration.engineVersion.replace(/[-+].*$/, '') + ) + ) { + errors.push( + `target engine ${backendTarget.engineVersion} does not match planned ` + + configuration.engineVersion + ); + } + if (backendTarget.executionBackend !== configuration.executionBackend) { + errors.push( + `target backend ${backendTarget.executionBackend} does not match planned ` + + configuration.executionBackend + ); + } + if (sqlSha && backendTarget.sqlSha && backendTarget.sqlSha !== sqlSha) { + errors.push(`target SQL SHA ${backendTarget.sqlSha} does not match planned ${sqlSha}`); + } + } + if (detectorTarget && detectorRead.value) { + for (const field of ['engineVersion', 'grammarHash', 'executionBackend']) { + if (detectorRead.value[field] !== detectorTarget[field]) { + errors.push( + `detector report ${field} ${JSON.stringify(detectorRead.value[field])} does not match ` + + `detector target ${JSON.stringify(detectorTarget[field])}` + ); + } + } + if (detectorRead.value.surface !== configuration.surface) { + errors.push( + `detector surface ${JSON.stringify(detectorRead.value.surface)} does not match planned ` + + configuration.surface + ); + } + } + if ( + bundleRead.value && + detectorTarget && + bundleRead.value.grammarHash !== detectorTarget.grammarHash + ) { + errors.push('runtime grammar bundle hash does not match detector target'); + } + + return { + dir, + errors: [...new Set(errors)], + backendTarget, + detectorTarget, + detectorReport: detectorRead.value, + detectorRows, + backendRows, + backendCommand: readOptionalText(path.join(dir, 'backend-command.txt')), + detectorCommand: readOptionalText(path.join(dir, 'detector-command.txt')), + }; +} + +function detectorActual(row, evidence) { + if (evidence.errors.length > 0 && !evidence.detectorReport) { + return { outcome: 'error', error: evidence.errors.join('; ') }; + } + if (!row) return { outcome: 'missing' }; + if (row.outcome === 'error') { + return { outcome: 'error', error: row.error || 'frontend execution failed' }; + } + if (row.outcome === 'not-applicable' || row.notApplicable) { + return { outcome: 'error', error: row.notApplicable || 'unexpected not-applicable row' }; + } + return { + outcome: 'observed', + count: row.actual, + severities: row.severities || [], + diagnostics: row.diagnostics || [], + }; +} + +function backendActual(row, evidence) { + if (evidence.errors.length > 0 && evidence.backendRows.size === 0) { + return { outcome: 'error', error: evidence.errors.join('; ') }; + } + if (!row) return { outcome: 'missing' }; + const state = classifyBackendReportRow(row); + if (state.status !== 'observed') { + return { + outcome: state.status === 'error' ? 'error' : 'error', + error: row.error || `backend outcome was ${state.status}`, + }; + } + const observed = row.observed || {}; + return { + outcome: 'observed', + rejected: state.rejected, + httpStatus: observed.httpStatus, + errorType: observed.type, + errorReason: observed.reason, + }; +} + +function expectedCase(spec, queryExpectation) { + const resolved = resolveBackendOracle(spec, queryExpectation, 'standard'); + if (resolved.status !== 'applicable') { + throw new Error(resolved.reason || 'standard backend oracle is not applicable'); + } + const expectedBackend = { + kind: resolved.oracle.kind, + httpStatus: resolved.oracle.httpStatus, + }; + const error = resolved.oracle.body && resolved.oracle.body.error; + if (error && error.type) expectedBackend.errorType = error.type; + if (error && error.reason) expectedBackend.errorReason = error.reason; + return { + detector: { + count: resolved.detector.count, + ...(resolved.detector.severity ? { severity: resolved.detector.severity } : {}), + ...(resolved.detector.matchMessage + ? { message: resolved.detector.matchMessage } + : {}), + ...(resolved.detector.messageEquals + ? { message: resolved.detector.messageEquals } + : {}), + }, + backend: expectedBackend, + frontend: resolved.frontend, + }; +} + +function compareDetector(expected, actual, row) { + if (actual.outcome !== 'observed') return []; + const differences = []; + if (actual.count !== expected.count) differences.push('detector.count'); + if (expected.severity && row.severityMatched === false) differences.push('detector.severity'); + if (expected.message && row.messageMatched === false) differences.push('detector.message'); + for (const field of [ + 'deterministicFixMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ]) { + if (row[field] === false) differences.push(`detector.${field}`); + } + for (const [field, matched] of Object.entries(row.assertions || {})) { + if (matched === false && !['count', 'severity'].includes(field)) { + differences.push(`detector.${field}`); + } + } + return [...new Set(differences)]; +} + +function compareBackend(expected, actual, row) { + if (actual.outcome !== 'observed') return []; + const differences = []; + const expectedRejected = expected.kind === 'rejection'; + if (actual.rejected !== expectedRejected) differences.push('backend.rejected'); + if ( + Number.isInteger(expected.httpStatus) && + actual.httpStatus !== expected.httpStatus + ) { + differences.push('backend.httpStatus'); + } + if (expected.errorType && actual.errorType !== expected.errorType) { + differences.push('backend.errorType'); + } + if (expected.errorReason && actual.errorReason !== expected.errorReason) { + differences.push('backend.errorReason'); + } + if (row.outcome === 'observed-mismatch' || row.error) { + differences.push('backend.result'); + } + return [...new Set(differences)]; +} + +function aggregateActual(cases, side) { + const actuals = cases.map((entry) => entry.actual[side]); + const errored = actuals.find((actual) => actual.outcome === 'error'); + if (errored) return { outcome: 'error', error: errored.error }; + if (actuals.some((actual) => actual.outcome === 'missing')) return { outcome: 'missing' }; + if (side === 'detector') { + return { + outcome: 'observed', + diagnosticCount: actuals.reduce((sum, actual) => sum + (actual.count || 0), 0), + }; + } + const rejected = actuals.filter((actual) => actual.rejected === true).length; + return { outcome: 'observed', rejected, observedCases: actuals.length }; +} + +function reasonForIncomplete(cases, evidence) { + for (const entry of cases) { + if (entry.actual.detector.outcome !== 'observed') { + return { + code: + entry.actual.detector.outcome === 'missing' + ? 'missing-detector-row' + : 'detector-error', + message: + entry.actual.detector.error || + `No detector result was produced for ${entry.ruleId}::${entry.queryName}.`, + }; + } + if (entry.actual.backend.outcome !== 'observed') { + return { + code: + entry.actual.backend.outcome === 'missing' + ? 'missing-backend-row' + : 'backend-error', + message: + entry.actual.backend.error || + `No backend result was produced for ${entry.ruleId}::${entry.queryName}.`, + }; + } + } + return { + code: 'invalid-leg-identity', + message: evidence.errors.join('; ') || 'The configuration did not produce trustworthy evidence.', + }; +} + +function classifyCell(cases) { + const triggerCases = cases.filter((entry) => entry.role === 'trigger'); + const rejectionTriggers = triggerCases.filter( + (entry) => entry.expected.backend.kind === 'rejection' + ); + const accepted = rejectionTriggers.filter( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.actual.backend.rejected === false + ); + const rejected = rejectionTriggers.filter( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.actual.backend.rejected === true + ); + const controls = cases.filter((entry) => entry.role !== 'trigger'); + const controlsProveSupport = controls.every( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.differences.every((difference) => !difference.startsWith('backend.')) + ); + const detectorDifferences = cases.flatMap((entry) => + entry.differences.filter((difference) => difference.startsWith('detector.')) + ); + const backendDifferences = cases.flatMap((entry) => + entry.differences.filter((difference) => difference.startsWith('backend.')) + ); + + const triggerSummary = { + contracted: rejectionTriggers.length, + acceptedByBackend: accepted.length, + rejectedByBackend: rejected.length, + missing: rejectionTriggers.filter( + (entry) => entry.actual.backend.outcome !== 'observed' + ).length, + }; + if ( + rejectionTriggers.length > 0 && + accepted.length === rejectionTriggers.length && + controlsProveSupport + ) { + return { classification: 'full-engine-relaxation', triggerSummary }; + } + if (accepted.length > 0 && rejected.length > 0) { + return { classification: 'partial-engine-relaxation', triggerSummary }; + } + if (backendDifferences.length === 0 && detectorDifferences.length > 0) { + return { classification: 'detector-regression', triggerSummary }; + } + if (backendDifferences.length > 0) { + return { classification: 'contract-drift', triggerSummary }; + } + return { classification: undefined, triggerSummary }; +} + +function remediation(classification) { + if (classification === 'detector-regression') { + return { + action: 'update-detector', + scope: 'detector-only', + detail: 'Backend behavior is unchanged; keep appliesTo unchanged.', + }; + } + if (classification === 'full-engine-relaxation') { + return { + action: 'scope-rule-version', + scope: 'appliesTo', + detail: + 'Every contracted trigger is accepted and controls remain supported; stop applying ' + + 'this rule to this version range.', + }; + } + if (classification === 'partial-engine-relaxation') { + return { + action: 'narrow-detector', + scope: 'detector-only', + detail: 'Keep the rule active for this version and narrow it to the forms the backend still rejects.', + }; + } + return { + action: 'update-contract', + scope: 'oracle', + detail: 'Confirm the backend behavior change is intentional before updating the pinned contract.', + }; +} + +function expectedDescription(spec) { + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const parts = []; + if (appliesTo.engine) { + parts.push(appliesTo.engine === 'calcite' ? 'Calcite' : appliesTo.engine); + } + if (appliesTo.minVersion && appliesTo.maxVersion) { + parts.push(`>= ${appliesTo.minVersion}, <= ${appliesTo.maxVersion}`); + } else if (appliesTo.minVersion) { + parts.push(`>= ${appliesTo.minVersion}`); + } else if (appliesTo.maxVersion) { + parts.push(`<= ${appliesTo.maxVersion}`); + } else { + parts.push('all versions'); + } + parts.push((spec.grammarSurface || 'runtime-bundle') === 'both' ? 'both' : 'runtime only'); + return parts.join('; '); +} + +function markdownCell(row) { + if (row.status === 'n/a') return `n/a (${row.expected.reason})`; + if (row.status === 'drift') return '**drift**'; + if (row.status === 'inconclusive') return '**inconclusive**'; + return 'compatible'; +} + +function renderMarkdown(report, contracts) { + const lines = [ + `## PPL lint compatibility: ${report.result.status.toUpperCase()}`, + '', + `SQL: \`${report.candidate.sqlSha.slice(0, 9)}\` `, + `OSD: \`${report.candidate.osd.repository} @ ${report.candidate.osd.sha || report.candidate.osd.ref}\` `, + `Rules: ${report.inventory.ruleCount} `, + `Configurations: ${report.configurations.length} `, + `Blocking results: ${report.result.drift} drift, ${report.result.inconclusive} inconclusive`, + '', + `| Rule | Expected compatibility | ${report.configurations + .map((configuration) => configuration.label) + .join(' | ')} |`, + `| --- | --- | ${report.configurations.map(() => '---').join(' | ')} |`, + ]; + for (const ruleId of ACTIVE_RULE_IDS) { + const spec = contracts.get(ruleId).spec; + const cells = report.configurations.map((configuration) => + markdownCell( + report.matrix.find( + (entry) => + entry.ruleId === ruleId && entry.configurationId === configuration.id + ) + ) + ); + lines.push( + `| \`${ruleId}\` | ${expectedDescription(spec)} | ${cells.join(' | ')} |` + ); + } + lines.push(''); + + const blocking = report.findings.filter((finding) => finding.blocking); + if (blocking.length > 0) { + lines.push('### Blocking findings', ''); + lines.push('| Rule | Configuration | Classification | Evidence | Action |'); + lines.push('| --- | --- | --- | --- | --- |'); + for (const finding of blocking) { + const evidence = + finding.reason?.message || + `${finding.evidence.triggerSummary.acceptedByBackend}/` + + `${finding.evidence.triggerSummary.contracted} contracted triggers accepted`; + lines.push( + `| \`${finding.ruleId}\` | ${finding.configurationLabel} | ` + + `${finding.classification || 'inconclusive'} | ${evidence.replace(/\|/g, '\\|')} | ` + + `${finding.remediation.detail.replace(/\|/g, '\\|')} |` + ); + } + lines.push(''); + } + lines.push('### Published evidence', ''); + lines.push('| Output | Location |'); + lines.push('| --- | --- |'); + lines.push('| Full table | `Aggregate rule compatibility` step summary |'); + lines.push('| Machine-readable report | `ppl-lint-multiversion-drift/drift-report.json` |'); + lines.push('| Detector logs and target identities | `ppl-lint-multiversion-evidence` |'); + return lines.join('\n'); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const plan = readRequiredJson(args.plan); + validatePlan(plan); + const contracts = loadContracts(args.contracts); + const evidenceByConfiguration = new Map( + plan.configurations.map((configuration) => [ + configuration.id, + loadEvidence(configuration, args.artifacts, plan.sqlSha), + ]) + ); + const configurations = plan.configurations.map((configuration) => { + const evidence = evidenceByConfiguration.get(configuration.id); + return { + id: configuration.id, + label: configuration.label, + engineVersion: + (evidence.backendTarget && evidence.backendTarget.engineVersion) || + configuration.engineVersion, + surface: configuration.surface, + executionBackend: configuration.executionBackend, + engineMode: configuration.engineMode, + grammar: { + source: + configuration.surface === 'runtime-bundle' + ? 'engine-runtime-bundle' + : 'osd-compiled', + hash: (evidence.detectorTarget && evidence.detectorTarget.grammarHash) || null, + }, + }; + }); + + const matrix = []; + const cases = []; + const findings = []; + + for (const ruleId of ACTIVE_RULE_IDS) { + const { spec } = contracts.get(ruleId); + for (const configuration of plan.configurations) { + const expected = expectedScope(spec, configuration); + if (!expected.applicable) { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'n/a', + expected, + actual: null, + }); + continue; + } + + const evidence = evidenceByConfiguration.get(configuration.id); + const expectation = selectExpectation(spec, configuration); + if (!expectation) { + const reason = { + code: 'missing-contract-expectation', + message: + `No unique expectation covers ${ruleId} on ${configuration.engineVersion} ` + + `(${configuration.engineMode}).`, + }; + const cell = { + ruleId, + configurationId: configuration.id, + status: 'inconclusive', + expected, + actual: { + detector: { outcome: 'missing' }, + backend: { outcome: 'missing' }, + }, + reason, + }; + matrix.push(cell); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + blocking: true, + reason, + remediation: { + action: 'fix-test-leg', + scope: 'contract', + detail: 'Add or correct the reviewed expectation, then rerun compatibility validation.', + }, + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + continue; + } + + const cellCases = []; + for (const [queryName, queryDefinition] of Object.entries(spec.queries || {})) { + const key = `${ruleId}::${queryName}`; + const detectorRow = evidence.detectorRows.get(key); + const backendRow = evidence.backendRows.get(key); + let expectedEvidence; + try { + expectedEvidence = expectedCase(spec, expectation.queries[queryName]); + } catch (error) { + fatal(`invalid ${ruleId}::${queryName} expectation: ${error.message}`); + } + const actual = { + detector: detectorActual(detectorRow, evidence), + backend: backendActual(backendRow, evidence), + }; + const differences = [ + ...compareDetector(expectedEvidence.detector, actual.detector, detectorRow || {}), + ...compareBackend(expectedEvidence.backend, actual.backend, backendRow || {}), + ]; + const entry = { + key: `${configuration.id}::${ruleId}::${queryName}`, + ruleId, + configurationId: configuration.id, + queryName, + role: queryDefinition.role || 'trigger', + query: String(queryDefinition.query || '').split('{{index}}').join(spec.index), + expected: { + detector: expectedEvidence.detector, + backend: expectedEvidence.backend, + }, + actual, + differences, + }; + cases.push(entry); + cellCases.push(entry); + } + + const actual = { + detector: aggregateActual(cellCases, 'detector'), + backend: aggregateActual(cellCases, 'backend'), + }; + const incomplete = + evidence.errors.length > 0 || + cellCases.some( + (entry) => + entry.actual.detector.outcome !== 'observed' || + entry.actual.backend.outcome !== 'observed' + ); + if (incomplete) { + const reason = reasonForIncomplete(cellCases, evidence); + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'inconclusive', + expected, + actual, + reason, + caseKeys: cellCases.map((entry) => entry.key), + }); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + blocking: true, + reason, + evidence: { caseKeys: cellCases.map((entry) => entry.key) }, + remediation: { + action: 'fix-test-leg', + scope: 'test-leg', + detail: 'Fix or rerun this test leg before recommending a product change.', + }, + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + continue; + } + + const classification = classifyCell(cellCases); + if (classification.classification) { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'drift', + expected, + actual, + classification: classification.classification, + triggerSummary: classification.triggerSummary, + caseKeys: cellCases.map((entry) => entry.key), + }); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + classification: classification.classification, + blocking: true, + evidence: { + caseKeys: cellCases.map((entry) => entry.key), + triggerSummary: classification.triggerSummary, + }, + remediation: remediation(classification.classification), + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + } else { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'compatible', + expected, + actual, + caseKeys: cellCases.map((entry) => entry.key), + }); + } + } + } + + const compatible = matrix.filter((entry) => entry.status === 'compatible').length; + const notApplicable = matrix.filter((entry) => entry.status === 'n/a').length; + const drift = matrix.filter((entry) => entry.status === 'drift').length; + const inconclusive = matrix.filter((entry) => entry.status === 'inconclusive').length; + const cellCount = matrix.length; + if ( + cellCount !== ACTIVE_RULE_IDS.length * configurations.length || + compatible + notApplicable + drift + inconclusive !== cellCount + ) { + fatal('internal matrix accounting invariant failed'); + } + + const report = { + schemaVersion: 3, + candidate: { + sqlSha: plan.sqlSha, + osd: { + repository: plan.osd.repository, + ref: plan.osd.ref, + sha: args.osdSha, + }, + }, + inventory: { + ruleCount: ACTIVE_RULE_IDS.length, + ruleIds: ACTIVE_RULE_IDS, + }, + configurations, + matrix, + cases, + findings, + result: { + status: drift + inconclusive === 0 ? 'pass' : 'fail', + cellCount, + compatible, + notApplicable, + drift, + inconclusive, + exitCode: drift + inconclusive === 0 ? 0 : 1, + }, + }; + + fs.writeFileSync(args.out, `${JSON.stringify(report, null, 2)}\n`); + const markdown = renderMarkdown(report, contracts); + process.stdout.write(`${markdown}\n`); + if (args.summary) fs.appendFileSync(args.summary, `${markdown}\n`); + if (report.result.exitCode !== 0) { + process.stderr.write( + `Rule compatibility validation failed after writing the complete report: ` + + `${drift} drift, ${inconclusive} inconclusive.\n` + ); + process.exitCode = report.result.exitCode; + } +} + +main(); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs new file mode 100644 index 00000000000..6f46f088a5e --- /dev/null +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -0,0 +1,1924 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Multi-version aggregation for the PPL lint contract. + * + * The single-version workflow answers "do the OSD detectors and this engine + * agree?". This script answers the question that actually protects users: "does + * every default-ERROR rule still agree with EVERY supported engine version, and + * if not, what should the linter engineer change?" + * + * Inputs: one `--leg =` per engine version, where holds that + * leg's `target.json`, `backend-report.json`, `detector-report.json` and + * `ppl-grammar-bundle.json` (the same four files the single-version jobs already + * produce — this script adds no new producer). + * + * Output: a `drift-report.json` plus a markdown remediation report. Exits + * non-zero when any ENFORCED rule drifted on any version, so the check is red + * exactly when a shipped default-error rule disagrees with a supported engine. + * + * Usage: + * node scripts/ppl-lint/aggregate-versions.mjs \ + * --contracts integ-test/src/test/resources/ppl-lint/contracts \ + * --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 --leg 3.8.0=legs/3.8.0 \ + * --out drift-report.json [--summary $GITHUB_STEP_SUMMARY] [--all-rules] + * + * By default only the manifest's `defaultError` set is enforced; `--all-rules` + * widens the report (still only enforcing `defaultError`) for nightly coverage. + */ + +import fs from 'fs'; +import path from 'path'; + +import { emitAnnotations } from './annotate.mjs'; +import { + assertContractSchema, + assertExactQueryCoverage, + assertExecutionBackend, + classifyBackendReportRow, + contractChannel, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; +import { + classifyDrift, + classifyExecutionBackendDivergence, + classifyGrammarDrift, + classifyRelaxationScope, + DRIFT_CLASSES, + formatDriftReport, + versionInAppliesTo, +} from './drift.mjs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-multiversion] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-multiversion] FATAL: ${message}`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { + legs: [], + contracts: '', + out: 'drift-report.json', + summary: '', + allRules: false, + observeAnalytics: false, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--leg') { + const raw = next(); + const eq = raw.indexOf('='); + if (eq <= 0) fatal(`--leg expects =, got "${raw}"`); + args.legs.push({ version: raw.slice(0, eq), dir: raw.slice(eq + 1) }); + } else if (arg === '--contracts') { + args.contracts = next(); + } else if (arg === '--out') { + args.out = next(); + } else if (arg === '--summary') { + args.summary = next(); + } else if (arg === '--all-rules') { + args.allRules = true; + } else if (arg === '--observe-analytics') { + args.observeAnalytics = true; + } else { + fatal(`unknown argument "${arg}"`); + } + } + if (args.legs.length === 0) fatal('at least one --leg = is required'); + if (!args.contracts) fatal('--contracts is required'); + return args; +} + +function readJson(file, { optional = false } = {}) { + if (!fs.existsSync(file)) { + if (optional) return undefined; + fatal(`expected file not found: ${file}`); + } + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + if (optional) return undefined; + fatal(`could not parse ${file}: ${error.message}`); + } + return undefined; +} + +function artifactFatal(file, error) { + fatal(`invalid ${file}: ${error.message}`); +} + +function reportRowKey(entry, label) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new TypeError(`${label} row must be a JSON object`); + } + if (typeof entry.ruleId !== 'string' || entry.ruleId.length === 0) { + throw new TypeError(`${label} row.ruleId must be a non-empty string`); + } + if (typeof entry.queryName !== 'string' || entry.queryName.length === 0) { + throw new TypeError(`${label} row.queryName must be a non-empty string`); + } + return `${entry.ruleId}::${entry.queryName}`; +} + +function rowExecutionBackend(entry, target, label, key) { + const hasIdentity = Object.prototype.hasOwnProperty.call(entry, 'executionBackend'); + if (!hasIdentity && !target.legacy) { + throw new Error(`${label} row ${key} is missing executionBackend for a schema-v2 target`); + } + const executionBackend = hasIdentity + ? assertExecutionBackend(entry.executionBackend, `${label} row ${key}.executionBackend`) + : 'standard'; + if (executionBackend !== target.executionBackend) { + throw new Error( + `${label} row ${key} executionBackend "${executionBackend}" does not match target ` + + `"${target.executionBackend}"` + ); + } + return executionBackend; +} + +function validateOptionalRowIdentity(entry, target, label, key) { + for (const field of ['engineVersion', 'grammarHash']) { + if ( + Object.prototype.hasOwnProperty.call(entry, field) && + entry[field] !== target[field] + ) { + throw new Error( + `${label} row ${key} ${field} ${JSON.stringify(entry[field])} does not match target ` + + `${JSON.stringify(target[field])}` + ); + } + } +} + +function normalizeDetectorReport(detector, target) { + if (!detector || typeof detector !== 'object' || Array.isArray(detector)) { + throw new TypeError('detector report must be a JSON object'); + } + + const hasIdentity = Object.prototype.hasOwnProperty.call(detector, 'executionBackend'); + if (!hasIdentity && !target.legacy) { + throw new Error('detector report is missing executionBackend for a schema-v2 target'); + } + const executionBackend = hasIdentity + ? assertExecutionBackend(detector.executionBackend, 'detector report.executionBackend') + : 'standard'; + if (executionBackend !== target.executionBackend) { + throw new Error( + `detector report executionBackend "${executionBackend}" does not match target ` + + `"${target.executionBackend}"` + ); + } + if (!target.legacy && detector.schemaVersion !== 2) { + throw new Error( + `detector report schemaVersion ${JSON.stringify(detector.schemaVersion)} does not match ` + + 'schema-v2 target' + ); + } + for (const field of ['engineVersion', 'grammarHash']) { + const hasField = Object.prototype.hasOwnProperty.call(detector, field); + if (!hasField && !target.legacy) { + throw new Error(`detector report is missing ${field} for a schema-v2 target`); + } + if (hasField && detector[field] !== target[field]) { + throw new Error( + `detector report ${field} ${JSON.stringify(detector[field])} does not match target ` + + `${JSON.stringify(target[field])}` + ); + } + } + if (!Array.isArray(detector.results)) { + throw new TypeError('detector report.results must be a JSON array'); + } + if (!['runtime-bundle', 'compiled-simplified'].includes(detector.surface)) { + throw new Error( + `detector report.surface must be "runtime-bundle" or "compiled-simplified", got ` + + `${JSON.stringify(detector.surface)}` + ); + } + if (!Array.isArray(detector.defaultErrorRules)) { + throw new TypeError('detector report.defaultErrorRules must be a JSON array'); + } + for (const field of ['enabledRules', 'requiredSyntaxFeatures', 'activeContractRules']) { + if (detector[field] === undefined) continue; + if (!Array.isArray(detector[field])) { + throw new TypeError(`detector report.${field} must be a JSON array`); + } + const values = new Set(); + for (const value of detector[field]) { + if (typeof value !== 'string' || value.length === 0) { + throw new TypeError(`detector report.${field} entries must be non-empty strings`); + } + if (values.has(value)) { + throw new Error(`detector report.${field} contains duplicate rule "${value}"`); + } + values.add(value); + } + } + const census = new Set(); + for (const ruleId of detector.defaultErrorRules) { + if (typeof ruleId !== 'string' || ruleId.length === 0) { + throw new TypeError('detector report.defaultErrorRules entries must be non-empty strings'); + } + if (census.has(ruleId)) { + throw new Error(`detector report.defaultErrorRules contains duplicate rule "${ruleId}"`); + } + census.add(ruleId); + } + + const results = new Map(); + for (const entry of detector.results) { + const key = reportRowKey(entry, 'detector report'); + rowExecutionBackend(entry, target, 'detector report', key); + validateOptionalRowIdentity(entry, target, 'detector report', key); + if (results.has(key)) { + throw new Error(`duplicate detector report key "${key}"`); + } + if (!entry.notApplicable && entry.outcome !== 'not-applicable') { + if (!Number.isInteger(entry.expected) || entry.expected < 0) { + throw new TypeError(`detector report row ${key}.expected must be a non-negative integer`); + } + if (!Number.isInteger(entry.actual) || entry.actual < 0) { + throw new TypeError(`detector report row ${key}.actual must be a non-negative integer`); + } + if (!Array.isArray(entry.severities)) { + throw new TypeError(`detector report row ${key}.severities must be a JSON array`); + } + if (typeof entry.severityMatched !== 'boolean') { + throw new TypeError(`detector report row ${key}.severityMatched must be a boolean`); + } + if (typeof entry.messageMatched !== 'boolean') { + throw new TypeError(`detector report row ${key}.messageMatched must be a boolean`); + } + for (const field of [ + 'deterministicFixMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ]) { + if (entry[field] !== undefined && typeof entry[field] !== 'boolean') { + throw new TypeError(`detector report row ${key}.${field} must be a boolean`); + } + } + if (entry.assertions !== undefined) { + if ( + !entry.assertions || + typeof entry.assertions !== 'object' || + Array.isArray(entry.assertions) + ) { + throw new TypeError(`detector report row ${key}.assertions must be a JSON object`); + } + for (const [field, matched] of Object.entries(entry.assertions)) { + if (typeof matched !== 'boolean') { + throw new TypeError( + `detector report row ${key}.assertions.${field} must be a boolean` + ); + } + } + } + if (entry.mismatches !== undefined && !Array.isArray(entry.mismatches)) { + throw new TypeError(`detector report row ${key}.mismatches must be a JSON array`); + } + } + results.set(key, entry); + } + return { ...detector, executionBackend, resultsByKey: results }; +} + +function makeLegKey({ label, version, surface, executionBackend }) { + return [label, version, surface, executionBackend] + .map((part) => encodeURIComponent(part)) + .join('::'); +} + +function legFields(leg) { + return { + version: leg.version, + leg: leg.label, + legKey: leg.key, + executionBackend: leg.executionBackend, + }; +} + +function findingKey(finding) { + const backend = Array.isArray(finding.executionBackends) + ? finding.executionBackends.join('-vs-') + : finding.executionBackend || 'standard'; + return [ + finding.legKey || finding.leg || finding.version, + backend, + finding.ruleId, + finding.queryName || '', + finding.driftClass, + ] + .map((part) => encodeURIComponent(String(part))) + .join('::'); +} + +function reportItemKey(item, kind) { + return [ + item.legKey || item.leg || item.version, + item.executionBackend || 'standard', + item.ruleId, + item.queryName || '', + kind, + ] + .map((part) => encodeURIComponent(String(part))) + .join('::'); +} + +/** Load the contract corpus, keyed by ruleId, plus the manifest's enforced sets. */ +function loadContracts(dir) { + const manifest = readJson(path.join(dir, 'manifest.json')); + const specs = new Map(); + const contractNames = manifest.contracts || []; + if (!Array.isArray(contractNames)) { + fatal(`${path.join(dir, 'manifest.json')} contracts must be an array`); + } + if (new Set(contractNames).size !== contractNames.length) { + fatal(`${path.join(dir, 'manifest.json')} contracts contains duplicate file names`); + } + for (const name of contractNames) { + const spec = readJson(path.join(dir, name)); + try { + assertContractSchema(spec); + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new TypeError(`[${spec.ruleId}] expectations must be a non-empty array`); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + for (const queryExpectation of Object.values(expectation.queries)) { + resolveBackendOracle(spec, queryExpectation, 'standard'); + resolveBackendOracle(spec, queryExpectation, 'analytics'); + } + } + } catch (error) { + artifactFatal(path.join(dir, name), error); + } + if (specs.has(spec.ruleId)) { + fatal(`contract manifest contains duplicate ruleId "${spec.ruleId}"`); + } + specs.set(spec.ruleId, { spec, file: name }); + } + // `defaultError` is the multi-version enforced set: every rule that ships + // enabled at error severity. Fall back to `enforced` for older manifests so + // this script still runs against an un-migrated corpus. + const enforcedFiles = new Set(manifest.defaultError || manifest.enforced || []); + for (const file of enforcedFiles) { + if (!contractNames.includes(file)) { + fatal(`manifest.defaultError references inactive or missing contract "${file}"`); + } + } + const requiredSyntaxFiles = manifest.requiredSyntaxFeatures || []; + if (!Array.isArray(requiredSyntaxFiles)) { + fatal('manifest.requiredSyntaxFeatures must be an array'); + } + if (new Set(requiredSyntaxFiles).size !== requiredSyntaxFiles.length) { + fatal('manifest.requiredSyntaxFeatures contains duplicate file names'); + } + for (const file of requiredSyntaxFiles) { + const entry = [...specs.values()].find((candidate) => candidate.file === file); + if (!entry) { + fatal(`manifest.requiredSyntaxFeatures references inactive or missing contract "${file}"`); + } + if (contractChannel(entry.spec) !== 'syntax') { + fatal(`manifest.requiredSyntaxFeatures entry "${file}" is not a syntax contract`); + } + } + const enforcedRules = new Set(); + for (const [ruleId, { file }] of specs) { + if (enforcedFiles.has(file)) enforcedRules.add(ruleId); + } + return { specs, enforcedRules, manifest }; +} + +/** + * Read one engine version's four artifacts. A leg whose backend never came up + * is fatal rather than skipped: silently dropping a version would turn a broken + * matrix into a green check, which is the failure mode this whole contract + * exists to prevent. + */ +function loadLeg({ version, dir }) { + const targetFile = path.join(dir, 'target.json'); + const detectorFile = path.join(dir, 'detector-report.json'); + const backendFile = path.join(dir, 'backend-report.json'); + const targetRaw = readJson(targetFile); + const detectorRaw = readJson(detectorFile); + const backendRaw = readJson(path.join(dir, 'backend-report.json')); + const bundle = readJson(path.join(dir, 'ppl-grammar-bundle.json'), { optional: true }); + + let target; + let detector; + let backend; + try { + target = normalizeTarget(targetRaw); + } catch (error) { + artifactFatal(targetFile, error); + } + try { + detector = normalizeDetectorReport(detectorRaw, target); + } catch (error) { + artifactFatal(detectorFile, error); + } + try { + backend = indexBackendReport(backendRaw, target); + for (const [key, entry] of backend) { + validateOptionalRowIdentity(entry, target, 'backend report', key); + } + } catch (error) { + artifactFatal(backendFile, error); + } + if ( + bundle && + Object.prototype.hasOwnProperty.call(bundle, 'grammarHash') && + bundle.grammarHash !== target.grammarHash + ) { + fatal( + `grammar bundle ${path.join(dir, 'ppl-grammar-bundle.json')} reports ` + + `${JSON.stringify(bundle.grammarHash)} but target reports ` + + `${JSON.stringify(target.grammarHash)}` + ); + } + + // The engine's self-reported version wins over the matrix label, so a matrix + // typo (asking for 3.7.0 and getting 3.8.0) cannot silently mislabel results. + const reported = target.engineVersion || ''; + if (reported && !reported.startsWith(version.split('-')[0])) { + log( + `WARN: leg "${version}" reported engineVersion "${reported}"; using the reported value for ` + + `version comparisons.` + ); + } + + const leg = { + version: reported || version, + label: version, + dir, + grammarHash: target.grammarHash || '', + sqlSha: target.sqlSha || '', + executionBackend: target.executionBackend, + targetSchemaVersion: target.schemaVersion, + legacyTarget: target.legacy, + // Which of OSD's two lint surfaces this leg validated. Older detector reports + // predate the field; they were all runtime-bundle runs. + surface: detector.surface || 'runtime-bundle', + parserRuleNames: bundle && Array.isArray(bundle.parserRuleNames) ? bundle.parserRuleNames : undefined, + detector, + backend, + }; + leg.key = makeLegKey(leg); + return leg; +} + +function pairBackendLegs(legs) { + const identities = new Set(); + for (const leg of legs) { + if (identities.has(leg.key)) { + fatal(`duplicate leg identity "${leg.key}"`); + } + identities.add(leg.key); + } + + const runtimeLegs = legs.filter((leg) => leg.surface === 'runtime-bundle'); + const standards = runtimeLegs.filter((leg) => leg.executionBackend === 'standard'); + const analyticsLegs = runtimeLegs.filter((leg) => leg.executionBackend === 'analytics'); + const usedStandards = new Set(); + const pairs = []; + const neutralLabel = (label) => String(label).replace(/[-_](?:standard|analytics)$/i, ''); + + for (const analytics of analyticsLegs) { + const labelPeers = standards.filter( + (standard) => neutralLabel(standard.label) === neutralLabel(analytics.label) + ); + const candidates = + labelPeers.length > 0 + ? labelPeers + : standards.filter((standard) => standard.version === analytics.version); + if (candidates.length === 0) { + if (standards.length > 0) { + fatal( + `analytics leg "${analytics.key}" has no standard peer for engine ` + + `${analytics.version}` + ); + } + continue; + } + + const sameLabel = candidates.filter((standard) => standard.label === analytics.label); + const sameGrammar = candidates.filter( + (standard) => standard.grammarHash === analytics.grammarHash + ); + let standard; + if (sameLabel.length === 1) { + standard = sameLabel[0]; + } else if (sameGrammar.length === 1) { + standard = sameGrammar[0]; + } else if (candidates.length === 1) { + standard = candidates[0]; + } else { + fatal( + `analytics leg "${analytics.key}" has ${candidates.length} possible standard peers for ` + + `${analytics.version}; use an unambiguous label/grammar identity` + ); + } + + if (standard.version !== analytics.version) { + fatal( + `paired standard/analytics legs report different engine versions: ` + + `${standard.label}=${JSON.stringify(standard.version)}, ` + + `${analytics.label}=${JSON.stringify(analytics.version)}` + ); + } + if (!standard.sqlSha || !analytics.sqlSha) { + fatal( + `paired standard/analytics legs must both report a non-empty SQL SHA: ` + + `${standard.label}=${JSON.stringify(standard.sqlSha)}, ` + + `${analytics.label}=${JSON.stringify(analytics.sqlSha)}` + ); + } + if (standard.sqlSha !== analytics.sqlSha) { + fatal( + `paired standard/analytics legs report different SQL SHAs: ` + + `${standard.label}=${JSON.stringify(standard.sqlSha)}, ` + + `${analytics.label}=${JSON.stringify(analytics.sqlSha)}` + ); + } + if (!standard.grammarHash || !analytics.grammarHash) { + fatal( + `paired standard/analytics legs for ${analytics.version} must both report a runtime grammar hash` + ); + } + if (standard.grammarHash !== analytics.grammarHash) { + fatal( + `paired standard/analytics legs for ${analytics.version} have different grammar hashes: ` + + `${standard.label}=${JSON.stringify(standard.grammarHash)}, ` + + `${analytics.label}=${JSON.stringify(analytics.grammarHash)}` + ); + } + if (usedStandards.has(standard.key)) { + fatal( + `standard leg "${standard.key}" matches more than one analytics leg; duplicate backend leg identity` + ); + } + usedStandards.add(standard.key); + const pair = { + key: `${standard.key}::${analytics.key}`, + standard, + analytics, + engineVersion: analytics.version, + grammarHash: analytics.grammarHash, + }; + assertDetectorParity(pair); + pairs.push(pair); + } + return pairs; +} + +function detectorParityValue(entry) { + return { + channel: entry.channel || 'lint', + role: entry.role || 'trigger', + query: entry.query || '', + expected: entry.expected, + actual: entry.actual, + severities: [...(entry.severities || [])].sort(), + severityMatched: + typeof entry.severityMatched === 'boolean' ? entry.severityMatched : undefined, + messageMatched: + typeof entry.messageMatched === 'boolean' ? entry.messageMatched : undefined, + fixMatched: typeof entry.fixMatched === 'boolean' ? entry.fixMatched : undefined, + rawMessageMatched: + typeof entry.rawMessageMatched === 'boolean' ? entry.rawMessageMatched : undefined, + totalErrorsMatched: + typeof entry.totalErrorsMatched === 'boolean' ? entry.totalErrorsMatched : undefined, + deterministicFixMatched: + typeof entry.deterministicFixMatched === 'boolean' + ? entry.deterministicFixMatched + : undefined, + assertions: entry.assertions, + mismatches: entry.mismatches, + deterministicFix: entry.deterministicFix, + code: entry.code, + codes: entry.codes, + totalErrors: entry.totalErrors, + }; +} + +/** + * Both detector passes use the same OSD checkout, grammar, contracts, and lint + * context. Any route-qualified difference is therefore a harness defect, not a + * backend observation. + */ +function assertDetectorParity(pair) { + const standard = pair.standard.detector.resultsByKey; + const analytics = pair.analytics.detector.resultsByKey; + const keys = new Set([...standard.keys(), ...analytics.keys()]); + for (const key of keys) { + const standardRow = standard.get(key); + const analyticsRow = analytics.get(key); + if (standardRow?.reportOnly === true || analyticsRow?.reportOnly === true) { + continue; + } + if (!standardRow || !analyticsRow) { + fatal( + `detector parity failed for ${key}: standard row=${!!standardRow}, ` + + `analytics row=${!!analyticsRow}` + ); + } + const standardValue = detectorParityValue(standardRow); + const analyticsValue = detectorParityValue(analyticsRow); + if (JSON.stringify(standardValue) !== JSON.stringify(analyticsValue)) { + fatal( + `detector parity failed for ${key}: standard=${JSON.stringify(standardValue)}, ` + + `analytics=${JSON.stringify(analyticsValue)}` + ); + } + } +} + +function backendVerdict(entry) { + if (!entry) { + return { + backendRejected: undefined, + backendType: undefined, + backendReason: undefined, + }; + } + const state = classifyBackendReportRow(entry); + const observedBackend = entry && entry.observed; + const rowRejected = + typeof entry.rejected === 'boolean' ? entry.rejected : undefined; + const observedRejected = + observedBackend && typeof observedBackend.rejected === 'boolean' + ? observedBackend.rejected + : undefined; + if ( + typeof rowRejected === 'boolean' && + typeof observedRejected === 'boolean' && + rowRejected !== observedRejected + ) { + fatal( + `backend report row ${reportRowKey(entry, 'backend report')} has conflicting ` + + `rejected verdicts` + ); + } + const explicitRejected = + typeof observedRejected === 'boolean' ? observedRejected : rowRejected; + const usableRawObservation = + state.status === 'observed' || state.status === 'coverage-missing'; + return { + backendRejected: + usableRawObservation && typeof explicitRejected === 'boolean' + ? explicitRejected + : undefined, + backendStatus: observedBackend ? observedBackend.httpStatus : undefined, + backendType: observedBackend ? observedBackend.type : undefined, + backendReason: observedBackend ? observedBackend.reason : undefined, + backendOutcome: entry.outcome, + backendMismatch: entry.error, + }; +} + +function indexDivergentCases(pairs) { + const cases = new Map(); + for (const pair of pairs) { + for (const [rowKey, standardEntry] of pair.standard.backend) { + const analyticsEntry = pair.analytics.backend.get(rowKey); + if (!analyticsEntry) continue; + const standardObserved = backendVerdict(standardEntry); + const analyticsObserved = backendVerdict(analyticsEntry); + if ( + typeof standardObserved.backendRejected !== 'boolean' || + typeof analyticsObserved.backendRejected !== 'boolean' || + standardObserved.backendRejected === analyticsObserved.backendRejected + ) { + continue; + } + const value = { pair, rowKey, standardObserved, analyticsObserved }; + cases.set(`${pair.standard.key}::${rowKey}`, value); + cases.set(`${pair.analytics.key}::${rowKey}`, value); + } + } + return cases; +} + +/** + * Compare the OSD catalog's default-error census (recorded by each detector leg) + * against the contracts this run knows about. Returns one entry per rule that + * ships enabled at error severity with no contract, or whose contract the + * manifest does not list under `defaultError`. + * + * Legs can disagree if they ran against different OSD checkouts, so the union is + * used: a rule that is default-error on ANY validated OSD ref must be accounted + * for. + */ +function auditDefaultErrorCensus(legs, specs, enforcedRules) { + const census = new Set(); + let sawCensus = false; + for (const leg of legs) { + const rules = leg.detector && leg.detector.defaultErrorRules; + if (!Array.isArray(rules)) continue; + sawCensus = true; + for (const ruleId of rules) census.add(ruleId); + } + if (!sawCensus) { + log( + "WARN: no detector leg reported a defaultErrorRules census, so the manifest's defaultError set " + + 'could not be cross-checked against the OSD catalog. Re-run with a detector build that emits it.' + ); + return []; + } + + const missing = []; + for (const ruleId of [...census].sort()) { + if (!specs.has(ruleId)) { + missing.push({ ruleId, reason: 'no contract file' }); + } else if (!enforcedRules.has(ruleId)) { + missing.push({ + ruleId, + reason: 'contract exists but is not listed under manifest.defaultError', + }); + } + } + for (const ruleId of [...enforcedRules].sort()) { + if (!census.has(ruleId)) { + missing.push({ + ruleId, + reason: 'listed under manifest.defaultError but not enabled at error severity in OSD', + }); + } + } + return missing; +} + +function setsEqual(left, right) { + return left.size === right.size && [...left].every((value) => right.has(value)); +} + +function auditShippingCensus(legs, specs, manifest) { + const reports = legs + .map((leg) => leg.detector) + .filter( + (detector) => + Array.isArray(detector.enabledRules) && + Array.isArray(detector.activeContractRules) && + Array.isArray(detector.requiredSyntaxFeatures) + ); + if (reports.length === 0) { + return { + available: false, + enforced: false, + passed: false, + problems: [ + 'detector reports predate the active shipping census; rerun with the channel-aware frontend runner', + ], + }; + } + + const activeRules = new Set(specs.keys()); + const activeLintRules = new Set( + [...specs.entries()] + .filter(([, { spec }]) => contractChannel(spec) === 'lint') + .map(([ruleId]) => ruleId) + ); + const activeSyntaxRules = new Set( + [...specs.entries()] + .filter(([, { spec }]) => contractChannel(spec) === 'syntax') + .map(([ruleId]) => ruleId) + ); + const requiredSyntaxRules = new Set( + (manifest.requiredSyntaxFeatures || []) + .map((file) => [...specs.entries()].find(([, entry]) => entry.file === file)) + .filter(Boolean) + .map(([ruleId]) => ruleId) + ); + const enabledRules = new Set(reports.flatMap((report) => report.enabledRules)); + const reportedActiveRules = new Set( + reports.flatMap((report) => report.activeContractRules) + ); + const reportedSyntaxRules = new Set( + reports.flatMap((report) => report.requiredSyntaxFeatures) + ); + const problems = []; + + if (activeLintRules.size !== 12) { + problems.push(`expected 12 active lint contracts, found ${activeLintRules.size}`); + } + if (requiredSyntaxRules.size !== 0) { + problems.push(`expected no required syntax features, found ${requiredSyntaxRules.size}`); + } + if (activeRules.size !== 12) { + problems.push(`expected 12 active contracts, found ${activeRules.size}`); + } + if (!setsEqual(activeLintRules, enabledRules)) { + problems.push( + `active lint rules ${JSON.stringify([...activeLintRules].sort())} do not equal enabled OSD ` + + `rules ${JSON.stringify([...enabledRules].sort())}` + ); + } + if (!setsEqual(activeSyntaxRules, requiredSyntaxRules)) { + problems.push( + `active syntax rules ${JSON.stringify([...activeSyntaxRules].sort())} do not equal manifest ` + + `required syntax features ${JSON.stringify([...requiredSyntaxRules].sort())}` + ); + } + if (!setsEqual(activeRules, reportedActiveRules)) { + problems.push('detector report activeContractRules does not match this manifest'); + } + if (!setsEqual(requiredSyntaxRules, reportedSyntaxRules)) { + problems.push('detector report requiredSyntaxFeatures does not match this manifest'); + } + + return { + available: true, + enforced: reports.some( + (report) => report.census && report.census.enforced === true + ), + enabledRules: [...enabledRules].sort(), + activeContractRules: [...activeRules].sort(), + activeLintRules: [...activeLintRules].sort(), + requiredSyntaxFeatures: [...requiredSyntaxRules].sort(), + passed: problems.length === 0, + problems, + }; +} + +/** + * Read one backend report entry into an observation, distinguishing "the engine + * accepted this" from "we never got an answer". + * + * This distinction is load-bearing. The IT marks a transport-level failure + * `outcome: "error"` and, having never received a verdict, writes no `rejected` + * field. Coercing that absence to `false` would report a timeout as an engine + * that now ACCEPTS a query it used to reject — which reads as an engine + * relaxation and would advise disabling a perfectly good rule. Anything that is + * not a real observed verdict becomes `undefined`, which the classifier treats as + * "not observed" rather than as acceptance. + * + * Returns `{ observed, usable }`: `usable` is false when this case produced no + * comparable engine verdict, so the caller can refuse to call it agreement. + */ +function readBackendObservation(backendEntry, detectorResult) { + const verdict = backendVerdict(backendEntry); + const hasVerdict = typeof verdict.backendRejected === 'boolean'; + const detectorUsable = !!detectorResult && detectorResult.outcome !== 'error'; + + return { + usable: hasVerdict && detectorUsable, + observed: { + detectorCount: detectorResult ? detectorResult.actual : 0, + severities: detectorResult ? detectorResult.severities || [] : [], + backendRejected: verdict.backendRejected, + backendStatus: verdict.backendStatus, + backendType: verdict.backendType, + backendReason: verdict.backendReason, + backendOutcome: verdict.backendOutcome, + backendMismatch: verdict.backendMismatch, + severityMatched: detectorResult ? detectorResult.severityMatched : undefined, + messageMatched: detectorResult ? detectorResult.messageMatched : undefined, + deterministicFixMatched: detectorResult + ? detectorResult.deterministicFixMatched + : undefined, + fixMatched: detectorResult ? detectorResult.fixMatched : undefined, + rawMessageMatched: detectorResult ? detectorResult.rawMessageMatched : undefined, + totalErrorsMatched: detectorResult + ? detectorResult.totalErrorsMatched + : undefined, + assertions: detectorResult ? detectorResult.assertions : undefined, + mismatches: detectorResult ? detectorResult.mismatches : undefined, + }, + }; +} + +function unusableObservationReason(detectorResult) { + if (!detectorResult) { + return 'no detector result'; + } + if (detectorResult.outcome === 'error') { + const message = + typeof detectorResult.error === 'string' && detectorResult.error.length > 0 + ? detectorResult.error + : 'unknown frontend execution error'; + return `frontend execution failed: ${message}`; + } + return 'no engine verdict'; +} + +function failedExtendedFrontendAssertions(entry, frontendOracle) { + if (!entry) { + return []; + } + const failures = new Set(); + for (const field of [ + 'deterministicFixMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ]) { + if (entry[field] === false) { + failures.add(field); + } + } + for (const [field, matched] of Object.entries(entry.assertions || {})) { + if ( + matched === false && + field !== 'count' && + field !== 'severity' && + (field !== 'message' || frontendOracle.messageEquals !== undefined) + ) { + failures.add(field); + } + } + for (const mismatch of entry.mismatches || []) { + const field = mismatch && mismatch.field; + if ( + typeof field === 'string' && + field !== 'count' && + field !== 'severity' && + (field !== 'message' || frontendOracle.messageEquals !== undefined) + ) { + failures.add(field); + } + } + return [...failures].sort(); +} + +/** + * Pick the contract expectation that applies to a version, reusing the same + * "exactly one must match" rule as the two single-version halves. Returns + * undefined when the corpus does not cover this version — reported separately as + * a coverage hole, not as behavioral drift. + * + * The `engine` filter matters as much as the version range: both single-version + * halves (`PplLintRuleValidationIT.selectExpectation` and + * `run-frontend-contract.mjs`) drop `engine: "calcite"` entries when Calcite is + * off. Omitting it here would make a contract that pins one range per engine match + * TWICE and be misreported as an uncovered version. Every leg in this workflow + * runs with Calcite enabled (the observation legs do not disable it), so a + * calcite-scoped expectation is in play; a contract's `frontendContext.isCalcite: + * false` opts out. + */ +function selectExpectation(spec, version, versionMatchesRange) { + const isCalcite = !((spec.frontendContext || {}).isCalcite === false); + const matches = (spec.expectations || []).filter((exp) => { + if (!versionMatchesRange(exp.version, version)) return false; + if (exp.engine === 'calcite' && !isCalcite) return false; + return true; + }); + return matches.length === 1 ? matches[0] : undefined; +} + +/** Minimal semver-range test, kept byte-compatible with the other two halves. */ +function makeRangeMatcher() { + const parse = (v) => { + const m = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(v || '')); + return m ? [Number(m[1]), Number(m[2] || 0), Number(m[3] || 0)] : undefined; + }; + const cmp = (a, b) => { + for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + return 0; + }; + return (range, version) => { + if (!range || !String(range).trim()) return true; + const have = parse(version); + if (!have) return true; + for (const token of String(range).trim().split(/\s+/)) { + let op = '='; + let ver = token; + for (const candidate of ['>=', '<=', '>', '<', '=']) { + if (token.startsWith(candidate)) { + op = candidate; + ver = token.slice(candidate.length); + break; + } + } + const c = cmp(have, parse(ver) || [0, 0, 0]); + const ok = + (op === '>=' && c >= 0) || + (op === '<=' && c <= 0) || + (op === '>' && c > 0) || + (op === '<' && c < 0) || + (op === '=' && c === 0); + if (!ok) return false; + } + return true; + }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const versionMatchesRange = makeRangeMatcher(); + const { specs, enforcedRules, manifest } = loadContracts(args.contracts); + const legs = args.legs.map(loadLeg); + const backendPairs = pairBackendLegs(legs); + const divergentCases = indexDivergentCases(backendPairs); + + log(`contracts=${specs.size} enforced(default-error)=${enforcedRules.size} legs=${legs.length}`); + for (const leg of legs) { + log( + ` leg ${leg.label} (${leg.executionBackend}): engine=${leg.version} ` + + `grammar=${(leg.grammarHash || '—').slice(0, 19)} ` + + `detectorResults=${(leg.detector.results || []).length} backendCases=${leg.backend.size}` + ); + } + + const drifts = []; + const coverageHoles = []; + // Rule/version pairs where no case could actually be compared (a leg that lost + // its engine verdicts or its detector rows). Tracked separately from drift + // because the answer is "re-run / fix the leg", not "edit the linter". + const inconclusive = []; + // Cases a leg's grammar surface cannot express (a `runtimeOnly` rule on a + // compiled-simplified leg). Recorded so the report can say WHY a cell is blank, + // but never a failure: the rule is inert there by design. + const notApplicable = []; + const matrix = []; // one row per rule × backend-qualified leg, for the summary table + const addDrift = (drift, leg, extra = {}) => { + const enriched = { + ...drift, + ...legFields(leg), + ...extra, + executionBackend: drift.executionBackend || leg.executionBackend, + }; + enriched.key = findingKey(enriched); + drifts.push(enriched); + return enriched; + }; + + // A rule that ships enabled at error severity but has no contract file is + // invisible to this whole check. Compare the manifest's declared set against + // the census each detector leg recorded from the OSD catalog it linted with, so + // a new default-error rule cannot land unvalidated. + const missingContracts = auditDefaultErrorCensus(legs, specs, enforcedRules); + const shippingCensus = auditShippingCensus(legs, specs, manifest); + const blockCensusDrift = !shippingCensus.available || shippingCensus.enforced; + shippingCensus.blocking = blockCensusDrift; + const blockingShippingCensusProblems = blockCensusDrift + ? shippingCensus.problems + : []; + for (const entry of missingContracts) { + entry.blocking = blockCensusDrift; + } + if (!shippingCensus.passed) { + for (const problem of shippingCensus.problems) { + log(`CENSUS ${blockCensusDrift ? 'ENFORCED' : 'REPORT-ONLY'}: ${problem}`); + } + } + + for (const [ruleId, { spec, file }] of specs) { + const isEnforced = enforcedRules.has(ruleId); + if (!isEnforced && !args.allRules) continue; + + // A rule the catalog does not apply to an engine version ships nothing to + // users there, so it needs no expectation for it. Only a rule that IS in + // scope and has no expectation is a genuine hole. + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + + for (const leg of legs) { + // A contract only speaks for the surface(s) it declares. Judge it on any + // other leg and every verdict is meaningless: a runtime-bundle contract on a + // compiled leg yields "the engine rejects but the rule is scoped away" + // (version-scope-too-narrow) and "no expectation covers this engine" + // (coverage hole) — both about a surface the contract never claimed to + // describe. Checked FIRST, before scope, grammar and coverage, because all + // three of those produce confident findings from an irrelevant comparison. + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + const legSurface = leg.surface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== legSurface) { + notApplicable.push({ + ruleId, + ...legFields(leg), + surface: legSurface, + reason: `contract declares grammarSurface "${contractSurface}"`, + }); + matrix.push({ + ruleId, + ...legFields(leg), + status: 'not-applicable', + drifts: 0, + }); + continue; + } + + const inScope = versionInAppliesTo(appliesTo, leg.version); + if (!inScope) { + notApplicable.push({ + ruleId, + ...legFields(leg), + surface: legSurface, + kind: 'applies-to', + reason: `wiring.appliesTo excludes engine ${leg.version}`, + }); + matrix.push({ + ruleId, + ...legFields(leg), + status: 'out-of-scope', + drifts: 0, + }); + continue; + } + + // A parser rule that vanished from the grammar is one fact about this + // rule on this engine, not one per query — raise it once and move on, so + // the report shows the single edit to make instead of the same paragraph + // repeated for every case. + if (inScope) { + const grammarDrift = classifyGrammarDrift({ + ruleId, + version: leg.version, + requiredParserRules: spec.requiredParserRules, + detectorPath: spec.detectorPath, + parserRuleNames: leg.parserRuleNames, + executionBackend: leg.executionBackend, + }); + if (grammarDrift) { + addDrift(grammarDrift, leg, { enforced: isEnforced, contractFile: file }); + matrix.push({ ruleId, ...legFields(leg), status: 'drift', drifts: 1 }); + continue; + } + } + + const expectation = selectExpectation(spec, leg.version, versionMatchesRange); + if (!expectation) { + // In scope on this engine but nothing pins its behavior there. + coverageHoles.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reason: 'no version expectation matches this engine', + }); + matrix.push({ ruleId, ...legFields(leg), status: 'uncovered', drifts: 0 }); + continue; + } + + let ruleDrifts = 0; + let compared = 0; + // Triggers are counted separately from controls. A trigger is the rule's + // entire behavioral claim ("this query is flagged"); a control only says the + // rule stays quiet nearby. So a rule that lost every trigger but kept one + // control has proven nothing about itself, even though `compared` is + // non-zero — flat-object-subfield has 3 triggers and 1 control, and would + // otherwise render `agree` off the control alone. + let triggersExpected = 0; + let triggersCompared = 0; + // Cases this leg's surface cannot express. Counted separately from `unusable` + // because the two need opposite advice: not-applicable is expected and needs + // no action, unusable means something did not answer and needs a re-run. + let ruleNotApplicable = 0; + let ruleCoverageHoles = 0; + const unusable = []; + // Per-trigger engine verdicts for this rule on this leg, so a relaxation can + // be judged across the WHOLE rule rather than one query at a time. A single + // relaxed trigger cannot distinguish a full engine fix (scope the rule away) + // from a partial one (narrow the detector), and those actions are opposites — + // acting on the per-query view ships a false negative in the partial case. + // `unobserved` is kept apart from `holding` on purpose: a trigger that never + // answered must not be counted as "still rejects", or a timed-out leg would + // read as a partial fix and send someone to narrow a healthy detector. + const relaxedTriggers = []; + const holdingTriggers = []; + const unobservedTriggers = []; + let relaxedDetectorFlagged = false; + const perQueryDrifts = []; + for (const [queryName, expected] of Object.entries(expectation.queries || {})) { + const queryDef = (spec.queries || {})[queryName]; + if (!queryDef) { + // The contract references a query it does not define. The single-version + // halves fail on this, but skipping it silently here would shrink the + // compared set without saying so. + unusable.push(`${queryName} (not defined in the contract's queries map)`); + continue; + } + const query = queryDef.query.split('{{index}}').join(spec.index); + const role = queryDef.role || 'trigger'; + if (role === 'trigger') { + triggersExpected++; + } + + let oracleSelection; + try { + oracleSelection = resolveBackendOracle(spec, expected, leg.executionBackend); + } catch (error) { + artifactFatal(`${file} query "${queryName}"`, error); + } + const rowKey = `${ruleId}::${queryName}`; + const detectorResult = leg.detector.resultsByKey.get(rowKey); + const backendEntry = leg.backend.get(rowKey); + if ( + detectorResult && + !detectorResult.notApplicable && + detectorResult.outcome !== 'not-applicable' + ) { + if (detectorResult.expected !== oracleSelection.detector.count) { + fatal( + `detector report row ${rowKey} expected=${JSON.stringify(detectorResult.expected)} ` + + `does not match contract detectorCount=${oracleSelection.detector.count}` + ); + } + if ((detectorResult.role || 'trigger') !== role) { + fatal( + `detector report row ${rowKey} role=${JSON.stringify(detectorResult.role)} ` + + `does not match contract role=${JSON.stringify(role)}` + ); + } + } + if (detectorResult && detectorResult.outcome === 'error') { + unusable.push(`${queryName} (${unusableObservationReason(detectorResult)})`); + if (role === 'trigger') { + unobservedTriggers.push(queryName); + } + continue; + } + + if (oracleSelection.status === 'coverage-missing') { + const { usable } = readBackendObservation(backendEntry, detectorResult); + if (!usable) { + unusable.push(`${queryName} (${unusableObservationReason(detectorResult)})`); + if (role === 'trigger') { + unobservedTriggers.push(queryName); + } + continue; + } + coverageHoles.push({ + ruleId, + queryName, + file, + ...legFields(leg), + enforced: isEnforced, + reason: oracleSelection.reason, + kind: 'backend-oracle', + }); + ruleCoverageHoles++; + continue; + } + if (oracleSelection.status === 'not-applicable') { + const backendState = backendEntry + ? classifyBackendReportRow(backendEntry) + : { status: 'error' }; + if (!detectorResult || backendState.status !== 'not-applicable') { + unusable.push( + `${queryName} (${ + !detectorResult + ? 'no detector result' + : 'backend did not report not-applicable' + })` + ); + continue; + } + notApplicable.push({ + ruleId, + queryName, + ...legFields(leg), + surface: leg.surface, + reason: oracleSelection.reason, + kind: 'backend-oracle', + }); + ruleNotApplicable++; + if (isEnforced) { + coverageHoles.push({ + ruleId, + queryName, + file, + ...legFields(leg), + enforced: true, + reason: + `default-error rule is not applicable on ${leg.executionBackend}: ` + + `${oracleSelection.reason}`, + kind: 'backend-oracle', + issue: oracleSelection.oracle.issue, + owner: oracleSelection.oracle.owner, + }); + ruleCoverageHoles++; + } + continue; + } + + // A case the surface cannot express at all (a `runtimeOnly` rule on a + // compiled-simplified leg) is excluded rather than compared. Its zero + // diagnostics are `lint_runner` deliberately skipping the rule, so + // comparing them against a non-zero expectation would classify a healthy + // rule as detector-silent and send someone to fix it. This is NOT the same + // as `inconclusive`: nothing went wrong, and there is nothing to re-run. + if (detectorResult && detectorResult.notApplicable) { + notApplicable.push({ + ruleId, + queryName, + ...legFields(leg), + surface: leg.surface, + reason: detectorResult.notApplicable, + }); + ruleNotApplicable++; + continue; + } + const failedFrontendAssertions = failedExtendedFrontendAssertions( + detectorResult, + oracleSelection.frontend + ); + if (failedFrontendAssertions.length > 0) { + addDrift( + { + ruleId, + version: leg.version, + driftVersion: leg.version, + queryName, + role, + query, + driftClass: 'frontend-contract-mismatch', + evidence: + `${ruleId} @ ${leg.version} [${queryName}]: frontend assertion(s) failed: ` + + failedFrontendAssertions.join(', '), + frontendAssertions: failedFrontendAssertions, + frontendMismatches: detectorResult.mismatches || [], + remediation: { + action: 'update-detector', + target: + spec.detectorPath || + `packages/osd-monaco/src/ppl/lint/rules/${ruleId.replace(/-/g, '_')}.ts`, + detail: + `Reproduce this contract query against the reported OSD commit and candidate ` + + `grammar. Restore the exact message/action/fix behavior, or update the contract ` + + `only after confirming an intentional product change.`, + }, + }, + leg, + { + enforced: isEnforced, + contractFile: file, + expectationRange: expectation.version, + expectationEngine: expectation.engine, + } + ); + ruleDrifts++; + compared++; + if (role === 'trigger') { + triggersCompared++; + } + continue; + } + const { observed, usable } = readBackendObservation(backendEntry, detectorResult); + if (!usable) { + // No comparable pair, so there is nothing to classify. Attempting it + // anyway would turn a dead leg into linter advice: a case with no engine + // verdict and no detector row looks exactly like "the detector went + // silent", and the report would tell the engineer to go fix a detector + // that is fine. Record it as not compared and move on. + unusable.push(`${queryName} (${unusableObservationReason(detectorResult)})`); + if (role === 'trigger') { + unobservedTriggers.push(queryName); + } + continue; + } + compared++; + const pairedDivergence = divergentCases.get(`${leg.key}::${rowKey}`); + if (role === 'trigger') { + triggersCompared++; + // Bucket this trigger by what the ENGINE did, but only where the contract + // pinned a rejection — a trigger pinned as accepted (an advisory rule like + // head-without-sort, whose queries are all valid PPL) never "relaxes", and + // counting it as relaxed would fabricate a full-fix verdict for a rule the + // engine was never rejecting in the first place. + const pinnedRejection = oracleSelection.oracle.kind === 'rejection'; + if ( + pinnedRejection && + (!pairedDivergence || leg.executionBackend === 'standard') + ) { + if (observed.backendRejected === false) { + relaxedTriggers.push(queryName); + if ((observed.detectorCount || 0) > 0) relaxedDetectorFlagged = true; + } else if (observed.backendRejected === true) { + holdingTriggers.push(queryName); + } + } + } + + const drift = + pairedDivergence && leg.executionBackend === 'analytics' + ? null + : classifyDrift({ + ruleId, + version: leg.version, + queryName, + role, + query, + expected: { + detectorCount: oracleSelection.detector.count, + severity: oracleSelection.detector.severity, + matchMessage: oracleSelection.detector.matchMessage, + backendKind: oracleSelection.oracle.kind, + }, + observed, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + parserRuleNames: leg.parserRuleNames, + requiredParserRules: spec.requiredParserRules, + expectedBackend: oracleSelection.oracle, + executionBackend: leg.executionBackend, + }); + + if (drift) { + // `expectationRange` is what the annotation anchors to: the version + // string identifies WHICH `expectations[]` entry produced this finding, + // so a `update-contract` annotation can land on that entry's line rather + // than at the top of the file. + // Buffered rather than pushed: a relaxation finding is only final once + // every trigger has been seen, because the rule-level rollup below + // replaces the per-query ones with a single full-vs-partial verdict. + perQueryDrifts.push({ + ...drift, + enforced: isEnforced, + contractFile: file, + expectationRange: expectation.version, + expectationEngine: expectation.engine, + }); + } + } + + // Every trigger has now been observed, so a relaxation can be judged for the + // rule as a whole. This supersedes the per-query `engine-relaxed` findings — + // they each said "scope this rule away from this version", which is the wrong + // action whenever another trigger still rejects. + const relaxationScope = + leg.executionBackend === 'standard' + ? classifyRelaxationScope({ + ruleId, + version: leg.version, + relaxedTriggers, + holdingTriggers, + unobservedTriggers, + detectorFlagged: relaxedDetectorFlagged, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + executionBackend: leg.executionBackend, + }) + : null; + const kept = relaxationScope + ? perQueryDrifts.filter((d) => d.supersededBy !== DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED) + : perQueryDrifts; + for (const drift of kept) { + addDrift(drift, leg); + ruleDrifts++; + } + if (relaxationScope) { + addDrift(relaxationScope, leg, { + enforced: isEnforced, + contractFile: file, + expectationRange: expectation.version, + expectationEngine: expectation.engine, + }); + ruleDrifts++; + } + // "agree" has to mean "we compared the rule's claim and it held". A rule + // whose every case lost its engine verdict (a timed-out leg) or its detector + // row (a runner that died mid-corpus) has proven nothing — and so has one + // that lost every TRIGGER while keeping a control, since the triggers are + // where the rule's behavior actually lives. Calling either agreement is the + // vacuous pass this check exists to prevent. + // A rule the surface cannot express at all is `n/a`, not `inconclusive`: + // nothing failed and there is nothing to re-run, so it must not fail the run. + // Checked BEFORE the inconclusive test, which would otherwise catch it + // (compared === 0) and demand a re-run that could never change the outcome. + if (unusable.length > 0) { + inconclusive.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reasons: unusable, + }); + matrix.push({ + ruleId, + ...legFields(leg), + status: 'inconclusive', + drifts: ruleDrifts, + }); + } else if (ruleCoverageHoles > 0) { + matrix.push({ + ruleId, + ...legFields(leg), + status: ruleDrifts > 0 ? 'drift' : 'uncovered', + drifts: ruleDrifts, + }); + } else if (compared === 0 && ruleNotApplicable > 0) { + matrix.push({ + ruleId, + ...legFields(leg), + status: 'not-applicable', + drifts: 0, + }); + } else if (compared === 0 || (triggersExpected > 0 && triggersCompared === 0)) { + inconclusive.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reasons: unusable, + }); + matrix.push({ + ruleId, + ...legFields(leg), + status: 'inconclusive', + drifts: ruleDrifts, + }); + } else { + matrix.push({ + ruleId, + ...legFields(leg), + status: ruleDrifts === 0 ? 'agree' : 'drift', + drifts: ruleDrifts, + }); + } + } + + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface === 'runtime-bundle' || contractSurface === 'both') { + for (const pair of backendPairs) { + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + const rowKey = `${ruleId}::${queryName}`; + const divergent = divergentCases.get(`${pair.analytics.key}::${rowKey}`); + if (!divergent || divergent.pair.key !== pair.key) continue; + + const query = queryDef.query.split('{{index}}').join(spec.index); + const drift = classifyExecutionBackendDivergence({ + ruleId, + version: pair.engineVersion, + queryName, + role: queryDef.role || 'trigger', + query, + standardObserved: divergent.standardObserved, + analyticsObserved: divergent.analyticsObserved, + standardLeg: pair.standard.label, + analyticsLeg: pair.analytics.label, + grammarHash: pair.grammarHash, + detectorPath: spec.detectorPath, + }); + if (!drift) continue; + + const expectation = selectExpectation(spec, pair.engineVersion, versionMatchesRange); + addDrift(drift, pair.analytics, { + enforced: isEnforced, + contractFile: file, + expectationRange: expectation && expectation.version, + expectationEngine: expectation && expectation.engine, + pairKey: pair.key, + standardLeg: pair.standard.label, + standardLegKey: pair.standard.key, + analyticsLeg: pair.analytics.label, + analyticsLegKey: pair.analytics.key, + }); + + for (const row of matrix) { + if ( + row.ruleId === ruleId && + (row.legKey === pair.standard.key || row.legKey === pair.analytics.key) + ) { + row.status = 'drift'; + row.drifts += 1; + } + } + } + } + } + } + + for (const collection of [drifts, coverageHoles, inconclusive, notApplicable, matrix]) { + for (const entry of collection) { + const contract = specs.get(entry.ruleId); + entry.channel = contract ? contractChannel(contract.spec) : 'lint'; + } + } + for (const drift of drifts) { + if (drift.channel !== 'syntax') continue; + drift.remediation = { + action: 'review-syntax-validation', + target: + 'OSD runtime_validation_core and the syntax contract expectation', + detail: + `Reproduce "${drift.ruleId}" with the candidate runtime grammar. Update the shared OSD ` + + `parser/listener core if UNKNOWN_COMMAND identity, suppression, or quick-fix behavior ` + + `regressed; update this contract only after confirming an intentional syntax UX change. ` + + `Do not change detector catalog appliesTo metadata for a syntax-channel failure.`, + }; + } + + const isObservedAnalyticsFinding = (entry) => + args.observeAnalytics && + (entry.executionBackend === 'analytics' || + (Array.isArray(entry.executionBackends) && + entry.executionBackends.includes('analytics'))) && + (entry.driftClass === DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE || + entry.driftClass === DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH || + entry.kind === 'backend-oracle'); + for (const drift of drifts) { + drift.blocking = + (!!drift.enforced || args.allRules) && !isObservedAnalyticsFinding(drift); + } + for (const hole of coverageHoles) { + hole.blocking = + (!!hole.enforced || args.allRules) && !isObservedAnalyticsFinding(hole); + } + const enforcedDrifts = drifts.filter((d) => d.blocking); + const enforcedHoles = coverageHoles.filter((h) => h.blocking); + // `--all-rules` makes drift, missing coverage, and inconclusive observations + // blocking for every active shipping rule in the manifest. + const enforcedInconclusive = inconclusive.filter((i) => i.enforced || args.allRules); + const blockingMissingContracts = missingContracts.filter((entry) => entry.blocking); + for (const row of matrix) { + row.key = reportItemKey(row, 'matrix'); + } + for (const hole of coverageHoles) { + hole.key = reportItemKey(hole, 'coverage-hole'); + } + for (const entry of inconclusive) { + entry.key = reportItemKey(entry, 'inconclusive'); + } + for (const entry of notApplicable) { + entry.key = reportItemKey(entry, 'not-applicable'); + } + + const report = { + schemaVersion: 2, + keyDimensions: ['leg', 'engineVersion', 'grammarSurface', 'executionBackend'], + legs: legs.map((l) => ({ + key: l.key, + label: l.label, + engineVersion: l.version, + grammarHash: l.grammarHash, + sqlSha: l.sqlSha, + surface: l.surface, + executionBackend: l.executionBackend, + })), + backendPairs: backendPairs.map((pair) => ({ + key: pair.key, + engineVersion: pair.engineVersion, + grammarHash: pair.grammarHash, + standardLegKey: pair.standard.key, + analyticsLegKey: pair.analytics.key, + })), + enforcedRules: [...enforcedRules].sort(), + missingContracts, + shippingCensus, + manifestDescription: manifest.description || '', + matrix, + drifts, + coverageHoles, + inconclusive, + notApplicable, + result: { + driftCount: drifts.length, + enforcedDriftCount: enforcedDrifts.length, + enforcedCoverageHoles: enforcedHoles.length, + observedAnalyticsFindings: + drifts.filter((d) => d.enforced && !d.blocking).length + + coverageHoles.filter((h) => h.enforced && !h.blocking).length, + missingContractCount: missingContracts.length, + blockingMissingContractCount: blockingMissingContracts.length, + blockingShippingCensusProblems: blockingShippingCensusProblems.length, + enforcedInconclusive: enforcedInconclusive.length, + // An inconclusive default-error rule fails too: "we could not check" must + // never render as "it is fine". + passed: + enforcedDrifts.length === 0 && + enforcedHoles.length === 0 && + blockingMissingContracts.length === 0 && + blockingShippingCensusProblems.length === 0 && + enforcedInconclusive.length === 0, + }, + }; + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + log(`wrote ${args.out}`); + + // Emitted BEFORE the summary on purpose. GitHub renders annotations at the top + // of the run page, which is where a developer looks first; without them the only + // thing above the summary is "Process completed with exit code 1" and the + // natural next click goes to raw logs instead of the remediation. When the + // contract is part of the PR's diff these also attach inline to the exact + // expectation that drifted. + emitAnnotations(report, { + contractsDir: args.contracts, + workspace: process.env.GITHUB_WORKSPACE, + }); + + const markdown = renderMarkdown(report, drifts, coverageHoles, legs, specs); + // eslint-disable-next-line no-console + console.log(markdown); + if (args.summary) { + try { + fs.appendFileSync(args.summary, markdown + '\n'); + } catch (error) { + log(`WARN: could not write summary to ${args.summary}: ${error.message}`); + } + } + + if (!report.result.passed) { + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-multiversion] FAIL: ${enforcedDrifts.length} drift(s), ` + + `${enforcedHoles.length} coverage hole(s), ${blockingMissingContracts.length} unvalidated ` + + `default-error rule(s), ${blockingShippingCensusProblems.length} shipping census problem(s), ` + + `and ${enforcedInconclusive.length} inconclusive rule/version pair(s).` + ); + process.exit(1); + } + log( + `PASS: every ${args.allRules ? 'active shipping' : 'default-error'} rule agrees with all ` + + `${legs.length} engine version(s)` + + (drifts.length > 0 ? ` (${drifts.length} non-enforced finding(s) reported)` : '') + + '.' + ); +} + +function expectedCompatibility(spec) { + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const scope = []; + if (appliesTo.engine) { + scope.push( + appliesTo.engine === 'calcite' + ? 'Calcite' + : appliesTo.engine.charAt(0).toUpperCase() + appliesTo.engine.slice(1) + ); + } + if (appliesTo.minVersion && appliesTo.maxVersion) { + scope.push(`>= ${appliesTo.minVersion}, <= ${appliesTo.maxVersion}`); + } else if (appliesTo.minVersion) { + scope.push(`>= ${appliesTo.minVersion}`); + } else if (appliesTo.maxVersion) { + scope.push(`<= ${appliesTo.maxVersion}`); + } else { + scope.push('all versions'); + } + return scope.join(', '); +} + +function actualCompatibility(row) { + if (!row) return 'not evaluated'; + if (row.status === 'agree') return 'compatible'; + if (row.status === 'out-of-scope') return 'expected n/a'; + if (row.status === 'drift') return row.drifts > 1 ? `**drift** (${row.drifts})` : '**drift**'; + if (row.status === 'uncovered' || row.status === 'inconclusive') { + return '**inconclusive**'; + } + if (row.status === 'not-applicable') return 'expected n/a'; + return `**${row.status}**`; +} + +/** Expected-vs-actual compatibility table followed by the detailed remediation report. */ +function renderMarkdown(report, drifts, coverageHoles, legs, specs) { + const lines = []; + lines.push('## PPL lint multi-version validation'); + lines.push(''); + // Every reason the run can be red belongs in the headline. Reporting only + // drifts and holes made a FAIL caused solely by inconclusive rules read as + // "0 drifts, 0 holes — FAIL", which looks like a reporting bug rather than the + // real cause. + const reasons = [ + `${report.result.enforcedDriftCount} enforced drift(s)`, + `${report.result.enforcedCoverageHoles} coverage hole(s)`, + ]; + if (report.result.observedAnalyticsFindings) { + reasons.push(`${report.result.observedAnalyticsFindings} analytics observation(s)`); + } + if (report.result.enforcedInconclusive) { + reasons.push(`${report.result.enforcedInconclusive} inconclusive`); + } + if (report.result.missingContractCount) { + reasons.push(`${report.result.missingContractCount} unvalidated rule(s)`); + } + if (report.result.blockingShippingCensusProblems) { + reasons.push( + `${report.result.blockingShippingCensusProblems} shipping census problem(s)` + ); + } + const compatibilityLegs = legs.filter( + (leg) => + leg.executionBackend === 'standard' && + (leg.surface || 'runtime-bundle') === 'runtime-bundle' + ); + lines.push( + `Standard runtime-bundle engines: ${ + compatibilityLegs + .map((leg) => + leg.label === leg.version + ? `\`${leg.version}\`` + : `\`${leg.label}\` → \`${leg.version}\`` + ) + .join(', ') || 'none' + } — **${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` + ); + lines.push(''); + + const columns = compatibilityLegs.map((leg) => ({ + key: leg.key, + heading: + leg.label === leg.version + ? `\`${leg.version}\` actual` + : `\`${leg.label}\` actual
(\`${leg.version}\`)`, + })); + const rules = [...specs.entries()] + .filter(([, entry]) => contractChannel(entry.spec) === 'lint') + .sort(([left], [right]) => left.localeCompare(right)); + lines.push( + `| Rule | Expected compatibility | ${columns.map((column) => column.heading).join(' | ')} |` + ); + lines.push(`| ---- | ---- | ${columns.map(() => '----').join(' | ')} |`); + for (const [ruleId, { spec }] of rules) { + const cells = columns.map((column) => { + const row = report.matrix.find((m) => m.ruleId === ruleId && m.legKey === column.key); + return actualCompatibility(row); + }); + lines.push( + `| \`${ruleId}\` | ${expectedCompatibility(spec)} | ${cells.join(' | ')} |` + ); + } + lines.push(''); + + if ((report.inconclusive || []).length > 0) { + lines.push('### Inconclusive (leg problem, not a linter problem)'); + lines.push(''); + for (const entry of report.inconclusive) { + lines.push( + `- \`${entry.ruleId}\` on engine \`${entry.version}\` (${entry.executionBackend}): ` + + `no case could be compared — ` + + `${entry.reasons.join('; ')}. This is NOT a lint finding: the engine or the detector run ` + + `did not answer, so nothing was validated. Check that leg's job logs (an unreachable ` + + `cluster, an index that failed to seed, or a detector runner that died mid-corpus) and ` + + `re-run. Do not edit the rule or the contract on the strength of this.` + ); + } + lines.push(''); + } + + if ((report.missingContracts || []).length > 0) { + lines.push('### Unvalidated default-error rules'); + lines.push(''); + for (const entry of report.missingContracts) { + lines.push( + `- \`${entry.ruleId}\` ships enabled at error severity but ${entry.reason}, so no engine ` + + `version validates it. FIX: add \`${entry.ruleId}.spec.json\` under ` + + `integ-test/src/test/resources/ppl-lint/contracts/ with a trigger + control query and list ` + + `it in manifest.json under \`defaultError\`. If the rule should not be default-error, lower ` + + `its severity or disable it in packages/osd-monaco/src/ppl/lint/rules_catalog.json.` + ); + } + lines.push(''); + } + + if (report.shippingCensus && !report.shippingCensus.passed) { + lines.push('### Shipping census'); + lines.push(''); + for (const problem of report.shippingCensus.problems || []) { + lines.push( + `- ${report.shippingCensus.blocking ? '**ENFORCED:**' : '**REPORT ONLY:**'} ${problem}. ` + + `Align \`manifest.json\` with the approved OSD shipping catalog.` + ); + } + lines.push(''); + } + + if (coverageHoles.length > 0) { + lines.push('### Coverage holes'); + lines.push(''); + for (const hole of coverageHoles) { + const query = hole.queryName ? ` query \`${hole.queryName}\`` : ''; + const fix = + hole.kind === 'backend-oracle' + ? `add a reviewed \`${hole.executionBackend}\` backend oracle for this query` + : `add an \`expectations[]\` entry whose \`version\` range covers \`${hole.version}\`, ` + + `or narrow the rule's \`appliesTo\` so it does not apply there`; + lines.push( + `- \`${hole.ruleId}\`${query} has no ${hole.executionBackend} coverage for engine ` + + `\`${hole.version}\`` + + `${hole.enforced ? ' (ENFORCED — this rule ships to users on that engine unpinned)' : ''}. ` + + `${hole.reason ? `${hole.reason}. ` : ''}` + + `FIX (${hole.file}): ${fix}.` + ); + } + lines.push(''); + } + + lines.push('### Remediation'); + lines.push(''); + lines.push('```'); + lines.push(formatDriftReport(drifts)); + lines.push('```'); + return lines.join('\n'); +} + +main(); diff --git a/scripts/ppl-lint/annotate.mjs b/scripts/ppl-lint/annotate.mjs new file mode 100644 index 00000000000..6cb2e031a9c --- /dev/null +++ b/scripts/ppl-lint/annotate.mjs @@ -0,0 +1,393 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * GitHub Actions annotations for the PPL lint required and multi-version checks. + * + * The drift report and the job summary already say exactly what to change. The + * problem is WHERE a developer looks first: GitHub renders workflow-command + * annotations at the top of the run page and, when a finding names a file in the + * pull request, inline on that file in the Files-changed view. Without them the + * only thing above the summary is "Process completed with exit code 1", so the + * natural next click leads into raw job logs instead of the remediation. + * + * This module turns findings into `::error file=…,line=…::` commands. Two rules + * govern everything here: + * + * 1. An annotation must point at a line the reader can act on, or carry no line + * at all. A confidently wrong line number sends someone to edit the wrong + * expectation, which is worse than making them find it themselves. + * 2. The annotation is a POINTER, not the report. It carries the finding and the + * one-line action; the summary keeps the full reasoning. Annotation text is + * truncated by the UI, so front-load the identity of the problem. + * + * Inconclusive findings are deliberately `::warning`, not `::error`: they mean + * "we could not check", and the run is already red from the exit code. Rendering + * them as errors next to real drift would invite exactly the response the + * classifier works to prevent — editing a rule because a leg timed out. + */ + +import fs from 'fs'; +import path from 'path'; + +/** Escape a workflow-command property value (file/title). */ +function escapeProperty(value) { + return String(value) + .replace(/%/g, '%25') + .replace(/\r/g, '%0D') + .replace(/\n/g, '%0A') + .replace(/:/g, '%3A') + .replace(/,/g, '%2C'); +} + +/** Escape a workflow-command message body; newlines must survive as %0A. */ +function escapeData(value) { + return String(value).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +} + +function backendLabel(entry) { + return Array.isArray(entry.executionBackends) && entry.executionBackends.length > 1 + ? entry.executionBackends.join(' vs ') + : entry.executionBackend || 'standard'; +} + +/** + * Line of the `expectations[]` entry whose `version` is `range`, 1-indexed. + * + * Deliberately a text scan rather than a JSON walk: JSON.parse discards line + * information, and every consumer of this number is a human reading the file in a + * browser. Returns undefined when the range is absent or ambiguous (appears more + * than once), because an annotation with no line still lands on the file while a + * wrong line actively misleads. + */ +export function findExpectationLine(contractText, range) { + if (!contractText || !range) return undefined; + const lines = contractText.split('\n'); + const needle = `"version": ${JSON.stringify(range)}`; + const hits = []; + for (let i = 0; i < lines.length; i++) { + // Match on the normalized form so incidental whitespace does not defeat it. + if (lines[i].replace(/\s+/g, ' ').includes(needle)) hits.push(i + 1); + } + return hits.length === 1 ? hits[0] : undefined; +} + +/** + * Line of the top-level `"ruleId"` key, used as the fallback anchor when the + * finding is about the rule as a whole (a renamed grammar rule) rather than one + * pinned expectation. + */ +export function findRuleIdLine(contractText) { + if (!contractText) return undefined; + const lines = contractText.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (/^\s*"ruleId"\s*:/.test(lines[i])) return i + 1; + } + return undefined; +} + +function findJsonKeyLine(jsonText, key) { + if (!jsonText || !key) return undefined; + const lines = jsonText.split('\n'); + const pattern = new RegExp(`^\\s*${JSON.stringify(key)}\\s*:`); + for (let i = 0; i < lines.length; i++) { + if (pattern.test(lines[i])) return i + 1; + } + return undefined; +} + +/** + * Repo-relative path of a contract file, for `file=`. + * + * GitHub only renders an annotation inline when the path is relative to the + * repository root AND the file is in the pull request's diff. `contractsDir` is + * an absolute path inside the workspace, so strip the workspace prefix. + */ +export function contractRepoPath(contractsDir, fileName, workspace) { + const absolute = path.join(contractsDir, fileName); + if (workspace && absolute.startsWith(workspace)) { + return path.relative(workspace, absolute); + } + return absolute; +} + +/** + * Build the annotation list for a drift report. Pure: returns descriptors so the + * caller decides where they are written and the tests can assert on them without + * capturing stdout. + */ +export function buildAnnotations(report, { contractsDir, workspace, readFile = readContract } = {}) { + const annotations = []; + const textCache = new Map(); + const contractText = (fileName) => { + if (!fileName) return undefined; + if (!textCache.has(fileName)) { + textCache.set(fileName, readFile(contractsDir, fileName)); + } + return textCache.get(fileName); + }; + + for (const drift of report.drifts || []) { + const file = drift.contractFile; + const text = contractText(file); + // `update-contract` findings are about one pinned expectation, so anchor + // there. Everything else is about the rule, so anchor at its identity line — + // the reader's next stop is the detector named in the message anyway. + const line = + (drift.expectationRange ? findExpectationLine(text, drift.expectationRange) : undefined) ?? + findRuleIdLine(text); + + annotations.push({ + level: (drift.blocking ?? drift.enforced) ? 'error' : 'warning', + file: file ? contractRepoPath(contractsDir, file, workspace) : undefined, + line, + title: + `PPL lint drift: ${drift.driftClass} ` + + `(${drift.ruleId} @ ${drift.version}, ${backendLabel(drift)})`, + // Message order matters: the UI truncates, so lead with what moved, then the + // action, then where. The summary carries the full rationale. + message: [ + drift.evidence, + `FIX (${drift.remediation.action}): ${drift.remediation.detail}`, + drift.query ? `QUERY: ${drift.query}` : undefined, + ] + .filter(Boolean) + .join('\n'), + }); + } + + for (const hole of report.coverageHoles || []) { + const text = contractText(hole.file); + annotations.push({ + level: (hole.blocking ?? hole.enforced) ? 'error' : 'warning', + file: hole.file ? contractRepoPath(contractsDir, hole.file, workspace) : undefined, + line: findRuleIdLine(text), + title: + `PPL lint coverage hole: ${hole.ruleId} @ ${hole.version}, ${backendLabel(hole)}`, + message: + (hole.reason + ? `${hole.reason}. ` + : `No expectation in this contract matches engine ${hole.version}. `) + + `Nothing pins "${hole.ruleId}" for the ${backendLabel(hole)} route there. Add a reviewed ` + + `${backendLabel(hole)} oracle or expectation; never use another route's oracle as fallback.`, + }); + } + + // Warning, not error: the linter is not what went wrong, and the run is already + // red from the exit code. See the module comment. + for (const entry of report.inconclusive || []) { + const text = contractText(entry.file); + annotations.push({ + level: 'warning', + file: entry.file ? contractRepoPath(contractsDir, entry.file, workspace) : undefined, + line: findRuleIdLine(text), + title: + `PPL lint inconclusive: ${entry.ruleId} @ ${entry.version}, ` + + `${backendLabel(entry)} (leg problem)`, + message: + `No case could be compared for "${entry.ruleId}" on engine ${entry.version} ` + + `(${backendLabel(entry)})` + + (entry.reasons && entry.reasons.length > 0 ? ` — ${entry.reasons.join('; ')}` : '') + + `. This is NOT a lint finding: the engine or the detector run did not answer, so ` + + `nothing was validated. Check that leg's job logs and re-run. Do not edit the rule or ` + + `the contract on the strength of this.`, + }); + } + + for (const missing of report.missingContracts || []) { + const ruleId = missing.ruleId || missing; + annotations.push({ + level: missing.blocking === false ? 'warning' : 'error', + // A rule with no contract has no file to point at; the manifest is where the + // reader's edit goes. + file: undefined, + title: `PPL lint unvalidated rule: ${ruleId}`, + message: + `"${ruleId}" ships enabled at error severity in OSD's rules_catalog.json but ` + + `${missing.reason || 'has no contract in this corpus'}. A default-error rule with no ` + + `contract is invisible to this check. Add a contract file and list it under ` + + `manifest.defaultError, or lower the rule's severity in OSD.` + + (missing.blocking === false + ? ' This compatibility phase reports the census mismatch without blocking until the paired OSD default-alignment change lands.' + : ''), + }); + } + + const shippingCensus = report.shippingCensus; + if (shippingCensus && shippingCensus.passed === false) { + const manifestText = contractText('manifest.json'); + for (const problem of shippingCensus.problems || []) { + const blocking = shippingCensus.blocking !== false; + annotations.push({ + level: blocking ? 'error' : 'warning', + file: contractRepoPath(contractsDir, 'manifest.json', workspace), + line: findJsonKeyLine(manifestText, manifestKeyFor(problem)), + title: 'PPL lint shipping census mismatch', + message: + `${problem}\n` + + (blocking + ? 'FIX: align the active SQL manifest with the approved OSD shipping catalog.' + : 'REPORT ONLY: align the active SQL manifest with the approved OSD shipping catalog before enabling census enforcement.'), + }); + } + } + + return annotations; +} + +function ruleIdentity(message) { + const bracketed = String(message).match(/^\[([A-Za-z0-9._-]+)(?:\/([A-Za-z0-9._-]+))?\]/); + if (bracketed && !['census', 'contracts', 'grammar-export', 'report'].includes(bracketed[1])) { + return { ruleId: bracketed[1], queryName: bracketed[2] }; + } + const row = String(message).match( + /\b(?:backend|detector) row ([A-Za-z0-9._-]+)::([A-Za-z0-9._-]+)\b/ + ); + return row ? { ruleId: row[1], queryName: row[2] } : {}; +} + +function loadContractFiles(contractsDir, readFile) { + const files = new Map(); + const manifestText = readFile(contractsDir, 'manifest.json'); + if (!manifestText) return { files, manifestText }; + try { + const manifest = JSON.parse(manifestText); + const names = [...(manifest.contracts || []), ...(manifest.dormantContracts || [])]; + for (const name of names) { + const text = readFile(contractsDir, name); + if (!text) continue; + try { + const spec = JSON.parse(text); + if (spec.ruleId) files.set(spec.ruleId, { name, text }); + } catch { + // Malformed contracts are reported by the schema/runner. Keep this helper + // best-effort so annotation generation never hides the original failure. + } + } + } catch { + // The manifest parse failure is itself annotated below without a line anchor. + } + return { files, manifestText }; +} + +function manifestKeyFor(message) { + if (/defaultError/.test(message)) return 'defaultError'; + if (/requiredSyntaxFeatures/.test(message)) return 'requiredSyntaxFeatures'; + if (/dormantContracts/.test(message)) return 'dormantContracts'; + return 'contracts'; +} + +/** + * Build annotations for the required single-version lane. + * + * Detector failures use their `[rule/query]` prefix to land on the owning + * contract. Census findings land on manifest.json. Job and artifact failures + * without a trustworthy repository location remain file-less. + */ +export function buildRequiredAnnotations( + report, + { contractsDir, workspace, readFile = readContract } = {} +) { + const annotations = []; + const { files, manifestText } = loadContractFiles(contractsDir, readFile); + const seen = new Set(); + + const addFailure = (message, source) => { + const text = String(message); + const { ruleId, queryName } = ruleIdentity(text); + const contract = ruleId ? files.get(ruleId) : undefined; + const census = source === 'census' || /^\[census\]/.test(text); + const file = census + ? contractRepoPath(contractsDir, 'manifest.json', workspace) + : contract + ? contractRepoPath(contractsDir, contract.name, workspace) + : undefined; + const line = census + ? findJsonKeyLine(manifestText, manifestKeyFor(text)) + : findRuleIdLine(contract?.text); + const level = census && report.censusEnforced !== true ? 'warning' : 'error'; + const title = census + ? 'PPL lint shipping census mismatch' + : ruleId + ? `PPL lint required validation: ${ruleId}${queryName ? `/${queryName}` : ''}` + : `PPL lint required validation: ${source}`; + const key = `${level}\0${file || ''}\0${line || ''}\0${title}\0${text}`; + if (seen.has(key)) return; + seen.add(key); + annotations.push({ + level, + file, + line, + title, + message: + census && report.censusEnforced !== true + ? `${text}\nREPORT ONLY: align the active SQL manifest with the approved OSD shipping catalog before enabling census enforcement.` + : text, + }); + }; + + for (const message of report.detectorFailures || []) addFailure(message, 'frontend'); + for (const message of report.artifactErrors || []) addFailure(message, 'artifact'); + for (const message of report.censusProblems || []) addFailure(message, 'census'); + + for (const [job, result] of [ + ['backend-validation', report.backendResult], + ['detector-validation', report.detectorResult], + ]) { + if (result && result !== 'success') { + addFailure( + `${job} finished with result "${result}". See that job's logs and uploaded artifacts for the underlying failure.`, + job + ); + } + } + return annotations; +} + +function readContract(contractsDir, fileName) { + try { + return fs.readFileSync(path.join(contractsDir, fileName), 'utf8'); + } catch { + // A contract we cannot read still deserves a file-less annotation. + return undefined; + } +} + +/** Render one descriptor as a workflow command line. */ +export function formatAnnotation(annotation) { + const props = []; + if (annotation.file) props.push(`file=${escapeProperty(annotation.file)}`); + if (annotation.line) props.push(`line=${annotation.line}`); + if (annotation.title) props.push(`title=${escapeProperty(annotation.title)}`); + const suffix = props.length > 0 ? ` ${props.join(',')}` : ''; + return `::${annotation.level}${suffix}::${escapeData(annotation.message)}`; +} + +/** + * Emit annotations for a report. No-op unless running under Actions (or forced), + * so a local run is not spammed with workflow-command noise. + */ +export function emitAnnotations(report, options = {}) { + const enabled = options.force || process.env.GITHUB_ACTIONS === 'true'; + if (!enabled) return []; + const annotations = buildAnnotations(report, options); + for (const annotation of annotations) { + // eslint-disable-next-line no-console + console.log(formatAnnotation(annotation)); + } + return annotations; +} + +/** Emit required-lane annotations under the same Actions-only policy. */ +export function emitRequiredAnnotations(report, options = {}) { + const enabled = options.force || process.env.GITHUB_ACTIONS === 'true'; + if (!enabled) return []; + const annotations = buildRequiredAnnotations(report, options); + for (const annotation of annotations) { + // eslint-disable-next-line no-console + console.log(formatAnnotation(annotation)); + } + return annotations; +} diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs new file mode 100644 index 00000000000..7047ffad28f --- /dev/null +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -0,0 +1,390 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Assemble the PPL lint validation run manifest and the compact per-rule PR + * summary in the result job (design §3.3, §4.4, T10). + * + * Inputs (env; identity fields may be empty on a partial run): + * SQL_SHA, OSD_REF, OSD_SHA, EVENT_NAME, SCHEDULE, + * BACKEND_RESULT, DETECTOR_RESULT, GITHUB_STEP_SUMMARY. + * Artifact files under ./artifacts (downloaded from both jobs): + * target.json (engineVersion + grammarHash), backend-report.json, + * detector-report.json. + * + * Outputs: + * run-manifest.json in the workspace root; a markdown table appended to + * $GITHUB_STEP_SUMMARY. + */ + +import fs from 'fs'; +import path from 'path'; + +import { + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, +} from './contract-schema.mjs'; +import { emitRequiredAnnotations } from './annotate.mjs'; + +const ARTIFACTS = 'artifacts'; +const CONTRACTS = path.resolve('integ-test/src/test/resources/ppl-lint/contracts'); + +function readJson(file, errors) { + try { + if (fs.existsSync(file)) { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } + errors.push(`required artifact is missing: ${file}`); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-manifest] could not parse ${file}: ${error.message}`); + errors.push(`required artifact is malformed: ${file}: ${error.message}`); + } + return undefined; +} + +function failedFrontendAssertions(entry) { + const failures = new Set(); + for (const [field, label] of [ + ['severityMatched', 'severity'], + ['messageMatched', 'message'], + ]) { + if (entry[field] !== true) { + failures.add(label); + } + } + for (const [field, label] of [ + ['deterministicFixMatched', 'deterministic-fix'], + ['fixMatched', 'syntax-fix'], + ['rawMessageMatched', 'raw-parser-error'], + ['totalErrorsMatched', 'total-error'], + ]) { + if (entry[field] === false) { + failures.add(label); + } + } + if ( + entry.assertions && + typeof entry.assertions === 'object' && + !Array.isArray(entry.assertions) + ) { + for (const [field, matched] of Object.entries(entry.assertions)) { + if (matched === false) { + failures.add(field); + } + } + } + for (const mismatch of Array.isArray(entry.mismatches) ? entry.mismatches : []) { + if (mismatch && typeof mismatch.field === 'string') { + failures.add(mismatch.field); + } + } + return [...failures].sort(); +} + +function main() { + const artifactErrors = []; + const targetRaw = readJson(path.join(ARTIFACTS, 'target.json'), artifactErrors) || {}; + const detector = + readJson(path.join(ARTIFACTS, 'detector-report.json'), artifactErrors) || {}; + const backend = + readJson(path.join(ARTIFACTS, 'backend-report.json'), artifactErrors) || []; + + let target = {}; + try { + target = normalizeTarget(targetRaw); + } catch (error) { + artifactErrors.push(`invalid target.json: ${error.message}`); + } + const executionBackend = target.executionBackend || ''; + if (target.schemaVersion !== 2 || target.legacy) { + artifactErrors.push( + `required workflow target schemaVersion must be 2, got ${JSON.stringify(target.schemaVersion)}` + ); + } + if (executionBackend !== 'standard') { + artifactErrors.push( + `required workflow target executionBackend must be "standard", got ${JSON.stringify(executionBackend)}` + ); + } + if (!target.sqlSha) { + artifactErrors.push('target sqlSha must be non-empty'); + } else if (process.env.SQL_SHA && target.sqlSha !== process.env.SQL_SHA) { + artifactErrors.push( + `target sqlSha ${JSON.stringify(target.sqlSha)} does not match workflow SQL_SHA ${JSON.stringify(process.env.SQL_SHA)}` + ); + } + if (detector.schemaVersion !== 2) { + artifactErrors.push( + `detector schemaVersion must be 2, got ${JSON.stringify(detector.schemaVersion)}` + ); + } + if (detector.executionBackend !== executionBackend) { + artifactErrors.push( + `detector executionBackend ${JSON.stringify(detector.executionBackend)} does not match target ${JSON.stringify(executionBackend)}` + ); + } + for (const field of ['engineVersion', 'grammarHash']) { + if (detector[field] !== target[field]) { + artifactErrors.push( + `detector ${field} ${JSON.stringify(detector[field])} does not match target ${JSON.stringify(target[field])}` + ); + } + } + if (!['runtime-bundle', 'compiled-simplified'].includes(detector.surface)) { + artifactErrors.push( + `detector surface must be "runtime-bundle" or "compiled-simplified", got ` + + `${JSON.stringify(detector.surface)}` + ); + } + if (!Array.isArray(detector.defaultErrorRules)) { + artifactErrors.push('detector defaultErrorRules must be an array'); + } + + let backendByKey = new Map(); + try { + backendByKey = indexBackendReport(backend, target); + } catch (error) { + artifactErrors.push(`invalid backend-report.json: ${error.message}`); + } + if (backendByKey.size === 0) { + artifactErrors.push('backend-report.json must be a non-empty array'); + } + if (!Array.isArray(detector.results) || detector.results.length === 0) { + artifactErrors.push('detector-report.json must contain a non-empty results array'); + } + const detectorKeys = new Set(); + const reportOnlyDetectorKeys = new Set(); + for (const entry of Array.isArray(detector.results) ? detector.results : []) { + const key = `${entry.ruleId}::${entry.queryName}`; + if (!entry.ruleId || !entry.queryName) { + artifactErrors.push(`detector-report.json contains an invalid row ${JSON.stringify(entry)}`); + continue; + } + if (detectorKeys.has(key)) { + artifactErrors.push(`detector-report.json contains duplicate row ${key}`); + } + detectorKeys.add(key); + if (entry.reportOnly === true) { + reportOnlyDetectorKeys.add(key); + continue; + } + if (entry.executionBackend !== executionBackend) { + artifactErrors.push( + `detector row ${key} executionBackend ${JSON.stringify(entry.executionBackend)} does not match target ${JSON.stringify(executionBackend)}` + ); + } + if (!backendByKey.has(key)) { + artifactErrors.push(`detector row ${key} has no matching backend row`); + } + if (entry.outcome === 'error') { + const message = + typeof entry.error === 'string' && entry.error.length > 0 + ? entry.error + : 'unknown frontend execution error'; + artifactErrors.push(`detector row ${key} execution failed: ${message}`); + continue; + } + if (!Number.isInteger(entry.expected) || !Number.isInteger(entry.actual)) { + artifactErrors.push(`detector row ${key} must contain integer expected/actual counts`); + } else if (entry.actual !== entry.expected) { + artifactErrors.push( + `detector row ${key} count mismatch: expected ${entry.expected}, got ${entry.actual}` + ); + } + if ( + entry.assertions !== undefined && + (!entry.assertions || + typeof entry.assertions !== 'object' || + Array.isArray(entry.assertions) || + Object.values(entry.assertions).some((matched) => typeof matched !== 'boolean')) + ) { + artifactErrors.push(`detector row ${key} assertions must contain only booleans`); + } + if (entry.mismatches !== undefined && !Array.isArray(entry.mismatches)) { + artifactErrors.push(`detector row ${key} mismatches must be an array`); + } + for (const assertion of failedFrontendAssertions(entry)) { + artifactErrors.push( + `detector row ${key} did not match its ${assertion} assertion` + ); + } + } + for (const [key, entry] of backendByKey) { + if (reportOnlyDetectorKeys.has(key)) { + continue; + } + if (!detectorKeys.has(key)) { + artifactErrors.push(`backend row ${key} has no matching detector row`); + } + const state = classifyBackendReportRow(entry); + if (state.status !== 'observed' || entry.outcome !== 'pass') { + artifactErrors.push( + `backend row ${key} did not pass its oracle (outcome=${JSON.stringify(entry.outcome)})` + ); + } + } + + const eventName = process.env.EVENT_NAME || ''; + const osdRef = process.env.OSD_REF || 'main'; + const osdRepo = process.env.OSD_REPO || 'opensearch-project/OpenSearch-Dashboards'; + const isUpstreamMain = osdRepo === 'opensearch-project/OpenSearch-Dashboards' && osdRef === 'main'; + const mode = + eventName === 'pull_request' + ? 'sql-pr-validation' + : eventName === 'schedule' + ? 'nightly' + : !isUpstreamMain + ? 'osd-branch-evidence' + : 'manual'; + + const backendResult = process.env.BACKEND_RESULT || 'unknown'; + const detectorResult = process.env.DETECTOR_RESULT || 'unknown'; + const passed = + backendResult === 'success' && detectorResult === 'success' && artifactErrors.length === 0; + + // The selected validation set is the set of rules the detector run actually + // evaluated (post schedule filtering). + const validationSet = Array.from( + new Set( + (detector.results || []) + .filter((row) => row.reportOnly !== true) + .map((row) => row.ruleId) + ) + ).sort(); + + const manifest = { + schemaVersion: 2, + mode, + // A workflow_dispatch osd_ref run is pre-merge evidence, never a + // branch-protection result (design §4.1.1, T11). + requiredCheck: eventName === 'pull_request', + event: eventName, + schedule: process.env.SCHEDULE || detector.schedule || 'pr', + sqlSha: process.env.SQL_SHA || '', + osdRepo, + osdRef, + osdSha: process.env.OSD_SHA || '', + engineVersion: target.engineVersion || detector.engineVersion || '', + grammarHash: target.grammarHash || detector.grammarHash || '', + executionBackend, + differential: !!detector.differential, + validationSet, + result: { + backend: backendResult, + detector: detectorResult, + artifactErrors, + passed, + }, + }; + + fs.writeFileSync('run-manifest.json', JSON.stringify(manifest, null, 2)); + + writeSummary(manifest, detector, backend); + + emitRequiredAnnotations( + { + artifactErrors, + detectorFailures: Array.isArray(detector.failures) ? detector.failures : [], + censusProblems: Array.isArray(detector.census?.problems) ? detector.census.problems : [], + censusEnforced: detector.census?.enforced === true, + backendResult, + detectorResult, + }, + { + contractsDir: CONTRACTS, + workspace: process.env.GITHUB_WORKSPACE || process.cwd(), + } + ); + + if (artifactErrors.length > 0) { + throw new Error(`invalid PPL lint artifacts:\n- ${artifactErrors.join('\n- ')}`); + } +} + +/** Compact per-rule PR summary: Rule | Version | Grammar | Detector | Backend | Result. */ +function writeSummary(manifest, detector, backend) { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (!summaryPath) { + return; + } + + const backendByKey = new Map(); + for (const e of Array.isArray(backend) ? backend : []) { + backendByKey.set(`${e.ruleId}::${e.queryName}`, e); + } + + const shortHash = (h) => (h ? String(h).replace(/^sha256:/, '').slice(0, 12) : '—'); + + const lines = []; + lines.push('## PPL lint rule validation'); + lines.push(''); + lines.push(`- Mode: \`${manifest.mode}\`${manifest.requiredCheck ? ' (required)' : ' (non-enforcing)'}`); + lines.push(`- SQL: \`${manifest.sqlSha || '—'}\``); + lines.push(`- OSD: \`${manifest.osdSha || '—'}\` (${manifest.osdRepo} @ \`${manifest.osdRef}\`)`); + lines.push(`- Backend version: \`${manifest.engineVersion || '—'}\``); + lines.push(`- Execution backend: \`${manifest.executionBackend || '—'}\``); + lines.push(`- Grammar: \`${shortHash(manifest.grammarHash)}\``); + lines.push( + `- Result: backend **${manifest.result.backend}**, detector **${manifest.result.detector}** → ` + + `**${manifest.result.passed ? 'PASS' : 'FAIL'}**` + ); + lines.push(''); + lines.push('| Rule | Query | Version | Grammar | Detector | Backend | Result |'); + lines.push('| ---- | ----- | ------- | ------- | -------- | ------- | ------ |'); + + for (const r of detector.results || []) { + const be = backendByKey.get(`${r.ruleId}::${r.queryName}`); + const detectorCell = + r.outcome === 'error' + ? 'Error' + : `${r.actual}/${r.expected}${ + r.severities && r.severities.length ? ` (${r.severities.join(',')})` : '' + }`; + const backendCell = !be + ? '—' + : typeof be.rejected !== 'boolean' + ? be.outcome || 'no verdict' + : be.rejected + ? `HTTP ${be.observed ? be.observed.httpStatus : '4xx'}` + : 'accepted'; + const ok = + r.reportOnly === true + ? undefined + : r.outcome !== 'error' && + r.actual === r.expected && + failedFrontendAssertions(r).length === 0 && + !!be && + be.outcome === 'pass'; + lines.push( + `| \`${r.ruleId}\` | \`${r.queryName}\` | \`${manifest.engineVersion || '—'}\` | ` + + `\`${shortHash(manifest.grammarHash)}\` | ${detectorCell} | ${backendCell} | ${ + ok === undefined ? 'Report only' : ok ? 'Pass' : 'Fail' + } |` + ); + } + + if ((detector.failures || []).length > 0) { + lines.push(''); + lines.push('
Failures'); + lines.push(''); + for (const f of detector.failures) { + lines.push(`- ${f}`); + } + lines.push(''); + lines.push('
'); + } + + lines.push(''); + try { + fs.appendFileSync(summaryPath, lines.join('\n') + '\n'); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-manifest] could not write step summary: ${error.message}`); + } +} + +main(); diff --git a/scripts/ppl-lint/contract-schema.mjs b/scripts/ppl-lint/contract-schema.mjs new file mode 100644 index 00000000000..3b8e4bf68f9 --- /dev/null +++ b/scripts/ppl-lint/contract-schema.mjs @@ -0,0 +1,745 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +const EXECUTION_BACKENDS = new Set(['standard', 'analytics']); +const CONTRACT_SCHEMA_VERSIONS = new Set([3, 4]); +const APPLICABLE_BACKEND_KINDS = new Set(['rejection', 'result-shape', 'advisory']); +const CONTRACT_CHANNELS = new Set(['lint', 'syntax']); +const QUERY_ROLES = new Set(['trigger', 'control', 'suppression-control']); +const LINT_V3_FRONTEND_FIELDS = new Set(['count', 'severity', 'matchMessage']); +const LINT_V4_FRONTEND_FIELDS = new Set([ + 'count', + 'severity', + 'messageEquals', + 'deterministicFix', +]); +const SYNTAX_FRONTEND_FIELDS = new Set([ + 'count', + 'code', + 'fixText', + 'matchMessage', + 'rawMessage', + 'totalErrors', +]); + +function isObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function describe(value) { + return typeof value === 'string' ? `"${value}"` : JSON.stringify(value); +} + +function requireObject(value, label) { + if (!isObject(value)) { + throw new TypeError(`${label} must be a JSON object.`); + } + return value; +} + +function requireNonEmptyString(value, label) { + if (typeof value !== 'string' || value.length === 0) { + throw new TypeError(`${label} must be a non-empty string.`); + } + return value; +} + +function requireString(value, label) { + if (typeof value !== 'string') { + throw new TypeError(`${label} must be a string.`); + } + return value; +} + +function requireNonNegativeInteger(value, label) { + if (!Number.isInteger(value) || value < 0) { + throw new TypeError(`${label} must be a non-negative integer.`); + } + return value; +} + +function assertOptionalString(value, label) { + if (value !== undefined && (typeof value !== 'string' || value.length === 0)) { + throw new TypeError(`${label} must be a non-empty string when present.`); + } +} + +function assertKnownKeys(value, allowed, label) { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) { + throw new Error(`${label}.${key} is not valid for this contract channel.`); + } + } +} + +function normalizeRange(value, label) { + const range = requireObject(value, label); + assertKnownKeys( + range, + new Set(['startLine', 'startColumn', 'endLine', 'endColumn']), + label + ); + for (const field of ['startLine', 'endLine']) { + if (!Number.isInteger(range[field]) || range[field] < 1) { + throw new TypeError(`${label}.${field} must be a positive integer.`); + } + } + for (const field of ['startColumn', 'endColumn']) { + requireNonNegativeInteger(range[field], `${label}.${field}`); + } + if ( + range.endLine < range.startLine || + (range.endLine === range.startLine && range.endColumn < range.startColumn) + ) { + throw new Error(`${label} must end at or after its start.`); + } + return { + startLine: range.startLine, + startColumn: range.startColumn, + endLine: range.endLine, + endColumn: range.endColumn, + }; +} + +function normalizeDeterministicFix(value, label) { + const fix = requireObject(value, label); + if (typeof fix.offered !== 'boolean') { + throw new TypeError(`${label}.offered must be a boolean.`); + } + const allowed = new Set([ + 'offered', + 'title', + 'text', + 'range', + 'expectedText', + 'appliedQuery', + ]); + assertKnownKeys(fix, allowed, label); + if (!fix.offered) { + if (Object.keys(fix).length !== 1) { + throw new Error(`${label} must contain only offered when no fix is expected.`); + } + return { offered: false }; + } + + const normalized = { + offered: true, + title: requireNonEmptyString(fix.title, `${label}.title`), + text: requireString(fix.text, `${label}.text`), + range: normalizeRange(fix.range, `${label}.range`), + ...(fix.expectedText === undefined + ? {} + : { expectedText: requireString(fix.expectedText, `${label}.expectedText`) }), + appliedQuery: requireString(fix.appliedQuery, `${label}.appliedQuery`), + }; + return normalized; +} + +export function contractChannel(spec) { + requireObject(spec, 'contract'); + const channel = spec.channel === undefined ? 'lint' : spec.channel; + if (!CONTRACT_CHANNELS.has(channel)) { + throw new Error( + `contract.channel must be "lint" or "syntax", got ${describe(channel)}.` + ); + } + return channel; +} + +/** + * Normalize legacy detector fields and channel-specific frontend assertions. + * + * Schema-v3 lint contracts retain substring messages. Schema-v4 lint contracts + * assert exact messages and deterministic fixes. Syntax contracts retain their + * legacy parser error, quick-fix, and raw-message assertions. + */ +export function normalizeFrontendOracle(spec, queryExpectation) { + const channel = contractChannel(spec); + requireObject(queryExpectation, `[${spec.ruleId}] query expectation`); + + const hasLegacy = Object.prototype.hasOwnProperty.call( + queryExpectation, + 'detectorCount' + ); + const hasFrontend = Object.prototype.hasOwnProperty.call( + queryExpectation, + 'frontend' + ); + if (hasLegacy && hasFrontend) { + throw new Error( + `[${spec.ruleId}] query expectation must use either detectorCount or frontend, not both.` + ); + } + + if (channel === 'lint') { + const frontend = hasFrontend + ? requireObject(queryExpectation.frontend, `[${spec.ruleId}] frontend`) + : { + count: queryExpectation.detectorCount, + severity: queryExpectation.severity, + ...(queryExpectation.matchMessage !== undefined + ? { matchMessage: queryExpectation.matchMessage } + : {}), + ...(queryExpectation.messageEquals !== undefined + ? { messageEquals: queryExpectation.messageEquals } + : {}), + ...(queryExpectation.deterministicFix !== undefined + ? { deterministicFix: queryExpectation.deterministicFix } + : {}), + }; + const allowed = + spec.schemaVersion === 3 ? LINT_V3_FRONTEND_FIELDS : LINT_V4_FRONTEND_FIELDS; + assertKnownKeys(frontend, allowed, `[${spec.ruleId}] frontend`); + requireNonNegativeInteger(frontend.count, `[${spec.ruleId}] frontend.count`); + assertOptionalString(frontend.severity, `[${spec.ruleId}] frontend.severity`); + if (spec.schemaVersion === 3) { + if ( + frontend.matchMessage !== undefined && + typeof frontend.matchMessage !== 'string' + ) { + throw new TypeError( + `[${spec.ruleId}] frontend.matchMessage must be a string when present.` + ); + } + } else { + assertOptionalString( + frontend.messageEquals, + `[${spec.ruleId}] frontend.messageEquals` + ); + } + if ( + hasFrontend && + ['severity', 'matchMessage', 'messageEquals', 'deterministicFix'].some( + (field) => queryExpectation[field] !== undefined + ) + ) { + throw new Error( + `[${spec.ruleId}] lint assertions must be nested under frontend when frontend is present.` + ); + } + return { + channel, + count: frontend.count, + severity: frontend.severity, + ...(spec.schemaVersion === 3 + ? { matchMessage: frontend.matchMessage } + : { + messageEquals: frontend.messageEquals, + deterministicFix: + frontend.deterministicFix === undefined + ? undefined + : normalizeDeterministicFix( + frontend.deterministicFix, + `[${spec.ruleId}] frontend.deterministicFix` + ), + }), + }; + } + + if (hasLegacy) { + throw new Error( + `[${spec.ruleId}] syntax contracts must use frontend instead of detectorCount.` + ); + } + const frontend = requireObject( + queryExpectation.frontend, + `[${spec.ruleId}] frontend` + ); + assertKnownKeys(frontend, SYNTAX_FRONTEND_FIELDS, `[${spec.ruleId}] frontend`); + requireNonNegativeInteger(frontend.count, `[${spec.ruleId}] frontend.count`); + requireNonEmptyString(frontend.code, `[${spec.ruleId}] frontend.code`); + if ( + frontend.fixText !== undefined && + frontend.fixText !== null && + (typeof frontend.fixText !== 'string' || frontend.fixText.length === 0) + ) { + throw new TypeError( + `[${spec.ruleId}] frontend.fixText must be a non-empty string or null when present.` + ); + } + if ( + frontend.matchMessage !== undefined && + typeof frontend.matchMessage !== 'string' + ) { + throw new TypeError( + `[${spec.ruleId}] frontend.matchMessage must be a string when present.` + ); + } + if (frontend.rawMessage !== undefined && typeof frontend.rawMessage !== 'boolean') { + throw new TypeError(`[${spec.ruleId}] frontend.rawMessage must be a boolean.`); + } + if (frontend.totalErrors !== undefined) { + requireNonNegativeInteger( + frontend.totalErrors, + `[${spec.ruleId}] frontend.totalErrors` + ); + } + for (const field of ['severity', 'matchMessage']) { + if (Object.prototype.hasOwnProperty.call(queryExpectation, field)) { + throw new Error( + `[${spec.ruleId}] syntax ${field} must be nested under frontend.` + ); + } + } + if (spec.wiring && frontend.code !== spec.wiring.code) { + throw new Error( + `[${spec.ruleId}] frontend.code ${describe(frontend.code)} does not match ` + + `contract.wiring.code ${describe(spec.wiring.code)}.` + ); + } + return { channel, ...frontend }; +} + +/** + * Active shipping contracts are stricter than dormant compatibility contracts: + * every lint finding pins its exact message and deterministic-fix behavior. + */ +export function assertShippingFrontendOracles(spec, expectation) { + if (spec.schemaVersion !== 4) { + throw new Error(`[${spec.ruleId}] active shipping contracts must use schemaVersion 4.`); + } + for (const queryName of assertExactQueryCoverage(spec, expectation)) { + const frontend = normalizeFrontendOracle(spec, expectation.queries[queryName]); + const label = `[${spec.ruleId}/${queryName}] frontend`; + + if (frontend.channel === 'syntax') { + if (frontend.fixText === undefined) { + throw new Error(`${label}.fixText must explicitly assert fix presence or absence.`); + } + if (frontend.rawMessage === undefined) { + throw new Error(`${label}.rawMessage must be explicitly asserted.`); + } + if (frontend.totalErrors === undefined) { + throw new Error(`${label}.totalErrors must be explicitly asserted.`); + } + if (frontend.count > 0 && frontend.matchMessage === undefined) { + throw new Error(`${label}.matchMessage is required for a syntax finding.`); + } + continue; + } + + if (frontend.deterministicFix === undefined) { + throw new Error(`${label}.deterministicFix must be explicitly asserted.`); + } + if (frontend.count > 0) { + if (frontend.severity === undefined) { + throw new Error(`${label}.severity is required for a lint finding.`); + } + if (frontend.messageEquals === undefined) { + throw new Error(`${label}.messageEquals is required for a lint finding.`); + } + } else if (frontend.deterministicFix.offered) { + throw new Error(`${label} must not offer a deterministic fix when no finding is expected.`); + } + } +} + +export function normalizeLintWiring(ruleId, wiring, label = 'wiring') { + requireNonEmptyString(ruleId, `${label}.id`); + requireObject(wiring, label); + const appliesTo = + wiring.appliesTo === undefined ? {} : requireObject(wiring.appliesTo, `${label}.appliesTo`); + const normalizedAppliesTo = {}; + for (const key of ['minVersion', 'maxVersion', 'engine']) { + assertOptionalString(appliesTo[key], `${label}.appliesTo.${key}`); + if (appliesTo[key] !== undefined) { + normalizedAppliesTo[key] = appliesTo[key]; + } + } + for (const key of [ + 'runtimeOnly', + 'needsContext', + 'needsExplain', + 'sourceScoped', + ]) { + if (wiring[key] !== undefined && typeof wiring[key] !== 'boolean') { + throw new TypeError(`${label}.${key} must be a boolean when present.`); + } + } + if (typeof wiring.enabled !== 'boolean') { + throw new TypeError(`${label}.enabled must be a boolean.`); + } + return { + id: ruleId, + detector: requireNonEmptyString(wiring.detector, `${label}.detector`), + enabled: wiring.enabled, + severity: requireNonEmptyString(wiring.severity, `${label}.severity`), + appliesTo: normalizedAppliesTo, + runtimeOnly: wiring.runtimeOnly === true, + needsContext: wiring.needsContext === true, + needsExplain: wiring.needsExplain === true, + sourceScoped: wiring.sourceScoped === true, + }; +} + +function assertBackendOracle(oracle, ruleId, executionBackend) { + const label = `[${ruleId}] ${executionBackend} backend oracle`; + requireObject(oracle, label); + const kind = requireNonEmptyString(oracle.kind, `${label}.kind`); + + if (kind === 'not-applicable') { + const reason = requireNonEmptyString(oracle.reason, `${label}.reason`); + if (reason.trim().length === 0) { + throw new TypeError(`${label}.reason must not be blank.`); + } + requireNonEmptyString(oracle.owner, `${label}.owner`); + requireNonEmptyString(oracle.issue, `${label}.issue`); + return kind; + } + if (!APPLICABLE_BACKEND_KINDS.has(kind)) { + throw new Error(`[${ruleId}] unknown ${executionBackend} backend oracle.kind "${kind}".`); + } + + if (!Number.isInteger(oracle.httpStatus) || oracle.httpStatus < 100 || oracle.httpStatus > 599) { + throw new TypeError(`${label}.httpStatus must be an integer from 100 through 599.`); + } + if ((kind === 'result-shape' || kind === 'advisory') && oracle.httpStatus !== 200) { + throw new TypeError(`${label}.httpStatus must be 200.`); + } + if (kind === 'rejection') { + const body = requireObject(oracle.body, `${label}.body`); + if (!Number.isInteger(body.status)) { + throw new TypeError(`${label}.body.status must be an integer.`); + } + if (body.status !== oracle.httpStatus) { + throw new TypeError(`${label}.httpStatus must equal ${label}.body.status.`); + } + if (body.error !== undefined) { + const error = requireObject(body.error, `${label}.body.error`); + assertOptionalString(error.type, `${label}.body.error.type`); + assertOptionalString(error.reason, `${label}.body.error.reason`); + } + } + if (kind === 'result-shape' && oracle.expect !== undefined) { + const expect = requireObject(oracle.expect, `${label}.expect`); + if ( + expect.datarowsNonEmpty !== undefined && + typeof expect.datarowsNonEmpty !== 'boolean' + ) { + throw new TypeError(`${label}.expect.datarowsNonEmpty must be a boolean.`); + } + if (expect.datarowsCount !== undefined) { + requireNonNegativeInteger(expect.datarowsCount, `${label}.expect.datarowsCount`); + } + assertOptionalString(expect.columnAllNull, `${label}.expect.columnAllNull`); + } + return kind; +} + +export function assertExecutionBackend(value, label = 'executionBackend') { + if (!EXECUTION_BACKENDS.has(value)) { + throw new Error( + `${label} must be "standard" or "analytics", got ${describe(value)}.` + ); + } + return value; +} + +/** + * Validate target.json and return the identity consumed by report readers. + * + * Execution identity is never inferred. Every producer in the current + * workflows writes schemaVersion 2, so an unversioned target is an incomplete + * artifact rather than a compatibility mode. + */ +export function normalizeTarget(target) { + requireObject(target, 'target'); + + const hasSchemaVersion = Object.prototype.hasOwnProperty.call(target, 'schemaVersion'); + if (!hasSchemaVersion) { + throw new Error('target.schemaVersion is required; expected 2.'); + } + + if (target.schemaVersion !== 2) { + throw new Error( + `Unsupported target schemaVersion ${describe(target.schemaVersion)}; expected 2.` + ); + } + const executionBackend = assertExecutionBackend( + target.executionBackend, + 'target.executionBackend' + ); + requireNonEmptyString(target.engineVersion, 'target.engineVersion'); + if (typeof target.grammarHash !== 'string') { + throw new TypeError('target.grammarHash must be a string.'); + } + if ( + Object.prototype.hasOwnProperty.call(target, 'grammarBundle') && + typeof target.grammarBundle !== 'string' + ) { + throw new TypeError('target.grammarBundle must be a string when present.'); + } + if ( + Object.prototype.hasOwnProperty.call(target, 'sqlSha') && + typeof target.sqlSha !== 'string' + ) { + throw new TypeError('target.sqlSha must be a string when present.'); + } + if (!Number.isInteger(target.shardCount) || target.shardCount < 1) { + throw new Error('target.shardCount must be a positive integer.'); + } + if (executionBackend === 'analytics') { + if (target.storage !== 'composite-parquet') { + throw new Error( + `analytics target.storage must be "composite-parquet", got ${describe(target.storage)}.` + ); + } + const analyticsStack = requireObject( + target.analyticsStack, + 'analytics target.analyticsStack' + ); + requireNonEmptyString( + analyticsStack.source, + 'analytics target.analyticsStack.source' + ); + const attestation = requireObject( + target.routeAttestation, + 'analytics target.routeAttestation' + ); + for (const check of [ + 'pluginsVerified', + 'clusterSettingsVerified', + 'fixtureIndicesVerified', + 'explainVerified', + 'profiledExecutionVerified', + ]) { + if (attestation[check] !== true) { + throw new Error(`analytics target.routeAttestation.${check} must be true.`); + } + } + } else if (target.storage !== 'lucene') { + throw new Error( + `standard target.storage must be "lucene", got ${describe(target.storage)}.` + ); + } + + return { + schemaVersion: 2, + executionBackend, + engineVersion: target.engineVersion, + grammarHash: target.grammarHash, + grammarBundle: target.grammarBundle || '', + sqlSha: target.sqlSha || '', + storage: target.storage || (executionBackend === 'standard' ? 'lucene' : ''), + shardCount: target.shardCount, + analyticsStack: target.analyticsStack, + routeAttestation: target.routeAttestation, + legacy: false, + }; +} + +export function assertContractSchema(spec) { + requireObject(spec, 'contract'); + if (!CONTRACT_SCHEMA_VERSIONS.has(spec.schemaVersion)) { + throw new Error( + `Unsupported contract schemaVersion ${describe(spec.schemaVersion)}; expected 3 or 4.` + ); + } + requireNonEmptyString(spec.ruleId, 'contract.ruleId'); + const channel = contractChannel(spec); + if (spec.wiring !== undefined) { + const wiring = requireObject(spec.wiring, `[${spec.ruleId}] contract.wiring`); + if (channel === 'syntax') { + const keys = Object.keys(wiring); + if (keys.length !== 1 || keys[0] !== 'code') { + throw new Error( + `[${spec.ruleId}] syntax wiring must contain only the stable error code.` + ); + } + requireNonEmptyString(wiring.code, `[${spec.ruleId}] contract.wiring.code`); + } else if (Object.prototype.hasOwnProperty.call(wiring, 'code')) { + throw new Error(`[${spec.ruleId}] lint wiring must not contain syntax code.`); + } + } + if (spec.queries !== undefined) { + requireObject(spec.queries, `[${spec.ruleId}] contract.queries`); + for (const [queryName, query] of Object.entries(spec.queries)) { + requireObject(query, `[${spec.ruleId}] contract.queries.${queryName}`); + const role = query.role === undefined ? 'trigger' : query.role; + if (!QUERY_ROLES.has(role)) { + throw new Error( + `[${spec.ruleId}] query "${queryName}" has invalid role ${describe(role)}.` + ); + } + if (role === 'suppression-control' && channel !== 'syntax') { + throw new Error( + `[${spec.ruleId}] suppression-control is valid only for syntax contracts.` + ); + } + } + } + return spec.schemaVersion; +} + +/** + * Require a selected expectation to cover every top-level query exactly once. + * JSON object keys are unique after parsing, so set equality establishes the + * one-to-one query identity needed by both backend and detector readers. + */ +export function assertExactQueryCoverage(spec, expectation) { + assertContractSchema(spec); + requireObject(spec.queries, `[${spec.ruleId}] contract.queries`); + requireObject(expectation, `[${spec.ruleId}] selected expectation`); + requireObject(expectation.queries, `[${spec.ruleId}] selected expectation.queries`); + + const contractKeys = Object.keys(spec.queries).sort(); + const expectationKeys = Object.keys(expectation.queries).sort(); + if (contractKeys.length === 0) { + throw new Error(`[${spec.ruleId}] contract.queries must not be empty.`); + } + const contractSet = new Set(contractKeys); + const expectationSet = new Set(expectationKeys); + const missing = contractKeys.filter((key) => !expectationSet.has(key)); + const extra = expectationKeys.filter((key) => !contractSet.has(key)); + + if (missing.length > 0 || extra.length > 0) { + const details = []; + if (missing.length > 0) { + details.push(`missing from expectation: ${missing.join(', ')}`); + } + if (extra.length > 0) { + details.push(`not present in contract.queries: ${extra.join(', ')}`); + } + throw new Error(`[${spec.ruleId}] query coverage must be exact (${details.join('; ')}).`); + } + return contractKeys; +} + +/** + * Resolve only the route-specific backend oracle. Detector count, severity, and + * message assertions remain on the shared query expectation and are returned + * unchanged for either execution backend. + */ +export function resolveBackendOracle(spec, queryExpectation, executionBackend) { + const schemaVersion = assertContractSchema(spec); + assertExecutionBackend(executionBackend); + const frontend = normalizeFrontendOracle(spec, queryExpectation); + const { channel: _channel, ...detector } = frontend; + + let oracle; + let missingReason; + if (schemaVersion === 3) { + if (executionBackend === 'standard') { + oracle = queryExpectation.backend; + missingReason = 'schema-v3 query has no backend oracle'; + } else { + missingReason = + 'schema-v3 backend oracles are standard-only; no analytics oracle is defined'; + } + } else { + if ( + Object.prototype.hasOwnProperty.call(queryExpectation, 'backends') && + !isObject(queryExpectation.backends) + ) { + throw new TypeError(`[${spec.ruleId}] backends must be a JSON object.`); + } + for (const backend of Object.keys(queryExpectation.backends || {})) { + assertExecutionBackend(backend, `[${spec.ruleId}] backends key`); + } + oracle = queryExpectation.backends && queryExpectation.backends[executionBackend]; + missingReason = `schema-v4 query has no ${executionBackend} backend oracle`; + } + + if (oracle === undefined) { + return { + status: 'coverage-missing', + executionBackend, + detector, + frontend, + oracle: undefined, + reason: missingReason, + }; + } + + const kind = assertBackendOracle(oracle, spec.ruleId, executionBackend); + if (kind === 'not-applicable') { + return { + status: 'not-applicable', + executionBackend, + detector, + frontend, + oracle, + reason: oracle.reason, + }; + } + + return { + status: 'applicable', + executionBackend, + detector, + frontend, + oracle, + reason: undefined, + }; +} + +export function backendReportKey(entry) { + requireObject(entry, 'backend report row'); + const ruleId = requireNonEmptyString(entry.ruleId, 'backend report row.ruleId'); + const queryName = requireNonEmptyString(entry.queryName, 'backend report row.queryName'); + return `${ruleId}::${queryName}`; +} + +/** + * Read the backend observation state without coercing a missing verdict to + * acceptance. Explicit infrastructure/coverage states take precedence even if + * a malformed row also happens to contain `rejected`. + */ +export function classifyBackendReportRow(entry) { + requireObject(entry, 'backend report row'); + if (entry.outcome === 'not-applicable' || entry.kind === 'not-applicable') { + return { status: 'not-applicable', rejected: undefined }; + } + if (entry.outcome === 'coverage-missing' || entry.kind === 'coverage-missing') { + return { status: 'coverage-missing', rejected: undefined }; + } + if (entry.outcome === 'error' || typeof entry.rejected !== 'boolean') { + return { status: 'error', rejected: undefined }; + } + return { status: 'observed', rejected: entry.rejected }; +} + +/** + * Validate and index the historical bare-array backend report. + * + * Every row carries the same explicit backend identity as its schema-v2 target. + */ +export function indexBackendReport(entries, targetIdentity) { + if (!Array.isArray(entries)) { + throw new TypeError('backend report must be a JSON array.'); + } + requireObject(targetIdentity, 'normalized target identity'); + assertExecutionBackend( + targetIdentity.executionBackend, + 'normalized target identity.executionBackend' + ); + + const byKey = new Map(); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]; + const key = backendReportKey(entry); + const hasIdentity = Object.prototype.hasOwnProperty.call(entry, 'executionBackend'); + if (!hasIdentity) { + throw new Error( + `backend report row ${key} is missing executionBackend for a schema-v2 target.` + ); + } + const rowBackend = assertExecutionBackend( + entry.executionBackend, + `backend report row ${key}.executionBackend` + ); + if (rowBackend !== targetIdentity.executionBackend) { + throw new Error( + `backend report row ${key} executionBackend "${rowBackend}" does not match ` + + `target "${targetIdentity.executionBackend}".` + ); + } + if (byKey.has(key)) { + throw new Error(`duplicate backend report key "${key}".`); + } + byKey.set(key, entry); + } + return byKey; +} diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs new file mode 100644 index 00000000000..560e683975c --- /dev/null +++ b/scripts/ppl-lint/drift.mjs @@ -0,0 +1,1001 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Drift classification for the multi-version PPL lint contract. + * + * The contract runs every default-error lint rule against SEVERAL engine + * versions. Detecting that a rule disagrees with an engine is only half the + * job — a bare "expected 1 diagnostic, got 0" tells an engineer that something + * moved but not what to do about it. This module turns one observed + * (expected vs detector vs backend) triple into: + * + * 1. a drift CLASS — which of the known ways engine/linter can diverge; and + * 2. a REMEDIATION — the concrete linter-side action, one of: + * disable-rule the engine no longer has the behavior; stop shipping + * the diagnostic (or scope it to older versions) + * version-scope-rule the behavior is version-bounded; fix appliesTo + * update-detector the detector's own logic/anchor is now wrong + * update-contract the linter is right; the pinned expectation is stale + * + * The classifier is deliberately pure and side-effect free so it can be unit + * tested without a cluster (see __tests__/drift.test.mjs). The runner supplies + * observations; this module decides nothing about how they were gathered. + * + * Naming: a "trigger" query is one the rule is supposed to flag; a "control" is + * a near-identical valid query it must stay silent on. `role` distinguishes them. + */ + +/** Every drift class this module can emit, with a stable one-line meaning. */ +export const DRIFT_CLASSES = { + GRAMMAR_RULE_MISSING: 'grammar-rule-missing', + EXECUTION_BACKEND_DIVERGENCE: 'execution-backend-divergence', + BACKEND_ORACLE_MISMATCH: 'backend-oracle-mismatch', + ENGINE_RELAXED: 'engine-relaxed', + ENGINE_PARTIALLY_RELAXED: 'engine-partially-relaxed', + ENGINE_TIGHTENED: 'engine-tightened', + ENGINE_MESSAGE_CHANGED: 'engine-message-changed', + DETECTOR_SILENT: 'detector-silent', + DETECTOR_NOISY: 'detector-noisy', + DETECTOR_COUNT_MISMATCH: 'detector-count-mismatch', + DETECTOR_MESSAGE_MISMATCH: 'detector-message-mismatch', + VERSION_SCOPE_TOO_NARROW: 'version-scope-too-narrow', + SEVERITY_MISMATCH: 'severity-mismatch', +}; + +/** Remediation actions, phrased as what the linter engineer changes. */ +export const REMEDIATIONS = { + ALIGN_EXECUTION_BACKENDS: 'align-execution-backends', + DISABLE_RULE: 'disable-rule', + VERSION_SCOPE_RULE: 'version-scope-rule', + UPDATE_DETECTOR: 'update-detector', + UPDATE_CONTRACT: 'update-contract', + REVIEW_BACKEND_ORACLE: 'review-backend-oracle', +}; + +/** OSD paths an engineer edits, kept in one place so a move is a one-line fix. */ +const OSD_PATHS = { + catalog: 'packages/osd-monaco/src/ppl/lint/rules_catalog.json', + ruleDir: 'packages/osd-monaco/src/ppl/lint/rules/', + ruleIndex: 'packages/osd-monaco/src/ppl/lint/rule_index.ts', +}; + +/** + * Path of the detector implementation for a rule. + * + * Most rules follow the snake_case-of-the-id convention, but not all: the + * catalog id `unsupported-window-function-in-eventstats` lives in + * `unsupported_window_function.ts`. A remediation that names a file the engineer + * cannot open is worse than one that names a directory, so a contract may pin the + * real path via `detectorPath` and we fall back to the convention otherwise. + */ +function detectorFile(ruleId, detectorPath) { + if (detectorPath) { + return detectorPath; + } + return `${OSD_PATHS.ruleDir}${String(ruleId).replace(/-/g, '_')}.ts`; +} + +/** + * Cheap edit-distance, used only to suggest "did the grammar rename X to Y?". + * Bounded by the shorter string, so it is O(n*m) on short identifiers. + */ +function editDistance(a, b) { + const m = a.length; + const n = b.length; + if (m === 0 || n === 0) return Math.max(m, n); + let prev = Array.from({ length: n + 1 }, (_, j) => j); + for (let i = 1; i <= m; i++) { + const row = [i]; + for (let j = 1; j <= n; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost); + } + prev = row; + } + return prev[n]; +} + +/** Split a camelCase parser rule name into lower-case tokens. */ +function camelTokens(name) { + return String(name) + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); +} + +/** + * Best candidates for a parser rule that vanished from the candidate grammar. + * + * Ranked by how a real ANTLR rename tends to look, strongest signal first: + * 0. containment `unionCommand` -> `unionCommandNew` + * 1. same leading token `unionCommand` -> `unionStatement` + * 2. near spelling `rexCommand` -> `regexCommand` + * + * The leading token carries the identity of the rule; the trailing one is + * usually a generic suffix (`Command`, `Clause`, `Expression`) shared by most of + * the grammar, so matching on it alone would suggest dozens of unrelated rules. + * That is why only the FIRST token counts, and why an unrelated rule returns + * nothing rather than a plausible-looking wrong guess. + */ +export function suggestParserRules(missingRule, availableRules, limit = 3) { + const missing = String(missingRule); + const lower = missing.toLowerCase(); + const missingHead = camelTokens(missing)[0]; + const scored = []; + for (const candidate of availableRules) { + const cl = String(candidate).toLowerCase(); + let score; + if (cl.includes(lower) || lower.includes(cl)) { + score = 0; // containment: strongest signal of a rename + } else if (missingHead && camelTokens(candidate)[0] === missingHead) { + score = 1; // same subject, renamed suffix + } else { + // Only very near spellings survive this tier. A looser budget scaled to + // name length lets long names match unrelated same-suffix rules + // (`unionCommand` vs `binCommand` differ by 4 edits but are unrelated), so + // the cap is absolute: typo-or-insertion distance, nothing more. + const distance = editDistance(lower, cl); + if (distance > 2) continue; + score = 1 + distance; + } + scored.push({ candidate, score }); + } + scored.sort((a, b) => a.score - b.score || String(a.candidate).localeCompare(String(b.candidate))); + return scored.slice(0, limit).map((s) => s.candidate); +} + +/** Parse "3.8.0-SNAPSHOT" / "3.7" into [major, minor, patch]; undefined if unparseable. */ +export function parseVersion(value) { + if (!value) return undefined; + const match = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(value)); + if (!match) return undefined; + return [Number(match[1]), Number(match[2] || 0), Number(match[3] || 0)]; +} + +export function compareVersion(a, b) { + for (let i = 0; i < 3; i++) { + if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + } + return 0; +} + +/** + * True when `version` falls inside the catalog's `appliesTo` window. An absent + * bound is open-ended, and an unparseable version is treated as in-range so an + * unrecognized engine build never silently drops coverage. + */ +export function versionInAppliesTo(appliesTo, version) { + const have = parseVersion(version); + if (!have) return true; + const min = parseVersion(appliesTo && appliesTo.minVersion); + const max = parseVersion(appliesTo && appliesTo.maxVersion); + if (min && compareVersion(have, min) < 0) return false; + // maxVersion is treated as inclusive, matching the OSD catalog's own reading. + if (max && compareVersion(have, max) > 0) return false; + return true; +} + +/** Human-readable one-liner for the observed pair, reused across messages. */ +function describeObservation(observed) { + const detector = observed.detectorCount > 0 ? `flagged (${observed.detectorCount})` : 'silent'; + let backend = 'accepted'; + if (observed.backendRejected) { + const type = observed.backendType ? ` ${observed.backendType}` : ''; + backend = `rejected${type}`; + } else if (observed.backendRejected === undefined) { + backend = 'not observed'; + } + return `detector ${detector}, engine ${backend}`; +} + +function describeBackendVerdict(observed) { + if (!observed || typeof observed.backendRejected !== 'boolean') { + return 'did not produce a verdict'; + } + if (!observed.backendRejected) { + return 'ACCEPTED'; + } + const type = observed.backendType ? ` (${observed.backendType})` : ''; + return `REJECTED${type}`; +} + +function executionBackendRemediation(ruleId, detectorPath) { + return { + action: REMEDIATIONS.ALIGN_EXECUTION_BACKENDS, + target: `analytics backend and ${detectorFile(ruleId, detectorPath)}`, + detail: + `Keep the OpenSearch version bounds unchanged. Review the route-specific backend oracles, ` + + `then either align analytics behavior with standard, narrow the detector to behavior common ` + + `to both routes, disable the rule for every route, or add a reliable execution-backend signal ` + + `to the lint context before emitting route-specific diagnostics.`, + }; +} + +/** + * Compare the two execution routes for one query on the same engine candidate. + * This is deliberately separate from product-version drift: a route difference + * cannot justify changing an OpenSearch version range. + */ +export function classifyExecutionBackendDivergence({ + ruleId, + version, + queryName, + role = 'trigger', + query, + standardObserved, + analyticsObserved, + standardLeg, + analyticsLeg, + grammarHash, + detectorPath, +}) { + if ( + !standardObserved || + !analyticsObserved || + typeof standardObserved.backendRejected !== 'boolean' || + typeof analyticsObserved.backendRejected !== 'boolean' || + standardObserved.backendRejected === analyticsObserved.backendRejected + ) { + return null; + } + + const where = `${ruleId} @ ${version} [${queryName}]`; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + executionBackend: 'analytics', + baselineExecutionBackend: 'standard', + executionBackends: ['standard', 'analytics'], + driftClass: DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE, + evidence: + `${where}: standard ${describeBackendVerdict(standardObserved)} while analytics ` + + `${describeBackendVerdict(analyticsObserved)} on the same engine candidate and runtime ` + + `grammar${grammarHash ? ` (${grammarHash})` : ''}` + + `${standardLeg || analyticsLeg ? `; legs ${standardLeg || 'standard'} / ${analyticsLeg || 'analytics'}` : ''}.`, + remediation: executionBackendRemediation(ruleId, detectorPath), + }; +} + +function backendOracleRemediation(executionBackend) { + return { + action: REMEDIATIONS.REVIEW_BACKEND_ORACLE, + target: `${executionBackend} backend and this contract file`, + detail: + `Review the captured raw response and determine whether the backend regressed or the reviewed ` + + `${executionBackend} oracle is stale. Restore the backend behavior when the status/result change ` + + `is unintended; update the oracle only after confirming the new behavior is intentional. Keep ` + + `the OpenSearch version bounds unchanged.`, + }; +} + +function classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected, + detectorPath, + reason, +}) { + const expected = expectedRejected ? 'REJECTION' : 'ACCEPTANCE'; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + executionBackend: 'analytics', + executionBackends: ['analytics'], + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${ruleId} @ ${version} [${queryName}]: analytics ${describeBackendVerdict(observed)}, ` + + `but ${reason || `its reviewed oracle requires ${expected}`}.`, + remediation: backendOracleRemediation('analytics'), + }; +} + +/** + * Report a parser rule the detector walks that the candidate grammar no longer + * defines. Exported so a caller can raise it ONCE per rule/version — the fact is + * a property of the grammar, not of any single query, and repeating it per query + * buries the one edit an engineer has to make. `classifyDrift` still calls it so + * a caller that does not hoist the check keeps the diagnosis. + */ +export function classifyGrammarDrift({ + ruleId, + version, + requiredParserRules, + parserRuleNames, + observed = {}, + queryName, + role = 'trigger', + query, + detectorPath, + executionBackend = 'standard', +}) { + if (!Array.isArray(requiredParserRules) || !Array.isArray(parserRuleNames)) { + return null; + } + const available = new Set(parserRuleNames); + const missing = requiredParserRules.filter((rule) => !available.has(rule)); + if (missing.length === 0) { + return null; + } + const missingList = missing.map((r) => `"${r}"`).join(', '); + const suggestions = [...new Set(missing.flatMap((rule) => suggestParserRules(rule, parserRuleNames)))]; + const at = queryName ? ` [${queryName}]` : ''; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + executionBackend, + driftClass: DRIFT_CLASSES.GRAMMAR_RULE_MISSING, + evidence: + `${ruleId} @ ${version} (${executionBackend})${at}: the candidate grammar has no parser rule(s) ${missingList}, ` + + `which this rule's detector walks.` + + (observed && observed.detectorCount !== undefined ? ` ${describeObservation(observed)}.` : ''), + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine grammar renamed or removed ${missingList}. ` + + (suggestions.length > 0 + ? `Closest rule(s) now in the grammar: ${suggestions.map((r) => `"${r}"`).join(', ')}. ` + : '') + + `Re-anchor the detector (and ${OSD_PATHS.ruleIndex} if the name is listed there) onto the ` + + `current rule name, then update this contract's requiredParserRules. If the command itself ` + + `is gone from the engine, disable the rule instead.`, + }, + }; +} + +/** + * Decide, for ONE rule on ONE engine version, whether an observed relaxation is + * total or partial — the difference between "scope the rule away from this + * version" and "narrow the detector". + * + * `classifyDrift` sees a single query, so it cannot tell these apart: one trigger + * that the engine now accepts looks identical whether the rule's other triggers + * still fail or not. Acting on that one query is actively harmful in the partial + * case, because scoping the rule out of the version drops the diagnostics that + * are STILL correct — turning a partial engine fix into a false negative on the + * shapes that remain broken. That is why this runs over the whole rule. + * + * Inputs are the per-trigger verdicts the caller already gathered: + * relaxed engine ACCEPTS a trigger the contract pinned as rejected + * holding engine still REJECTS the trigger + * Triggers with no usable verdict are passed as neither, and are reported as the + * reason a verdict is being withheld rather than silently treated as `holding` + * (which would read a dead leg as a partial fix and narrow a healthy detector). + * + * Returns null when nothing relaxed — the caller's per-query drifts stand on + * their own. Otherwise returns ONE rule-level drift that supersedes the + * per-query `engine-relaxed` findings, so the report shows a single decision + * instead of one "scope this away" paragraph per trigger. + * + * @param {object} input + * @param {string[]} input.relaxedTriggers trigger names the engine now accepts + * @param {string[]} input.holdingTriggers trigger names the engine still rejects + * @param {string[]} [input.unobservedTriggers] triggers with no comparable verdict + * @param {boolean} [input.detectorFlagged] did the detector fire on any relaxed trigger + */ +export function classifyRelaxationScope({ + ruleId, + version, + relaxedTriggers = [], + holdingTriggers = [], + unobservedTriggers = [], + detectorFlagged = false, + wiring, + detectorPath, + executionBackend = 'standard', +}) { + if (relaxedTriggers.length === 0) { + return null; + } + + const where = `${ruleId} @ ${version}`; + const base = { + ruleId, + version, + driftVersion: version, + role: 'trigger', + executionBackend, + scope: { + relaxed: [...relaxedTriggers], + holding: [...holdingTriggers], + unobserved: [...unobservedTriggers], + }, + }; + // How thin is the basis for a "fully relaxed" claim? A rule with ONE pinned + // trigger that relaxes proves only that one shape changed; calling that "the + // behavior is gone" is a much bigger inference than the data supports. The + // count goes in the evidence either way so the reader can judge it, rather + // than the tool quietly presenting 1-of-1 as though it were 5-of-5. + const observed = relaxedTriggers.length + holdingTriggers.length; + const basis = `${relaxedTriggers.length} of ${observed} observed trigger(s) relaxed`; + + if (executionBackend === 'analytics') { + return { + ...base, + driftClass: DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE, + executionBackends: ['analytics'], + evidence: + `${where}: analytics accepted ${basis}; this is route-specific behavior, not product-version drift.`, + remediation: executionBackendRemediation(ruleId, detectorPath), + }; + } + + // --- Partial: some triggers relaxed, others still rejected ------------------ + // The engine fixed part of the condition. Scoping the rule out of this version + // would ship a false negative on everything in `holding`, so the action is to + // narrow the detector to the shapes that still fail. + if (holdingTriggers.length > 0) { + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS ${relaxedTriggers.length} of this rule's triggers ` + + `(${relaxedTriggers.join(', ')}) but still REJECTS ${holdingTriggers.length} ` + + `(${holdingTriggers.join(', ')}) — a PARTIAL fix, ${basis}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Do NOT scope "${ruleId}" away from ${version}: the engine still rejects ` + + `${holdingTriggers.join(', ')}, so a maxVersion below ${version} would drop diagnostics that ` + + `are still correct and ship a false NEGATIVE there. Narrow the detector so it stops matching ` + + `the now-valid shape(s) (${relaxedTriggers.join(', ')}) while still flagging the rest, then ` + + `re-pin the ${version} expectation for the relaxed trigger(s) to detectorCount 0.`, + }, + }; + } + + // --- Full: every observed trigger relaxed ----------------------------------- + // Nothing the rule claims is still true on this engine. Version-scoping is now + // the right action — with the caveat that "every observed trigger" is only as + // strong as the trigger count, which the evidence states. + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS every observed trigger for this rule ` + + `(${relaxedTriggers.join(', ')}) — a FULL fix, ${basis}` + + (unobservedTriggers.length > 0 + ? `; ${unobservedTriggers.length} trigger(s) produced no verdict (${unobservedTriggers.join(', ')}) ` + + `and were NOT counted` + : '') + + '.', + remediation: detectorFlagged + ? { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `Every trigger this contract pins is now valid on ${version}, so "${ruleId}" is a FALSE ` + + `POSITIVE there. Set appliesTo.maxVersion just below ${version} to keep protecting users on ` + + `older engines; if no supported engine rejects any trigger any more, set "enabled": false and ` + + `drop the detector. Then re-pin the ${version} expectations to detectorCount 0.` + + (observed < 2 + ? ` NOTE: this rule pins only ${observed} trigger, so "fully relaxed" rests on a single ` + + `observation — confirm with more shapes of the same condition before scoping the rule away.` + : '') + + (unobservedTriggers.length > 0 + ? ` NOTE: ${unobservedTriggers.length} trigger(s) produced no verdict on this leg; re-run it ` + + `before acting, since one of them may still reject.` + : ''), + } + : { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already stays silent on ${version}, so no linter change is needed. Re-pin the ` + + `${version} expectations to detectorCount 0 / backend.kind "result-shape" to record the ` + + `engine's new behavior.`, + }, + }; +} + +/** + * Classify one query's outcome on one engine version. + * + * Returns `null` when the detector, the engine and the pinned expectation all + * agree — the overwhelmingly common case. Otherwise returns a single drift + * object; checks run most-specific-first so the reported cause is the root one + * (a renamed grammar rule explains a silent detector, not the other way round). + * + * @param {object} input + * @param {string} input.ruleId + * @param {string} input.version engine version under test, e.g. "3.7.0" + * @param {string} input.queryName + * @param {string} input.role 'trigger' | 'control' + * @param {string} input.query the query as sent to both halves + * @param {object} input.expected { detectorCount, severity, backendKind } + * @param {object} input.observed { detectorCount, severities, backendRejected, backendType, backendReason } + * @param {object} [input.wiring] OSD catalog entry (appliesTo, runtimeOnly, ...) + * @param {string[]} [input.parserRuleNames] candidate grammar's parser rule names + * @param {string[]} [input.requiredParserRules] grammar rules the detector walks + * @param {object} [input.expectedBackend] contract's pinned rejection body + * @param {boolean} [input.controlAlsoRejected] true when this rule's control query was + * ALSO rejected on this engine, i.e. the command itself is unsupported here + */ +export function classifyDrift(input) { + const { + ruleId, + version, + queryName, + role = 'trigger', + query, + expected = {}, + observed = {}, + wiring, + parserRuleNames, + requiredParserRules, + expectedBackend, + detectorPath, + controlAlsoRejected, + executionBackend = 'standard', + } = input; + + const detectorFlagged = (observed.detectorCount || 0) > 0; + const expectFlagged = (expected.detectorCount || 0) > 0; + const backendRejected = observed.backendRejected; + const where = `${ruleId} @ ${version} (${executionBackend}) [${queryName}]`; + const base = { ruleId, version, queryName, role, query, driftVersion: version, executionBackend }; + + // --- 1. Did the grammar move out from under the detector? ------------------- + // A detector that walks a parser rule the candidate grammar no longer defines + // cannot fire at all. This is the root cause of an otherwise baffling silent + // detector, so it is checked before any behavioral comparison. + const grammarDrift = classifyGrammarDrift({ + ruleId, + version, + requiredParserRules, + parserRuleNames, + observed, + queryName, + role, + query, + detectorPath, + executionBackend, + }); + if (grammarDrift) { + return grammarDrift; + } + + // --- 2. Is the rule even in scope for this engine version? ------------------ + // A rule whose appliesTo excludes this version is intentionally inert here. + // That is only correct if the engine also does not exhibit the behavior; if the + // engine rejects the trigger, the version window is too narrow and users on + // this version get no diagnostic. + const inScope = versionInAppliesTo(wiring && wiring.appliesTo, version); + const expectRejection = expected.backendKind === 'rejection'; + if (!inScope) { + // A trigger the engine rejects normally means the version window is too + // narrow. But if the rule's CONTROL — a valid query using the same command — + // is rejected too, the command itself does not exist on this engine yet, and + // the rejection says nothing about the rule's specific condition. Widening + // appliesTo there would ship a diagnostic that claims a precise cause for + // what is really "unsupported command", so that case is correctly silent: + // the version window is doing its job. + if (role === 'trigger' && backendRejected === true && controlAlsoRejected !== true) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: false, + detectorPath, + reason: 'the standard product-version rule is inactive for this engine candidate', + }); + } + return { + ...base, + driftClass: DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW, + evidence: + `${where}: engine ${version} rejects this trigger, but the rule's appliesTo ` + + `(${JSON.stringify((wiring && wiring.appliesTo) || {})}) excludes ${version}, so no diagnostic ` + + `is shown to users on that version.`, + remediation: { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `Widen "${ruleId}".appliesTo to include ${version} (lower minVersion / raise maxVersion) so the ` + + `diagnostic reaches users on engines that actually reject the query.`, + }, + }; + } + // Out of scope but the detector fired anyway. `appliesTo` is applied by OSD's + // version filter, which runs a rule when the cluster version is UNKNOWN — so a + // user whose version could not be resolved sees a diagnostic the catalog says + // does not apply to them. If the engine accepts the query, that is a false + // positive reaching exactly the users the version window was meant to protect. + if (detectorFlagged) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_NOISY, + evidence: + `${where}: the rule's appliesTo (${JSON.stringify((wiring && wiring.appliesTo) || {})}) ` + + `excludes ${version}, yet the detector emitted ${observed.detectorCount} diagnostic(s) ` + + `(${describeObservation(observed)}).`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `A rule out of scope for ${version} must stay silent there. Check that the detector honors ` + + `the version context rather than deciding on its own, and remember OSD's version filter runs ` + + `a rule when the cluster version is unknown — so this also fires for users whose version ` + + `could not be resolved.` + + (backendRejected === true && executionBackend === 'standard' + ? ` The engine does reject this query, so widening appliesTo in ${OSD_PATHS.catalog} may be` + + ` the right fix instead.` + : ''), + }, + }; + } + // Out of scope and the engine agrees it is a non-issue: nothing to report. + return null; + } + + // --- 3. Behavioral flips: the engine changed its verdict -------------------- + + // 3a. The engine now ACCEPTS what the contract pinned as a rejection. Any + // diagnostic the linter still emits is a false positive shipped to users — + // the single most damaging drift, so it is reported even when the detector + // count happens to match the stale expectation. + if (role === 'trigger' && expectRejection && backendRejected === false) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: true, + detectorPath, + }); + } + return { + ...base, + // A single relaxed trigger cannot tell a full fix from a partial one, and the + // two need OPPOSITE actions (scope the rule away vs narrow the detector). The + // caller aggregates every trigger through `classifyRelaxationScope` and drops + // the findings carrying this marker in favour of that one rule-level verdict. + // Kept as a finding rather than returning null so a caller that does not + // aggregate still reports the relaxation instead of silently passing. + supersededBy: DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED, + driftClass: DRIFT_CLASSES.ENGINE_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS a query the contract pinned as rejected ` + + `(${describeObservation(observed)}). The engine gained support for this construct.`, + remediation: detectorFlagged + ? { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `"${ruleId}" is now a FALSE POSITIVE on ${version}. Bound it to the versions that still ` + + `reject: set appliesTo.maxVersion just below ${version}. If no supported engine rejects it ` + + `any more, set "enabled": false (disable-rule) and drop the detector. Then re-pin this ` + + `contract's ${version} expectation to detectorCount 0 / backend.kind "result-shape".`, + } + : { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already stays silent on ${version}, so no linter change is needed. Re-pin the ` + + `${version} expectation to detectorCount 0 / backend.kind "result-shape" to record the ` + + `engine's new behavior.`, + }, + }; + } + + // 3b. The engine now REJECTS what the contract pinned as valid. A control that + // started failing means the linter is silently missing a real error. + if (!expectRejection && backendRejected === true) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: false, + detectorPath, + }); + } + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_TIGHTENED, + evidence: + `${where}: engine ${version} now REJECTS a query the contract pinned as valid ` + + `(${observed.backendType || 'error'}: ${observed.backendReason || 'no reason'}). ` + + `${describeObservation(observed)}.`, + remediation: detectorFlagged + ? { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already flags this, so the linter is correct and only the pinned expectation ` + + `is stale. Re-pin the ${version} expectation to backend.kind "rejection" with the observed ` + + `error.type/reason, and pick a genuinely valid query for the control.`, + } + : { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine rejects this but the linter is silent — a false NEGATIVE on ${version}. Extend ` + + `the detector to cover this shape (or, if the rejection belongs to a different rule, add a ` + + `contract case under that rule). Then re-pin this expectation.`, + }, + }; + } + + // --- 4. Same verdict, different wording ------------------------------------ + // The engine still rejects, but the error type/reason moved. The pinned body — + // and any detector text that quotes the engine wording — is stale. Worth + // flagging because linter messages and quick-fix copy are written against these + // strings. + // + // Requires the detector to still agree with the expectation (`detectorMatches`). + // Without that condition a reworded message would MASK a detector that went + // silent at the same time: the report would say "the detector's verdict is + // unaffected, no rule change required", the engineer would re-pin the string, + // and the check would go green over a rule that no longer fires. When both moved + // at once, the silent detector is the more serious story and step 5 tells it. + const detectorMatches = (observed.detectorCount || 0) === (expected.detectorCount || 0); + if (backendRejected === true && expectRejection && expectedBackend && detectorMatches) { + const expectedError = (expectedBackend.body && expectedBackend.body.error) || {}; + const statusChanged = + expectedBackend.httpStatus !== undefined && + observed.backendStatus !== undefined && + expectedBackend.httpStatus !== observed.backendStatus; + const typeChanged = + expectedError.type !== undefined && + observed.backendType !== undefined && + expectedError.type !== observed.backendType; + const reasonChanged = + expectedError.reason !== undefined && + observed.backendReason !== undefined && + expectedError.reason !== observed.backendReason; + if (statusChanged) { + return { + ...base, + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${where}: the backend still rejects the query, but its HTTP status changed from ` + + `${expectedBackend.httpStatus} to ${observed.backendStatus}.`, + remediation: backendOracleRemediation(executionBackend), + }; + } + if (typeChanged || reasonChanged) { + const parts = []; + if (typeChanged) parts.push(`error.type "${expectedError.type}" -> "${observed.backendType}"`); + if (reasonChanged) + parts.push(`error.reason "${expectedError.reason}" -> "${observed.backendReason}"`); + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED, + evidence: + `${where}: engine ${version} still rejects the query but reworded the failure — ${parts.join('; ')}.`, + remediation: { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector's verdict is unaffected, so no rule change is required. Update the ${version} ` + + `expectation's backend.body to the observed wording. Also check whether "${ruleId}"'s message ` + + `or quick-fix copy in ${detectorFile(ruleId, detectorPath)} quotes the old engine wording.`, + }, + }; + } + } + + // Result-shape assertions and other detailed backend oracles can change while + // the coarse accepted/rejected verdict stays the same. The Java observer + // records that assertion failure explicitly; it must not be treated as + // agreement merely because a boolean verdict is still available. + if (observed.backendOutcome === 'observed-mismatch' || observed.backendOutcome === 'fail') { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: expectRejection, + detectorPath, + reason: `its reviewed backend oracle did not match: ${ + observed.backendMismatch || 'unspecified assertion mismatch' + }`, + }); + } + return { + ...base, + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${where}: the backend kept the same coarse verdict but failed its detailed oracle — ` + + `${observed.backendMismatch || 'unspecified assertion mismatch'}.`, + remediation: backendOracleRemediation(executionBackend), + }; + } + + // --- 5. Detector-only disagreements ---------------------------------------- + // The engine behaved as pinned, so any mismatch is on the linter side. + if (expectFlagged && !detectorFlagged) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_SILENT, + evidence: + `${where}: expected ${expected.detectorCount} diagnostic(s) but the detector produced none, ` + + `while the engine behaved as pinned (${describeObservation(observed)}).`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine still exhibits the behavior, so the rule is still wanted — the detector regressed. ` + + `Check, in order: (1) appliesTo/minVersion vs engine ${version}; (2) runtimeOnly — a runtimeOnly ` + + `rule only fires when the lint context's grammarSurface is "runtime-bundle"; (3) required lint ` + + `context (fields/typeMap) that the detector self-suppresses without; (4) the detector's own ` + + `traversal. Do NOT re-pin the expectation to 0 — that would hide a false negative.`, + }, + }; + } + + if (!expectFlagged && detectorFlagged) { + const engineAgrees = backendRejected === true; + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_NOISY, + evidence: + `${where}: expected no diagnostic but the detector emitted ${observed.detectorCount} ` + + `(${describeObservation(observed)}).`, + remediation: engineAgrees + ? { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The engine rejects this query too, so the diagnostic is arguably correct and the ` + + `expectation is what is wrong. Re-pin the ${version} expectation, or choose a control query ` + + `the engine actually accepts.`, + } + : { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine ACCEPTS this query, so the diagnostic is a false positive on ${version}. Narrow ` + + `the detector so it stops matching this shape; if the whole rule no longer applies to any ` + + `supported engine, disable it in ${OSD_PATHS.catalog}.`, + }, + }; + } + + if (observed.detectorCount !== expected.detectorCount) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_COUNT_MISMATCH, + evidence: + `${where}: expected exactly ${expected.detectorCount} diagnostic(s), but the detector ` + + `emitted ${observed.detectorCount} while the backend behaved as pinned.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Restore the detector to emit exactly ${expected.detectorCount} diagnostic(s) for this ` + + `query, or re-pin detectorCount only after confirming the changed multiplicity is intentional.`, + }, + }; + } + + // --- 6. Right verdict, wrong severity/message ------------------------------ + if ( + expected.severity && + detectorFlagged && + (observed.severityMatched === false || + (Array.isArray(observed.severities) && + observed.severities.length > 0 && + !observed.severities.every((s) => s === expected.severity))) + ) { + return { + ...base, + driftClass: DRIFT_CLASSES.SEVERITY_MISMATCH, + evidence: + `${where}: expected severity "${expected.severity}" but the detector emitted ` + + `${JSON.stringify(observed.severities)}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: OSD_PATHS.catalog, + detail: + `Restore "${ruleId}".severity to "${expected.severity}" in the catalog, or — if the downgrade was ` + + `deliberate — re-pin this contract and note that the rule left the enforced default-error set.`, + }, + }; + } + + if (expected.matchMessage && observed.messageMatched !== true) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_MESSAGE_MISMATCH, + evidence: + `${where}: the detector diagnostic no longer contains the contracted message fragment ` + + `${JSON.stringify(expected.matchMessage)}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Restore the diagnostic message asserted by this contract, or update matchMessage only ` + + `after reviewing the new user-facing wording.`, + }, + }; + } + + return null; +} + +/** + * Render drifts as the PR-facing remediation report. Grouped by remediation + * action so the reader sees the decision first ("two rules need version + * scoping") rather than a flat wall of query failures. + */ +export function formatDriftReport(drifts) { + if (drifts.length === 0) { + return 'No engine/linter drift detected.'; + } + const byAction = new Map(); + for (const drift of drifts) { + const action = drift.remediation.action; + if (!byAction.has(action)) byAction.set(action, []); + byAction.get(action).push(drift); + } + + const lines = [ + `PPL lint drift: ${drifts.length} finding(s) across ${new Set(drifts.map((d) => d.version)).size} engine version(s).`, + '', + ]; + // Most urgent action first: a false positive already reaching users outranks a + // stale pinned string. + const order = [ + REMEDIATIONS.ALIGN_EXECUTION_BACKENDS, + REMEDIATIONS.DISABLE_RULE, + REMEDIATIONS.VERSION_SCOPE_RULE, + REMEDIATIONS.UPDATE_DETECTOR, + REMEDIATIONS.REVIEW_BACKEND_ORACLE, + REMEDIATIONS.UPDATE_CONTRACT, + ]; + for (const action of order) { + const group = byAction.get(action); + if (!group || group.length === 0) continue; + lines.push(`## ${action} (${group.length})`); + for (const drift of group) { + const backend = + Array.isArray(drift.executionBackends) && drift.executionBackends.length > 1 + ? drift.executionBackends.join(' vs ') + : drift.executionBackend || 'standard'; + lines.push(`- [${drift.driftClass}] [${backend}] ${drift.evidence}`); + lines.push(` FIX (${drift.remediation.target}): ${drift.remediation.detail}`); + // A rule-level finding (e.g. a grammar rename) has no single query behind it. + if (drift.query) { + lines.push(` QUERY: ${drift.query}`); + } + } + lines.push(''); + } + return lines.join('\n'); +} diff --git a/scripts/ppl-lint/harvest-queries.mjs b/scripts/ppl-lint/harvest-queries.mjs new file mode 100644 index 00000000000..4a19a6b9b50 --- /dev/null +++ b/scripts/ppl-lint/harvest-queries.mjs @@ -0,0 +1,696 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Harvest PPL queries out of OSD's own lint test suite into a discovery corpus. + * + * ## Why this exists + * + * The enforced contract corpus is hand-written and therefore small — around one + * trigger and one control per rule. That is right for an enforced contract (every + * expectation is a reviewed claim) but it is too thin to answer the question that + * decides a remediation: when an engine starts accepting a query a rule flags, is + * the behavior FULLY gone on that version, or only PARTIALLY? + * + * `classifyRelaxationScope` in drift.mjs needs several triggers per rule to tell + * those apart, because they need opposite actions — version-scope the rule away + * (full) versus narrow the detector (partial). With one pinned trigger, a partial + * engine fix is indistinguishable from a total one, and the advice that follows + * ships a false negative. + * + * OSD's lint tests already contain that variety: whoever wrote each detector wrote + * several queries that should fire and several that should not, grouped by rule. + * `invalid-capture-group-name` has three distinct reasons a name is invalid + * (hyphen, leading digit, all digits) in OSD's tests versus one in the contract. + * Harvesting them costs nothing and is exactly the input the rollup needs. + * + * ## What this does NOT do + * + * It does not produce expectations, and the discovery corpus never fails a build. + * A harvested query carries no pinned verdict — `label-discovery.mjs` derives its + * role by running the real detectors, and the engine supplies the other half. That + * is deliberate: auto-deriving an expectation from current behavior can only ever + * confirm current behavior, locking in whatever the detector does today, bugs and + * all. Promotion into the enforced corpus stays a human writing a spec entry. + * + * ## Attribution + * + * A query is attributed to a rule by the innermost enclosing `describe('')` + * whose title is a known catalog rule id (OSD's tests are organized that way, see + * `__tests__/silent_failure_rules.test.ts`). A query with no such ancestor is + * recorded with `ruleId: null` and skipped by the labeler unless `--keep-unowned` + * is passed — guessing an owner from a filename would attribute queries to the + * wrong rule, which is worse than dropping them. + * + * Usage: + * node scripts/ppl-lint/harvest-queries.mjs \ + * --osd \ + * --catalog-rules \ + * --index opensearch-sql_test_index_account \ + * --out discovery-corpus.json + */ + +import fs from 'fs'; +import path from 'path'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-harvest] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-harvest] FATAL: ${message}`); + process.exit(2); +} + +const LINT_TEST_ROOT = 'packages/osd-monaco/src/ppl/lint'; + +/** + * Benchmarks and repro captures are excluded. Bench files hold deliberately + * pathological queries built to be slow rather than to be right or wrong, and + * they would dominate the corpus with near-duplicates. + */ +const EXCLUDED_FILE_PATTERNS = [/\.bench\.test\.ts$/, /\.verify\.test\.ts$/]; +const EXCLUDED_DIRS = new Set([ + '__fixtures__', + '__snapshots__', + 'fixtures', + 'generated', + 'target', +]); + +function parseArgs(argv) { + const args = { + osd: '', + out: 'discovery-corpus.json', + index: '', + catalogRules: [], + keepUnowned: false, + maxPerRule: 40, + specsOut: '', + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--osd') args.osd = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--index') args.index = next(); + else if (arg === '--catalog-rules') args.catalogRules = readRuleList(next()); + else if (arg === '--keep-unowned') args.keepUnowned = true; + else if (arg === '--max-per-rule') args.maxPerRule = Number(next()); + else if (arg === '--specs-out') args.specsOut = next(); + else fatal(`unknown argument "${arg}"`); + } + if (!args.osd) fatal('--osd is required'); + return args; +} + +/** Rule ids either inline (`a,b,c`) or from a file (`@path`), one per line or JSON. */ +function readRuleList(value) { + if (!value.startsWith('@')) { + return value + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + } + const file = value.slice(1); + if (!fs.existsSync(file)) fatal(`--catalog-rules file not found: ${file}`); + const raw = fs.readFileSync(file, 'utf8'); + try { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) { + // Accept both a bare id list and OSD's rules_catalog.json shape. + return parsed.map((e) => (typeof e === 'string' ? e : e && e.id)).filter(Boolean); + } + } catch { + // not JSON; fall through to line-delimited + } + return raw + .split('\n') + .map((s) => s.trim()) + .filter((s) => s && !s.startsWith('#')); +} + +/** Every lint test file under the lint package, excluding generated test data. */ +export function findTestFiles(osdRoot) { + const files = []; + const root = path.join(osdRoot, LINT_TEST_ROOT); + const visit = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + if (!EXCLUDED_DIRS.has(entry.name)) { + visit(path.join(dir, entry.name)); + } + continue; + } + const name = entry.name; + if (!name.endsWith('.test.ts') && !name.endsWith('.test.tsx')) continue; + if (EXCLUDED_FILE_PATTERNS.some((re) => re.test(name))) continue; + files.push(path.join(dir, name)); + } + }; + if (fs.existsSync(root)) visit(root); + return files.sort(); +} + +/** + * Track which `describe(...)` blocks enclose a given offset, so a query can be + * attributed to the rule whose block it sits in. + * + * Brace counting is enough here and a real TS parser is not worth the dependency: + * these are test files whose describes are conventional `describe('x', () => {` + * calls. The failure mode of miscounting is a query attributed to an outer block + * (or to none), which the `ruleId: null` path already handles safely — never a + * query attributed to a rule that does not own it, because titles must match a + * known catalog id. + */ +function buildDescribeScopes(source) { + const scopes = []; + const describeRe = /\bdescribe(?:\.\w+)?\s*\(\s*(['"`])((?:\\.|(?!\1).)*)\1/g; + let match; + while ((match = describeRe.exec(source)) !== null) { + const title = match[2]; + // Find the block's opening brace after the describe call, then its matching + // close, ignoring braces inside strings and comments. + const braceStart = source.indexOf('{', match.index + match[0].length); + if (braceStart === -1) continue; + const end = matchBrace(source, braceStart); + scopes.push({ title, start: braceStart, end: end === -1 ? source.length : end }); + } + return scopes; +} + +/** Index of the `}` matching the `{` at `open`, or -1. Skips strings/comments. */ +function matchBrace(source, open) { + let depth = 0; + for (let i = open; i < source.length; i++) { + const ch = source[i]; + if (ch === '/' && source[i + 1] === '/') { + const nl = source.indexOf('\n', i); + i = nl === -1 ? source.length : nl; + continue; + } + if (ch === '/' && source[i + 1] === '*') { + const close = source.indexOf('*/', i + 2); + i = close === -1 ? source.length : close + 1; + continue; + } + if (ch === "'" || ch === '"' || ch === '`') { + i = skipString(source, i); + continue; + } + if (ch === '{') depth++; + else if (ch === '}') { + depth--; + if (depth === 0) return i; + } + } + return -1; +} + +/** Index of the closing quote of the string starting at `start`. */ +function skipString(source, start) { + const quote = source[start]; + for (let i = start + 1; i < source.length; i++) { + if (source[i] === '\\') { + i++; + continue; + } + if (source[i] === quote) return i; + } + return source.length; +} + +/** + * PPL query literals. Anchored on the commands that can open a PPL statement, so + * arbitrary strings in a test file are not mistaken for queries. `search` and + * `source=`/`index=` are the real openers; `describe` is deliberately absent + * because it collides with the test function of the same name. + */ +const QUERY_RE = /(['"`])((?:source\s*=|index\s*=|search\s+)(?:\\.|(?!\1).)*)\1/g; + +/** + * A harvested literal is a JS string literal, so its escapes are JS-level. The + * detectors want the RUNTIME string: `'(?\\\\d+)'` in a test source is + * the four characters `\\d+` on the wire... which is itself a regex escape the + * engine sees. Unescaping the JS layer (and only that layer) is what makes a + * harvested query identical to what the test actually linted. + */ +function unescapeJsString(raw) { + return raw.replace(/\\(u\{[0-9a-fA-F]+\}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2}|.)/g, (all, esc) => { + switch (esc[0]) { + case 'n': + return '\n'; + case 't': + return '\t'; + case 'r': + return '\r'; + case 'b': + return '\b'; + case 'f': + return '\f'; + case 'v': + return '\v'; + case '0': + return '\0'; + case 'x': + return String.fromCharCode(parseInt(esc.slice(1), 16)); + case 'u': + return esc[1] === '{' + ? String.fromCodePoint(parseInt(esc.slice(2, -1), 16)) + : String.fromCharCode(parseInt(esc.slice(1), 16)); + default: + // Covers \\ \' \" \` and any other single-character escape. + return esc; + } + }); +} + +/** + * Rewrite the test's index onto the backend fixture's index. + * + * OSD's unit tests lint against invented sources (`source=logs`, `search + * accounts`) that do not exist in the SQL integ-test cluster. A harvested query + * must name a real index or the engine rejects it for a reason that has nothing to + * do with the rule — which would read as "the engine rejects this" and be counted + * as a trigger holding. + * + * Only the leading source/index/search clause is rewritten. Subsearch sources + * inside the query body are rewritten too, since those are equally invented. + * Returns null when no rewrite was possible, so the caller can drop the query + * rather than send an unresolvable index to the cluster. + */ +export function remapIndex(query, targetIndex) { + if (!targetIndex) return query; + // A WILDCARD source is left alone. `wildcard-source-zero-match` exists precisely + // to flag a pattern matching no visible index, so rewriting `source=\`nope-*\`` + // to a concrete index destroys the only thing the rule detects — the query became + // a control and the rule reported zero triggers. More generally, a wildcard is + // part of the query's meaning rather than an incidental index name, and the + // engine resolves a non-matching pattern on its own without erroring. + if (/\bsource\s*=\s*`?[^\s`|,]*\*/.test(query) || /\bindex\s*=\s*`?[^\s`|,]*\*/.test(query)) { + return query; + } + let out = query + .replace(/\bsource\s*=\s*`[^`]+`/g, `source=${targetIndex}`) + .replace(/\bsource\s*=\s*[A-Za-z_][\w.*-]*/g, `source=${targetIndex}`) + .replace(/\bindex\s*=\s*`[^`]+`/g, `index=${targetIndex}`) + .replace(/\bindex\s*=\s*[A-Za-z_][\w.*-]*/g, `index=${targetIndex}`); + // `search ` — the bare-index form. Only the opener, and only when the + // token is not already a keyword-led clause. + out = out.replace(/^(\s*search\s+)(?!source\s*=|index\s*=)([A-Za-z_][\w.*-]*)/, `$1${targetIndex}`); + return out; +} + +/** + * Does this query reference fields the backend fixture will not have? + * + * A harvested query naming `durationNano` against the `account` index is rejected + * for an unknown field, not for the rule's condition. Counting that as "the engine + * still rejects" would fake a partial fix and send someone to narrow a healthy + * detector — the same class of vacuous result the enforced contract's + * control-also-rejected guard exists to prevent. + * + * This cannot be decided statically, so it is not decided here: the query is + * harvested with the field names it mentions recorded, and the labeler drops the + * ones the fixture cannot satisfy. Extracting identifiers is best-effort and + * deliberately over-broad (it will include command keywords), because the labeler + * intersects against the fixture's real field list rather than trusting this. + */ +export function referencedIdentifiers(query) { + const ids = new Set(); + for (const match of query.matchAll(/\b([A-Za-z_][\w.]*)\b/g)) { + ids.add(match[1]); + } + return [...ids]; +} + +/** + * The rule a `describe(...)` title names, or null. + * + * OSD's titles are not bare ids — a rule-scoped suite reads + * `describe('rex-scan-cost (compiled surface)')` or + * `describe('field-validation alternate-source suppression')`. Requiring an exact + * match dropped ~75% of harvestable queries, all of them from files dedicated to a + * single rule, so the title is matched as a PREFIX at a word boundary. + * + * Prefix-only is the point: matching a rule id anywhere in the title would let + * `describe('does not fire on rex-scan-cost candidates')`, nested under a + * different rule, steal the attribution. A title that merely mentions another rule + * mid-sentence is not that rule's suite. Longest match wins, so + * `field-validation-shape` is preferred over `field-validation` when both exist. + */ +export function ruleFromDescribeTitle(title, knownRules) { + const text = String(title || ''); + let best = null; + for (const ruleId of knownRules) { + if (!text.startsWith(ruleId)) continue; + // Must end at a word boundary: `head-without-sorting` is not `head-without-sort`. + const after = text.charAt(ruleId.length); + if (after && /[\w-]/.test(after)) continue; + if (!best || ruleId.length > best.length) best = ruleId; + } + return best; +} + +/** + * Harvest the lint CONTEXT a test file declares, not just its queries. + * + * Seven of nineteen rules are `needsContext: true` — they self-suppress without a + * `typeMap`, and `enabled-false-object` additionally needs `disabledObjectFields`. + * Harvesting their queries without their context produced 26 `rex-scan-cost` + * queries and zero triggers: the detector never ran, which is indistinguishable in + * the report from a rule that fired on nothing. + * + * The context is taken from the file rather than invented because the OSD test + * author wrote it to make exactly these queries fire. A hand-written substitute + * would be a guess about which field types each query depends on, and a wrong guess + * silently suppresses the detector again. + * + * Parses the conventional shapes those files use: + * const typeMap = new Map([ ['age', 'long'], ... ]); + * disabledObjectFields: new Set(['raw']), + * + * Regex rather than a TS parser for the same reason as `buildDescribeScopes`: these + * are conventional declarations, and the failure mode is an empty context, which + * leaves the rule visibly at zero triggers rather than producing a wrong verdict. + */ +export function harvestContext(source) { + const typeMap = {}; + // Every `['name', 'type']` pair inside a `new Map...([ ... ])` initializer. Scoped + // to Map literals so unrelated tuple arrays in the file are not picked up. + for (const mapMatch of source.matchAll(/new Map\s*(?:<[^>]*>)?\s*\(\s*\[([\s\S]*?)\]\s*\)/g)) { + for (const pair of mapMatch[1].matchAll(/\[\s*'([^']+)'\s*,\s*'([^']+)'\s*\]/g)) { + typeMap[pair[1]] = pair[2]; + } + } + + const disabledObjectFields = []; + for (const match of source.matchAll(/disabledObjectFields:\s*new Set\s*\(\s*\[([^\]]*)\]/g)) { + for (const item of match[1].matchAll(/'([^']+)'/g)) { + disabledObjectFields.push(item[1]); + } + } + + return { + typeMap, + disabledObjectFields: [...new Set(disabledObjectFields)], + }; +} + +/** Harvest one file into `{ ruleId, query, source }` records. */ +export function harvestFile(source, { file, knownRules, index }) { + const context = harvestContext(source); + const scopes = buildDescribeScopes(source); + const known = new Set(knownRules || []); + const out = []; + for (const match of source.matchAll(QUERY_RE)) { + const raw = match[2]; + const query = unescapeJsString(raw); + // Template literals with interpolation are not real queries — the `${...}` is + // a placeholder the test fills at runtime, and sending it to the engine tests + // nothing. Dropped rather than guessed at. + if (/\$\{/.test(query)) continue; + // A query must have at least one pipe or be a bare source read; anything + // shorter is usually a fragment asserted against, not a lintable statement. + if (query.trim().length < 8) continue; + + // Innermost enclosing describe whose title is a known rule id. + const at = match.index; + const enclosing = scopes + .filter((s) => at > s.start && at < s.end) + .sort((a, b) => b.start - a.start); + // Innermost first: a query inside `describe('flat-object-subfield')` nested in + // `describe('silent-failure rules')` belongs to the specific rule, not the file. + let owner = null; + for (const scope of enclosing) { + owner = ruleFromDescribeTitle(scope.title, known); + if (owner) break; + } + const line = source.slice(0, at).split('\n').length; + + out.push({ + ruleId: owner, + query: index ? remapIndex(query, index) : query, + originalQuery: query, + identifiers: referencedIdentifiers(query), + source: `${file}:${line}`, + // Carried per query, not per rule: two files can test the same rule with + // different field types, and merging them would give a query a typeMap its + // own test never used. + context, + }); + } + return out; +} + +/** + * Emit the harvested corpus as spec files the EXISTING detector runner can consume. + * + * `run-frontend-contract.mjs` is expectation-driven: it walks `expectations[]`, + * scores each query against a pinned `detectorCount`, and records the real `actual` + * count in its report either way. Discovery needs only that `actual`, so rather + * than teach the runner a second mode — which would risk changing how the ENFORCED + * check behaves — the corpus is written out as ordinary specs whose expectations are + * deliberately arbitrary. + * + * Two consequences, both intended: + * - The runner will report failures for every query whose real count differs from + * the placeholder. Those are meaningless here and the caller discards the exit + * code; only `detector-report.json` is read. This is why discovery must never + * be wired to a required check. + * - `grammarSurface: 'both'` so a rule is scored on whichever surface the leg + * ran, and `schedule: 'nightly'` to match how the aggregate legs invoke it. + * + * One spec per rule, because the runner keys wiring checks off `spec.ruleId`. + */ +export function toRunnerSpecs(corpus) { + // Grouped by rule AND by harvested context. A `needsContext` rule self-suppresses + // without a typeMap, so a query has to be scored under the context its own test + // declared — merging two files' contexts into one spec would hand a query field + // types its test never used, and the resulting verdict would describe a scenario + // nobody wrote. + const byRule = new Map(); + for (const [i, entry] of (corpus.queries || []).entries()) { + if (!entry.ruleId) continue; + const contextKey = JSON.stringify(entry.context || {}); + const key = `${entry.ruleId}${contextKey}`; + if (!byRule.has(key)) { + byRule.set(key, { ruleId: entry.ruleId, context: entry.context, entries: [] }); + } + byRule.get(key).entries.push({ ...entry, name: entry.name || `discovery-${i}` }); + } + + // A rule with more than one distinct context needs more than one spec file, so + // names are suffixed only when that happens — keeping the common case readable. + const groupCount = new Map(); + for (const { ruleId } of byRule.values()) { + groupCount.set(ruleId, (groupCount.get(ruleId) || 0) + 1); + } + const seenPerRule = new Map(); + + const specs = []; + for (const [, group] of [...byRule].sort((a, b) => a[0].localeCompare(b[0]))) { + const { ruleId, context } = group; + const entries = group.entries; + const queries = {}; + const expected = {}; + for (const entry of entries) { + // Every query is declared a trigger: the runner needs SOME role, and the real + // role is derived later from the detector's actual output. Calling them all + // triggers keeps the runner from applying its control-specific cross-checks, + // whose failures would be pure noise on a corpus with no pinned verdicts. + queries[entry.name] = { role: 'trigger', query: entry.query }; + expected[entry.name] = { detectorCount: 0 }; + } + const ordinal = (seenPerRule.get(ruleId) || 0) + 1; + seenPerRule.set(ruleId, ordinal); + const suffix = groupCount.get(ruleId) > 1 ? `.${ordinal}` : ''; + + const typeMap = (context && context.typeMap) || {}; + const disabledObjectFields = (context && context.disabledObjectFields) || []; + const frontendContext = { isCalcite: true }; + if (Object.keys(typeMap).length > 0) { + // `deriveFromMapping` is what the runner turns into `fields` + `typeMap`, the + // context every `needsContext` rule requires before it will emit anything. + frontendContext.deriveFromMapping = typeMap; + } + // Always supplied, independent of the typeMap. `wildcard-source-zero-match` + // reads ONLY `visibleIndices` and self-suppresses on an empty list (otherwise + // every wildcard would false-fire "matched 0 of 0") — and its test file declares + // no typeMap, so keying this off the mapping left the rule permanently inert. + frontendContext.visibleIndices = ['{{index}}']; + if (disabledObjectFields.length > 0) { + frontendContext.disabledObjectFields = disabledObjectFields; + } + // Two rules ship `enabled: false` and only run when the host overrides them. + // Without this they are inert and every harvested query reads as a control. + frontendContext.forceEnable = true; + + specs.push({ + fileName: `${ruleId}${suffix}.discovery.spec.json`, + spec: { + schemaVersion: 3, + ruleId, + grammarSurface: 'both', + schedule: 'nightly', + // No `wiring` block: the runner deep-equals it against the catalog when + // present, and a mismatch there would fail the run for a reason that has + // nothing to do with discovery. + index: corpus.index || undefined, + frontendContext, + queries, + // A single open expectation so exactly one entry matches every engine + // version; the pinned counts are placeholders (see the note above). + expectations: [{ version: '', queries: expected }], + }, + }); + } + return specs; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const files = findTestFiles(args.osd); + if (files.length === 0) { + fatal(`no lint test files found under ${args.osd}; is --osd an OSD checkout root?`); + } + + const records = []; + for (const file of files) { + const source = fs.readFileSync(file, 'utf8'); + const rel = path.relative(args.osd, file); + records.push( + ...harvestFile(source, { file: rel, knownRules: args.catalogRules, index: args.index }) + ); + } + + // Dedupe on (ruleId, query): the same query legitimately appears in several + // tests, and running it repeatedly against the cluster buys nothing. + const seen = new Set(); + const unique = []; + for (const record of records) { + const key = `${record.ruleId}::${record.query}`; + if (seen.has(key)) continue; + seen.add(key); + unique.push(record); + } + + const owned = unique.filter((r) => r.ruleId); + const unowned = unique.filter((r) => !r.ruleId); + + // Cap per rule so one heavily-tested rule cannot dominate a leg's runtime. The + // drop is LOGGED rather than silent: a truncated corpus that reads as complete + // is how a coverage gap hides. + const byRule = new Map(); + for (const record of owned) { + if (!byRule.has(record.ruleId)) byRule.set(record.ruleId, []); + byRule.get(record.ruleId).push(record); + } + const kept = []; + for (const [ruleId, group] of [...byRule].sort()) { + if (group.length > args.maxPerRule) { + log( + `NOTE: ${ruleId} harvested ${group.length} queries; keeping the first ${args.maxPerRule} ` + + `(--max-per-rule). ${group.length - args.maxPerRule} dropped.` + ); + } + kept.push(...group.slice(0, args.maxPerRule)); + } + + // Names are assigned ONCE, here, and every downstream artifact keys off them. If + // the detector runner and the engine probe derived names independently they could + // disagree, and every row would silently lose its counterpart — the whole corpus + // would read as unobserved rather than as a bug. + kept.forEach((entry, i) => { + entry.name = `discovery-${i}`; + }); + + const corpus = { + schemaVersion: 1, + kind: 'discovery', + // Stated in the artifact itself so no downstream consumer can mistake this for + // the enforced corpus and start failing builds on it. + enforced: false, + note: + 'Auto-harvested from OSD lint tests. Roles are assigned by label-discovery.mjs from real ' + + 'detector output; there are no pinned expectations and this corpus must never fail a build.', + index: args.index || null, + sourceFiles: files.map((f) => path.relative(args.osd, f)), + ruleCoverage: args.catalogRules + .map((ruleId) => { + const entries = kept.filter((entry) => entry.ruleId === ruleId); + return { + ruleId, + filesScanned: [ + ...new Set(entries.map((entry) => entry.source.split(':')[0])), + ].sort(), + ownedQueryCount: entries.length, + explicitException: null, + }; + }) + .sort((a, b) => a.ruleId.localeCompare(b.ruleId)), + exceptions: [], + queries: kept, + unowned: args.keepUnowned ? unowned : [], + stats: { + files: files.length, + harvested: records.length, + unique: unique.length, + owned: kept.length, + unowned: unowned.length, + rules: byRule.size, + }, + }; + + fs.writeFileSync(args.out, JSON.stringify(corpus, null, 2)); + log( + `wrote ${args.out}: ${kept.length} owned query(s) across ${byRule.size} rule(s) from ` + + `${files.length} file(s); ${unowned.length} unattributed` + + (args.keepUnowned ? ' (kept)' : ' (dropped)') + ); + + // Optional: the same corpus as spec files, so the existing detector runner can + // produce real diagnostic counts for it without being modified. + if (args.specsOut) { + fs.mkdirSync(args.specsOut, { recursive: true }); + const specs = toRunnerSpecs(corpus); + for (const { fileName, spec } of specs) { + fs.writeFileSync(path.join(args.specsOut, fileName), JSON.stringify(spec, null, 2)); + } + fs.writeFileSync( + path.join(args.specsOut, 'manifest.json'), + JSON.stringify( + { + schemaVersion: 3, + description: + 'AUTO-GENERATED discovery corpus. No reviewed expectations; the pinned counts are ' + + 'placeholders. Never list these under defaultError and never wire them to a required check.', + contracts: specs.map((s) => s.fileName), + // Empty on purpose: `defaultError` is the ENFORCED set, and nothing here is + // enforced. A non-empty value would make the aggregator fail the build on + // auto-generated expectations. + defaultError: [], + }, + null, + 2 + ) + ); + log(`wrote ${specs.length} runner spec(s) to ${args.specsOut}`); + } + for (const [ruleId, group] of [...byRule].sort()) { + log(` ${ruleId}: ${Math.min(group.length, args.maxPerRule)}`); + } +} + +// Importable for unit tests; only runs the CLI when executed directly. +if (process.argv[1] && path.resolve(process.argv[1]).endsWith('harvest-queries.mjs')) { + main(); +} diff --git a/scripts/ppl-lint/label-discovery.mjs b/scripts/ppl-lint/label-discovery.mjs new file mode 100644 index 00000000000..487975c2fa3 --- /dev/null +++ b/scripts/ppl-lint/label-discovery.mjs @@ -0,0 +1,491 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Assign roles to harvested queries and report detector/engine disagreements. + * + * ## Why roles are derived, not authored + * + * A trigger is a query the rule's detector fires on; a control is one it stays + * silent on. That is mechanically checkable, so it is checked rather than declared: + * the enforced corpus's hand-set `role` field carries a reviewer's intent, but for + * a harvested corpus of 100+ queries hand-labelling is both the bottleneck and a + * source of error. This script reads the detector report the OSD runner already + * produces and labels from it. + * + * Deriving roles is safe. Deriving EXPECTATIONS would not be: an expectation + * auto-set from current behavior can only ever confirm current behavior, locking in + * whatever the detector does today including its bugs. So this script pins nothing + * and never fails a build. It emits findings. + * + * ## The finding it exists for + * + * With both halves observed, the interesting cell needs no expected output at all: + * + * detector engine meaning + * silent rejects possible FALSE NEGATIVE + * fires accepts possible FALSE POSITIVE + * fires rejects agreement + * silent accepts agreement + * + * "Possible", not "confirmed", and the asymmetry is deliberate. A false positive + * is nearly conclusive: the engine ran the query fine and the linter called it + * broken. A false negative is much weaker — the engine may have rejected the query + * for a reason that has nothing to do with this rule (an unknown field, an index + * that does not exist, a command the version predates), in which case the linter + * was right to stay quiet. Both are reported, ranked, and neither is ever asserted. + * + * ## What this feeds + * + * `classifyRelaxationScope` needs several triggers per rule to tell a FULL engine + * fix (version-scope the rule away) from a PARTIAL one (narrow the detector). This + * corpus is where that trigger variety comes from. For that use it needs only + * "does any trigger still get rejected" — a single counterexample settles the + * question, which is why no pinned verdict is required. + * + * Usage: + * node scripts/ppl-lint/label-discovery.mjs \ + * --corpus discovery-corpus.json \ + * --detector discovery-detector-report.json \ + * --backend discovery-backend-report.json \ + * --fixture-fields account-fields.json \ + * --out discovery-findings.json [--summary $GITHUB_STEP_SUMMARY] + */ + +import fs from 'fs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-discovery] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-discovery] FATAL: ${message}`); + process.exit(2); +} + +/** Roles a harvested query can be assigned. */ +export const ROLES = { + TRIGGER: 'trigger', + CONTROL: 'control', + UNKNOWN: 'unknown', +}; + +/** Finding kinds, ranked by how conclusive they are. */ +export const FINDINGS = { + FALSE_POSITIVE: 'possible-false-positive', + FALSE_NEGATIVE: 'possible-false-negative', +}; + +function parseArgs(argv) { + const args = { + corpus: '', + detector: '', + backend: '', + fixtureFields: '', + out: 'discovery-findings.json', + summary: '', + version: '', + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--corpus') args.corpus = next(); + else if (arg === '--detector') args.detector = next(); + else if (arg === '--backend') args.backend = next(); + else if (arg === '--fixture-fields') args.fixtureFields = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--summary') args.summary = next(); + else if (arg === '--version') args.version = next(); + else fatal(`unknown argument "${arg}"`); + } + if (!args.corpus) fatal('--corpus is required'); + if (!args.detector) fatal('--detector is required'); + return args; +} + +function readJson(file, { optional = false } = {}) { + if (!fs.existsSync(file)) { + if (optional) return undefined; + fatal(`expected file not found: ${file}`); + } + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + if (optional) return undefined; + fatal(`could not parse ${file}: ${error.message}`); + } + return undefined; +} + +/** + * Reasons an engine rejection tells us nothing about the rule under test. + * + * This is the guard that keeps the false-negative side honest. A harvested query + * mentions whatever fields its original OSD unit test invented, and those rarely + * exist in the SQL integ-test index. The engine then rejects for an unknown field + * — and reading that as "the engine rejects this, so the silent detector is a false + * negative" would generate a finding per harvested query and bury the real ones. + * + * Also excluded: a command the engine version does not have. Same logic as the + * enforced corpus's control-also-rejected guard — "unsupported command" is not + * evidence about a rule's specific condition. + */ +export const UNINFORMATIVE_REJECTION_PATTERNS = [ + { re: /can't resolve|cannot resolve|unknown field|no such field|field \[[^\]]+\] not found/i, why: 'unknown field' }, + { re: /IndexNotFoundException|no such index|index \[[^\]]+\] (does not exist|not found)/i, why: 'missing index' }, + { re: /unsupported (command|operation)|is not supported|not yet supported/i, why: 'unsupported command' }, + { re: /SyntaxCheckException|ParseException|mismatched input|extraneous input/i, why: 'syntax error' }, +]; + +/** + * Is this rejection informative about the rule, or an artifact of the harvested + * query not fitting the fixture? Returns the reason it is uninformative, or null. + * + * A syntax error counts as uninformative on purpose. A harvested query that the + * grammar cannot even parse says nothing about a semantic rule — and after index + * remapping some harvested queries genuinely are malformed (a join whose right-hand + * index was a bare identifier the remap could not reach). Treating those as + * evidence would be the vacuous-finding equivalent of a timed-out leg. + */ +export function uninformativeRejection(backendType, backendReason) { + const text = `${backendType || ''} ${backendReason || ''}`; + for (const { re, why } of UNINFORMATIVE_REJECTION_PATTERNS) { + if (re.test(text)) return why; + } + return null; +} + +/** + * Label one query and decide whether it is a finding. + * + * `detectorCount` and `backendRejected` come from the two observation halves. + * `backendRejected === undefined` means no verdict arrived, which is a third state: + * the query is labelled but produces no finding, because a leg that did not answer + * must never generate linter advice. + */ +export function labelQuery({ + ruleId, + query, + detectorCount, + backendRejected, + backendType, + backendReason, + severities = [], +}) { + const fired = (detectorCount || 0) > 0; + // An advisory diagnostic is one the engine will never contradict: `info` severity + // marks cost or non-determinism, not an error the engine would refuse. Read from + // the severities the detector actually EMITTED rather than from the catalog, so a + // rule that emits mixed severities is judged on what this query produced. + const advisory = fired && severities.length > 0 && severities.every((s) => s === 'info'); + const role = fired ? ROLES.TRIGGER : ROLES.CONTROL; + const base = { + ruleId, + query, + role, + detectorCount: detectorCount || 0, + backendRejected, + severities, + }; + + // No engine verdict: label the role (which only needs the detector) but claim + // nothing about correctness. + if (typeof backendRejected !== 'boolean') { + return { ...base, role: fired ? ROLES.TRIGGER : ROLES.UNKNOWN, finding: null, unobserved: true }; + } + + // Detector fires, engine accepts → the linter called a working query broken. + // Nearly conclusive: nothing about the fixture can make a query the engine RAN + // into a rule violation. + // + // EXCEPT for advisory rules. `head-without-sort` (info) and `rex-scan-cost` (info) + // flag non-determinism and cost — things the engine executes happily and will + // never reject. For those, "engine accepts + detector fires" is the rule working + // exactly as designed, not a false positive. Without this the report is dominated + // by every advisory rule's every trigger, and the real findings are unreadable. + // + // Severity is the discriminator because it already encodes the distinction: an + // error/warning rule asserts the engine will refuse or mishandle the query, and + // only such a claim can be contradicted by the engine accepting it. + if (fired && backendRejected === false) { + if (advisory) { + return { ...base, finding: null, advisory: true }; + } + return { + ...base, + finding: { + kind: FINDINGS.FALSE_POSITIVE, + evidence: + `the engine ACCEPTED this query but "${ruleId}" emitted ${detectorCount} diagnostic(s). ` + + `A user running this query sees an error marker on a query that works.`, + }, + }; + } + + // Detector silent, engine rejects → possible missed diagnostic, but only if the + // rejection is about something this rule could have caught. + if (!fired && backendRejected === true) { + const uninformative = uninformativeRejection(backendType, backendReason); + if (uninformative) { + return { ...base, finding: null, suppressed: uninformative }; + } + return { + ...base, + finding: { + kind: FINDINGS.FALSE_NEGATIVE, + evidence: + `the engine REJECTED this query (${backendType || 'error'}: ${backendReason || 'no reason'}) ` + + `and "${ruleId}" stayed silent. VERIFY the rejection is this rule's condition before acting — ` + + `a rejection for an unrelated reason is not a missed diagnostic.`, + }, + }; + } + + return { ...base, finding: null }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const corpus = readJson(args.corpus); + const detectorReport = readJson(args.detector); + const backendReport = readJson(args.backend, { optional: true }); + + if (corpus.enforced) { + // A corpus that claims to be enforced does not belong on this path: this script + // pins nothing and exits zero, so running it over the enforced corpus would + // look like validation while asserting nothing. + fatal('--corpus is marked enforced; this script only labels the discovery corpus.'); + } + + const detectorByKey = new Map(); + for (const row of detectorReport.results || []) { + detectorByKey.set(`${row.ruleId}::${row.queryName}`, row); + } + const backendByKey = new Map(); + for (const row of Array.isArray(backendReport) ? backendReport : []) { + backendByKey.set(`${row.ruleId}::${row.queryName}`, row); + } + + const labelled = []; + for (const [i, entry] of (corpus.queries || []).entries()) { + const queryName = entry.name || `discovery-${i}`; + const key = `${entry.ruleId}::${queryName}`; + const detectorRow = detectorByKey.get(key); + const backendRow = backendByKey.get(key); + // Same three-state read as the enforced aggregator: `outcome: "error"` carries + // no verdict, and coercing that absence to "accepted" would manufacture a + // false-positive finding out of a network blip. + const hasVerdict = + !!backendRow && + backendRow.outcome !== 'error' && + (typeof backendRow.rejected === 'boolean' || !!backendRow.observed); + + labelled.push({ + ...labelQuery({ + ruleId: entry.ruleId, + query: entry.query, + detectorCount: detectorRow ? detectorRow.actual : undefined, + severities: (detectorRow && detectorRow.severities) || [], + backendRejected: hasVerdict ? !!backendRow.rejected : undefined, + backendType: backendRow && backendRow.observed ? backendRow.observed.type : undefined, + backendReason: backendRow && backendRow.observed ? backendRow.observed.reason : undefined, + }), + queryName, + source: entry.source, + noDetectorRow: !detectorRow, + }); + } + + const findings = labelled.filter((l) => l.finding); + const byRule = new Map(); + for (const coverage of corpus.ruleCoverage || []) { + byRule.set(coverage.ruleId, { + triggers: [], + controls: [], + unknown: [], + suppressed: 0, + files: new Set(coverage.filesScanned || []), + explicitException: coverage.explicitException || null, + }); + } + for (const row of labelled) { + if (!byRule.has(row.ruleId)) { + byRule.set(row.ruleId, { + triggers: [], + controls: [], + unknown: [], + suppressed: 0, + files: new Set(), + explicitException: null, + }); + } + const bucket = byRule.get(row.ruleId); + if (row.source) bucket.files.add(row.source.split(':')[0]); + if (row.suppressed) bucket.suppressed++; + if (row.role === ROLES.TRIGGER) bucket.triggers.push(row.queryName); + else if (row.role === ROLES.CONTROL) bucket.controls.push(row.queryName); + else bucket.unknown.push(row.queryName); + } + + const report = { + schemaVersion: 1, + kind: 'discovery-findings', + enforced: false, + engineVersion: args.version || detectorReport.engineVersion || null, + surface: detectorReport.surface || null, + // Whether an engine half was supplied at all. Without it the run yields trigger + // counts but structurally cannot yield findings, and the report has to say which + // of those two it is. + differential: backendByKey.size > 0, + stats: { + queries: labelled.length, + triggers: labelled.filter((l) => l.role === ROLES.TRIGGER).length, + controls: labelled.filter((l) => l.role === ROLES.CONTROL).length, + unknown: labelled.filter((l) => l.role === ROLES.UNKNOWN).length, + suppressed: labelled.filter((l) => l.suppressed).length, + // Advisory triggers the engine accepted. Counted so the number is visible: it + // is the single largest category the filter removes, and a silent removal + // would make the corpus look smaller than it is. + advisory: labelled.filter((l) => l.advisory).length, + findings: findings.length, + falsePositives: findings.filter((f) => f.finding.kind === FINDINGS.FALSE_POSITIVE).length, + falseNegatives: findings.filter((f) => f.finding.kind === FINDINGS.FALSE_NEGATIVE).length, + }, + // Per-rule trigger counts are the payload the relaxation rollup consumes: a + // rule with several triggers can distinguish a partial engine fix from a full + // one, and a rule with one cannot. + triggerCoverage: [...byRule] + .map(([ruleId, b]) => ({ + ruleId, + filesScanned: [...b.files].sort(), + triggers: b.triggers.length, + controls: b.controls.length, + unknown: b.unknown.length, + suppressed: b.suppressed, + unattributedQueryCount: (corpus.stats && corpus.stats.unowned) || 0, + explicitException: b.explicitException, + coverageSatisfied: + b.triggers.length + b.controls.length + b.unknown.length > 0 || + !!b.explicitException, + // Below two triggers, "every trigger relaxed" is a single observation and + // cannot support a version-scoping decision. Flagged so the gap is visible + // rather than implied by a number nobody reads. + sufficientForScopeDecision: b.triggers.length >= 2, + })) + .sort((a, b) => a.ruleId.localeCompare(b.ruleId)), + findings: findings.map((f) => ({ + ruleId: f.ruleId, + kind: f.finding.kind, + evidence: f.finding.evidence, + query: f.query, + source: f.source, + })), + labelled, + }; + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + log(`wrote ${args.out}`); + + const markdown = renderMarkdown(report); + // eslint-disable-next-line no-console + console.log(markdown); + if (args.summary) { + try { + fs.appendFileSync(args.summary, markdown + '\n'); + } catch (error) { + log(`WARN: could not write summary to ${args.summary}: ${error.message}`); + } + } + + // ALWAYS exit zero. This corpus has no reviewed expectations, so a finding here + // is a lead to investigate, not a proven defect — failing the build on it would + // block unrelated PRs on the strength of an auto-generated guess. + log( + `discovery: ${report.stats.findings} finding(s) from ${report.stats.queries} query(s) ` + + `(${report.stats.falsePositives} possible false positive(s), ` + + `${report.stats.falseNegatives} possible false negative(s)); ` + + `${report.stats.suppressed} rejection(s) suppressed as uninformative, ` + + `${report.stats.advisory} advisory trigger(s) excluded. Not enforced.` + ); +} + +export function renderMarkdown(report) { + const lines = []; + lines.push('## PPL lint discovery corpus (not enforced)'); + lines.push(''); + lines.push( + `Engine \`${report.engineVersion || 'unknown'}\`${report.surface ? ` (${report.surface})` : ''} — ` + + `${report.stats.queries} harvested query(s): ${report.stats.triggers} trigger, ` + + `${report.stats.controls} control, ${report.stats.unknown} unknown. ` + + `**${report.stats.findings} finding(s)** — these are LEADS, not failures.` + ); + lines.push(''); + // Without an engine half there is nothing to disagree WITH, so the run can only + // count triggers. Saying so beats printing "0 finding(s)" next to a large corpus, + // which reads as "everything agrees" when in fact nothing was compared. + if (!report.differential) { + lines.push( + '> No engine verdicts were supplied, so no agreement was checked and no finding can be ' + + 'produced. Trigger counts below are still valid — they come from the detector alone.' + ); + lines.push(''); + } + + if (report.stats.findings > 0) { + lines.push('### Findings'); + lines.push(''); + // False positives first: a query the engine ran successfully but the linter + // marked broken is nearly conclusive, while a false negative may be a rejection + // for an unrelated reason. + const order = [FINDINGS.FALSE_POSITIVE, FINDINGS.FALSE_NEGATIVE]; + for (const kind of order) { + const group = report.findings.filter((f) => f.kind === kind); + if (group.length === 0) continue; + lines.push(`#### ${kind} (${group.length})`); + for (const f of group) { + lines.push(`- \`${f.ruleId}\`: ${f.evidence}`); + lines.push(` QUERY: \`${f.query}\``); + if (f.source) lines.push(` HARVESTED FROM: ${f.source}`); + } + lines.push(''); + } + } + + lines.push('### Trigger coverage'); + lines.push(''); + lines.push('Whether each rule has enough triggers to tell a PARTIAL engine fix from a FULL one.'); + lines.push(''); + lines.push('| Rule | Triggers | Controls | Enough for a scope decision? |'); + lines.push('| ---- | -------- | -------- | ---------------------------- |'); + for (const row of report.triggerCoverage) { + // Zero triggers and one trigger are different problems and must not read the + // same. No trigger at all means this corpus proves nothing about the rule — + // usually that the harvested queries are all controls, or that the detector + // never fired because it is gated off on this surface. One trigger means the + // rule is observable but a "fully relaxed" verdict would rest on a single case. + let verdict; + if (row.triggers === 0) { + verdict = '**none — no trigger observed**'; + } else if (row.triggers === 1) { + verdict = '**no — 1 trigger only**'; + } else { + verdict = 'yes'; + } + lines.push(`| \`${row.ruleId}\` | ${row.triggers} | ${row.controls} | ${verdict} |`); + } + lines.push(''); + return lines.join('\n'); +} + +// Importable for unit tests; only runs the CLI when executed directly. +if (process.argv[1] && process.argv[1].endsWith('label-discovery.mjs')) { + main(); +} diff --git a/scripts/ppl-lint/plan-compatibility.mjs b/scripts/ppl-lint/plan-compatibility.mjs new file mode 100644 index 00000000000..bdd77e06581 --- /dev/null +++ b/scripts/ppl-lint/plan-compatibility.mjs @@ -0,0 +1,187 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +function fail(message) { + throw new Error(message); +} + +export function parseVersion(value) { + const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(String(value || '').trim()); + if (!match) return undefined; + return { + normalized: `${match[1]}.${match[2]}.${match[3]}`, + parts: match.slice(1, 4).map(Number), + }; +} + +function compareVersions(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] - right[index]; + } + return 0; +} + +export function readPrTarget(buildFile) { + const source = fs.readFileSync(buildFile, 'utf8'); + const match = + /opensearch_version\s*=\s*System\.getProperty\(\s*["']opensearch\.version["']\s*,\s*["']([^"']+)["']\s*\)/.exec( + source + ); + if (!match) { + fail(`could not resolve the default opensearch.version from ${buildFile}`); + } + const parsed = parseVersion(match[1]); + if (!parsed) { + fail(`default opensearch.version ${JSON.stringify(match[1])} is not semantic version X.Y.Z`); + } + return { raw: match[1], normalized: parsed.normalized, parts: parsed.parts }; +} + +export function releaseVersions(text) { + const versions = new Map(); + for (const line of String(text || '').split(/\r?\n/)) { + const refMatch = /refs\/tags\/([^\s^]+)$/.exec(line.trim()); + const candidate = refMatch ? refMatch[1] : line.trim(); + if (!/^\d+\.\d+\.\d+$/.test(candidate)) continue; + const parsed = parseVersion(candidate); + versions.set(parsed.normalized, parsed.parts); + } + return [...versions.entries()] + .map(([version, parts]) => ({ version, parts })) + .sort((left, right) => compareVersions(left.parts, right.parts)); +} + +export function selectLatestGaAtOrBelow(tags, target) { + const eligible = releaseVersions(tags).filter( + (release) => compareVersions(release.parts, target.parts) <= 0 + ); + if (eligible.length === 0) { + fail(`no official GA release tag exists at or below ${target.normalized}`); + } + return eligible.at(-1).version; +} + +function parseArgs(argv) { + const args = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) fail(`unexpected argument ${JSON.stringify(key)}`); + const value = argv[++index]; + if (value === undefined) fail(`${key} requires a value`); + args[key.slice(2)] = value; + } + for (const required of [ + 'build-file', + 'release-tags', + 'compiled-version', + 'sql-sha', + 'osd-repository', + 'osd-ref', + 'out', + ]) { + if (!args[required]) fail(`--${required} is required`); + } + return args; +} + +export function createPlan({ + buildFile, + releaseTags, + compiledVersion, + sqlSha, + osdRepository, + osdRef, +}) { + const target = readPrTarget(buildFile); + const compiled = parseVersion(compiledVersion); + if (!compiled || compiled.normalized !== compiledVersion) { + fail(`compiled version ${JSON.stringify(compiledVersion)} must be exact semantic version X.Y.Z`); + } + const latestGa = selectLatestGaAtOrBelow(releaseTags, target); + + const configurations = [ + { + id: `${compiledVersion}-compiled`, + label: `${compiledVersion} compiled`, + engineVersion: compiledVersion, + surface: 'compiled-simplified', + executionBackend: 'standard', + engineMode: 'legacy', + artifactName: `ppl-lint-observation-${compiledVersion}-compiled`, + exportRuntimeBundle: false, + }, + { + id: 'latest-release-runtime', + label: `Latest release (${latestGa}) runtime`, + engineVersion: latestGa, + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-latest-release-runtime', + exportRuntimeBundle: true, + }, + { + id: 'pr-build-runtime', + label: 'PR runtime', + engineVersion: target.raw, + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-pr-build-runtime', + exportRuntimeBundle: true, + }, + ]; + + return { + schemaVersion: 1, + sqlSha, + prTargetVersion: target.raw, + normalizedPrTarget: target.normalized, + latestEligibleGa: latestGa, + osd: { + repository: osdRepository, + ref: osdRef, + }, + configurations, + releasedTargets: { + include: configurations.slice(0, 2).map((configuration) => ({ + version: configuration.engineVersion, + configuration_id: configuration.id, + surface: configuration.surface, + label: configuration.id.endsWith('-compiled') ? 'compiled' : 'runtime', + export_runtime_bundle: configuration.exportRuntimeBundle, + artifact_name: configuration.artifactName, + })), + }, + }; +} + +function main() { + try { + const args = parseArgs(process.argv.slice(2)); + const plan = createPlan({ + buildFile: args['build-file'], + releaseTags: fs.readFileSync(args['release-tags'], 'utf8'), + compiledVersion: args['compiled-version'], + sqlSha: args['sql-sha'], + osdRepository: args['osd-repository'], + osdRef: args['osd-ref'], + }); + fs.mkdirSync(path.dirname(path.resolve(args.out)), { recursive: true }); + fs.writeFileSync(args.out, `${JSON.stringify(plan, null, 2)}\n`); + process.stdout.write(`${JSON.stringify(plan)}\n`); + } catch (error) { + process.stderr.write(`[ppl-lint-plan] ${error.message}\n`); + process.exitCode = 2; + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/scripts/ppl-lint/probe-discovery-backend.mjs b/scripts/ppl-lint/probe-discovery-backend.mjs new file mode 100644 index 00000000000..dc5d5282a8a --- /dev/null +++ b/scripts/ppl-lint/probe-discovery-backend.mjs @@ -0,0 +1,197 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Run the discovery corpus against a live engine and record each verdict. + * + * This is the engine half of the discovery pipeline, and it deliberately does NOT + * go through `PplLintRuleValidationIT`. The IT is an assertion harness built around + * reviewed contract files: it selects a pinned expectation per query and compares + * against it. Discovery queries have no pinned expectation by design, so there is + * nothing for the IT to assert and no reason to pay for a Gradle test-cluster run. + * All that is needed is `POST /_plugins/_ppl` per query and the verdict recorded — + * which is what this does, in the same report shape the aggregator and the labeler + * already read. + * + * Emits `[{ ruleId, queryName, executionBackend, rejected, outcome, observed: + * { httpStatus, type, reason } }]`, matching `backend-report.json` so + * `label-discovery.mjs` can read either source without a special case. + * + * The `outcome` field carries the distinction everything downstream depends on: + * + * observed the engine answered; `rejected` is a real verdict + * error no answer arrived (timeout, connection refused, unparseable body) + * + * Never collapse `error` into `rejected: false`. That coercion is what turns a + * network blip into "the engine now ACCEPTS this query" and generates a + * false-positive finding against a healthy rule. + * + * Usage: + * node scripts/ppl-lint/probe-discovery-backend.mjs \ + * --corpus discovery-corpus.json \ + * --endpoint http://localhost:9200 \ + * --out discovery-backend-report.json [--timeout-ms 15000] [--concurrency 4] + */ + +import fs from 'fs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-probe] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-probe] FATAL: ${message}`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { + corpus: '', + endpoint: 'http://localhost:9200', + out: 'discovery-backend-report.json', + timeoutMs: 15000, + concurrency: 4, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--corpus') args.corpus = next(); + else if (arg === '--endpoint') args.endpoint = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--timeout-ms') args.timeoutMs = Number(next()); + else if (arg === '--concurrency') args.concurrency = Math.max(1, Number(next())); + else fatal(`unknown argument "${arg}"`); + } + if (!args.corpus) fatal('--corpus is required'); + return args; +} + +/** + * Read one PPL response into a verdict. + * + * A 2xx is acceptance. A 4xx/5xx is rejection, and the engine's `error.type` / + * `error.reason` are extracted because the labeler's uninformative-rejection filter + * keys on them — a rejection for an unknown field must not be read as evidence + * about a lint rule. + * + * Exported so the mapping is unit-testable without a cluster. + */ +export function readResponse({ status, bodyText }) { + let body; + try { + body = bodyText ? JSON.parse(bodyText) : undefined; + } catch { + body = undefined; + } + const error = (body && body.error) || {}; + const rejected = status >= 400; + return { + outcome: 'observed', + rejected, + observed: { + httpStatus: status, + rejected, + ...(rejected + ? { + type: error.type || undefined, + // Truncated: engine reasons can embed a whole stack trace, and the full + // text bloats the report without adding signal for the filter. + reason: typeof error.reason === 'string' ? error.reason.slice(0, 500) : undefined, + } + : {}), + }, + }; +} + +/** One query against the engine. Never throws: a failure becomes `outcome: error`. */ +async function probeOne({ endpoint, query, timeoutMs }) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(`${endpoint.replace(/\/$/, '')}/_plugins/_ppl`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ query }), + signal: controller.signal, + }); + const bodyText = await response.text(); + return readResponse({ status: response.status, bodyText }); + } catch (error) { + // No verdict. Recorded as such rather than guessed at — see the header note. + return { + outcome: 'error', + observed: undefined, + error: String((error && error.message) || error), + }; + } finally { + clearTimeout(timer); + } +} + +/** Run `tasks` with at most `limit` in flight, preserving input order. */ +async function mapLimit(items, limit, fn) { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + for (;;) { + const index = next++; + if (index >= items.length) return; + results[index] = await fn(items[index], index); + } + }); + await Promise.all(workers); + return results; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const corpus = JSON.parse(fs.readFileSync(args.corpus, 'utf8')); + const queries = corpus.queries || []; + if (queries.length === 0) fatal(`corpus ${args.corpus} has no queries`); + + log(`probing ${queries.length} query(s) against ${args.endpoint} (concurrency ${args.concurrency})`); + + const report = await mapLimit(queries, args.concurrency, async (entry, i) => { + const verdict = await probeOne({ + endpoint: args.endpoint, + query: entry.query, + timeoutMs: args.timeoutMs, + }); + return { + ruleId: entry.ruleId, + // Must match the name `label-discovery.mjs` derives, or every row misses its + // detector counterpart and the whole corpus reads as unobserved. + queryName: entry.name || `discovery-${i}`, + role: 'discovery', + query: entry.query, + executionBackend: 'standard', + ...verdict, + }; + }); + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + const errors = report.filter((r) => r.outcome === 'error').length; + const rejected = report.filter((r) => r.rejected === true).length; + log( + `wrote ${args.out}: ${report.length} probed, ${rejected} rejected, ` + + `${report.length - rejected - errors} accepted, ${errors} unobserved` + ); + if (errors > 0) { + // A warning, not a failure: discovery is best-effort and the labeler already + // withholds findings for unobserved queries. Saying nothing would let a leg + // that mostly failed look like a leg that mostly agreed. + log(`WARN: ${errors} query(s) produced no verdict; those yield no findings.`); + } +} + +if (process.argv[1] && process.argv[1].endsWith('probe-discovery-backend.mjs')) { + main().catch((error) => fatal(String((error && error.stack) || error))); +} diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs new file mode 100644 index 00000000000..628bd707ff4 --- /dev/null +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -0,0 +1,1626 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * SQL-owned detector-validation runner for the PPL lint rule validation CI. + * + * This script is executed from inside an OpenSearch-Dashboards (OSD) checkout, + * for example: + * + * cd .ci/OpenSearch-Dashboards + * PPL_LINT_CONTRACT_DIR= \ + * PPL_LINT_SCHEDULE=pr \ + * PPL_LINT_GRAMMAR_BUNDLE= \ + * PPL_LINT_TARGET_MANIFEST= \ + * PPL_LINT_BACKEND_REPORT= \ + * PPL_LINT_REPORT= \ + * node -r ./src/setup_node_env \ + * "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + * + * `node -r ./src/setup_node_env` installs OSD's process-wide auto-transpilation + * hook (`@osd/optimizer`'s `registerNodeAutoTranspilation`), which transpiles + * `src/plugins/**` and `packages/osd-monaco/src/**` TypeScript on `require()` + * regardless of where the entry script lives. That is what lets this SQL-owned + * `.mjs` load OSD's Node-safe headless lint API without OSD's own Jest. + * + * This is the detector half of a schema-v3/v4 cross-repository differential + * contract (see integ-test/src/test/resources/ppl-lint/contracts/*.spec.json). + * Unlike the earlier PoC — which linted with the compiled analyzer or a + * hand-rolled reparse against OSD `main`'s checked-in grammar — it lints against + * the *candidate* runtime grammar bundle the SQL backend job exported, via OSD's + * production headless API (`headless_ppl_lint`). Both halves therefore validate + * the exact same candidate grammar (design §4.3). + * + * It asserts, per contract: + * 1. Wiring: the OSD catalog entry deep-equals the contract's `wiring` block, + * so a silently removed/retyped/re-gated/re-severitied detector reds the + * build. + * 2. Detector: for the single version expectation that matches the candidate + * backend version, each query emits exactly the contracted number of + * `ruleId` diagnostics at the contracted severity. + * 3. Differential (when PPL_LINT_BACKEND_REPORT is supplied): the observed + * backend behavior for each query agrees with the observed detector output + * — a trigger the detector flags is one the backend rejected; a control the + * detector passes is one the backend accepted (design §3.2, §4.3). + * 4. Coverage census: records whether every enabled catalog rule has a + * contract file. Census drift is report-only unless explicitly enforced. + * + * ## Two grammar surfaces + * + * OSD ships lint on TWO surfaces, and a user gets whichever one their session + * resolves to: + * + * runtime-bundle the candidate grammar the engine exported. Requires + * `GET /_plugins/_ppl/_grammar`, which landed in 3.6. + * compiled-simplified OSD's own checked-in grammar, used whenever the runtime + * bundle is unavailable — no dataset selected, an engine + * below 3.6, or a bundle that has not loaded yet. This is + * `lintRuntimePPLQuery`'s fallback path, and it runs + * detector logic the runtime path does not (see + * field_validation's text-side pass). + * + * `PPL_LINT_SURFACE` selects which one this run validates; it defaults to + * `runtime-bundle`, so the required check is unchanged. The compiled surface is + * an EXPLICIT opt-in, never a silent fallback: the whole point of the required + * check is that a missing bundle is a hard failure rather than a quiet + * downgrade to OSD's own grammar (which would validate the wrong thing). + * + * The compiled surface is what makes pre-3.6 engine legs meaningful. It also + * carries a mandatory caveat: `runtimeOnly` rules (multisearch/union/replace + * arity) are SKIPPED on it by `lint_runner` because the productions they walk do + * not exist in the compiled grammar. A compiled leg therefore reports them as + * `not-applicable` rather than as zero diagnostics, so the aggregator cannot + * mistake a deliberately-inert rule for a detector that regressed. + */ + +import fs from 'fs'; +import path from 'path'; +import { createRequire } from 'module'; +import { fileURLToPath } from 'url'; + +import { + assertContractSchema, + assertExactQueryCoverage, + assertShippingFrontendOracles, + classifyBackendReportRow, + contractChannel, + indexBackendReport, + normalizeLintWiring, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; + +// OSD's Node-safe headless lint API (design §4.3). Deep-path module; resolved +// against the OSD checkout root, not this script's SQL-repo location. +const HEADLESS_MODULE = 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint'; +const SYNTAX_MODULE = + 'src/plugins/data/public/antlr/opensearch_ppl/runtime_validation_core'; +// The COMPILED-simplified surface: OSD's own checked-in grammar, used when the +// engine cannot export a runtime bundle. See `PPL_LINT_SURFACE` below. +const ANALYZER_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; +// The Monaco-free engine barrel (@osd/monaco/ppl-lint) exposes the catalog; the +// detector registry is a deep import used only for the wiring registration check. +const CATALOG_MODULE = 'packages/osd-monaco/ppl-lint'; +// Source-path fallback for the catalog. The `ppl-lint` subpath is a built export +// that only exists on checkouts that ship it; the compiled surface deliberately +// supports older checkouts (that is the coverage it adds), so fall back to the +// source module, which `setup_node_env` transpiles on require anyway. +const CATALOG_SOURCE_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; +const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/target/ppl/lint/detector_registry.js'; + +/** + * Which grammar surface this run validates. Defaults to `runtime-bundle` so the + * required check's behavior is unchanged; `compiled-simplified` is an explicit + * opt-in for legs whose engine cannot export a bundle. + */ +const SURFACE = (() => { + const requested = process.env.PPL_LINT_SURFACE || 'runtime-bundle'; + if (requested !== 'runtime-bundle' && requested !== 'compiled-simplified') { + // A typo must not silently select the default: that would report compiled + // results under a runtime-bundle label, or vice versa. + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-frontend] FATAL: PPL_LINT_SURFACE must be "runtime-bundle" or ` + + `"compiled-simplified", got "${requested}".` + ); + process.exit(2); + } + return requested; +})(); + +const APPLICABLE_ONLY = process.env.PPL_LINT_APPLICABLE_ONLY === '1'; +const ENGINE_MODE = (() => { + const requested = process.env.PPL_LINT_ENGINE_MODE; + if (requested === undefined || requested === '') { + if (APPLICABLE_ONLY) { + // eslint-disable-next-line no-console + console.error( + '[ppl-lint-frontend] FATAL: PPL_LINT_ENGINE_MODE is required when ' + + 'PPL_LINT_APPLICABLE_ONLY=1.' + ); + process.exit(2); + } + return undefined; + } + if (!['calcite', 'legacy'].includes(requested)) { + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-frontend] FATAL: PPL_LINT_ENGINE_MODE must be "calcite" or "legacy", ` + + `got "${requested}".` + ); + process.exit(2); + } + return requested; +})(); + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-detector-contract] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-detector-contract] FATAL: ${message}`); + process.exit(2); +} + +function loadContractFile(file) { + try { + const spec = JSON.parse(fs.readFileSync(file, 'utf8')); + assertContractSchema(spec); + const grammarSurface = spec.grammarSurface || 'runtime-bundle'; + if (!['runtime-bundle', 'compiled-simplified', 'both'].includes(grammarSurface)) { + throw new Error( + `[${spec.ruleId}] grammarSurface must be "runtime-bundle", ` + + `"compiled-simplified", or "both", got ${JSON.stringify(grammarSurface)}.` + ); + } + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new TypeError(`[${spec.ruleId}] expectations must be a non-empty array.`); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + for (const queryExpectation of Object.values(expectation.queries)) { + // Validate every declared oracle, including ranges not selected by this + // target. Missing route coverage is a supported state; malformed route + // names and oracle kinds are not. + resolveBackendOracle(spec, queryExpectation, 'standard'); + resolveBackendOracle(spec, queryExpectation, 'analytics'); + } + } + return { file, spec }; + } catch (error) { + fatal(`Invalid contract ${file}: ${error.message}`); + } + return undefined; // unreachable +} + +export function assertActiveShippingContracts(contracts, { discovery = false } = {}) { + if (discovery) { + return; + } + for (const { file, spec } of contracts) { + for (const expectation of spec.expectations) { + try { + assertShippingFrontendOracles(spec, expectation); + } catch (error) { + throw new Error(`Invalid active shipping contract ${file}: ${error.message}`); + } + } + } +} + +export function selectManifestContractNames(manifest, includeDormant = false) { + if (!Array.isArray(manifest.contracts)) { + throw new TypeError('manifest.json must have a "contracts" array of file names.'); + } + if (new Set(manifest.contracts).size !== manifest.contracts.length) { + throw new Error('manifest.json "contracts" contains duplicate file names.'); + } + const active = manifest.contracts.map((name) => ({ name, reportOnly: false })); + if (!includeDormant) { + return active; + } + if (!Array.isArray(manifest.dormantContracts)) { + throw new TypeError( + 'PPL_LINT_INCLUDE_DORMANT=1 requires manifest.json "dormantContracts" to be an array.' + ); + } + if (new Set(manifest.dormantContracts).size !== manifest.dormantContracts.length) { + throw new Error('manifest.json "dormantContracts" contains duplicate file names.'); + } + const activeNames = new Set(manifest.contracts); + for (const name of manifest.dormantContracts) { + if (activeNames.has(name)) { + throw new Error( + `manifest.json contract "${name}" cannot be both active and dormant.` + ); + } + } + return [ + ...active, + ...manifest.dormantContracts.map((name) => ({ name, reportOnly: true })), + ]; +} + +/** Load every *.spec.json under the contract dir, honoring manifest.json if present. */ +function loadContracts() { + const dir = process.env.PPL_LINT_CONTRACT_DIR; + const single = process.env.PPL_LINT_CONTRACT_FILE; + + if (single) { + if (!fs.existsSync(single)) { + fatal(`Contract file not found: ${single}`); + } + const contract = loadContractFile(single); + try { + assertActiveShippingContracts([contract], { + discovery: process.env.PPL_LINT_DISCOVERY === '1', + }); + } catch (error) { + fatal(error.message); + } + return { + contracts: [{ ...contract, reportOnly: false }], + activeContracts: [contract], + manifest: { contracts: [path.basename(single)] }, + manifestPath: '', + }; + } + + if (!dir) { + fatal('Set PPL_LINT_CONTRACT_DIR (a directory of *.spec.json) or PPL_LINT_CONTRACT_FILE.'); + } + if (!fs.existsSync(dir)) { + fatal(`Contract directory not found: ${dir}`); + } + + const manifestPath = path.join(dir, 'manifest.json'); + let files; + let selectedFiles; + let manifest; + if (fs.existsSync(manifestPath)) { + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + } catch (error) { + fatal(`Invalid contract manifest ${manifestPath}: ${error.message}`); + } + try { + selectedFiles = selectManifestContractNames( + manifest, + process.env.PPL_LINT_INCLUDE_DORMANT === '1' + ); + } catch (error) { + fatal(error.message); + } + files = selectedFiles.map(({ name }) => path.join(dir, name)); + } else { + files = fs + .readdirSync(dir) + .filter((f) => f.endsWith('.spec.json')) + .sort() + .map((f) => path.join(dir, f)); + selectedFiles = files.map((file) => ({ + name: path.basename(file), + reportOnly: false, + })); + } + + const contracts = files.map((file, index) => { + if (!fs.existsSync(file)) { + fatal(`Contract referenced by manifest not found: ${file}`); + } + return { + ...loadContractFile(file), + reportOnly: selectedFiles[index].reportOnly, + }; + }); + const activeContracts = contracts + .filter(({ reportOnly }) => !reportOnly) + .map(({ file, spec }) => ({ file, spec })); + try { + assertActiveShippingContracts(activeContracts, { + discovery: process.env.PPL_LINT_DISCOVERY === '1', + }); + } catch (error) { + fatal(error.message); + } + return { + contracts, + activeContracts, + manifest: manifest || { contracts: files.map(path.basename) }, + manifestPath, + }; +} + +function loadOsd() { + const osdRoot = process.cwd(); + const require = createRequire(path.join(osdRoot, 'noop.js')); + + const resolveOsd = (relativeModule, { optional = false } = {}) => { + const absolute = path.join(osdRoot, relativeModule); + const exists = + fs.existsSync(absolute) || + fs.existsSync(`${absolute}.ts`) || + fs.existsSync(`${absolute}.js`); + if (!exists) { + if (optional) { + return undefined; + } + fatal( + `Expected OSD module not found under the checkout root: ${relativeModule}\n` + + `Resolved OSD root: ${osdRoot}\n` + + `Run this script from the OSD checkout (e.g. cd .ci/OpenSearch-Dashboards) after bootstrap.` + ); + } + try { + return require(absolute); + } catch (error) { + if (optional) { + return undefined; + } + throw error; + } + }; + + // Prefer the built subpath (what the required check uses); fall back to source + // so a checkout without the built export can still run the compiled surface. + const catalogModule = + resolveOsd(CATALOG_MODULE, { optional: true }) || resolveOsd(CATALOG_SOURCE_MODULE); + const { getBundledCatalog } = catalogModule; + const registry = resolveOsd(DETECTOR_REGISTRY_MODULE, { optional: true }); + if (typeof getBundledCatalog !== 'function') { + fatal(`getBundledCatalog not found in ${CATALOG_MODULE} or ${CATALOG_SOURCE_MODULE}.`); + } + const getDetector = registry && registry.getDetector; + + // On the compiled surface the headless bundle API is not needed at all, and + // requiring it would make this mode unusable on an OSD checkout that predates + // it — precisely the older-version coverage the mode exists to provide. + if (SURFACE === 'compiled-simplified') { + const { PPLLanguageAnalyzer } = resolveOsd(ANALYZER_MODULE); + if (typeof PPLLanguageAnalyzer !== 'function') { + fatal(`PPLLanguageAnalyzer not found in ${ANALYZER_MODULE}.`); + } + const analyzer = new PPLLanguageAnalyzer(); + return { + surface: SURFACE, + // Same (query, grammar, context) shape as the bundle path so the main loop + // does not branch per surface; `grammar` is unused here. + lintQuery: (query, _grammar, context) => { + const analysis = analyzer.analyzeLint(query, context); + return (analysis && analysis.result) || { diagnostics: [] }; + }, + getBundledCatalog, + getDetector, + osdRoot, + }; + } + + const headless = resolveOsd(HEADLESS_MODULE); + const { deserializeBundleOrThrow, lintQueryWithBundle } = headless; + if (typeof deserializeBundleOrThrow !== 'function' || typeof lintQueryWithBundle !== 'function') { + fatal( + `Headless lint API not found in ${HEADLESS_MODULE}. ` + + `Expected exports deserializeBundleOrThrow + lintQueryWithBundle. ` + + `Is the OSD checkout on a branch that ships the headless API (design §4.3)?` + ); + } + const syntaxModule = resolveOsd(SYNTAX_MODULE, { optional: true }); + const validateSyntax = + syntaxModule && typeof syntaxModule.validateQueryWithBundle === 'function' + ? syntaxModule.validateQueryWithBundle + : undefined; + + return { + surface: SURFACE, + deserializeBundleOrThrow, + lintQuery: lintQueryWithBundle, + validateSyntax, + getBundledCatalog, + getDetector, + osdRoot, + }; +} + +/** Load the candidate grammar bundle + deserialize it once (fail loud; CI has no fallback). */ +function loadCandidateGrammar(osd, target) { + const bundlePath = process.env.PPL_LINT_GRAMMAR_BUNDLE; + if (!bundlePath) { + fatal( + 'PPL_LINT_GRAMMAR_BUNDLE is not set. Detector validation lints against the candidate ' + + 'runtime grammar bundle exported by the backend job; there is no compiled fallback.' + ); + } + if (!fs.existsSync(bundlePath)) { + fatal(`Candidate grammar bundle not found: ${bundlePath}`); + } + let bundle; + try { + bundle = JSON.parse(fs.readFileSync(bundlePath, 'utf8')); + } catch (error) { + fatal(`Could not parse grammar bundle ${bundlePath}: ${error.message}`); + } + if (bundle.grammarHash !== target.grammarHash) { + fatal( + `Candidate grammar hash ${JSON.stringify(bundle.grammarHash)} does not match target ` + + `${JSON.stringify(target.grammarHash)}.` + ); + } + if (target.grammarBundle && path.basename(bundlePath) !== target.grammarBundle) { + fatal( + `Candidate grammar filename "${path.basename(bundlePath)}" does not match target ` + + `"${target.grammarBundle}".` + ); + } + try { + return osd.deserializeBundleOrThrow(bundle); + } catch (error) { + fatal(`Could not deserialize candidate grammar bundle: ${error.message}`); + } + return undefined; // unreachable +} + +/** Read and validate the target identity written beside the grammar bundle. */ +function loadTarget() { + const targetPath = process.env.PPL_LINT_TARGET_MANIFEST; + if (!targetPath) { + fatal('PPL_LINT_TARGET_MANIFEST is required.'); + } + if (!fs.existsSync(targetPath)) { + fatal(`Target manifest not found: ${targetPath}`); + } + try { + return normalizeTarget(JSON.parse(fs.readFileSync(targetPath, 'utf8'))); + } catch (error) { + fatal(`Invalid target manifest ${targetPath}: ${error.message}`); + } + return undefined; // unreachable +} + +/** Index the backend report by `${ruleId}::${queryName}` for the differential. */ +function loadBackendReport(target) { + const reportPath = process.env.PPL_LINT_BACKEND_REPORT; + if (!reportPath) { + return undefined; + } + if (!fs.existsSync(reportPath)) { + fatal(`Backend report not found: ${reportPath}`); + } + let entries; + try { + entries = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + } catch (error) { + fatal(`Could not parse backend report ${reportPath}: ${error.message}`); + } + try { + return indexBackendReport(entries, target); + } catch (error) { + fatal(`Invalid backend report ${reportPath}: ${error.message}`); + } + return undefined; // unreachable +} + +/** Coerce "3.8.0-SNAPSHOT" / "3.8" to a comparable [major, minor, patch]. */ +function parseVersion(v) { + if (!v) return undefined; + const m = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(v)); + if (!m) return undefined; + return [Number(m[1]), Number(m[2] || 0), Number(m[3] || 0)]; +} + +function compareVersion(a, b) { + for (let i = 0; i < 3; i++) { + if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + } + return 0; +} + +/** + * Test a space-separated semver range (e.g. ">=3.6.0 <3.8.0") against the + * candidate backend version. An empty range or an unknown version matches (do + * not over-filter). Mirrors PplLintRuleValidationIT.versionMatchesRange. + */ +function versionMatchesRange(range, version) { + if (!range || !range.trim()) return true; + const have = parseVersion(version); + if (!have) return true; + for (const token of range.trim().split(/\s+/)) { + let op = '='; + let ver = token; + if (token.startsWith('>=')) { + op = '>='; + ver = token.slice(2); + } else if (token.startsWith('<=')) { + op = '<='; + ver = token.slice(2); + } else if (token.startsWith('>')) { + op = '>'; + ver = token.slice(1); + } else if (token.startsWith('<')) { + op = '<'; + ver = token.slice(1); + } else if (token.startsWith('=')) { + op = '='; + ver = token.slice(1); + } + const cmp = compareVersion(have, parseVersion(ver) || [0, 0, 0]); + const ok = + (op === '>=' && cmp >= 0) || + (op === '<=' && cmp <= 0) || + (op === '>' && cmp > 0) || + (op === '<' && cmp < 0) || + (op === '=' && cmp === 0); + if (!ok) return false; + } + return true; +} + +export function compatibilityExclusion(spec, version, surface, engineMode) { + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== surface) { + return { + reason: 'surface', + detail: `grammarSurface=${contractSurface}, running ${surface}`, + }; + } + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const have = parseVersion(version); + const min = parseVersion(appliesTo.minVersion); + const max = parseVersion(appliesTo.maxVersion); + if ( + have && + ((min && compareVersion(have, min) < 0) || + (max && compareVersion(have, max) > 0)) + ) { + return { + reason: 'version', + detail: `wiring.appliesTo excludes ${version || 'unknown version'}`, + }; + } + if (appliesTo.engine && appliesTo.engine !== engineMode) { + return { + reason: 'engine', + detail: `wiring.appliesTo.engine=${appliesTo.engine}, running ${engineMode}`, + }; + } + return undefined; +} + +/** + * Select the single expectation that applies to the candidate version + engine. + * Exactly one must match (design §5.3): zero means the rule test does not cover + * this version; more than one means overlapping ranges. Both fail. + */ +function selectExpectation(spec, version, isCalcite, failures, { allowMissing = false } = {}) { + const expectations = spec.expectations || []; + const matches = expectations.filter((exp) => { + if (!versionMatchesRange(exp.version, version)) return false; + if (exp.engine === 'calcite' && isCalcite !== true) return false; + return true; + }); + if (matches.length === 1) { + return matches[0]; + } + const label = version || 'unknown'; + if (matches.length === 0) { + if (!allowMissing) { + failures.push(`[${spec.ruleId}] no version expectation matches backend version ${label}.`); + } + } else { + failures.push( + `[${spec.ruleId}] ${matches.length} expectations match backend version ${label} ` + + '(exactly one required).' + ); + } + return undefined; +} + +/** + * Assert the OSD catalog entry deep-equals the contract's `wiring` block. This is + * the primary OSD-drift tripwire: if a detector is removed, retyped, re-gated or + * its severity changed, this fails before any query runs. + */ +function checkWiring(spec, catalog, getDetector, failures) { + const { ruleId, wiring } = spec; + if (contractChannel(spec) === 'syntax') { + if (!wiring) { + failures.push(`[${ruleId}] contract.wiring is required for strict syntax wiring.`); + } + return { id: ruleId, syntaxCode: wiring && wiring.code }; + } + const entry = catalog.find((c) => c.id === ruleId); + if (!entry) { + failures.push(`[${ruleId}] not present in the OSD bundled catalog.`); + return undefined; + } + if (!wiring) { + failures.push(`[${ruleId}] contract.wiring is required for strict catalog comparison.`); + return entry; + } + + let expected; + let actual; + try { + expected = normalizeLintWiring(ruleId, wiring, `[${ruleId}] contract.wiring`); + actual = normalizeLintWiring(ruleId, entry, `[${ruleId}] catalog`); + } catch (error) { + failures.push(error.message); + return entry; + } + if (JSON.stringify(expected) !== JSON.stringify(actual)) { + failures.push( + `[${ruleId}] normalized wiring mismatch: contract=${JSON.stringify(expected)} ` + + `catalog=${JSON.stringify(actual)}.` + ); + } + + if (wiring.detector && typeof getDetector === 'function' && typeof getDetector(wiring.detector) !== 'function') { + failures.push(`[${ruleId}] has no registered detector "${wiring.detector}".`); + } + + return entry; +} + +/** + * Build the per-contract lint context passed to `lintQueryWithBundle`. Derives + * `fields`/`typeMap` from the `deriveFromMapping` block (a single source shared + * with the backend seeding), pins `dataSourceVersion`/`knownVersion` to the + * candidate backend version so version filtering matches the backend, and sets + * an enable override for default-off rules that declare `forceEnable`. + */ +function buildContext(spec, engineVersion, engineMode) { + const fc = spec.frontendContext || {}; + const context = { + isCalcite: engineMode ? engineMode === 'calcite' : fc.isCalcite !== false, + dataSourceVersion: engineVersion || undefined, + // Pin the "latest verified engine" to the candidate version rather than the + // hardcoded OSD_KNOWN_VERSION ('3.7.0'), which can mis-filter rules near a + // version boundary (design §4.3, D-version). + knownVersion: engineVersion || undefined, + }; + + const mapping = fc.deriveFromMapping; + if (mapping && typeof mapping === 'object') { + const fields = new Set(); + const typeMap = new Map(); + for (const [name, type] of Object.entries(mapping)) { + fields.add(name); + typeMap.set(name, type); + } + context.fields = fields; + context.typeMap = typeMap; + } + if (Array.isArray(fc.disabledObjectFields) && fc.disabledObjectFields.length > 0) { + context.disabledObjectFields = new Set(fc.disabledObjectFields); + } + if (Array.isArray(fc.visibleIndices) && fc.visibleIndices.length > 0) { + context.visibleIndices = fc.visibleIndices.map((i) => i.split('{{index}}').join(spec.index)); + } + if (fc.settings && typeof fc.settings === 'object') { + context.settings = fc.settings; + } + if (fc.forceEnable) { + context.overrides = { [spec.ruleId]: { enabled: true } }; + } + return context; +} + +function rangeOffsets(query, range) { + const lineStarts = [0]; + for (let index = 0; index < query.length; index += 1) { + if (query[index] === '\n') { + lineStarts.push(index + 1); + } + } + const offset = (line, column) => { + const lineStart = lineStarts[line - 1]; + if (lineStart === undefined) { + throw new Error(`range line ${line} is outside a ${lineStarts.length}-line query`); + } + const lineEnd = lineStarts[line] === undefined ? query.length : lineStarts[line] - 1; + if (lineStart + column > lineEnd) { + throw new Error(`range column ${column} is outside query line ${line}`); + } + return lineStart + column; + }; + return { + start: offset(range.startLine, range.startColumn), + end: offset(range.endLine, range.endColumn), + }; +} + +function materializeDeterministicFix(query, diagnostic) { + if (!diagnostic.fix) { + return undefined; + } + const range = diagnostic.fix.range || diagnostic.range; + const { start, end } = rangeOffsets(query, range); + const sourceText = query.slice(start, end); + const expectedTextMatchesSource = + diagnostic.fix.expectedText === undefined || + diagnostic.fix.expectedText === sourceText; + return { + offered: true, + title: diagnostic.fix.title, + text: diagnostic.fix.text, + range: { + startLine: range.startLine, + startColumn: range.startColumn, + endLine: range.endLine, + endColumn: range.endColumn, + }, + ...(diagnostic.fix.expectedText !== undefined + ? { expectedText: diagnostic.fix.expectedText } + : {}), + ...(!expectedTextMatchesSource + ? { expectedTextMatchesSource: false } + : {}), + appliedQuery: query.slice(0, start) + diagnostic.fix.text + query.slice(end), + }; +} + +function exactEqual(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +export function evaluateFrontendAssertions({ + channel, + query, + matches, + allFrontendFindings = matches, + frontendOracle, +}) { + const assertions = {}; + const mismatches = []; + const record = (field, matched, expected, actual) => { + assertions[field] = matched; + if (!matched) { + mismatches.push({ field, expected, actual }); + } + return matched; + }; + + const severityMatched = + channel === 'syntax' || + !frontendOracle.severity || + matches.length === 0 || + matches.every((finding) => finding.severity === frontendOracle.severity); + if (channel === 'lint' && frontendOracle.severity !== undefined) { + record( + 'severity', + severityMatched, + frontendOracle.severity, + matches.map((finding) => finding.severity) + ); + } + + let messageMatched = true; + if (frontendOracle.matchMessage !== undefined) { + messageMatched = matches.some((finding) => + String(finding.message || '').includes(frontendOracle.matchMessage) + ); + record( + 'message', + messageMatched, + { contains: frontendOracle.matchMessage }, + matches.map((finding) => finding.message) + ); + } else if (frontendOracle.messageEquals !== undefined) { + messageMatched = + matches.length > 0 && + matches.every((finding) => finding.message === frontendOracle.messageEquals); + record( + 'message', + messageMatched, + { equals: frontendOracle.messageEquals }, + matches.map((finding) => finding.message) + ); + } + + let deterministicFixMatched = true; + let deterministicFixActual; + if (frontendOracle.deterministicFix !== undefined) { + const fixes = matches + .map((diagnostic) => materializeDeterministicFix(query, diagnostic)) + .filter(Boolean); + deterministicFixActual = + fixes.length === 0 + ? { offered: false } + : fixes.length === 1 + ? fixes[0] + : { offered: true, count: fixes.length, fixes }; + deterministicFixMatched = record( + 'deterministicFix', + exactEqual(frontendOracle.deterministicFix, deterministicFixActual), + frontendOracle.deterministicFix, + deterministicFixActual + ); + } + + let syntaxFixMatched = true; + if (channel === 'syntax' && frontendOracle.fixText !== undefined) { + const fixes = allFrontendFindings + .filter((finding) => finding.fix) + .map((finding) => finding.fix.text); + const expected = + frontendOracle.fixText === null + ? { offered: false } + : { offered: true, text: frontendOracle.fixText }; + const actual = + fixes.length === 0 + ? { offered: false } + : fixes.length === 1 + ? { offered: true, text: fixes[0] } + : { offered: true, count: fixes.length, texts: fixes }; + syntaxFixMatched = record('syntaxFix', exactEqual(expected, actual), expected, actual); + } + + let rawMessageMatched = true; + if (channel === 'syntax' && frontendOracle.rawMessage !== undefined) { + const rawMessages = allFrontendFindings + .map((finding) => finding.rawMessage) + .filter((message) => typeof message === 'string' && message.length > 0); + const actual = rawMessages.length > 0; + rawMessageMatched = record( + 'rawParserError', + actual === frontendOracle.rawMessage, + frontendOracle.rawMessage, + actual + ); + } + + let totalErrorsMatched = true; + if (channel === 'syntax' && frontendOracle.totalErrors !== undefined) { + totalErrorsMatched = record( + 'totalErrors', + allFrontendFindings.length === frontendOracle.totalErrors, + frontendOracle.totalErrors, + allFrontendFindings.length + ); + } + + return { + assertions, + mismatches, + severityMatched, + messageMatched, + deterministicFixMatched, + deterministicFixActual, + syntaxFixMatched, + rawMessageMatched, + totalErrorsMatched, + }; +} + +export function buildFrontendExecutionError({ + ruleId, + channel, + queryName, + role, + query, + expected = 0, + surface, + executionBackend, + error, + reportOnly = false, +}) { + const message = error instanceof Error ? error.message : String(error); + return { + ruleId, + channel, + queryName, + role, + query, + expected: Number.isInteger(expected) ? expected : 0, + actual: 0, + severities: [], + severityMatched: true, + messageMatched: true, + assertions: { execution: false }, + mismatches: [ + { + field: 'execution', + expected: 'completed', + actual: message, + }, + ], + outcome: 'error', + error: message, + surface, + executionBackend, + backendOracleStatus: 'error', + ...(reportOnly ? { reportOnly: true } : {}), + }; +} + +function equalSets(left, right) { + return left.size === right.size && [...left].every((value) => right.has(value)); +} + +export function buildCensus(contracts, manifest, catalog) { + const problems = []; + const byFile = new Map( + contracts.map(({ file, spec }) => [path.basename(file), spec]) + ); + const resolveManifestRules = (field) => { + const names = manifest[field] || []; + if (!Array.isArray(names)) { + problems.push(`manifest.${field} must be an array.`); + return []; + } + if (new Set(names).size !== names.length) { + problems.push(`manifest.${field} contains duplicate file names.`); + } + const rules = []; + for (const name of names) { + const spec = byFile.get(name); + if (!spec) { + problems.push(`manifest.${field} references inactive or missing contract "${name}".`); + } else { + rules.push(spec.ruleId); + } + } + return rules; + }; + + const activeContractRules = contracts.map(({ spec }) => spec.ruleId).sort(); + const duplicateRuleIds = activeContractRules.filter( + (ruleId, index) => activeContractRules.indexOf(ruleId) !== index + ); + if (duplicateRuleIds.length > 0) { + problems.push(`active contracts contain duplicate rule IDs: ${duplicateRuleIds.join(', ')}.`); + } + + const activeLintRules = contracts + .filter(({ spec }) => contractChannel(spec) === 'lint') + .map(({ spec }) => spec.ruleId) + .sort(); + const activeSyntaxRules = contracts + .filter(({ spec }) => contractChannel(spec) === 'syntax') + .map(({ spec }) => spec.ruleId) + .sort(); + const catalogRuleIds = catalog.map((rule) => rule.id).sort(); + const duplicateCatalogRuleIds = catalogRuleIds.filter( + (ruleId, index) => catalogRuleIds.indexOf(ruleId) !== index + ); + if (duplicateCatalogRuleIds.length > 0) { + problems.push( + `catalog contains duplicate rule IDs: ${[ + ...new Set(duplicateCatalogRuleIds), + ].join(', ')}.` + ); + } + const syntaxRulesInCatalog = activeSyntaxRules.filter((ruleId) => + catalogRuleIds.includes(ruleId) + ); + if (syntaxRulesInCatalog.length > 0) { + problems.push( + `syntax features must remain outside the detector catalog: ${syntaxRulesInCatalog.join( + ', ' + )}.` + ); + } + const enabledRules = catalog + .filter((rule) => rule.enabled) + .map((rule) => rule.id) + .sort(); + const defaultErrorRules = catalog + .filter((rule) => rule.enabled && rule.severity === 'error') + .map((rule) => rule.id) + .sort(); + const manifestDefaultErrorRules = resolveManifestRules('defaultError').sort(); + const requiredSyntaxFeatures = resolveManifestRules('requiredSyntaxFeatures').sort(); + + if (activeLintRules.length !== 12) { + problems.push(`expected 12 active lint contracts, found ${activeLintRules.length}.`); + } + if (requiredSyntaxFeatures.length !== 0) { + problems.push( + `required syntax features must be empty, found ` + + `${JSON.stringify(requiredSyntaxFeatures)}.` + ); + } + if (activeContractRules.length !== 12) { + problems.push(`expected 12 active contracts, found ${activeContractRules.length}.`); + } + if (!equalSets(new Set(activeSyntaxRules), new Set(requiredSyntaxFeatures))) { + problems.push( + `active syntax contracts ${JSON.stringify(activeSyntaxRules)} do not equal ` + + `manifest.requiredSyntaxFeatures ${JSON.stringify(requiredSyntaxFeatures)}.` + ); + } + if (!equalSets(new Set(activeLintRules), new Set(enabledRules))) { + problems.push( + `active lint contracts ${JSON.stringify(activeLintRules)} do not equal enabled catalog ` + + `rules ${JSON.stringify(enabledRules)}.` + ); + } + if (!equalSets(new Set(manifestDefaultErrorRules), new Set(defaultErrorRules))) { + problems.push( + `manifest.defaultError rules ${JSON.stringify(manifestDefaultErrorRules)} do not equal ` + + `enabled error catalog rules ${JSON.stringify(defaultErrorRules)}.` + ); + } + + return { + enabledRules, + defaultErrorRules, + requiredSyntaxFeatures, + activeContractRules, + activeLintRules, + activeSyntaxRules, + manifestDefaultErrorRules, + passed: problems.length === 0, + problems, + }; +} + +function main() { + const schedule = process.env.PPL_LINT_SCHEDULE || 'pr'; + const reportPath = process.env.PPL_LINT_REPORT; + const target = loadTarget(); + const backendReport = loadBackendReport(target); + const { contracts, activeContracts, manifest, manifestPath } = loadContracts(); + + const osd = loadOsd(); + const { + getBundledCatalog, + getDetector, + lintQuery, + validateSyntax, + osdRoot, + surface, + } = osd; + const catalog = getBundledCatalog(); + const census = buildCensus(activeContracts, manifest, catalog); + + // The compiled surface lints with OSD's own checked-in grammar, so there is no + // candidate bundle to load. On the runtime surface a missing bundle stays a hard + // failure — never a quiet downgrade to the compiled grammar. + const grammar = + surface === 'compiled-simplified' ? undefined : loadCandidateGrammar(osd, target); + const engineVersion = target.engineVersion; + const executionBackend = target.executionBackend; + const observeAnalytics = process.env.PPL_LINT_OBSERVE_ANALYTICS === '1'; + const observeOnly = + process.env.PPL_LINT_OBSERVE_ONLY === '1' || observeAnalytics; + if (observeAnalytics && executionBackend !== 'analytics') { + fatal('PPL_LINT_OBSERVE_ANALYTICS=1 requires an analytics target.'); + } + + const failures = []; + const reportOnlyFailures = []; + // Contracts this surface did not score, recorded so the report says a rule was + // skipped for surface rather than leaving its absence unexplained. + const skippedForSurface = []; + const report = { + schemaVersion: 2, + executionBackend, + osdRoot, + schedule, + engineVersion, + // Which of OSD's two lint surfaces produced these results. The aggregator + // needs this to interpret them: a compiled leg legitimately has no verdict for + // `runtimeOnly` rules, and mixing the two surfaces under one label would + // report a deliberately-inert rule as a regression. + surface, + // Contracts whose declared `grammarSurface` excludes this run, so a reader can + // see WHY a rule has no scored cases here. + skippedForSurface, + grammarHash: target.grammarHash || '', + observeAnalytics, + observeOnly, + differential: !!backendReport, + includedDormant: process.env.PPL_LINT_INCLUDE_DORMANT === '1', + reportOnlyFailures, + // Census of the rules that ship enabled at ERROR severity, read from the OSD + // catalog this run linted with. The multi-version aggregator compares its + // `defaultError` manifest set against this list without needing its own OSD + // checkout. Drift remains report-only unless census enforcement is enabled. + defaultErrorRules: catalog + .filter((rule) => rule.enabled && rule.severity === 'error') + .map((rule) => rule.id) + .sort(), + enabledRules: census.enabledRules, + requiredSyntaxFeatures: census.requiredSyntaxFeatures, + activeContractRules: census.activeContractRules, + census: { + enforced: process.env.PPL_LINT_ENFORCE_CENSUS === '1', + manifest: manifestPath, + ...census, + }, + results: [], + }; + if (!census.passed) { + for (const problem of census.problems) { + log(`CENSUS REPORT-ONLY: ${problem}`); + } + if (process.env.PPL_LINT_ENFORCE_CENSUS === '1') { + failures.push(...census.problems.map((problem) => `[census] ${problem}`)); + } + } + + log(`OSD root: ${osdRoot}`); + log( + `schedule=${schedule} engineVersion=${engineVersion} executionBackend=${executionBackend} ` + + `grammarHash=${target.grammarHash || '(unset)'} differential=${!!backendReport} ` + + `contracts=${contracts.length}` + ); + + const recordExecutionError = ({ + spec, + channel, + queryName, + queryDef, + expected, + error, + reportOnly, + scoringFailures, + }) => { + const query = (queryDef.query || '').split('{{index}}').join(spec.index); + const message = error instanceof Error ? error.message : String(error); + log(` FAIL ${spec.ruleId}/${queryName}: execution error — ${message}`); + scoringFailures.push( + `[${spec.ruleId}/${queryName}] frontend.execution failed: ${message}` + ); + report.results.push( + buildFrontendExecutionError({ + ruleId: spec.ruleId, + channel, + queryName, + role: queryDef.role || 'trigger', + query, + expected, + surface, + executionBackend, + error: message, + reportOnly, + }) + ); + }; + + for (const { file, spec, reportOnly = false } of contracts) { + const ruleId = spec.ruleId; + const index = spec.index; + const channel = contractChannel(spec); + const scoringFailures = reportOnly ? reportOnlyFailures : failures; + if (APPLICABLE_ONLY) { + const exclusion = compatibilityExclusion(spec, engineVersion, surface, ENGINE_MODE); + if (exclusion) { + log(`SKIP ${ruleId} (${exclusion.detail}) — ${path.basename(file)}`); + if (exclusion.reason === 'surface') { + skippedForSurface.push({ + ruleId, + contractSurface: spec.grammarSurface || 'runtime-bundle', + }); + } + continue; + } + } + const entry = checkWiring(spec, catalog, getDetector, scoringFailures); + if (!entry) { + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error: `OSD catalog entry "${ruleId}" is unavailable`, + reportOnly, + scoringFailures, + }); + } + continue; + } + + // A contract runs on PR only when scheduled for PR; nightly runs everything. + const contractSchedule = spec.schedule || 'pr'; + if (!reportOnly && schedule === 'pr' && contractSchedule !== 'pr') { + log(`SKIP ${ruleId} (schedule=${contractSchedule}, running ${schedule}) — ${path.basename(file)}`); + continue; + } + + // A contract declares the surface its expectations were verified against. + // Until now that field was decorative; honoring it keeps a runtime-bundle + // contract from being scored on a compiled leg, where its rule may legitimately + // behave differently. `both` opts into being checked on either surface. + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== surface) { + log( + `SKIP ${ruleId} (grammarSurface=${contractSurface}, running ${surface}) — ` + + `${path.basename(file)}` + ); + skippedForSurface.push({ ruleId, contractSurface }); + // Emit an explicit not-applicable row per query rather than dropping the rule. + // Dropping it leaves the aggregator with no rows at all, which it correctly + // reads as `inconclusive` — "we could not check" — and fails on. But nothing + // went wrong here and there is nothing to re-run: this contract simply does + // not describe this surface. + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + report.results.push({ + ruleId, + channel, + queryName, + role: queryDef.role || 'trigger', + query: (queryDef.query || '').split('{{index}}').join(index), + surface, + executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), + outcome: 'not-applicable', + notApplicable: `contract declares grammarSurface "${contractSurface}"`, + }); + } + continue; + } + + const context = buildContext(spec, engineVersion, ENGINE_MODE); + const expectation = selectExpectation(spec, engineVersion, context.isCalcite, scoringFailures, { + allowMissing: observeOnly, + }); + if (!expectation) { + if (!observeOnly) { + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error: `no unique expectation matches backend version ${engineVersion || 'unknown'}`, + reportOnly, + scoringFailures, + }); + } + continue; + } + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + const role = queryDef.role || 'trigger'; + const query = queryDef.query.split('{{index}}').join(index); + if (surface === 'compiled-simplified' && entry.runtimeOnly) { + report.results.push({ + ruleId, + channel, + queryName, + role, + query, + surface, + executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), + outcome: 'not-applicable', + notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', + }); + continue; + } + try { + if (channel === 'syntax' && typeof validateSyntax !== 'function') { + throw new Error( + `syntax validation requires validateQueryWithBundle from ${SYNTAX_MODULE}` + ); + } + const result = + channel === 'syntax' + ? validateSyntax(query, grammar) + : lintQuery(query, grammar, context); + const matches = + channel === 'syntax' + ? result.errors || [] + : (result.diagnostics || []).filter((d) => d.ruleId === ruleId); + report.results.push({ + ruleId, + channel, + queryName, + role, + query, + surface, + executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), + expected: 0, + actual: matches.length, + severities: matches.map((m) => m.severity), + severityMatched: true, + messageMatched: true, + backendOracleStatus: 'coverage-missing', + expectationStatus: 'coverage-missing', + }); + } catch (error) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error, + reportOnly, + scoringFailures, + }); + } + } + continue; + } + + const queries = spec.queries || {}; + const expectedQueries = expectation.queries || {}; + try { + assertExactQueryCoverage(spec, expectation); + } catch (error) { + for (const [queryName, queryDef] of Object.entries(queries)) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error: `invalid contract ${file}: ${error.message}`, + reportOnly, + scoringFailures, + }); + } + continue; + } + for (const queryName of Object.keys(queries)) { + const queryDef = queries[queryName]; + const role = queryDef.role || 'trigger'; + const query = queryDef.query.split('{{index}}').join(index); + const expected = expectedQueries[queryName]; + let oracleSelection; + try { + oracleSelection = resolveBackendOracle(spec, expected, executionBackend); + } catch (error) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error: `invalid contract ${file} query "${queryName}": ${error.message}`, + reportOnly, + scoringFailures, + }); + continue; + } + const frontendOracle = oracleSelection.frontend; + const expectedCount = frontendOracle.count; + + // A `runtimeOnly` rule walks grammar productions that exist only in the + // runtime bundle, so `lint_runner` skips it on the compiled surface. Its + // zero diagnostics here mean "deliberately inert", NOT "the detector went + // silent" — reporting them as a count would make the aggregator classify a + // healthy rule as detector-silent drift and send someone to fix it. Mark the + // case not-applicable and let the aggregator exclude it. + if (surface === 'compiled-simplified' && entry && entry.runtimeOnly) { + log( + ` SKIP ${ruleId}/${queryName} (${role}): runtimeOnly rule is inert on the ` + + `compiled-simplified surface.` + ); + report.results.push({ + ruleId, + channel, + queryName, + role, + query, + surface, + executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), + outcome: 'not-applicable', + notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', + }); + continue; + } + + try { + let result; + if (channel === 'syntax' && typeof validateSyntax !== 'function') { + throw new Error( + `syntax validation requires validateQueryWithBundle from ${SYNTAX_MODULE}` + ); + } + result = + channel === 'syntax' + ? validateSyntax(query, grammar) + : lintQuery(query, grammar, context); + const allFrontendFindings = + channel === 'syntax' ? result.errors || [] : result.diagnostics || []; + const matches = + channel === 'syntax' + ? allFrontendFindings.filter((finding) => finding.code === frontendOracle.code) + : allFrontendFindings.filter((finding) => finding.ruleId === ruleId); + const actual = matches.length; + const ok = actual === expectedCount; + + log( + ` ${ok ? 'PASS' : 'FAIL'} ${ruleId}/${queryName} (${role}): ` + + `expected ${expectedCount}, got ${actual} — ${query}` + ); + + const evaluated = evaluateFrontendAssertions({ + channel, + query, + matches, + allFrontendFindings, + frontendOracle, + }); + const assertions = { count: ok, ...evaluated.assertions }; + const mismatches = [ + ...(ok + ? [] + : [ + { + field: 'count', + expected: expectedCount, + actual, + }, + ]), + ...evaluated.mismatches, + ]; + + const resultEntry = { + ruleId, + channel, + queryName, + role, + query, + expected: expectedCount, + actual, + severities: matches.map((m) => m.severity).filter(Boolean), + severityMatched: evaluated.severityMatched, + messageMatched: evaluated.messageMatched, + deterministicFixMatched: evaluated.deterministicFixMatched, + fixMatched: evaluated.syntaxFixMatched, + rawMessageMatched: evaluated.rawMessageMatched, + totalErrorsMatched: evaluated.totalErrorsMatched, + assertions, + mismatches, + ...(evaluated.deterministicFixActual !== undefined + ? { deterministicFix: evaluated.deterministicFixActual } + : {}), + ...(channel === 'syntax' + ? { + code: frontendOracle.code, + codes: allFrontendFindings.map((finding) => finding.code).filter(Boolean), + totalErrors: allFrontendFindings.length, + } + : {}), + ...(reportOnly ? { reportOnly: true } : {}), + executionBackend, + backendOracleStatus: oracleSelection.status, + }; + + for (const mismatch of mismatches) { + scoringFailures.push( + `[${ruleId}/${queryName}] frontend.${mismatch.field} mismatch: ` + + `expected ${JSON.stringify(mismatch.expected)}, got ` + + `${JSON.stringify(mismatch.actual)} for: ${query}` + ); + } + + if (oracleSelection.status === 'not-applicable') { + // Only the backend fixture is non-applicable. The detector still ran above and its + // count/severity/message assertions remain ordinary, comparable frontend evidence. + resultEntry.backendOracleReason = oracleSelection.reason; + } else if (oracleSelection.status === 'coverage-missing') { + resultEntry.outcome = 'coverage-missing'; + resultEntry.coverage = 'missing'; + resultEntry.reason = oracleSelection.reason; + resultEntry.coverageMissing = oracleSelection.reason; + if (!observeAnalytics) { + scoringFailures.push( + `[${ruleId}/${queryName}] ${executionBackend} backend coverage missing: ${oracleSelection.reason}.` + ); + } + } + + // Differential: the observed backend behavior must agree with the observed + // detector output through the shared contract (design §3.2, §4.3). A + // rejection-kind query the backend rejected must be one the detector flags; + // a success/advisory query the backend accepted must be one the detector + // passes. This catches drift the two halves would otherwise hide by both + // pinning to the same JSON. + if (backendReport) { + const be = backendReport.get(`${ruleId}::${queryName}`); + if (!be) { + scoringFailures.push( + `[${ruleId}/${queryName}] no backend report entry (backend did not run this query).` + ); + } else { + const backendObservation = classifyBackendReportRow(be); + if (oracleSelection.status !== 'applicable') { + // A missing or non-applicable oracle is never an acceptance claim. Keep + // any backend observation visible, but do not coerce a missing verdict + // through `!!be.rejected` or score a differential against another route. + resultEntry.backendOutcome = backendObservation.status; + } else if (backendObservation.status !== 'observed') { + resultEntry.backendOutcome = backendObservation.status; + scoringFailures.push( + `[${ruleId}/${queryName}] backend report has no accepted/rejected verdict ` + + `(outcome=${JSON.stringify(backendObservation.status)}).` + ); + } else { + const backendKind = oracleSelection.oracle.kind; + const expectRejected = backendKind === 'rejection'; + const backendRejected = backendObservation.rejected; + resultEntry.backendRejected = backendRejected; + if (backendRejected !== expectRejected) { + scoringFailures.push( + `[${ruleId}/${queryName}] differential: backend ${backendRejected ? 'rejected' : 'accepted'} ` + + `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` + ); + } + // Pair detector and backend rejection only for rejection rules. + // Advisory rules intentionally flag queries the backend accepts. + const detectorFlagged = actual > 0; + if ( + role === 'trigger' && + expectRejected && + detectorFlagged !== backendRejected + ) { + scoringFailures.push( + `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `but backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + if (role === 'control' && (detectorFlagged || backendRejected)) { + scoringFailures.push( + `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `and backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + if ( + role === 'suppression-control' && + (detectorFlagged || !backendRejected) + ) { + scoringFailures.push( + `[${ruleId}/${queryName}] differential: suppression control must retain a backend ` + + `syntax rejection without a "${frontendOracle.code}" suggestion, but frontend ` + + `${detectorFlagged ? 'suggested a rewrite' : 'did not suggest a rewrite'} and ` + + `backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + } + } + } + + report.results.push(resultEntry); + } catch (error) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + expected: expectedCount, + error, + reportOnly, + scoringFailures, + }); + } + } + } + + if (reportPath) { + report.failures = failures; + try { + fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); + log(`wrote report to ${reportPath}`); + } catch (error) { + fatal(`Could not write detector report ${reportPath}: ${error.message}`); + } + } + + if (failures.length > 0) { + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-detector-contract] FAIL: ${failures.length} problem(s):\n- ${failures.join('\n- ')}` + ); + process.exit(1); + } + + if (reportOnlyFailures.length > 0) { + log( + `REPORT-ONLY: ${reportOnlyFailures.length} dormant contract problem(s):\n- ` + + reportOnlyFailures.join('\n- ') + ); + } + log(`PASS: all contracts agreed with the OSD detectors on the candidate bundle (schedule=${schedule}).`); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/settings.gradle b/settings.gradle index 7fc93fe1725..5e04e144468 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,6 +6,7 @@ pluginManagement { repositories { maven { url "https://ci.opensearch.org/maven2/" } + maven { url "https://ci.opensearch.org/m2/" } gradlePluginPortal() mavenCentral() } @@ -17,6 +18,7 @@ include 'opensearch-sql-plugin' project(':opensearch-sql-plugin').projectDir = file('plugin') include 'api' include 'ppl' +include 'ppl-rest-spi' include 'common' include 'opensearch' include 'core'