feat(ci): add PPL lint rule validation check (eventstats PoC)

.github/workflows/analytics-engine-compat.yml

@@ -15,7 +15,7 @@ on:
 
 jobs:
   Get-CI-Image-Tag:
-    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     with:
       product: opensearch
 

.github/workflows/backport.yml

@@ -1,30 +1,12 @@
+---
 name: Backport
 on:
   pull_request_target:
-    types:
-      - closed
-      - labeled
+    types: [closed, labeled]
 
 jobs:
   backport:
-    if: github.event.pull_request.merged == true
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    name: Backport
-    steps:
-      - name: GitHub App token
-        id: github_app_token
-        uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0
-        with:
-          app_id: ${{ secrets.APP_ID }}
-          private_key: ${{ secrets.APP_PRIVATE_KEY }}
-          installation_id: 22958780
-
-      - name: Backport
-        uses: VachaShah/backport@142d3b8a8c70dc54db515e653e5ed3c3fac64100 # v2.2.0
-        with:
-          github_token: ${{ steps.github_app_token.outputs.token }}
-          head_template: backport/backport-<%= number %>-to-<%= base %>
-          failure_labels: backport-failed
+    if: github.repository == 'opensearch-project/sql'
+    uses: opensearch-project/opensearch-build/.github/workflows/backport-pr.yml@main
+    secrets:
+      OPENSEARCH_CI_BOT_TOKEN: ${{ secrets.OPENSEARCH_CI_BOT_TOKEN }}

.github/workflows/integ-tests-with-security.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   Get-CI-Image-Tag:
-    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     with:
       product: opensearch
 

.github/workflows/issue-dedupe.yml

@@ -23,7 +23,7 @@ on:
 jobs:
   detect:
     if: (github.event_name == 'issues' && github.event.issue.user.type != 'Bot') || (github.event_name == 'workflow_dispatch' && inputs.job == 'detect')
-    uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-detect.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-detect.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     permissions:
       contents: read
       issues: write
@@ -36,7 +36,7 @@ jobs:
 
   auto-close:
     if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.job == 'auto-close')
-    uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-autoclose.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-autoclose.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     permissions:
       issues: write
     with:

.github/workflows/ppl-lint-rule-validation.yml

@@ -0,0 +1,157 @@
+name: PPL lint rule validation
+
+# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rule
+# `unsupported-window-function-in-eventstats` and the SQL backend must agree.
+#
+# Frontend half (frontend job): a SQL-owned Node script loads the compiled OSD
+# analyzer from an OSD checkout and asserts the rule's diagnostic counts. This
+# runs on a bare ubuntu-latest runner because OSD requires a modern Node whose
+# prebuilt binary needs a newer glibc than the OpenSearch CI container (Amazon
+# Linux 2) provides.
+# Backend half (backend job): a Gradle integration test sends the same queries
+# to the live `/_plugins/_ppl` endpoint of the SQL plugin built from this
+# checkout. This runs inside the OpenSearch CI container because the Gradle
+# test cluster needs it.
+#
+# Both jobs are required; a failure on either side fails the SQL PR check.
+#
+# The OSD detector is loaded from `main` by default (PR and nightly runs), so a
+# removed or changed detector is detected. `workflow_dispatch` can target a
+# specific OSD ref to reproduce a run or test an unmerged OSD branch.
+
+on:
+  pull_request:
+  schedule:
+    - cron: '0 10 * * *'
+  workflow_dispatch:
+    inputs:
+      osd_ref:
+        description: OSD commit or branch to test instead of main
+        required: false
+        type: string
+
+jobs:
+  frontend:
+    name: Frontend contract (OSD analyzer)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout SQL pull request
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Resolve OSD ref
+        id: osd-ref
+        env:
+          REQUESTED_REF: ${{ inputs.osd_ref }}
+        run: echo "ref=${REQUESTED_REF:-main}" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout OpenSearch-Dashboards
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          repository: opensearch-project/OpenSearch-Dashboards
+          ref: ${{ steps.osd-ref.outputs.ref }}
+          path: .ci/OpenSearch-Dashboards
+
+      - name: Record OSD revision
+        run: |
+          sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD)
+          echo "OSD revision: \`$sha\` (ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY"
+
+      # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding
+      # it, so an OSD toolchain bump does not silently drift this job.
+      - name: Set up Node from OSD .nvmrc
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4
+        with:
+          node-version-file: .ci/OpenSearch-Dashboards/.nvmrc
+
+      - name: Pin Yarn from OSD engines
+        working-directory: .ci/OpenSearch-Dashboards
+        run: |
+          yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)")
+          # Take the lower bound of the engines.yarn range (e.g. "^1.22.10" -> "1.22.10").
+          yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g')
+          npm install -g "yarn@${yarn_version}"
+
+      - name: Cache OSD Yarn dependencies
+        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4
+        with:
+          path: |
+            ~/.cache/yarn
+          key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-osd-yarn-
+
+      - name: Bootstrap OpenSearch-Dashboards
+        working-directory: .ci/OpenSearch-Dashboards
+        run: yarn osd bootstrap
+
+      # The Gradle test cluster runs the version from build.gradle's
+      # `opensearch.version` default (e.g. 3.8.0-SNAPSHOT). Export the release
+      # portion (3.8.0) as PPL_SQL_VERSION so the frontend applies the same
+      # version filtering the backend does, without maintaining a second string.
+      - name: Resolve OpenSearch version
+        id: os-version
+        run: |
+          raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | sed -E 's/.*"opensearch.version", "([^"]+)"/\1/')
+          version="${raw%%-*}"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+          echo "OpenSearch version: \`$version\` (from \`$raw\`)" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Run frontend contract
+        working-directory: .ci/OpenSearch-Dashboards
+        env:
+          PPL_LINT_CONTRACT_FILE: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json
+          PPL_SQL_VERSION: ${{ steps.os-version.outputs.version }}
+        run: |
+          node -r ./src/setup_node_env \
+            "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \
+            | tee "$GITHUB_WORKSPACE/frontend-contract.log"
+
+      - name: Upload frontend log
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        continue-on-error: true
+        with:
+          name: ppl-lint-frontend-contract-log
+          path: frontend-contract.log
+
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
+    with:
+      product: opensearch
+
+  backend:
+    name: Backend integration test (live /_plugins/_ppl)
+    needs: Get-CI-Image-Tag
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - name: Checkout SQL pull request
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+
+      # OpenSearch refuses to start as root, so run Gradle as a non-root user.
+      - name: Run backend integration test
+        run: |
+          chown -R 1000:1000 "$(pwd)"
+          su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT"
+
+      - name: Upload failure artifacts
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        continue-on-error: true
+        with:
+          name: ppl-lint-backend-artifacts
+          path: |
+            integ-test/build/reports/**
+            integ-test/build/testclusters/*/logs/*

.github/workflows/pr_review.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   Code-Diff-Analyzer:
-    uses: opensearch-project/opensearch-build/.github/workflows/code-diff-analyzer.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/code-diff-analyzer.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     if: github.repository == 'opensearch-project/sql'
     permissions:
       id-token: write  # github oidc to assume aws roles
@@ -18,7 +18,7 @@ jobs:
       update_pr_comment_with_analyzer_report: true
 
   Code-Diff-Reviewer:
-    uses: opensearch-project/opensearch-build/.github/workflows/code-diff-reviewer.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/code-diff-reviewer.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     needs: Code-Diff-Analyzer
     if: github.repository == 'opensearch-project/sql'
     permissions:

.github/workflows/sql-pitest.yml

@@ -12,7 +12,7 @@ run-name:
 
 jobs:
   Get-CI-Image-Tag:
-    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     with:
       product: opensearch
 

.github/workflows/sql-test-and-build-workflow.yml

@@ -20,7 +20,7 @@ on:
 
 jobs:
   Get-CI-Image-Tag:
-    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     with:
       product: opensearch
 

.github/workflows/sql-test-workflow.yml

@@ -12,7 +12,7 @@ run-name:
 
 jobs:
   Get-CI-Image-Tag:
-    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main
     with:
       product: opensearch
 

.lycheeignore

@@ -0,0 +1,12 @@
+# example opensearch/dashboards ports
+http://localhost:5601
+http://localhost:9200
+
+# sites that block scraping
+https://hg.openjdk.org/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/
+https://www.linkedin.com/in/*/
+
+# we have many pull requests links due to changelogs, skip these as they're unlikely to break and it saves API usage
+https://github.com/opendistro-for-elasticsearch/sql/pull/*
+https://github.com/opensearch-project/sql/pull/*
+

README.md

@@ -55,8 +55,6 @@ The following projects are related to the SQL plugin, but stored in the differen
 | [![untriaged open][untriaged-badge]][untriaged-link]           |
 | [![nolabel open][nolabel-badge]][nolabel-link]                 |
 
-[dco-badge]: https://github.com/opensearch-project/sql/actions/workflows/dco.yml/badge.svg
-[dco-badge-link]: https://github.com/opensearch-project/sql/actions/workflows/dco.yml
 [link-check-badge]: https://github.com/opensearch-project/sql/actions/workflows/link-checker.yml/badge.svg
 [link-check-link]: https://github.com/opensearch-project/sql/actions/workflows/link-checker.yml
 [bwc-tests-badge]: https://img.shields.io/badge/BWC%20tests-in%20progress-yellow

api/build.gradle

@@ -14,13 +14,13 @@ plugins {
 dependencies {
     api project(':ppl')
     api project(':sql')
-    api group: 'org.apache.calcite', name: 'calcite-babel', version: '1.41.0'
+    api group: 'org.apache.calcite', name: 'calcite-babel', version: "${calcite_version}"
 
     testImplementation testFixtures(project(':api'))
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
     testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"
-    testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0'
+    testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: "${calcite_version}"
 
     testFixturesApi group: 'junit', name: 'junit', version: '4.13.2'
     testFixturesApi group: 'org.hamcrest', name: 'hamcrest', version: "${hamcrest_version}"

api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java

@@ -262,7 +262,7 @@ public UnifiedQueryContext build() {
     private UnifiedQueryParser<?> createParser(CalcitePlanContext planContext, Settings settings) {
       return switch (queryType) {
         case PPL -> new PPLQueryParser(settings);
-        case SQL -> new SqlV2QueryParser();
+        case SQL -> new SqlV2QueryParser(settings);
       };
     }
 

api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java

@@ -27,6 +27,7 @@
 import org.opensearch.sql.calcite.CalciteRelNodeVisitor;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.common.error.ErrorReport;
+import org.opensearch.sql.exception.CalciteUnsupportedException;
 import org.opensearch.sql.exception.QueryEngineException;
 import org.opensearch.sql.exception.SemanticCheckException;
 
@@ -73,6 +74,10 @@ public RelNode plan(String query) {
             }
             return plan;
           });
+    } catch (CalciteUnsupportedException e) {
+      // Unsupported feature (e.g. table functions) is an invalid query, i.e. a client error.
+      // Must precede the QueryEngineException branch as it is a subclass.
+      throw new SemanticCheckException(e.getMessage(), e);
     } catch (SyntaxCheckException
         | QueryEngineException
         | UnsupportedOperationException