diff --git a/.konflux/artifacts.lock.yaml b/.konflux/artifacts.lock.yaml index 7dc6c0d5c..4583281fb 100644 --- a/.konflux/artifacts.lock.yaml +++ b/.konflux/artifacts.lock.yaml @@ -3,6 +3,6 @@ metadata: version: "1.0" artifacts: - - download_url: https://github.com/lightspeed-core/lightspeed-providers/archive/8cd1b3d3bdd841ea99d31b334ae00a275581661c.zip - checksum: sha256:6e660a30babcd1c00eef1686b251f9368410e03e08a7c0d4a680db00693ac448 + - download_url: https://github.com/lightspeed-core/lightspeed-providers/archive/faf6a89a3ad7856e2e7a934324f31d146108acdb.zip + checksum: sha256:c5ef9158146de2193daa16b1c3a0b338b20d136b6d3bb69da32ad592ad626f36 filename: lightspeed-providers.zip diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml index c3060e079..c06144f57 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml @@ -212,6 +212,10 @@ spec: - name: quay-aipcc-password secret: secretName: quay-aipcc-password + # Red Hat registry pull secret (docker-registry type) - will be copied to test namespace + - name: redhat-registry-pull-secret-source + secret: + secretName: redhat-registry-io-pull-secret - name: credentials emptyDir: {} steps: @@ -251,6 +255,9 @@ spec: mountPath: /var/run/quay-aipcc-name - name: quay-aipcc-password mountPath: /var/run/quay-aipcc-password + # Red Hat registry pull secret (docker-registry type) + - name: redhat-registry-pull-secret-source + mountPath: /var/run/redhat-registry-pull-secret - name: credentials mountPath: /credentials env: @@ -274,6 +281,11 @@ spec: value: "true" - name: OTEL_ANONYMIZATION_SECRET value: "lightspeed-stack-otel-anonymization-dev-default" + # PipelineRun metadata for ownerReference in secret creation + - name: TEKTON_PIPELINERUN_NAME + value: "$(context.pipelineRun.name)" + - name: TEKTON_PIPELINERUN_UID + value: "$(context.pipelineRun.uid)" image: registry.access.redhat.com/ubi9/ubi-minimal script: | set +e diff --git a/Makefile b/Makefile index 2fa421ada..1338d6097 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ SHELL := /bin/bash #TODO: We need to rename all those python and config files as well +# Define comma for use in $(if) expressions (where commas are argument separators) +COMMA := , + ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results) PATH_TO_PLANTUML := ~/bin @@ -16,6 +19,7 @@ OGX_CONFIG ?= run.yaml OGX_CONTAINER_NAME ?= lightspeed-ogx OGX_IMAGE ?= lightspeed-ogx:local OGX_PORT ?= 8321 +LIGHTSPEED_PROVIDERS_DIR ?= $(shell [ -d ../lightspeed-providers ] && cd ../lightspeed-providers && pwd) CONTAINER_RUNTIME ?= $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null) .PHONY: run \ @@ -86,6 +90,9 @@ start-ogx-container: build-ogx-image ## Start OGX container -v $(PWD)/$(CONFIG):/opt/app-root/lightspeed-stack.yaml:ro,z \ -v $(PWD)/scripts/ogx-entrypoint.sh:/opt/app-root/enrich-entrypoint.sh:ro,z \ -v $(PWD)/src/ogx_configuration.py:/opt/app-root/ogx_configuration.py:ro,z \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/lightspeed_stack_providers:/opt/app-root/providers/lightspeed_stack_providers:ro$(COMMA)z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-v $(LIGHTSPEED_PROVIDERS_DIR)/resources/external_providers:/opt/app-root/src/.llama/providers.d:ro$(COMMA)z) \ + $(if $(LIGHTSPEED_PROVIDERS_DIR),-e EXTERNAL_PROVIDERS_DIR=/opt/app-root/src/.llama/providers.d) \ -e OPENAI_API_KEY \ -e BRAVE_SEARCH_API_KEY \ -e TAVILY_SEARCH_API_KEY \ @@ -164,7 +171,7 @@ test-e2e-local: ## Run end to end tests for the service (no script wrapper) # Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g. # E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local -E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_shields or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified) +E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_shields or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified or @cfg_okp) test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*) script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt" diff --git a/docs/user_doc/okp_guide.md b/docs/user_doc/okp_guide.md index 60136d892..6ff800acb 100644 --- a/docs/user_doc/okp_guide.md +++ b/docs/user_doc/okp_guide.md @@ -51,12 +51,20 @@ validating that queries return referenced chunks. ## Step 1: Launch OKP -Start the OKP RAG service with Podman: +Start the OKP RAG service with Podman or Docker: +**Using Podman:** ```bash +podman login registry.redhat.io podman run --rm -d -p 8081:8080 registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest ``` +**Using Docker:** +```bash +docker login registry.redhat.io +docker run --rm -d -p 8081:8080 registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest +``` + > **Note:** Remove `-d` to run in the foreground. * The service listens on **port 8081** on the host (mapped from 8080 in the container). Lightspeed Stack itself listens on `8080`, so this avoids port conflicts. @@ -105,41 +113,61 @@ Edit your Lightspeed Stack config file (e.g. `lightspeed-stack.yaml`) and add the following top-level sections so that OKP is used for either inline or tool RAG: -Inline RAG: +**Inline RAG:** ```yaml # RAG configuration rag: - inline: - - okp -okp: - rhokp_url: ${env.RH_SERVER_OKP} - offline: true + retrieval: + inline: + sources: + - okp + max_chunks: 3 # Final tight cap on chunks returned to user + okp: + rhokp_url: ${env.RH_SERVER_OKP} + offline: true + max_chunks: 5 # Initial generous fetch from OKP ``` -Tool RAG: +**Tool RAG:** ```yaml # RAG configuration rag: - tool: - - okp -okp: - rhokp_url: ${env.RH_SERVER_OKP} - offline: true + retrieval: + tool: + sources: + - okp + max_chunks: 10 # Tool RAG limit + okp: + rhokp_url: ${env.RH_SERVER_OKP} + offline: true + max_chunks: 5 ``` -* **`rag.inline`** and **`rag.tool`**: Enable OKP as the RAG source for inline context injection and for the RAG tool. Tool rag means the LLM will be provided a search tool it can choose to invoke to find relevant content and augment the user prompt. The tool may or may not be invoked. Inline means a rag search and prompt augmentation will always occur. +* **`rag.retrieval.inline`** and **`rag.retrieval.tool`**: Enable OKP as the RAG source for inline context injection and for the RAG tool. Tool RAG means the LLM will be provided a search tool it can choose to invoke to find relevant content and augment the user prompt. The tool may or may not be invoked. Inline RAG means a RAG search and prompt augmentation will always occur. * **`okp.offline`**: When `true`, source URLs use `parent_id` (offline/Mimir-style). When `false`, use `reference_url` (online). +* **`okp.search_mode`**: Default Solr search strategy for OKP queries. Controls *how* Solr ranks chunks, not *which* documents are eligible (that is `chunk_filter_query` and per-request `solr.filters`). Allowed values: + - **`hybrid`** (default when the field is omitted): combines vector similarity with keyword (BM25) text search. Requires an embedding model. This is the global default. + - **`semantic`**: pure vector search against Solr embeddings. Requires an embedding model. Best when the query is conceptual rather than matching document wording. + - **`keyword`**: BM25 text search only. No embedding model is needed, so it is the right choice for air-gapped or embedding-free deployments, and it is typically faster than hybrid. +* **`max_chunks` behavior**: + - **Inline RAG**: `rag.retrieval.inline.max_chunks` is the final tight cap on chunks returned in the response, while `rag.okp.max_chunks` is the initial generous fetch from OKP before filtering and ranking. + - **Tool RAG**: `rag.retrieval.tool.max_chunks` sets the limit on chunks available to the tool. If you want to filter the docs to a specific product, you can include a static query filter such as: ```yaml okp: offline: true + search_mode: keyword chunk_filter_query: "product:*openshift* AND product_version:4.21" ``` +A per-request `solr.mode` on `/v1/query` (or streaming/responses) overrides this YAML default for that call. Per-request `lexical` is treated as `keyword`. + +> **Important:** When running OGX in **Server mode**, changes to static OKP settings such as `chunk_filter_query` and `search_mode` require an OGX restart to take effect. They are applied when Lightspeed Stack enriches OGX configuration at startup and will not propagate automatically during runtime. + When you launch Lightspeed Stack it will augment the OGX configuration (the synthesized run.yaml in unified mode, or your external run.yaml in the deprecated legacy mode) with configuration for OKP. @@ -258,7 +286,7 @@ curl -sX POST http://localhost:8080/v1/query \ - **Static filters preserved:** The configured `chunk_filter_query` (e.g., `"product:*openshift*"`) is always applied - **Dynamic filters added:** Request filters are combined with static filters using AND logic - **String escaping:** Special Solr characters in filter values are automatically escaped -- **Works with all search modes:** Filters apply to `semantic`, `hybrid`, and `lexical` search modes +- **Works with all search modes:** Filters apply whether the search mode is `semantic`, `hybrid`, or `keyword` (per-request `lexical` is an alias of `keyword`). --- @@ -312,9 +340,9 @@ Example response excerpt: ], ``` -> **Note:** The first time you query the system the response may take -> additional time because it must first download the necessary embedding model -> to perform the vector search. +> **Note:** The first time you query the system, `hybrid` or `semantic` search may take +> additional time because OGX must download the embedding model used for vector +> search. `keyword` search skips that download. If you see no RAG context, verify: diff --git a/src/ogx_configuration.py b/src/ogx_configuration.py index 23d58201b..db7089e20 100644 --- a/src/ogx_configuration.py +++ b/src/ogx_configuration.py @@ -819,7 +819,7 @@ def enrich_vector_store( # ============================================================================= -def enrich_solr( # pylint: disable=too-many-locals,too-many-statements +def enrich_solr( # pylint: disable=too-many-locals,too-many-statements,too-many-branches ls_config: dict[str, Any], rag_config: dict[str, Any], okp_config: dict[str, Any], @@ -860,6 +860,15 @@ def enrich_solr( # pylint: disable=too-many-locals,too-many-statements logger.info("Enriching OGX config with OKP") + # run-ci.yaml comments this out; Solr is a remote provider and needs providers.d. + if "external_providers_dir" not in ls_config: + ls_config["external_providers_dir"] = ( + "${env.EXTERNAL_PROVIDERS_DIR:=/opt/app-root/providers.d}" + ) + logger.info( + "Added external_providers_dir to OGX config for remote provider resolution" + ) + # Add vector_io provider for Solr if "providers" not in ls_config: ls_config["providers"] = {} diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/ogx-openai.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/ogx-openai.yaml index 20ec60a8b..f7b848ac3 100644 --- a/tests/e2e-prow/rhoai/manifests/lightspeed/ogx-openai.yaml +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/ogx-openai.yaml @@ -43,12 +43,22 @@ spec: # Fast-path: PVC already has a valid venv from a previous pod creation in this pipeline run. # TLS scenarios delete+recreate this pod up to 16 times; skipping the expensive install # reduces per-restart time from ~6-15 min to ~30-90 s (just RAG seed refresh + chown). + # Check if cached revision matches requested revision + CACHED_REVISION="" + if [[ -f /opt/app-root/.cached-revision ]]; then + CACHED_REVISION=$(cat /opt/app-root/.cached-revision) + fi + REPO_REVISION="${REPO_REVISION:-main}" + if [[ -d /opt/app-root/.venv ]] \ && /opt/app-root/.venv/bin/python --version >/dev/null 2>&1 \ && [[ -d /opt/app-root/src ]] \ && [[ -f /opt/app-root/scripts/ogx-entrypoint.sh ]] \ - && [[ -f /opt/app-root/src/ogx_configuration.py ]]; then - echo "PVC cache hit: app-root already provisioned — skipping full install" + && [[ -f /opt/app-root/src/ogx_configuration.py ]] \ + && [[ -d /opt/app-root/lightspeed_stack_providers ]] \ + && [[ -d /opt/app-root/providers.d ]] \ + && [[ "$CACHED_REVISION" == "$REPO_REVISION" ]]; then + echo "PVC cache hit: app-root already provisioned at $REPO_REVISION — skipping full install" mkdir -p /opt/app-root/.e2e-rag-seed /opt/app-root/.e2e-rag-work /opt/app-root/src/.ogx /opt/app-root/src/.llama/storage/files if [[ -f /rag-seed/kv_store.db.gz ]]; then gzip -dc /rag-seed/kv_store.db.gz > /opt/app-root/.e2e-rag-seed/kv_store.db @@ -81,6 +91,19 @@ spec: (cd /opt/app-root/repo && tar cf - .) | (cd /opt/app-root && tar xf -) rm -rf /opt/app-root/repo sed -i 's|/opt/app-root/repo/.venv|/opt/app-root/.venv|g' /opt/app-root/.venv/bin/* 2>/dev/null || true + # Download lightspeed-providers (external provider code + specs) — matches Containerfile + # Default matches deploy/lightspeed-stack/Containerfile ARG LIGHTSPEED_PROVIDERS_COMMIT + PROVIDERS_COMMIT="${PROVIDERS_COMMIT:-faf6a89a3ad7856e2e7a934324f31d146108acdb}" + ZIP_PATH="/tmp/lightspeed-providers.zip" + EXTRACT_DIR="/tmp/providers" + curl -fL --retry 2 --retry-delay 5 "https://github.com/lightspeed-core/lightspeed-providers/archive/${PROVIDERS_COMMIT}.zip" -o "${ZIP_PATH}" + mkdir -p "${EXTRACT_DIR}" + export ZIP_PATH EXTRACT_DIR + ROOT_DIR=$(python3 -c 'import os, zipfile; z=zipfile.ZipFile(os.environ["ZIP_PATH"]); print(z.namelist()[0].split("/")[0])') + python3 -c 'import os, zipfile; zipfile.ZipFile(os.environ["ZIP_PATH"]).extractall(os.environ["EXTRACT_DIR"])' + mv "${EXTRACT_DIR}/${ROOT_DIR}/lightspeed_stack_providers" /opt/app-root/ + mv "${EXTRACT_DIR}/${ROOT_DIR}/resources/external_providers" /opt/app-root/providers.d + rm -rf "${ZIP_PATH}" "${EXTRACT_DIR}" mkdir -p /opt/app-root/.e2e-rag-seed /opt/app-root/.e2e-rag-work /opt/app-root/src/.ogx /opt/app-root/src/.llama/storage/files if [[ ! -f /rag-seed/kv_store.db.gz ]]; then echo "FATAL: missing /rag-seed/kv_store.db.gz (ConfigMap rag-data key kv_store.db.gz)" @@ -96,6 +119,8 @@ spec: cp -f /opt/app-root/.e2e-rag-seed/kv_store.db /opt/app-root/.e2e-rag-work/kv_store.db cp -f /opt/app-root/scripts/ogx-entrypoint.sh /opt/app-root/enrich-entrypoint.sh cp -f /opt/app-root/src/ogx_configuration.py /opt/app-root/ogx_configuration.py + # Save the provisioned revision for cache validation + echo "${REPO_REVISION:-main}" > /opt/app-root/.cached-revision chmod 755 /opt/app-root/enrich-entrypoint.sh chmod -R 775 /opt/app-root && chown -R 1001:0 /opt/app-root volumeMounts: @@ -138,7 +163,9 @@ spec: - name: PATH value: "/opt/app-root/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - name: PYTHONPATH - value: "/opt/app-root/src" + value: "/opt/app-root:/opt/app-root/src" + - name: EXTERNAL_PROVIDERS_DIR + value: "/opt/app-root/providers.d" - name: HOME value: "/opt/app-root/src" # Match GitHub Actions docker-compose + ogx-entrypoint.sh: @@ -188,6 +215,10 @@ spec: secretKeyRef: name: llama-stack-ip-secret key: key + # In-cluster OKP Solr Service. Used when LCS YAML lists okp as a RAG source + # and OGX enriches run.yaml. Offline doc_url rewriting stays on LCS (localhost:8081 default). + - name: RH_SERVER_OKP + value: "http://okp-solr-service-svc:8080" command: - /bin/bash - -c diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/ogx-prow.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/ogx-prow.yaml index 2efea832c..1c7e2c43a 100644 --- a/tests/e2e-prow/rhoai/manifests/lightspeed/ogx-prow.yaml +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/ogx-prow.yaml @@ -84,7 +84,9 @@ spec: workingDir: /opt/app-root env: - name: PYTHONPATH - value: "/opt/app-root/src" + value: "/opt/app-root:/opt/app-root/src" + - name: EXTERNAL_PROVIDERS_DIR + value: "/opt/app-root/providers.d" - name: HOME value: "/opt/app-root/src" # Prevent OGX from shutil.move(~/.llama → ~/.ogx) which would steal the fixture. diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml new file mode 100644 index 000000000..bc730230c --- /dev/null +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/okp-solr.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Pod +metadata: + name: okp-solr-service + labels: + app: okp-solr +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + imagePullSecrets: + - name: redhat-registry-pull-secret + containers: + - name: okp-solr + image: registry.redhat.io/offline-knowledge-portal/rhokp-rhel9:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /solr + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /solr + port: 8080 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" +--- +apiVersion: v1 +kind: Service +metadata: + name: okp-solr-service-svc +spec: + selector: + app: okp-solr + ports: + - port: 8080 + targetPort: 8080 + name: http diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 14020dbcf..cea38f5d2 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -110,6 +110,115 @@ oc create secret docker-registry quay-lightspeed-pull-secret \ # Link the secret to default service account for image pulls oc secrets link default quay-lightspeed-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" +# Create Red Hat registry pull secret for OKP images +# Option 1: Use mounted docker-registry secret (preferred - simpler) +if [[ -f /var/run/redhat-registry-pull-secret/.dockerconfigjson ]]; then + log "Creating Red Hat registry pull secret from mounted docker-registry secret..." + + DOCKERCONFIG_BASE64=$(cat /var/run/redhat-registry-pull-secret/.dockerconfigjson | base64 -w0) + + # Use PipelineRun metadata for ownerReference (provided by Tekton context) + # This ensures automatic cleanup when the PipelineRun completes + if [[ -n "${TEKTON_PIPELINERUN_NAME:-}" && -n "${TEKTON_PIPELINERUN_UID:-}" ]]; then + log "Setting ownerReference to PipelineRun: $TEKTON_PIPELINERUN_NAME (UID: ${TEKTON_PIPELINERUN_UID:0:8}...)" + + # Create secret with ownerReference using YAML (ensures automatic cleanup) + cat </dev/null || echo "⚠️ Secret already linked to default SA" + else + # Fallback: create without ownerReference (requires manual cleanup) + log "⚠️ TEKTON_PIPELINERUN_NAME/UID not set - creating secret without ownerReference" + log "⚠️ Manual cleanup required after test completion" + + cat </dev/null || echo "⚠️ Secret already linked to default SA" + fi + +# Option 2: Fallback to username/password mounted separately (legacy approach) +elif [[ -d /var/run/redhat-registry-username ]] && [[ -d /var/run/redhat-registry-password ]]; then + log "Creating Red Hat registry pull secret from username/password..." + REDHAT_USERNAME="" + REDHAT_PASSWORD="" + + # Read username + shopt -s nullglob + for _f in /var/run/redhat-registry-username/*; do + [[ -f "$_f" ]] && REDHAT_USERNAME="$(cat "$_f")" && break + done + + # Read password + for _f in /var/run/redhat-registry-password/*; do + [[ -f "$_f" ]] && REDHAT_PASSWORD="$(cat "$_f")" && break + done + shopt -u nullglob + + if [[ -n "$REDHAT_USERNAME" ]] && [[ -n "$REDHAT_PASSWORD" ]]; then + # Use PipelineRun metadata for ownerReference (provided by Tekton context) + if [[ -n "${TEKTON_PIPELINERUN_NAME:-}" && -n "${TEKTON_PIPELINERUN_UID:-}" ]]; then + log "Setting ownerReference to PipelineRun: $TEKTON_PIPELINERUN_NAME (UID: ${TEKTON_PIPELINERUN_UID:0:8}...)" + + # Create secret with ownerReference (oc handles JSON encoding safely) + oc create secret docker-registry redhat-registry-pull-secret \ + --docker-server=registry.redhat.io \ + --docker-username="$REDHAT_USERNAME" \ + --docker-password="$REDHAT_PASSWORD" \ + -n "$NAMESPACE" \ + --dry-run=client -o json | \ + jq --arg name "$TEKTON_PIPELINERUN_NAME" --arg uid "$TEKTON_PIPELINERUN_UID" \ + '.metadata.ownerReferences = [{"apiVersion":"tekton.dev/v1beta1","kind":"PipelineRun","name":$name,"uid":$uid,"controller":false,"blockOwnerDeletion":false}]' | \ + oc apply -f - + log "✅ Red Hat registry pull secret created with ownerReference" + else + # Fallback: create without ownerReference + log "⚠️ TEKTON_PIPELINERUN_NAME/UID not set - creating secret without ownerReference" + log "⚠️ Manual cleanup required after test completion" + + oc create secret docker-registry redhat-registry-pull-secret \ + --docker-server=registry.redhat.io \ + --docker-username="$REDHAT_USERNAME" \ + --docker-password="$REDHAT_PASSWORD" \ + -n "$NAMESPACE" 2>/dev/null && log "✅ Red Hat registry pull secret created" || log "⚠️ Secret exists or creation failed" + fi + + # Link to default service account + oc secrets link default redhat-registry-pull-secret --for=pull -n "$NAMESPACE" 2>/dev/null || echo "⚠️ Secret already linked to default SA" + else + log "⚠️ Red Hat registry credentials not found in /var/run - OKP image pull may fail" + fi +else + log "⚠️ Red Hat registry credential mounts not found - OKP image pull may fail" + log " (This is OK if not testing OKP features)" +fi + #======================================== # 4. DEPLOY MOCK SERVERS (JWKS & MCP) @@ -146,6 +255,10 @@ oc wait pod/mock-jwks pod/mock-mcp \ } log "✅ Mock servers deployed" +# OKP Solr is not part of cluster setup. okp_rag.feature (@cfg_okp) deploys it from +# before_feature via e2e-ops deploy-okp-solr (7GB image, ~10-15 min first pull). +# No other e2e feature depends on OKP. +# # e2e-tunnel-proxy and e2e-interception-proxy are deployed from proxy.feature steps # (see tests/e2e/features/steps/proxy.py + e2e-ops deploy-e2e-*-proxy). @@ -269,14 +382,29 @@ e2e_echo_pod_logs() { done < <(oc logs llama-stack-service -n "$NAMESPACE" --tail="$n" 2>&1) || true } -progress "Waiting for lightspeed-stack and llama-stack pods" -if ! oc wait pod/lightspeed-stack-service pod/llama-stack-service \ - -n "$NAMESPACE" --for=condition=Ready --timeout=600s; then - progress "❌ One or both service pods failed to become ready within timeout" - e2e_echo_pod_logs 200 - exit 1 -fi -log "✅ Both service pods are ready" +progress "Waiting for lightspeed-stack and llama-stack pods (up to 10 min)" +for i in $(seq 1 60); do + lcs_ready=$(oc get pod lightspeed-stack-service -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") + llama_ready=$(oc get pod llama-stack-service -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") + + if [[ "$lcs_ready" == "True" ]] && [[ "$llama_ready" == "True" ]]; then + log "✅ Both service pods are ready after $(( i * 10 ))s" + break + fi + + if [ $((i % 6)) -eq 0 ]; then + lcs_status=$(oc get pod lightspeed-stack-service -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "unknown") + llama_status=$(oc get pod llama-stack-service -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "unknown") + progress "[$(( i * 10 ))s] lightspeed-stack: $lcs_status ($lcs_ready), llama-stack: $llama_status ($llama_ready)" + fi + + if [ $i -eq 60 ]; then + progress "❌ One or both service pods failed to become ready within 600s timeout" + e2e_echo_pod_logs 200 + exit 1 + fi + sleep 10 +done if [ "$QUIET" = "1" ]; then e2e_echo_pod_logs 80 @@ -297,8 +425,10 @@ fi # Debug hook/port churn: export E2E_OPS_VERBOSE=1 before running pipeline.sh export E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" export E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" +export E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" rm -f "$E2E_LSC_PORT_FORWARD_PID_FILE" rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" +rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" oc label pod lightspeed-stack-service pod=lightspeed-stack-service -n $NAMESPACE @@ -319,7 +449,7 @@ kill_listeners_on_ports() { fi done } -kill_listeners_on_ports 8080 8000 8321 +kill_listeners_on_ports 8080 8000 8321 8081 # Start port-forward for lightspeed-stack progress "Starting port-forward, then E2E tests" @@ -339,6 +469,9 @@ oc port-forward svc/llama-stack-service-svc 8321:8321 -n $NAMESPACE & PF_LLAMA_PID=$! echo "$PF_LLAMA_PID" >"$E2E_LLAMA_PORT_FORWARD_PID_FILE" +# OKP Solr port-forward (localhost:8081) is started by e2e-ops deploy-okp-solr +# when okp_rag.feature runs — do not start it here. + # Wait for port-forward to be usable (app may not be listening immediately; port-forward can drop) log "Waiting for port-forward to lightspeed-stack to be ready..." for i in $(seq 1 36); do @@ -448,6 +581,13 @@ if [[ -n "${E2E_LLAMA_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_LLAMA_PORT_FORWARD_P fi rm -f "$E2E_LLAMA_PORT_FORWARD_PID_FILE" fi +if [[ -n "${E2E_OKP_PORT_FORWARD_PID_FILE:-}" && -f "$E2E_OKP_PORT_FORWARD_PID_FILE" ]]; then + read -r _okp_pf <"$E2E_OKP_PORT_FORWARD_PID_FILE" 2>/dev/null || true + if [[ "${_okp_pf:-}" =~ ^[0-9]+$ ]]; then + kill -9 "$_okp_pf" 2>/dev/null || true + fi + rm -f "$E2E_OKP_PORT_FORWARD_PID_FILE" +fi kill $PF_LCS_PID 2>/dev/null || true kill $PF_JWKS_PID 2>/dev/null || true diff --git a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh index 955d8463d..95763defe 100755 --- a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh +++ b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh @@ -16,6 +16,10 @@ # - restart-lightspeed ensures Llama is running before LCS recreate when needed. # - restart-both-services is available explicitly; restart-lightspeed / restart-llama-stack # do not auto-trigger a full stack restart on failure. +# - OKP Solr is not deployed in pipeline setup. Features tagged @cfg_okp call +# deploy-okp-solr from before_feature (7GB image, first pull ~10-15 min) and +# start the localhost:8081 port-forward there. Background "OKP is running" +# then GETs Solr from inside the OGX pod (in-cluster Service), not runner :8081. # # Commands: # restart-lightspeed - Restart lightspeed-stack pod and port-forward @@ -23,6 +27,7 @@ # restart-both-services - Full OGX then lightspeed-stack restart (explicit only) # restart-port-forward - Re-establish port-forward for lightspeed # restart-llama-port-forward - Re-establish port-forward for OGX (8321) +# restart-okp-port-forward - Re-establish port-forward for OKP Solr (8081) # wait-for-pod [attempts] - Wait for a pod to be ready # update-configmap - Update ConfigMap from file # get-configmap-content - Get ConfigMap content (outputs to stdout) @@ -33,6 +38,11 @@ # delete-e2e-mock-tls-inference - Remove mock TLS pod + Service (manual cleanup) # restart-e2e-mock-tls-inference - Delete then deploy mock TLS (manual / recovery) # sync-mock-tls-certs-secret - Copy mock /certs into Secret for OGX mount +# check-okp-solr-from-llama - GET Solr /solr/ from inside the OGX pod +# deploy-okp-solr - Deploy OKP Solr (idempotent; @cfg_okp before_feature) +# delete-okp-solr - Delete OKP Solr pod +# disrupt-okp-solr - Delete OKP Solr pod to disrupt connection +# restore-okp-solr - Restore OKP Solr pod set -e @@ -43,6 +53,7 @@ MANIFEST_DIR="$SCRIPT_DIR/../manifests/lightspeed" E2E_LSC_PORT_FORWARD_PID_FILE="${E2E_LSC_PORT_FORWARD_PID_FILE:-/tmp/e2e-lightspeed-port-forward.pid}" E2E_LLAMA_PORT_FORWARD_PID_FILE="${E2E_LLAMA_PORT_FORWARD_PID_FILE:-/tmp/e2e-llama-port-forward.pid}" E2E_JWKS_PORT_FORWARD_PID_FILE="${E2E_JWKS_PORT_FORWARD_PID_FILE:-/tmp/e2e-jwks-port-forward.pid}" +E2E_OKP_PORT_FORWARD_PID_FILE="${E2E_OKP_PORT_FORWARD_PID_FILE:-/tmp/e2e-okp-port-forward.pid}" # ============================================================================ # Helper functions @@ -208,6 +219,24 @@ kill_stale_jwks_forward() { free_local_tcp_port "$port" } +# Kill anything likely to hold the OKP Solr local forward (localhost:8081). +kill_stale_okp_forward() { + local port="${1:-8081}" + local saved_pf + if [[ -f "$E2E_OKP_PORT_FORWARD_PID_FILE" ]]; then + read -r saved_pf <"$E2E_OKP_PORT_FORWARD_PID_FILE" 2>/dev/null || true + if [[ "$saved_pf" =~ ^[0-9]+$ ]]; then + kill -9 "$saved_pf" 2>/dev/null || true + fi + fi + pkill -9 -f "port-forward.*okp-solr-service-svc.*8081:8080" 2>/dev/null || true + pkill -9 -f "oc port-forward svc/okp-solr-service-svc 8081:8080" 2>/dev/null || true + pkill -9 -f "port-forward pod/okp-solr-service.*8081:8080" 2>/dev/null || true + free_local_tcp_port "$port" + sleep 1 + free_local_tcp_port "$port" +} + # After oc port-forward dies in <2s, show recent oc stderr from the log file. e2e_ops_emit_port_forward_immediate_failure_diag() { echo "[e2e-ops] /tmp/port-forward.log (tail 25):" @@ -554,6 +583,26 @@ verify_llama_local_forward() { return 1 } +verify_okp_connectivity() { + local max_attempts="${1:-15}" + local local_port="${2:-8081}" + local http_code="" + local attempt + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + http_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://localhost:${local_port}/solr" 2>/dev/null) || http_code="000" + # OKP Solr returns various 200-399 codes for /solr endpoint + if [[ "$http_code" =~ ^[23][0-9][0-9]$ ]]; then + return 0 + fi + if [[ $attempt -lt $max_attempts ]]; then + sleep 2 + fi + done + echo "OKP Solr localhost:${local_port} connectivity check failed (HTTP: ${http_code:-unknown})" + return 1 +} + cmd_restart_llama_port_forward() { local local_port="${LOCAL_LLAMA_PORT:-8321}" local remote_port="${REMOTE_LLAMA_PORT:-8321}" @@ -676,6 +725,67 @@ cmd_restart_jwks_port_forward() { return 1 } +cmd_restart_okp_port_forward() { + local local_port="${LOCAL_OKP_PORT:-8081}" + local remote_port="${REMOTE_OKP_PORT:-8080}" + local max_attempts=6 + local pf_pid + local pf_resource + local okp_pf_log="/tmp/port-forward-okp.log" + + echo "Re-establishing OKP Solr port-forward on $local_port:$remote_port..." + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + kill_stale_okp_forward "$local_port" + sleep 3 + + if [[ $attempt -le 2 ]]; then + pf_resource="svc/okp-solr-service-svc" + else + pf_resource="pod/okp-solr-service" + fi + echo "OKP port-forward attempt $attempt/$max_attempts -> $pf_resource" + + : >"$okp_pf_log" + nohup oc port-forward "$pf_resource" "$local_port:$remote_port" -n "$NAMESPACE" \ + "$okp_pf_log" 2>&1 & + pf_pid=$! + disown "$pf_pid" 2>/dev/null || true + sleep 3 + + if ! kill -0 "$pf_pid" 2>/dev/null; then + echo "OKP port-forward process exited immediately:" + if [[ -s "$okp_pf_log" ]]; then + tail -25 "$okp_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true + fi + kill_stale_okp_forward "$local_port" + sleep 2 + continue + fi + sleep 4 + + if verify_okp_connectivity 12 "$local_port"; then + echo "$pf_pid" >"$E2E_OKP_PORT_FORWARD_PID_FILE" + echo "✓ OKP Solr port-forward established (PID: $pf_pid)" + return 0 + fi + + if [[ $attempt -lt $max_attempts ]]; then + echo "OKP forward attempt $attempt failed (connectivity check failed), retrying..." + kill -9 "$pf_pid" 2>/dev/null || true + kill_stale_okp_forward "$local_port" + sleep 3 + fi + done + + echo "Failed to establish OKP Solr port-forward after $max_attempts attempts" + if [[ -s "$okp_pf_log" ]]; then + echo "Port-forward log (tail 30):" + tail -30 "$okp_pf_log" 2>/dev/null | sed 's/^/[e2e-ops] /' || true + fi + return 1 +} + cmd_wait_for_pod() { local pod_name="${1:?Pod name required}" local max_attempts="${2:-24}" @@ -989,6 +1099,111 @@ cmd_disrupt_llama_stack() { fi } +# Prove OGX can reach Solr in-cluster. Runner localhost:8081 only tests port-forward. +cmd_check_okp_solr_from_llama() { + local pod="llama-stack-service" + local ctr="llama-stack-container" + local url="http://okp-solr-service-svc:8080/solr/" + + echo "Checking OKP Solr from OGX pod at ${url}..." + if oc exec -n "$NAMESPACE" "$pod" -c "$ctr" -- \ + curl -sf --max-time 10 -o /dev/null "$url" 2>/dev/null; then + echo "✓ OGX pod can reach OKP Solr" + return 0 + fi + if oc exec -n "$NAMESPACE" "$pod" -c "$ctr" -- \ + /opt/app-root/.venv/bin/python -c \ + 'import urllib.request; urllib.request.urlopen("http://okp-solr-service-svc:8080/solr/", timeout=10).read()'; then + echo "✓ OGX pod can reach OKP Solr" + return 0 + fi + echo "ERROR: OGX pod cannot reach OKP Solr at $url" + return 1 +} + +cmd_deploy_okp_solr() { + local pod_name="okp-solr-service" + # First pull of the ~7GB OKP image is typically 10-15 min. 300 attempts × 3s = 900s. + local wait_attempts=300 + local ready + + ready=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null || echo "false") + if [[ "$ready" == "true" ]]; then + echo "✓ OKP Solr already ready — skipping deploy" + cmd_restart_okp_port_forward + return 0 + fi + + echo "Deploying OKP Solr service in namespace $NAMESPACE..." + if oc get secret redhat-registry-pull-secret -n "$NAMESPACE" &>/dev/null; then + echo "✓ redhat-registry-pull-secret exists" + else + echo "WARNING: redhat-registry-pull-secret NOT found — image pull will fail" + oc get secrets -n "$NAMESPACE" --field-selector type=kubernetes.io/dockerconfigjson -o name 2>/dev/null || \ + echo "No dockerconfigjson secrets found" + fi + + oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" + echo "Waiting for OKP Solr to be ready (${wait_attempts} attempts, ~$((wait_attempts * 3))s for 7GB image pull)..." + if ! wait_for_pod "$pod_name" "$wait_attempts"; then + echo "==========================================" + echo "OKP Solr not ready — diagnostics" + echo "==========================================" + oc get pod "$pod_name" -n "$NAMESPACE" -o wide || true + oc get pod "$pod_name" -n "$NAMESPACE" \ + -o jsonpath='{.status.containerStatuses[*].state}' && echo "" || true + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' \ + --field-selector involvedObject.name="$pod_name" \ + --limit=30 2>/dev/null || echo "No events found for $pod_name" + oc describe pod "$pod_name" -n "$NAMESPACE" || true + if oc get secret redhat-registry-pull-secret -n "$NAMESPACE" &>/dev/null; then + echo "redhat-registry-pull-secret: present" + else + echo "redhat-registry-pull-secret: NOT FOUND" + fi + echo "OKP Solr failed to become ready (7GB image — check node network to registry.redhat.io)" + return 1 + fi + echo "✓ OKP Solr service deployed and ready" + cmd_restart_okp_port_forward +} + +cmd_delete_okp_solr() { + echo "Deleting OKP Solr pod from namespace $NAMESPACE..." + timeout 60 oc delete pod okp-solr-service -n "$NAMESPACE" --ignore-not-found=true --wait=true 2>/dev/null || { + oc delete pod okp-solr-service -n "$NAMESPACE" --ignore-not-found=true --force --grace-period=0 2>/dev/null || true + sleep 2 + } + echo "✓ OKP Solr pod deleted" +} + +cmd_disrupt_okp_solr() { + local pod_name="okp-solr-service" + + local phase + phase=$(oc get pod "$pod_name" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") + + if [[ "$phase" == "Running" ]]; then + oc delete pod "$pod_name" -n "$NAMESPACE" --wait=true + sleep 2 + echo "OKP Solr connection disrupted successfully (pod deleted)" + exit 0 + else + echo "OKP Solr pod was not running (phase: $phase)" + exit 2 + fi +} + +cmd_restore_okp_solr() { + echo "Restoring OKP Solr service in namespace $NAMESPACE..." + oc apply -n "$NAMESPACE" -f "$MANIFEST_DIR/okp-solr.yaml" + wait_for_pod "okp-solr-service" 60 + echo "✓ OKP Solr pod restored and ready" + + # Restart port-forward since pod was replaced + cmd_restart_okp_port_forward +} + # ============================================================================ # Main command dispatcher # ============================================================================ @@ -1012,6 +1227,9 @@ case "$COMMAND" in restart-jwks-port-forward) cmd_restart_jwks_port_forward ;; + restart-okp-port-forward) + cmd_restart_okp_port_forward + ;; restart-port-forward) cmd_restart_port_forward ;; @@ -1060,6 +1278,21 @@ case "$COMMAND" in dump-pod-logs) cmd_dump_pod_logs "$@" ;; + check-okp-solr-from-llama) + cmd_check_okp_solr_from_llama + ;; + deploy-okp-solr) + cmd_deploy_okp_solr + ;; + delete-okp-solr) + cmd_delete_okp_solr + ;; + disrupt-okp-solr) + cmd_disrupt_okp_solr + ;; + restore-okp-solr) + cmd_restore_okp_solr + ;; *) echo "Usage: $0 [args...]" echo "" @@ -1068,6 +1301,7 @@ case "$COMMAND" in echo " restart-llama-stack - Restart/restore llama-stack pod" echo " restart-both-services - Full llama-stack + lightspeed-stack restart (explicit)" echo " restart-llama-port-forward - Re-establish port-forward for OGX (8321)" + echo " restart-okp-port-forward - Re-establish port-forward for OKP Solr (8081)" echo " restart-port-forward - Re-establish port-forward for lightspeed" echo " wait-for-pod [attempts] - Wait for a pod to be ready" echo " update-configmap - Update ConfigMap from file" @@ -1081,6 +1315,11 @@ case "$COMMAND" in echo " deploy-e2e-interception-proxy - Deploy in-cluster interception proxy pod" echo " deploy-e2e-mock-tls-inference - Deploy mock HTTPS inference (tls-*.feature)" echo " delete-e2e-mock-tls-inference - Remove mock TLS pod + Service" + echo " check-okp-solr-from-llama - GET Solr /solr/ from inside the OGX pod" + echo " deploy-okp-solr - Deploy OKP Solr (idempotent; @cfg_okp before_feature)" + echo " delete-okp-solr - Delete OKP Solr pod" + echo " disrupt-okp-solr - Delete OKP Solr pod to disrupt connection" + echo " restore-okp-solr - Restore OKP Solr pod" echo " restart-e2e-mock-tls-inference - Delete then deploy mock TLS (recovery)" echo " sync-mock-tls-certs-secret - Publish mock TLS /certs to Secret" echo " dump-pod-logs [tail-lines] - Print init + container logs" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-negative.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-negative.yaml new file mode 100644 index 000000000..d6ddf1426 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-negative.yaml @@ -0,0 +1,28 @@ +# @cfg_okp +# OKP RAG disabled — okp is not listed in retrieval sources. +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml new file mode 100644 index 000000000..bc6015887 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP inline RAG — offline mode. +# Chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 1 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml new file mode 100644 index 000000000..55de7674a --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-online.yaml @@ -0,0 +1,42 @@ +# @cfg_okp +# OKP inline RAG — online mode. +# Chunks use reference_url-based URLs (docs.redhat.com). +# search_mode: keyword — the OKP image's httpd proxy times out hybrid-search +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + inline: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 3 + search_mode: keyword + chunk_filter_query: "product:openshift_container_platform AND product_version:4.21" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml new file mode 100644 index 000000000..82c981216 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-offline.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — offline mode. +# LLM calls file_search on demand; chunks use parent_id-based URLs (localhost:8081). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 2 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: true + max_chunks: 5 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml new file mode 100644 index 000000000..ec6c258e2 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-okp-tool-online.yaml @@ -0,0 +1,39 @@ +# @cfg_okp +# OKP tool RAG — online mode. +# LLM calls file_search on demand; chunks use reference_url-based URLs (access.redhat.com). +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +rag: + retrieval: + tool: + sources: + - okp + max_chunks: 3 + okp: + rhokp_url: ${env.RH_SERVER_OKP:=http://localhost:8081/solr} + offline: false + max_chunks: 5 diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 4e449f86b..a2cf35833 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -34,11 +34,13 @@ ) from tests.e2e.utils.ogx_utils import register_shield from tests.e2e.utils.prow_utils import ( + ensure_okp_solr_ready, restart_pod, restore_ogx_pod, run_e2e_ops, ) from tests.e2e.utils.utils import ( + is_konflux_environment, is_prow_environment, remove_config_backup, restart_container, @@ -208,10 +210,11 @@ def before_scenario(context: Context, scenario: Scenario) -> None: resetting per-scenario Lightspeed override tracking and skip-restart flags. Skips the scenario if it has the `skip` tag, if it has the `local` tag - while the test run is not in local mode, if it has `skip-in-library-mode` - when running in library mode, or if it has `skip-in-server-mode` when running - in server mode. Scenario-specific Lightspeed YAML is applied in the feature - files (``The service uses the ... configuration`` steps). + while the test run is not in local mode, if it has `konflux-only` when + ``E2E_KONFLUX_E2E`` is not ``1``, if it has `skip-in-library-mode` when + running in library mode, or if it has `skip-in-server-mode` when running + in server mode. Scenario-specific Lightspeed YAML is applied in the + feature files (``The service uses the ... configuration`` steps). """ if "skip" in scenario.effective_tags: scenario.skip("Marked with @skip") @@ -219,6 +222,9 @@ def before_scenario(context: Context, scenario: Scenario) -> None: if "local" in scenario.effective_tags and not context.local: scenario.skip("Marked with @local") return + if "konflux-only" in scenario.effective_tags and not is_konflux_environment(): + scenario.skip("Skipped outside Konflux (requires E2E_KONFLUX_E2E=1)") + return # Skip scenarios that require separate OGX container in library mode if context.is_library_mode and "skip-in-library-mode" in scenario.effective_tags: @@ -271,8 +277,11 @@ def _dump_pod_logs_on_failure( pods: tuple[str, ...] = ("llama-stack-service", "lightspeed-stack-service") feature = getattr(context, "feature", None) feat_file = getattr(feature, "filename", "") or "" if feature else "" + feat_tags = getattr(feature, "tags", []) if feature else [] if is_tls_feature_file(feat_file): pods = (*pods, "e2e-mock-tls-inference") + if "cfg_okp" in feat_tags: + pods = (*pods, "okp-solr-service") print(f"--- scenario failed: {scenario.name!r} — pod logs ---", flush=True) for pod in pods: try: @@ -304,6 +313,8 @@ def after_scenario(context: Context, scenario: Scenario) -> None: running before the scenario. - hostname_llama, port_llama (str/int, optional): host and port used for the OGX health check. + - okp_was_running (bool, optional): whether OKP server was running + before it was stopped by the scenario. scenario (Scenario): Behave scenario (unused; shield restore uses context flags). """ if is_prow_environment(): @@ -329,6 +340,12 @@ def after_scenario(context: Context, scenario: Scenario) -> None: except (TypeError, ValueError, RuntimeError, KeyboardInterrupt) as e: print(f"Warning: Could not re-register shield: {e}") + # Restore OKP Solr if it was stopped during the scenario (Konflux only). + if getattr(context, "okp_was_running", False) and is_konflux_environment(): + from tests.e2e.utils.prow_utils import restore_okp_solr_pod + + restore_okp_solr_pod() + def _print_ogx_diagnostics() -> None: """Print container state, health, and recent logs to diagnose why OGX did not recover.""" @@ -448,6 +465,19 @@ def _restore_llama_stack() -> None: _print_ogx_diagnostics() +def _ensure_okp_solr_for_feature() -> None: + """Deploy OKP Solr for ``@cfg_okp`` features on Konflux only. + + Classic Prow and local Docker are no-ops. GitHub Actions / Prow skip the + feature via ``@konflux-only`` before scenarios run. + """ + if not is_konflux_environment(): + return + print("[okp_rag.feature] Konflux: ensuring OKP Solr is deployed...") + ensure_okp_solr_ready() + print("[okp_rag.feature] OKP Solr ready", flush=True) + + def before_feature(context: Context, feature: Feature) -> None: """Run before each feature file is exercised. @@ -463,16 +493,23 @@ def before_feature(context: Context, feature: Feature) -> None: ``max_attempts`` times before accepting failure. The cap defaults to ``_E2E_FLAKY_MAX_ATTEMPTS`` and can be overridden with the ``E2E_FLAKY_MAX_ATTEMPTS`` environment variable. + + Features tagged ``@cfg_okp`` deploy OKP Solr on Konflux only (idempotent) + via ``e2e-ops deploy-okp-solr`` before scenarios run. Classic Prow and + local Docker are no-ops. """ setattr(feature, _E2E_FEATURE_PERF_START_ATTR, time.perf_counter()) context.feature_config = None context.scenario_lightspeed_override_active = False context.active_lightspeed_stack_config_basename = None + # One real Llama disruption per feature (module-level flag; survives context resets) reset_llama_stack_disrupt_once_tracking() if feature.filename and is_tls_feature_file(feature.filename): reset_tls_prow_state() prepare_tls_feature_entry_on_prow(feature.filename) + if "cfg_okp" in feature.tags: + _ensure_okp_solr_for_feature() try: max_flaky = int(os.getenv("E2E_FLAKY_MAX_ATTEMPTS", _E2E_FLAKY_MAX_ATTEMPTS)) diff --git a/tests/e2e/features/okp_rag.feature b/tests/e2e/features/okp_rag.feature index fb5c6654f..478567c69 100644 --- a/tests/e2e/features/okp_rag.feature +++ b/tests/e2e/features/okp_rag.feature @@ -1,11 +1,16 @@ -@cfg_okp @skip +@cfg_okp @konflux-only Feature: OKP(Solr) RAG retrieval tests + # Server-mode tests: OGX is a separate pod and re-enriches Solr/OKP from the + # Lightspeed ConfigMap on restart. Restart Llama Stack after each YAML change + # so the Solr provider picks up inline/tool/offline/online settings. + # Konflux only: before_feature deploys OKP Solr (7GB image) once. + # Background "OKP is running" checks Solr from the OGX pod (not runner :8081). + # @konflux-only: skipped on GitHub Actions, classic Prow, and local Docker. # Offline Knowledge Portal (OKP) provides a Solr-backed RAG source to LSC. # Tests verify that Lightspeed Stack can use OKP for both Inline RAG # (context injected before the LLM request) and Tool RAG (context # retrieved on demand via file_search), in both offline and online modes. - Background: Given The service is started locally And The system is in default state @@ -18,6 +23,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario: Offline mode query with inline RAG returns rag_chunks and referenced_documents Given The service uses the lightspeed-stack-okp-offline.yaml configuration + And Llama Stack is restarted And The service is restarted When I use "query" to ask question with authorization header """ @@ -58,6 +64,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario: Query with inline RAG with dynamic semantic filter returns rag_chunks and referenced_documents Given The service uses the lightspeed-stack-okp-offline.yaml configuration + And Llama Stack is restarted And The service is restarted When I use "query" to ask question with authorization header """ @@ -75,7 +82,9 @@ Feature: OKP(Solr) RAG retrieval tests } """ Then The status code of the response is 200 - And The response contains "security best practices" + And The response contains following fragments + | Fragments in LLM response | + | security best practices | And The number of rag_chunk returned is 1 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" @@ -90,6 +99,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario: Offline query API with OKP tool RAG has rag_chunk and referenced_documents returned Given The service uses the lightspeed-stack-okp-tool-offline.yaml configuration + And Llama Stack is restarted And The service is restarted When I use "query" to ask question with authorization header """ @@ -118,6 +128,7 @@ Feature: OKP(Solr) RAG retrieval tests Scenario: Online responses API with OKP tool RAG has rag results returned Given The service uses the lightspeed-stack-okp-tool-online.yaml configuration + And Llama Stack is restarted And The service is restarted When I use "responses" to ask question with authorization header """ @@ -130,18 +141,18 @@ Feature: OKP(Solr) RAG retrieval tests """ Then The status code of the response is 200 And The responses output includes an item with type "file_search_call" - And The response contains non-empty tool_calls - And A tool_call has type "file_search" And The response contains non-empty results And The number of results returned is 3 And Each rag_chunk has a non-empty score And Each rag_chunk source is "okp" - And Each rag_chunk reference_url contains "access.redhat.com" + And Each rag_chunk reference_url contains "docs.redhat.com" # ── OKP Server Unavailable — Graceful Error Handling ── - + @skip + # https://redhat.atlassian.net/browse/LCORE-4022 Scenario: Query succeeds with empty rag_chunks when OKP server is unavailable Given The service uses the lightspeed-stack-okp-online.yaml configuration + And Llama Stack is restarted And The service is restarted And The OKP(Solr) server is stopped When I use "query" to ask question with authorization header @@ -153,7 +164,8 @@ Feature: OKP(Solr) RAG retrieval tests And The response contains no referenced_documents Scenario: Streaming query succeeds with empty referenced_documents when OKP server is unavailable - Given The service uses the lightspeed-stack-okp-online.yaml configuration + Given The service uses the lightspeed-stack-okp-tool-online.yaml configuration + And Llama Stack is restarted And The service is restarted And The OKP(Solr) server is stopped When I use "streaming_query" to ask question with authorization header @@ -162,4 +174,30 @@ Feature: OKP(Solr) RAG retrieval tests """ Then The status code of the response is 200 And I wait for the response to be completed - And The response contains no referenced_documents \ No newline at end of file + And The response contains no referenced_documents + + # ── OKP RAG Disabled (okp not in rag.retrieval.inline.sources) ───── + + Scenario: Query returns no rag_chunks and no reference_documents when OKP is disabled + Given The service uses the lightspeed-stack-okp-negative.yaml configuration + And Llama Stack is restarted + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response contains no rag_chunks + And The response contains no referenced_documents + + Scenario: Streaming query returns no referenced_documents when OKP is disabled + Given The service uses the lightspeed-stack-okp-negative.yaml configuration + And Llama Stack is restarted + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I wait for the response to be completed + And The response contains no referenced_documents diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index 67fd86322..f7ee63172 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -320,26 +320,4 @@ Scenario: Check if LLM responds for query request with error for missing query Given shields are disabled for this scenario When I use "query" to ask question with too-long query and authorization header Then The status code of the response is 413 - And The body of the response contains Prompt is too long - - # # ── OKP RAG Disabled (okp not in rag.retrieval.inline.sources) ───── - @cfg_okp - @skip - Scenario: Query returns no rag_chunks and no reference_documents when OKP OKP is disabled - When I use "query" to ask question with authorization header - """ - {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The response contains no rag_chunks - And The response contains no referenced_documents - @cfg_okp - @skip - Scenario: Streaming query returns no referenced_documents when OKP is disabled - When I use "streaming_query" to ask question with authorization header - """ - {"query": "configure remote desktop using gnome", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And I wait for the response to be completed - And The response contains no referenced_documents + And The body of the response contains Prompt is too long \ No newline at end of file diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index e6c5e135b..89f073b12 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -9,6 +9,7 @@ from behave.runner import Context from tests.e2e.utils.utils import ( + RESPONSE_TOOL_OUTPUT_ITEM_TYPES, parse_responses_sse_final_response_object, replace_placeholders, request_with_transient_retry, @@ -17,17 +18,6 @@ # Longer timeout for Prow/OpenShift with CPU-based vLLM DEFAULT_LLM_TIMEOUT = 180 if os.getenv("RUNNING_PROW") else 120 -# Responses API ``output`` item types that indicate tool listing or invocation. -_RESPONSE_TOOL_OUTPUT_ITEM_TYPES = frozenset( - { - "file_search_call", - "mcp_call", - "mcp_list_tools", - "function_call", - "web_search_call", - } -) - def _response_contains_fragment(text: str, fragment: str) -> bool: """Return whether *fragment* occurs in *text* as a substring (case-insensitive).""" @@ -46,7 +36,7 @@ def responses_output_should_not_include_tool_items(context: Context) -> None: assert context.response is not None, "Request needs to be performed first" response_json = cast(dict[str, Any], context.response.json()) types_found = _collect_output_item_types(response_json) - bad = [t for t in types_found if t in _RESPONSE_TOOL_OUTPUT_ITEM_TYPES] + bad = [t for t in types_found if t in RESPONSE_TOOL_OUTPUT_ITEM_TYPES] assert not bad, ( "Expected no tool-related output items, but found types " f"{bad!r} among all output types {types_found!r}" @@ -95,7 +85,14 @@ def responses_output_should_include_one_of_types(context: Context) -> None: @step("I wait for the response to be completed") def wait_for_complete_response(context: Context) -> None: """Wait for the response to be complete.""" - context.response_data = _parse_streaming_response(context.response.text) + # Reuse cached parse result if it was created for the current response + if not ( + hasattr(context, "response_data") + and hasattr(context, "_response_data_source") + and context._response_data_source is context.response + ): + context.response_data = _parse_streaming_response(context.response.text) + context._response_data_source = context.response context.response.raise_for_status() assert context.response_data["finished"] is True context.use_streaming_response_data = True @@ -156,6 +153,33 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: body = _read_streamed_response(resp) resp._content = body.encode(resp.encoding or "utf-8") context.response = resp + # Parse SSE events and make data available for assertions + # Preserve existing conversation_id if the new response doesn't have one + # (e.g., 403 errors won't have a 'start' event with conversation_id) + old_conversation_id = ( + context.response_data.get("conversation_id") + if hasattr(context, "response_data") + else None + ) + context.response_data = _parse_streaming_response(body) + # Extract conversation from terminal event if not found in start event (Responses API) + if not context.response_data.get("conversation_id"): + if old_conversation_id: + # Preserve from previous response (for 403 error scenarios) + context.response_data["conversation_id"] = old_conversation_id + else: + # Try to extract from terminal event (Responses API streaming format) + try: + terminal = parse_responses_sse_final_response_object(body) + context.response_data["conversation"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get( + "conversation" + ) + except AssertionError: + pass # No terminal event found (e.g., error responses) + # Mark that this parsed data is from the current response (for cache reuse) + context._response_data_source = resp + context.use_streaming_response_data = True else: context.response = request_with_transient_retry( method="POST", @@ -164,6 +188,8 @@ def ask_question_authorized(context: Context, endpoint: str) -> None: headers=context.auth_headers, timeout=DEFAULT_LLM_TIMEOUT, ) + # Reset flag so assertions read from response.json(), not stale response_data + context.use_streaming_response_data = False # Query length chosen to exceed typical model context windows (e.g. 128k tokens) @@ -186,15 +212,35 @@ def ask_question_too_long_authorized(context: Context, endpoint: str) -> None: @step("I store conversation details") def store_conversation_details(context: Context) -> None: - """Store details about the conversation.""" + """Store details about the conversation. + + Reuses cached parse result if it was created for the current response. + """ + # Reuse if already parsed for this response + if ( + hasattr(context, "response_data") + and hasattr(context, "_response_data_source") + and context._response_data_source is context.response + ): + return + try: context.response_data = json.loads(context.response.text) + context._response_data_source = context.response except json.JSONDecodeError: + # Streaming response - parse it context.response_data = _parse_streaming_response(context.response.text) + # Extract conversation from terminal event if not in start event (Responses API) if not context.response_data.get("conversation_id"): - terminal = parse_responses_sse_final_response_object(context.response.text) - context.response_data["conversation"] = terminal.get("conversation") - context.response_data["conversation_id"] = terminal.get("conversation") + try: + terminal = parse_responses_sse_final_response_object( + context.response.text + ) + context.response_data["conversation"] = terminal.get("conversation") + context.response_data["conversation_id"] = terminal.get("conversation") + except AssertionError: + pass # No terminal event found + context._response_data_source = context.response @step('I use "{endpoint}" to ask question with same conversation_id') @@ -220,7 +266,10 @@ def ask_question_in_same_conversation(context: Context, endpoint: str) -> None: def check_rag_chunks_present(context: Context) -> None: """Check that the response contains non-empty rag_chunks from inline RAG.""" assert context.response is not None - response_json = context.response.json() + if getattr(context, "use_streaming_response_data", False): + response_json = context.response_data + else: + response_json = context.response.json() assert "rag_chunks" in response_json, "rag_chunks field missing from response" assert ( len(response_json["rag_chunks"]) > 0 @@ -231,7 +280,10 @@ def check_rag_chunks_present(context: Context) -> None: def check_referenced_documents_present(context: Context) -> None: """Check that the response contains non-empty referenced_documents.""" assert context.response is not None - response_json = context.response.json() + if getattr(context, "use_streaming_response_data", False): + response_json = context.response_data + else: + response_json = context.response.json() assert ( "referenced_documents" in response_json ), "referenced_documents field missing from response" @@ -379,6 +431,7 @@ def _parse_streaming_response(response_text: str) -> dict: full_response_split: list[str] = [] tool_calls: list[dict[str, Any]] = [] tool_results: list[dict[str, Any]] = [] + referenced_documents: list[dict[str, Any]] = [] finished = False stream_error = ( None # {"status_code": int, "response": str, "cause": str} if event "error" @@ -403,6 +456,10 @@ def _parse_streaming_response(response_text: str) -> dict: full_response = data["data"]["token"] case "end": finished = True + end_data = data.get("data") or {} + referenced_documents = end_data.get( + "referenced_documents", referenced_documents + ) case "error": stream_error = data.get("data") or {} except json.JSONDecodeError: @@ -414,6 +471,7 @@ def _parse_streaming_response(response_text: str) -> dict: "response_complete": full_response, "tool_calls": tool_calls, "tool_results": tool_results, + "referenced_documents": referenced_documents, "finished": finished, "stream_error": stream_error, } diff --git a/tests/e2e/features/steps/okp_rag.py b/tests/e2e/features/steps/okp_rag.py new file mode 100644 index 000000000..9027d3c3a --- /dev/null +++ b/tests/e2e/features/steps/okp_rag.py @@ -0,0 +1,431 @@ +"""Step definitions for OKP(Solr) RAG retrieval tests.""" + +from typing import Any + +from behave import given, then # pyright: ignore[reportAttributeAccessIssue] +from behave.runner import Context + +from tests.e2e.utils.utils import ( + RESPONSE_TOOL_OUTPUT_ITEM_TYPES, + is_konflux_environment, +) + +# ── Response Body Extraction ── + + +def _get_response_body(context: Context) -> dict[str, Any]: + """Return the response body dict, handling both JSON and streaming formats.""" + if getattr(context, "use_streaming_response_data", False): + return context.response_data + return context.response.json() + + +# ── Data Extractors ── + + +def _get_file_search_results(context: Context) -> list[dict[str, Any]]: + """Extract file_search results from Responses API ``output`` items. + + Query API bodies have no ``output`` array, so this returns an empty list + there. Use ``_get_rag_chunks`` when either endpoint is possible. + """ + body = _get_response_body(context) + results: list[dict[str, Any]] = [] + for item in body.get("output", []): + if item.get("type") == "file_search_call": + results.extend(item.get("results") or []) + return results + + +def _get_rag_chunks(context: Context) -> list[dict[str, Any]]: + """Extract RAG chunks from a query or Responses API body. + + Query and streaming_query put chunks on ``rag_chunks``. The Responses API + has no such field; tool RAG chunks are ``file_search_call.results`` in + ``output``, so this falls through to ``_get_file_search_results``. + """ + body = _get_response_body(context) + if "rag_chunks" in body: + return body["rag_chunks"] + return _get_file_search_results(context) + + +def _get_referenced_documents(context: Context) -> list[dict[str, Any]]: + """Extract referenced_documents from response body.""" + body = _get_response_body(context) + return body.get("referenced_documents", []) + + +def _get_tool_calls(context: Context) -> list[dict[str, Any]]: + """Extract tool calls from query response or output items from Responses API.""" + body = _get_response_body(context) + if "tool_calls" in body: + return body["tool_calls"] + # Responses API: extract tool-type items from output + return [ + item + for item in body.get("output", []) + if item.get("type") in RESPONSE_TOOL_OUTPUT_ITEM_TYPES + ] + + +# ── Generic Field Accessors ── + + +def _get_nested_field(item: dict[str, Any], field_path: str) -> Any: + """Get a field from item, supporting nested access via dot notation. + + Examples: + _get_nested_field(chunk, "score") -> chunk.get("score") + _get_nested_field(chunk, "attributes.reference_url") + -> chunk.get("attributes", {}).get("reference_url") + + Parameters: + item: Dictionary to extract field from. + field_path: Field path, using dots for nested access. + + Returns: + Field value or None if not found. + """ + keys = field_path.split(".") + value: Any = item + for key in keys: + if isinstance(value, dict): + value = value.get(key) + else: + return None + return value + + +# ── Generic Assertion Helpers ── + + +def _assert_count_matches(items: list, expected_count: int, item_type: str) -> None: + """Assert the number of items matches the expected count. + + Parameters: + items: List of items to check. + expected_count: Expected number of items. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If count doesn't match. + """ + actual_count = len(items) + assert ( + actual_count == expected_count + ), f"Expected {expected_count} {item_type}, but found {actual_count}" + + +def _assert_not_empty(items: list, item_type: str) -> None: + """Assert the collection is not empty. + + Parameters: + items: List of items to check. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If collection is empty. + """ + assert len(items) > 0, f"{item_type} is empty — no items were found" + + +def _assert_empty(items: list, item_type: str) -> None: + """Assert the collection is empty. + + Parameters: + items: List of items to check. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If collection is not empty. + """ + assert len(items) == 0, f"Expected no {item_type}, but found {len(items)}" + + +def _assert_field_not_empty( + items: list[dict[str, Any]], field_path: str, item_type: str +) -> None: + """Assert every item has a non-empty value for the specified field. + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item has empty or missing field. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + value = _get_nested_field(item, field_path) + assert value is not None and value != "", ( + f"Expected non-empty {field_path} in {item_type}[{i}], " + f"but found {value!r}" + ) + + +def _assert_field_contains( + items: list[dict[str, Any]], field_path: str, substring: str, item_type: str +) -> None: + """Assert every item's field contains the expected substring (case-insensitive). + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + substring: Expected substring. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item's field doesn't contain substring. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + value = _get_nested_field(item, field_path) + assert substring.lower() in str(value).lower(), ( + f"Expected {substring!r} in {item_type}[{i}].{field_path}, " + f"but found {value!r}" + ) + + +def _assert_field_matches( + items: list[dict[str, Any]], field_path: str, expected: Any, item_type: str +) -> None: + """Assert every item's field matches the expected value. + + For fields that might be nested (e.g., source in attributes), checks both + the direct field and the attributes.field path. + + Parameters: + items: List of items to check. + field_path: Field path to check (supports dot notation). + expected: Expected value. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item's field doesn't match expected value. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + actual = _get_nested_field(item, field_path) + # Fallback: check if field exists in attributes + if actual is None and "." not in field_path: + actual = _get_nested_field(item, f"attributes.{field_path}") + assert actual == expected, ( + f"Expected {field_path}={expected!r} in {item_type}[{i}], " + f"but found {actual!r}" + ) + + +def _assert_has_fields( + items: list[dict[str, Any]], required_fields: set[str], item_type: str +) -> None: + """Assert every item has all required fields. + + Parameters: + items: List of items to check. + required_fields: Set of required field names. + item_type: Human-readable item type for error messages. + + Raises: + AssertionError: If any item is missing required fields. + """ + assert items, f"No {item_type} to check" + for i, item in enumerate(items): + missing = required_fields - set(item.keys()) + assert not missing, ( + f"Expected fields {required_fields} in {item_type}[{i}], " + f"but missing {missing}. Available fields: {list(item.keys())}" + ) + + +# ── Given steps ── + + +@given("OKP(Solr) server is running") +def okp_server_is_running(context: Context) -> None: + """Verify Solr is reachable from the OGX pod (Konflux in-cluster Service).""" + if not is_konflux_environment(): + raise RuntimeError( + "OKP Solr health check is only supported in Konflux " + "(in-cluster GET via e2e-ops check-okp-solr-from-llama)." + ) + + from tests.e2e.utils.prow_utils import assert_okp_reachable_from_ogx + + assert_okp_reachable_from_ogx() + + +@given("The OKP(Solr) server is stopped") +def okp_server_is_stopped(context: Context) -> None: + """Stop the OKP Solr pod to simulate unavailability (Konflux only).""" + context.okp_was_running = False + + if not is_konflux_environment(): + raise RuntimeError( + "Stopping OKP Solr is only supported in Konflux " + "(pod disruption via e2e-ops disrupt-okp-solr)." + ) + + from tests.e2e.utils.prow_utils import disrupt_okp_solr_pod + + was_running = disrupt_okp_solr_pod() + if was_running: + context.okp_was_running = True + print("✓ OKP Solr pod disrupted in Konflux environment") + else: + print("✓ OKP Solr pod was not running") + + +# ── Then Steps: rag_chunks Assertions ── + + +@then("The number of rag_chunk returned is {count:d}") +def check_rag_chunk_count(context: Context, count: int) -> None: + """Assert the number of rag_chunks matches the expected count.""" + chunks = _get_rag_chunks(context) + _assert_count_matches(chunks, count, "rag_chunks") + + +@then("Each rag_chunk has a non-empty score") +def check_rag_chunk_scores(context: Context) -> None: + """Assert every rag_chunk has a non-empty score.""" + chunks = _get_rag_chunks(context) + _assert_field_not_empty(chunks, "score", "rag_chunks") + + +@then('Each rag_chunk source is "{source}"') +def check_rag_chunk_source(context: Context, source: str) -> None: + """Assert every rag_chunk has the expected source.""" + chunks = _get_rag_chunks(context) + _assert_field_matches(chunks, "source", source, "rag_chunks") + + +@then('Each rag_chunk reference_url contains "{domain}"') +def check_rag_chunk_reference_url(context: Context, domain: str) -> None: + """Assert every rag_chunk's reference_url contains the expected domain.""" + chunks = _get_rag_chunks(context) + # Check both possible field paths for reference_url + assert chunks, "No rag_chunks to check" + for i, chunk in enumerate(chunks): + attrs = chunk.get("attributes") or {} + ref_url = attrs.get("reference_url") or attrs.get("doc_url") or "" + assert domain in str(ref_url), ( + f"Expected {domain!r} in rag_chunks[{i}].attributes.reference_url, " + f"but found {ref_url!r}" + ) + + +# ── Then Steps: referenced_documents Assertions ── + + +@then("Each referenced_document has fields doc_url, doc_title, source, and document_id") +def check_referenced_document_fields(context: Context) -> None: + """Assert every referenced_document has the required fields.""" + docs = _get_referenced_documents(context) + required_fields = {"doc_url", "doc_title", "source", "document_id"} + _assert_has_fields(docs, required_fields, "referenced_documents") + + +@then('Each referenced_document doc_url contains "{domain}"') +def check_referenced_document_doc_url(context: Context, domain: str) -> None: + """Assert every referenced_document doc_url contains the expected domain.""" + docs = _get_referenced_documents(context) + _assert_field_contains(docs, "doc_url", domain, "referenced_documents") + + +@then("Each referenced_document doc_title is not empty") +def check_referenced_document_doc_title(context: Context) -> None: + """Assert every referenced_document has a non-empty doc_title.""" + docs = _get_referenced_documents(context) + _assert_field_not_empty(docs, "doc_title", "referenced_documents") + + +@then('Each referenced_document doc_title contains "{substring}"') +def check_referenced_document_doc_title_contains( + context: Context, substring: str +) -> None: + """Assert every referenced_document doc_title contains the expected substring. + + Matching is case-insensitive. + """ + docs = _get_referenced_documents(context) + _assert_field_contains(docs, "doc_title", substring, "referenced_documents") + + +@then("The number of referenced_document returned is {count:d}") +def check_referenced_document_count(context: Context, count: int) -> None: + """Assert the number of referenced_documents matches the expected count.""" + docs = _get_referenced_documents(context) + _assert_count_matches(docs, count, "referenced_documents") + + +@then('Each referenced_document source is "{source}"') +def check_referenced_document_source(context: Context, source: str) -> None: + """Assert every referenced_document has the expected source.""" + docs = _get_referenced_documents(context) + _assert_field_matches(docs, "source", source, "referenced_documents") + + +@then("Each referenced_document has a non-empty document_id") +def check_referenced_document_id(context: Context) -> None: + """Assert every referenced_document has a non-empty document_id.""" + docs = _get_referenced_documents(context) + _assert_field_not_empty(docs, "document_id", "referenced_documents") + + +# ── Then Steps: tool_calls Assertions ── + + +@then("The response contains non-empty tool_calls") +def check_tool_calls_present(context: Context) -> None: + """Assert the response contains at least one tool call.""" + tool_calls = _get_tool_calls(context) + _assert_not_empty(tool_calls, "tool_calls") + + +@then('A tool_call has name "{name}"') +def check_tool_call_name(context: Context, name: str) -> None: + """Assert at least one tool call has the expected name.""" + tool_calls = _get_tool_calls(context) + assert tool_calls, "No tool_calls to check" + names = [tc.get("name") for tc in tool_calls] + assert name in names, ( + f"Expected tool_call with name {name!r}, " f"but found names {names!r}" + ) + + +# ── Then Steps: Content and Results Assertions ── + + +@then("The response contains non-empty results") +def check_results_present(context: Context) -> None: + """Assert the Responses API output contains non-empty file_search results.""" + results = _get_file_search_results(context) + _assert_not_empty(results, "file_search results") + + +@then("The number of results returned is {count:d}") +def check_results_count(context: Context, count: int) -> None: + """Assert the number of file_search results matches the expected count.""" + results = _get_file_search_results(context) + _assert_count_matches(results, count, "file_search results") + + +# ── Then Steps: Empty Response Assertions ── + + +@then("The response contains no rag_chunks") +def check_no_rag_chunks(context: Context) -> None: + """Assert the response has no rag_chunks (empty or absent).""" + body = _get_response_body(context) + chunks = body.get("rag_chunks", []) + _assert_empty(chunks, "rag_chunks") + + +@then("The response contains no referenced_documents") +def check_no_referenced_documents(context: Context) -> None: + """Assert the response has no referenced_documents (empty or absent).""" + body = _get_response_body(context) + docs = body.get("referenced_documents", []) + _assert_empty(docs, "referenced_documents") diff --git a/tests/e2e/features/steps/proxy.py b/tests/e2e/features/steps/proxy.py index eb2e3ced3..9bfeeb866 100644 --- a/tests/e2e/features/steps/proxy.py +++ b/tests/e2e/features/steps/proxy.py @@ -323,7 +323,16 @@ def restore_if_modified(context: Context) -> None: @given("Llama Stack is restarted") def restart_ogx(context: Context) -> None: - """Restart the OGX container.""" + """Restart the OGX container. + + No-ops when ``configure_service`` set ``lightspeed_stack_skip_restart`` + (same Lightspeed YAML as the previous scenario). Does not clear that + flag — ``The service is restarted`` still consumes it. + """ + if getattr(context, "lightspeed_stack_skip_restart", False): + print("Skipping Llama Stack restart (Lightspeed config unchanged)") + return + from tests.e2e.features.steps.tls import ( is_tls_configuration_feature, restart_llama_for_tls_feature, diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index 9e19f0874..bc6808159 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -20,6 +20,7 @@ def get_namespace() -> str: _POD_NAME_MAP = { "lightspeed-stack": "lightspeed-stack-service", "llama-stack": "llama-stack-service", + "okp-solr": "okp-solr-service", } @@ -346,3 +347,86 @@ def update_config_configmap( finally: if temp_path and os.path.exists(temp_path): os.remove(temp_path) + + +def assert_okp_reachable_from_ogx() -> None: + """Fail unless the OGX pod can HTTP GET in-cluster OKP Solr. + + This is the Konflux health check: runner localhost:8081 only proves + the e2e port-forward, not OGX → Solr. + + Raises: + subprocess.CalledProcessError: If the in-cluster GET fails. + subprocess.TimeoutExpired: If the operation times out. + """ + result = run_e2e_ops("check-okp-solr-from-llama", timeout=60) + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr, end="") + raise subprocess.CalledProcessError( + result.returncode, "check-okp-solr-from-llama", result.stderr + ) + + +def ensure_okp_solr_ready() -> None: + """Deploy OKP Solr on Konflux if it is not already Ready. + + Idempotent. First image pull can take ~10-15 minutes (7GB). Starts the + localhost:8081 port-forward after the pod is ready. + + Raises: + subprocess.CalledProcessError: If deploy-okp-solr fails. + subprocess.TimeoutExpired: If the operation times out. + """ + # 300 wait_for_pod attempts × 3s = 900s, plus oc apply and port-forward. + result = run_e2e_ops("deploy-okp-solr", timeout=1080) + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr, end="") + raise subprocess.CalledProcessError( + result.returncode, "deploy-okp-solr", result.stderr + ) + print("✓ OKP Solr is ready") + + +def disrupt_okp_solr_pod() -> bool: + """Disrupt OKP Solr in the Konflux test namespace. + + Returns: + True if the pod was running and has been disrupted, False otherwise. + """ + try: + result = run_e2e_ops("disrupt-okp-solr", timeout=60) + print(result.stdout, end="") + + # Exit code 0 = disrupted (was running), exit code 2 = was not running + if result.returncode == 0: + return True + elif result.returncode == 2: + return False + else: + print(result.stderr, end="") + return False + + except subprocess.TimeoutExpired: + print("Warning: Timeout while disrupting OKP Solr connection") + return False + + +def restore_okp_solr_pod() -> None: + """Restore the OKP Solr pod in the Konflux test namespace. + + Raises: + subprocess.CalledProcessError: If oc/e2e-ops restore fails. + subprocess.TimeoutExpired: If the operation times out. + """ + # restore-okp-solr can spend 180 seconds in wait_for_pod, plus oc apply time. + # Use a timeout with margin (240s) so we fail instead of hanging forever. + result = run_e2e_ops("restore-okp-solr", timeout=240) + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr, end="") + raise subprocess.CalledProcessError( + result.returncode, "restore-okp-solr", result.stderr + ) + print("✓ OKP Solr pod restored successfully") diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index 378dbde12..4cb23a458 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -56,10 +56,23 @@ def absolute_repo_path(repo_relative: str) -> str: def is_prow_environment() -> bool: - """Check if running in Prow/OpenShift environment.""" + """Check if running in Prow/OpenShift environment. + + True for both classic Prow (``pipeline.sh``) and Konflux + (``pipeline-konflux.sh``), which also sets ``RUNNING_PROW``. + """ return os.getenv("RUNNING_PROW") is not None +def is_konflux_environment() -> bool: + """Check if running in the Konflux integration E2E pipeline. + + Konflux sets ``E2E_KONFLUX_E2E=1`` in ``pipeline-konflux.sh``. Classic Prow + does not. Use this to gate tests that must not run on Prow (e.g. OKP RAG). + """ + return os.getenv("E2E_KONFLUX_E2E") == "1" + + # Transient connection resets (e.g. errno 104) after container restarts in CI/Docker. E2E_HTTP_TRANSIENT_MAX_ATTEMPTS: int = 3 E2E_HTTP_TRANSIENT_DELAY_S: float = 0.5 @@ -310,6 +323,17 @@ def validate_json_partially(actual: Any, expected: Any) -> None: {"response.completed", "response.incomplete", "response.failed"} ) +# Responses API ``output`` item types that indicate tool listing or invocation. +RESPONSE_TOOL_OUTPUT_ITEM_TYPES = frozenset( + { + "file_search_call", + "function_call", + "mcp_call", + "mcp_list_tools", + "web_search_call", + } +) + def parse_responses_sse_final_response_object(text: str) -> dict[str, Any]: """Return the ``response`` object from the last terminal LCORE ``/responses`` SSE event.""" diff --git a/tests/unit/test_ogx_configuration.py b/tests/unit/test_ogx_configuration.py index 19be9278f..3f76c2c18 100644 --- a/tests/unit/test_ogx_configuration.py +++ b/tests/unit/test_ogx_configuration.py @@ -1006,6 +1006,30 @@ def test_enrich_solr_preserves_existing_vector_stores() -> None: ) +def test_enrich_solr_adds_external_providers_dir_when_missing() -> None: + """Test enrich_solr injects external_providers_dir for remote Solr resolution.""" + ls_config: dict[str, Any] = {} + enrich_solr(ls_config, _OKP_RAG_CONFIG, {}) + + assert ( + ls_config["external_providers_dir"] + == "${env.EXTERNAL_PROVIDERS_DIR:=/opt/app-root/providers.d}" + ) + + +def test_enrich_solr_preserves_existing_external_providers_dir() -> None: + """Test enrich_solr does not overwrite an existing external_providers_dir.""" + ls_config: dict[str, Any] = { + "external_providers_dir": "${env.EXTERNAL_PROVIDERS_DIR:=~/.llama/providers.d}" + } + enrich_solr(ls_config, _OKP_RAG_CONFIG, {}) + + assert ( + ls_config["external_providers_dir"] + == "${env.EXTERNAL_PROVIDERS_DIR:=~/.llama/providers.d}" + ) + + # ============================================================================= # Test enrich_vector_store # =============================================================================