diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionGuard.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionGuard.java new file mode 100644 index 0000000000..6159d3fb89 --- /dev/null +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionGuard.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.gandiva.evaluator; + +import java.util.ArrayDeque; +import java.util.Deque; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.ipc.GandivaTypes; + +/** + * Pre-flight check for Gandiva expression trees. + * + *

Gandiva compiles each expression into a single LLVM-emitted function. Two AST shapes can + * crash the JVM in native code: + * + *

+ * + *

This class walks the protobuf form of an expression iteratively (no Java-side recursion, so + * the guard itself never overflows on pathological input) and rejects trees whose depth or + * node-count exceed configured limits, converting what would be a JVM crash into a recoverable + * {@link GandivaException}. + * + *

Limits can be overridden via the system properties {@value #MAX_DEPTH_PROPERTY} and {@value + * #MAX_NODES_PROPERTY}; the defaults are sized to admit any plausible hand-written expression + * while rejecting the planner-generated pathological shapes that have been observed to crash. + */ +final class ExpressionGuard { + + static final String MAX_DEPTH_PROPERTY = "org.apache.arrow.gandiva.expr.max_depth"; + static final String MAX_NODES_PROPERTY = "org.apache.arrow.gandiva.expr.max_nodes"; + + static final int DEFAULT_MAX_DEPTH = 100; + static final int DEFAULT_MAX_NODES = 10_000; + + private ExpressionGuard() {} + + static int maxDepth() { + return Integer.getInteger(MAX_DEPTH_PROPERTY, DEFAULT_MAX_DEPTH); + } + + static int maxNodes() { + return Integer.getInteger(MAX_NODES_PROPERTY, DEFAULT_MAX_NODES); + } + + /** Validates a Condition's root tree. */ + static void check(GandivaTypes.Condition condition) throws GandivaException { + if (condition.hasRoot()) { + check(condition.getRoot()); + } + } + + /** + * Validates every expression root in an ExpressionList independently. Gandiva's + * {@code LLVMGenerator::Add} compiles each {@code Projector} expression into its own LLVM + * function ({@code expr__}), so the per-function spill-slot budget — which is what + * the node-count limit defends — applies per expression, not in aggregate. A Projector with + * many small expressions is fine even if their combined node count exceeds the limit. + */ + static void check(GandivaTypes.ExpressionList exprs) throws GandivaException { + for (GandivaTypes.ExpressionRoot root : exprs.getExprsList()) { + if (root.hasRoot()) { + check(root.getRoot()); + } + } + } + + /** Walks a single tree iteratively, throwing if depth or node-count exceed the limits. */ + static void check(GandivaTypes.TreeNode root) throws GandivaException { + final int maxDepth = maxDepth(); + final int maxNodes = maxNodes(); + + // Pair each node with its depth on a work-stack. ArrayDeque is the JDK's recommended + // non-recursive stack; per-entry cost is tiny so we can hold ~maxNodes entries without + // approaching the heap budget. + Deque stack = new ArrayDeque<>(); + stack.push(new Frame(root, 1)); + + int nodes = 0; + while (!stack.isEmpty()) { + Frame frame = stack.pop(); + nodes++; + if (nodes > maxNodes) { + throw new GandivaException( + "Gandiva expression exceeds node-count limit: > " + + maxNodes + + " nodes (override with -D" + + MAX_NODES_PROPERTY + + "=N)"); + } + if (frame.depth > maxDepth) { + throw new GandivaException( + "Gandiva expression exceeds depth limit: depth " + + frame.depth + + " > " + + maxDepth + + " (override with -D" + + MAX_DEPTH_PROPERTY + + "=N)"); + } + + GandivaTypes.TreeNode node = frame.node; + int childDepth = frame.depth + 1; + + if (node.hasIfNode()) { + GandivaTypes.IfNode ifNode = node.getIfNode(); + if (ifNode.hasCond()) { + stack.push(new Frame(ifNode.getCond(), childDepth)); + } + if (ifNode.hasThenNode()) { + stack.push(new Frame(ifNode.getThenNode(), childDepth)); + } + if (ifNode.hasElseNode()) { + stack.push(new Frame(ifNode.getElseNode(), childDepth)); + } + } + if (node.hasAndNode()) { + for (GandivaTypes.TreeNode child : node.getAndNode().getArgsList()) { + stack.push(new Frame(child, childDepth)); + } + } + if (node.hasOrNode()) { + for (GandivaTypes.TreeNode child : node.getOrNode().getArgsList()) { + stack.push(new Frame(child, childDepth)); + } + } + if (node.hasFnNode()) { + for (GandivaTypes.TreeNode child : node.getFnNode().getInArgsList()) { + stack.push(new Frame(child, childDepth)); + } + } + if (node.hasInNode() && node.getInNode().hasNode()) { + stack.push(new Frame(node.getInNode().getNode(), childDepth)); + } + // Leaf nodes (field, literals) have no children to enqueue. + } + } + + private static final class Frame { + final GandivaTypes.TreeNode node; + final int depth; + + Frame(GandivaTypes.TreeNode node, int depth) { + this.node = node; + this.depth = depth; + } + } +} diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java index 6c8540cde8..363dfd4072 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java @@ -115,6 +115,7 @@ public static synchronized Filter make(Schema schema, Condition condition, long throws GandivaException { // Invoke the JNI layer to create the LLVM module representing the filter. GandivaTypes.Condition conditionBuf = condition.toProtobuf(); + ExpressionGuard.check(conditionBuf); GandivaTypes.Schema schemaBuf = ArrowTypeHelper.arrowSchemaToProtobuf(schema); JniWrapper wrapper = JniLoader.getInstance().getWrapper(); long moduleId = diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java index 9c5b22d659..23ce3710cc 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java @@ -199,6 +199,8 @@ public static synchronized Projector make( for (ExpressionTree expr : exprs) { builder.addExprs(expr.toProtobuf()); } + GandivaTypes.ExpressionList exprList = builder.build(); + ExpressionGuard.check(exprList); // Invoke the JNI layer to create the LLVM module representing the expressions GandivaTypes.Schema schemaBuf = ArrowTypeHelper.arrowSchemaToProtobuf(schema); @@ -206,7 +208,7 @@ public static synchronized Projector make( long moduleId = wrapper.buildProjector( schemaBuf.toByteArray(), - builder.build().toByteArray(), + exprList.toByteArray(), selectionVectorType.getNumber(), configurationId); logger.debug("Created module for the projector with id {}", moduleId); diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CustomerUnpivotFilterReproTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CustomerUnpivotFilterReproTest.java new file mode 100644 index 0000000000..daca25c2c6 --- /dev/null +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CustomerUnpivotFilterReproTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.gandiva.evaluator; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.Lists; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.expression.Condition; +import org.apache.arrow.gandiva.expression.TreeBuilder; +import org.apache.arrow.gandiva.expression.TreeNode; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +/** + * Faithful reproduction of the customer's failing Gandiva Filter. + * + *

Built from the post-planner Filter expression text the customer's Dremio emitted (saved at + * {@code gandiva/filter.txt} during the incident). The expression has the shape + * + * 

+ *   OR(
+ *     AND(=(key, lit_0), IS NOT NULL(v_0)),
+ *     AND(=(key, lit_1), <>(key, lit_0), IS NOT NULL(v_1)),
+ *     AND(=(key, lit_2), <>(key, lit_0), <>(key, lit_1), IS NOT NULL(v_2)),
+ *     ...
+ *     AND(=(key, lit_{N-1}), <>(key, lit_0), ..., <>(key, lit_{N-2}), IS NOT NULL(v_{N-1}))
+ *   )
+ *
+ * + *

For N=393 (the customer's count) that's 393 AND arms, 77 028 {@code <>} comparisons, + * 393 {@code IS NOT NULL} checks over a schema of 1 key column + 393 nullable VARCHAR(65536) + * value columns. + * + *

Two test methods: + * + *

    + *
  1. {@link #customerFilterIsRejectedByGuard()} — runs unconditionally. Asserts that the + * arrow-java guard rejects the expression with a {@code node-count} message instead of + * letting it reach native code. This is the production behaviour. + *
  2. {@link #customerFilterRunsAgainstGandivaIfGuardDisabled()} — opt-in via {@code + * -Dgandiva.repro.runUnguarded=1}. Raises the guard limits via system properties so the + * expression actually reaches {@code Filter.make} / {@code Filter.evaluate}. Will likely + * crash the forked JVM with SIGSEGV on builds where the runtime JIT-frame overflow + * reproduces; otherwise will either compile-and-evaluate cleanly (on newer LLVM that + * deduplicates the comparisons) or take a long time. Run in isolation because the + * JVM exit may take other tests with it. + *
+ */ +public class CustomerUnpivotFilterReproTest extends BaseEvaluatorTest { + + private static final String LITERALS_RESOURCE = "/unpivot_393_literals.txt"; + + /** Loads the 393 literal strings extracted from the customer's filter.txt. */ + private static List loadLiterals() { + List out = new ArrayList<>(393); + try (InputStream in = + CustomerUnpivotFilterReproTest.class.getResourceAsStream(LITERALS_RESOURCE); + BufferedReader r = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + if (!line.isEmpty()) { + out.add(line); + } + } + } catch (Exception e) { + throw new RuntimeException("Failed to load " + LITERALS_RESOURCE, e); + } + if (out.size() != 393) { + throw new IllegalStateException("Expected 393 literals, got " + out.size()); + } + return out; + } + + /** + * Builds the customer's filter expression with the first {@code numArms} literals. + * + *

Total node count grows as N + N*(N-1)/2 (positive eqs + cumulative neqs) + N (isnotnull) + + * N (and) + 1 (or) ≈ N²/2 + 3N + 1. For N=393 ≈ 78 600 nodes. + * + * @return the bundle containing the constructed schema and the Condition wrapping the OR root. + */ + private static FilterBundle buildCustomerFilter(int numArms) { + ArrowType utf8 = new ArrowType.Utf8(); + ArrowType bool = new ArrowType.Bool(); + List literals = loadLiterals(); + if (numArms > literals.size()) { + throw new IllegalArgumentException( + "numArms=" + numArms + " exceeds available literals " + literals.size()); + } + + // Schema: column 0 is the key, columns 1..numArms are the per-arm values. + Field keyField = Field.nullable("key", utf8); + TreeNode keyNode = TreeBuilder.makeField(keyField); + List schemaFields = new ArrayList<>(); + schemaFields.add(keyField); + List valueNodes = new ArrayList<>(numArms); + for (int i = 0; i < numArms; i++) { + Field vi = Field.nullable("v" + i, utf8); + schemaFields.add(vi); + valueNodes.add(TreeBuilder.makeField(vi)); + } + + // Precompute the positive `=(key, lit_i)` nodes once each (they're reused as `<>(...)` siblings + // — but Gandiva tree nodes are immutable values; reuse here doesn't actually share structure + // in the resulting protobuf, since each AND arm is built fresh from literal strings). + List eqNodes = new ArrayList<>(numArms); + for (int i = 0; i < numArms; i++) { + eqNodes.add( + TreeBuilder.makeFunction( + "equal", + Lists.newArrayList(keyNode, TreeBuilder.makeStringLiteral(literals.get(i))), + bool)); + } + + // Build each AND arm: =(key, lit_i), <>(key, lit_0), ..., <>(key, lit_{i-1}), isnotnull(v_i) + List orArgs = new ArrayList<>(numArms); + for (int i = 0; i < numArms; i++) { + List andArgs = new ArrayList<>(i + 2); + andArgs.add(eqNodes.get(i)); + for (int j = 0; j < i; j++) { + andArgs.add( + TreeBuilder.makeFunction( + "not_equal", + Lists.newArrayList( + keyNode, TreeBuilder.makeStringLiteral(literals.get(j))), + bool)); + } + andArgs.add( + TreeBuilder.makeFunction("isnotnull", Lists.newArrayList(valueNodes.get(i)), bool)); + orArgs.add(TreeBuilder.makeAnd(andArgs)); + } + TreeNode root = TreeBuilder.makeOr(orArgs); + + return new FilterBundle(new Schema(schemaFields), TreeBuilder.makeCondition(root)); + } + + /** + * With the guard at its default settings, the customer's 393-arm filter must be rejected with a + * GandivaException naming the node-count limit, before any native code runs. + */ + @Test + public void customerFilterIsRejectedByGuard() { + FilterBundle bundle = buildCustomerFilter(393); + GandivaException ex = + assertThrows( + GandivaException.class, () -> Filter.make(bundle.schema, bundle.condition)); + assertTrue( + ex.getMessage().contains("node-count") || ex.getMessage().contains("depth"), + ex.getMessage()); + } + + /** + * Opt-in runtime reproduction. Raises guard limits via the test JVM's system properties + * (caller is responsible for setting both — see test class Javadoc), then attempts to compile + * AND evaluate the customer's filter. + * + *

To run: 

+   *   mvn -Parrow-jni -pl gandiva test \
+   *     -Dtest='CustomerUnpivotFilterReproTest#customerFilterRunsAgainstGandivaIfGuardDisabled' \
+   *     -Dsurefire.add-opens.argLine="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED \
+   *         -Xss1m -Dgandiva.repro.runUnguarded=1 \
+   *         -Dorg.apache.arrow.gandiva.expr.max_depth=1000000 \
+   *         -Dorg.apache.arrow.gandiva.expr.max_nodes=10000000"
+   *
+ * + *

Expected outcomes (none are "test pass" — this method is an investigation tool): + * + *

+ */ + @Test + @EnabledIfSystemProperty(named = "gandiva.repro.runUnguarded", matches = ".+") + public void customerFilterRunsAgainstGandivaIfGuardDisabled() throws GandivaException { + int n = Integer.parseInt(System.getProperty("gandiva.repro.n", "393")); + boolean optimize = Boolean.parseBoolean(System.getProperty("gandiva.repro.optimize", "true")); + FilterBundle bundle = buildCustomerFilter(n); + + System.out.println("[probe] before Filter.make n=" + n + " optimize=" + optimize); + System.out.flush(); + ConfigurationBuilder.ConfigOptions configOptions = + new ConfigurationBuilder.ConfigOptions().withOptimize(optimize); + Filter filter = Filter.make(bundle.schema, bundle.condition, configOptions); + System.out.println("[probe] after Filter.make n=" + n); + System.out.flush(); + + int numRows = 1; + int numCols = n + 1; + ArrowFieldNode fieldNode = new ArrowFieldNode(numRows, 0); + List fieldNodes = new ArrayList<>(numCols); + List buffers = new ArrayList<>(numCols * 3); + for (int i = 0; i < numCols; i++) { + fieldNodes.add(fieldNode); + ArrowBuf validity = allocator.buffer(1); + validity.writeByte((byte) 0xFF); + ArrowBuf offsets = allocator.buffer(8); + offsets.writeInt(0); + offsets.writeInt(0); + ArrowBuf data = allocator.buffer(0); + buffers.add(validity); + buffers.add(offsets); + buffers.add(data); + } + ArrowRecordBatch batch = new ArrowRecordBatch(numRows, fieldNodes, buffers); + ArrowBuf selectionBuffer = allocator.buffer(numRows * 2L); + SelectionVectorInt16 selectionVector = new SelectionVectorInt16(selectionBuffer); + try { + System.out.println("[probe] before Filter.evaluate n=" + n); + System.out.flush(); + filter.evaluate(batch, selectionVector); + System.out.println("[probe] after Filter.evaluate n=" + n); + System.out.flush(); + } finally { + selectionBuffer.close(); + releaseRecordBatch(batch); + filter.close(); + } + } + + private static final class FilterBundle { + final Schema schema; + final Condition condition; + + FilterBundle(Schema schema, Condition condition) { + this.schema = schema; + this.condition = condition; + } + } +} diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ExpressionGuardTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ExpressionGuardTest.java new file mode 100644 index 0000000000..a5571cac68 --- /dev/null +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ExpressionGuardTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.gandiva.evaluator; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.List; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.expression.Condition; +import org.apache.arrow.gandiva.expression.ExpressionTree; +import org.apache.arrow.gandiva.expression.TreeBuilder; +import org.apache.arrow.gandiva.expression.TreeNode; +import org.apache.arrow.gandiva.ipc.GandivaTypes; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; + +/** + * Focused tests for the {@link ExpressionGuard} depth and node-count limits as wired into {@link + * Filter#make(Schema, Condition)} and {@link Projector#make(Schema, java.util.List)}. Each test + * builds the smallest expression that violates one limit, calls the relevant make() variant, and + * asserts the right kind of {@link GandivaException} is raised before reaching the JNI layer. + */ +public class ExpressionGuardTest extends BaseEvaluatorTest { + + /** + * Builds a chain of nested {@link TreeBuilder#makeIf} nodes that is {@code numIfs} deep. With + * {@code numIfs > DEFAULT_MAX_DEPTH} the guard must reject this with a depth message. + */ + private static TreeNode nestedIfChain(int numIfs, Field cond, Field then, Field els) { + TreeNode condNode = TreeBuilder.makeField(cond); + TreeNode thenNode = TreeBuilder.makeField(then); + TreeNode current = TreeBuilder.makeField(els); + ArrowType retType = els.getType(); + for (int i = 0; i < numIfs; i++) { + current = TreeBuilder.makeIf(condNode, thenNode, current, retType); + } + return current; + } + + /** + * Builds an OR of {@code numArgs} {@code isnotnull(field_i)} comparisons — a flat tree that + * pushes the node count up without adding any depth. + */ + private static TreeNode wideOr(List fields) { + ArrowType bool = new ArrowType.Bool(); + List args = new ArrayList<>(fields.size()); + for (Field f : fields) { + args.add( + TreeBuilder.makeFunction( + "isnotnull", Lists.newArrayList(TreeBuilder.makeField(f)), bool)); + } + return TreeBuilder.makeOr(args); + } + + // ---------- depth limit ---------- + + @Test + public void filterRejectsExpressionAboveDepthLimit() { + Field cond = Field.nullable("c", new ArrowType.Bool()); + Field then = Field.nullable("t", new ArrowType.Bool()); + Field els = Field.nullable("e", new ArrowType.Bool()); + Schema schema = new Schema(Lists.newArrayList(cond, then, els)); + + TreeNode root = + nestedIfChain(ExpressionGuard.DEFAULT_MAX_DEPTH + 50, cond, then, els); + Condition condition = TreeBuilder.makeCondition(root); + + GandivaException ex = + assertThrows(GandivaException.class, () -> Filter.make(schema, condition)); + assertTrue(ex.getMessage().contains("depth"), ex.getMessage()); + } + + @Test + public void projectorRejectsExpressionAboveDepthLimit() { + Field cond = Field.nullable("c", new ArrowType.Bool()); + Field then = Field.nullable("t", new ArrowType.Bool()); + Field els = Field.nullable("e", new ArrowType.Bool()); + Schema schema = new Schema(Lists.newArrayList(cond, then, els)); + + TreeNode root = + nestedIfChain(ExpressionGuard.DEFAULT_MAX_DEPTH + 50, cond, then, els); + ExpressionTree expr = + TreeBuilder.makeExpression(root, Field.nullable("res", new ArrowType.Bool())); + + GandivaException ex = + assertThrows( + GandivaException.class, () -> Projector.make(schema, Lists.newArrayList(expr))); + assertTrue(ex.getMessage().contains("depth"), ex.getMessage()); + } + + // ---------- node-count limit ---------- + + @Test + public void filterRejectsExpressionAboveNodeCountLimit() { + List fields = new ArrayList<>(); + // numFields chosen so total nodes (1 OR + N functions + N fields) is comfortably past limit. + int numFields = ExpressionGuard.DEFAULT_MAX_NODES; + for (int i = 0; i < numFields; i++) { + fields.add(Field.nullable("f" + i, new ArrowType.Bool())); + } + Schema schema = new Schema(fields); + Condition condition = TreeBuilder.makeCondition(wideOr(fields)); + + GandivaException ex = + assertThrows(GandivaException.class, () -> Filter.make(schema, condition)); + assertTrue(ex.getMessage().contains("node-count"), ex.getMessage()); + } + + @Test + public void projectorRejectsExpressionAboveNodeCountLimit() { + List fields = new ArrayList<>(); + int numFields = ExpressionGuard.DEFAULT_MAX_NODES; + for (int i = 0; i < numFields; i++) { + fields.add(Field.nullable("f" + i, new ArrowType.Bool())); + } + Schema schema = new Schema(fields); + ExpressionTree expr = + TreeBuilder.makeExpression( + wideOr(fields), Field.nullable("res", new ArrowType.Bool())); + + GandivaException ex = + assertThrows( + GandivaException.class, () -> Projector.make(schema, Lists.newArrayList(expr))); + assertTrue(ex.getMessage().contains("node-count"), ex.getMessage()); + } + + // ---------- sanity: small expressions still compile ---------- + + @Test + public void smallExpressionPassesGuard() throws GandivaException { + Field a = Field.nullable("a", new ArrowType.Bool()); + Schema schema = new Schema(Lists.newArrayList(a)); + Condition condition = + TreeBuilder.makeCondition( + TreeBuilder.makeFunction( + "isnotnull", + Lists.newArrayList(TreeBuilder.makeField(a)), + new ArrowType.Bool())); + Filter filter = Filter.make(schema, condition); + filter.close(); + } + + // ---------- boundary tests ---------- + // The guard fires on `depth > maxDepth` and `nodes > maxNodes`, so the exact-limit value + // must pass and limit+1 must fail. These tests pin that boundary by calling the guard + // directly on a hand-built protobuf tree — going through Filter.make would also compile the + // expression natively, which for the node-count boundary (10 000-arg AND) takes ~9 minutes. + // The Filter.make/Projector.make wiring is covered by the other tests in this class. + + /** Builds a protobuf TreeNode that is exactly {@code depth} deep. */ + private static GandivaTypes.TreeNode protoChainOfDepth(int depth) throws GandivaException { + Field f = Field.nullable("a", new ArrowType.Bool()); + TreeNode current = TreeBuilder.makeField(f); + for (int i = 1; i < depth; i++) { + current = + TreeBuilder.makeFunction("not", Lists.newArrayList(current), new ArrowType.Bool()); + } + return current.toProtobuf(); + } + + /** Builds a protobuf TreeNode with exactly {@code totalNodes} nodes (flat AND of fields). */ + private static GandivaTypes.TreeNode protoFlatAndOfSize(int totalNodes) throws GandivaException { + int numArgs = totalNodes - 1; // 1 AndNode + numArgs field nodes + List args = new ArrayList<>(numArgs); + for (int i = 0; i < numArgs; i++) { + args.add(TreeBuilder.makeField(Field.nullable("f" + i, new ArrowType.Bool()))); + } + return TreeBuilder.makeAnd(args).toProtobuf(); + } + + @Test + public void guardPassesAtDepthOneBelowLimit() throws GandivaException { + ExpressionGuard.check(protoChainOfDepth(ExpressionGuard.DEFAULT_MAX_DEPTH - 1)); + } + + @Test + public void guardPassesAtDepthExactlyAtLimit() throws GandivaException { + ExpressionGuard.check(protoChainOfDepth(ExpressionGuard.DEFAULT_MAX_DEPTH)); + } + + @Test + public void guardRejectsAtDepthOneAboveLimit() throws GandivaException { + GandivaTypes.TreeNode root = protoChainOfDepth(ExpressionGuard.DEFAULT_MAX_DEPTH + 1); + GandivaException ex = + assertThrows(GandivaException.class, () -> ExpressionGuard.check(root)); + assertTrue(ex.getMessage().contains("depth"), ex.getMessage()); + } + + @Test + public void guardPassesAtNodeCountExactlyAtLimit() throws GandivaException { + ExpressionGuard.check(protoFlatAndOfSize(ExpressionGuard.DEFAULT_MAX_NODES)); + } + + @Test + public void guardRejectsAtNodeCountOneAboveLimit() throws GandivaException { + GandivaTypes.TreeNode root = protoFlatAndOfSize(ExpressionGuard.DEFAULT_MAX_NODES + 1); + GandivaException ex = + assertThrows(GandivaException.class, () -> ExpressionGuard.check(root)); + assertTrue(ex.getMessage().contains("node-count"), ex.getMessage()); + } + + // ---------- system-property override ---------- + + /** + * Raising the depth limit via {@link ExpressionGuard#MAX_DEPTH_PROPERTY} must let through a + * tree that would otherwise be rejected. Property is restored in finally so subsequent tests + * in the same JVM fork see the original behaviour. + */ + @Test + public void depthLimitHonoursSystemPropertyOverride() throws GandivaException { + GandivaTypes.TreeNode root = protoChainOfDepth(ExpressionGuard.DEFAULT_MAX_DEPTH + 1); + // Without override: rejected. + assertThrows(GandivaException.class, () -> ExpressionGuard.check(root)); + + String previous = System.getProperty(ExpressionGuard.MAX_DEPTH_PROPERTY); + try { + System.setProperty( + ExpressionGuard.MAX_DEPTH_PROPERTY, + Integer.toString(ExpressionGuard.DEFAULT_MAX_DEPTH + 10)); + // With override: accepted. + ExpressionGuard.check(root); + } finally { + if (previous == null) { + System.clearProperty(ExpressionGuard.MAX_DEPTH_PROPERTY); + } else { + System.setProperty(ExpressionGuard.MAX_DEPTH_PROPERTY, previous); + } + } + } +} diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/UnpivotCaseStackOverflowTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/UnpivotCaseStackOverflowTest.java new file mode 100644 index 0000000000..f573641cd6 --- /dev/null +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/UnpivotCaseStackOverflowTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.gandiva.evaluator; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.List; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.expression.Condition; +import org.apache.arrow.gandiva.expression.ExpressionTree; +import org.apache.arrow.gandiva.expression.TreeBuilder; +import org.apache.arrow.gandiva.expression.TreeNode; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; + +/** + * Regression tests for the UNPIVOT-style nested-CASE expression that originally crashed the JVM + * in native Gandiva code. The expression shape is + * + * 
+ *   CAST(CASE WHEN key = 'lit_0'   THEN v0
+ *             WHEN key = 'lit_1'   THEN v1
+ *             ...
+ *             WHEN key = 'lit_392' THEN v392
+ *             ELSE NULL
+ *        END AS VARCHAR(65536))
+ *
+ * + *

Calcite/Gandiva lower this into nested {@code if/else} nodes, one per WHEN branch. Without a + * Java-side guard the resulting 393-deep AST blows the native stack during {@code Projector.make} + * / {@code Filter.make}. With {@link ExpressionGuard} in place these deep variants must be + * rejected cleanly with a {@link GandivaException} naming the depth limit; shallow variants must + * still compile and evaluate normally. + */ +public class UnpivotCaseStackOverflowTest extends BaseEvaluatorTest { + + private static ProjectionBundle buildNestedCaseProjection(int numBranches) { + ArrowType utf8 = new ArrowType.Utf8(); + + Field keyField = Field.nullable("key", utf8); + TreeNode keyNode = TreeBuilder.makeField(keyField); + + List schemaFields = new ArrayList<>(); + schemaFields.add(keyField); + + List valueNodes = new ArrayList<>(numBranches); + List literals = new ArrayList<>(numBranches); + for (int i = 0; i < numBranches; i++) { + Field vi = Field.nullable("v" + i, utf8); + schemaFields.add(vi); + valueNodes.add(TreeBuilder.makeField(vi)); + literals.add("peg-krones.ns=3;s=V:0/3/" + i + ".value"); + } + + TreeNode current = TreeBuilder.makeNull(utf8); + for (int i = numBranches - 1; i >= 0; i--) { + TreeNode cond = + TreeBuilder.makeFunction( + "equal", + Lists.newArrayList(keyNode, TreeBuilder.makeStringLiteral(literals.get(i))), + new ArrowType.Bool()); + current = TreeBuilder.makeIf(cond, valueNodes.get(i), current, utf8); + } + + TreeNode casted = + TreeBuilder.makeFunction( + "castVARCHAR", + Lists.newArrayList(current, TreeBuilder.makeLiteral(65536L)), + utf8); + + ExpressionTree expr = TreeBuilder.makeExpression(casted, Field.nullable("value", utf8)); + return new ProjectionBundle(new Schema(schemaFields), expr); + } + + private static FilterBundle buildNestedCaseFilter(int numBranches) { + ProjectionBundle proj = buildNestedCaseProjection(numBranches); + // The projector tree is utf8-typed; wrap its root in isnotnull(...) so the whole expression + // becomes a Bool condition usable by Filter. The if-chain depth — not the result type — is + // what blows the native stack. + TreeNode root = + TreeBuilder.makeFunction( + "isnotnull", + Lists.newArrayList(rebuildNestedCaseRoot(numBranches)), + new ArrowType.Bool()); + return new FilterBundle(proj.schema, TreeBuilder.makeCondition(root)); + } + + /** Identical to the body of buildNestedCaseProjection but exposes the raw root TreeNode. */ + private static TreeNode rebuildNestedCaseRoot(int numBranches) { + ArrowType utf8 = new ArrowType.Utf8(); + Field keyField = Field.nullable("key", utf8); + TreeNode keyNode = TreeBuilder.makeField(keyField); + + List valueNodes = new ArrayList<>(numBranches); + for (int i = 0; i < numBranches; i++) { + valueNodes.add(TreeBuilder.makeField(Field.nullable("v" + i, utf8))); + } + + TreeNode current = TreeBuilder.makeNull(utf8); + for (int i = numBranches - 1; i >= 0; i--) { + TreeNode cond = + TreeBuilder.makeFunction( + "equal", + Lists.newArrayList( + keyNode, + TreeBuilder.makeStringLiteral( + "peg-krones.ns=3;s=V:0/3/" + i + ".value")), + new ArrowType.Bool()); + current = TreeBuilder.makeIf(cond, valueNodes.get(i), current, utf8); + } + return TreeBuilder.makeFunction( + "castVARCHAR", Lists.newArrayList(current, TreeBuilder.makeLiteral(65536L)), utf8); + } + + /** + * Reproduces the failing plan exactly: 393 WHEN branches (the count in the Dremio plan that + * blew up LLVM). Without the guard this crashes the JVM with a native SIGSEGV during + * {@code Projector.make}; with the guard wired in it must be rejected with a + * {@link GandivaException} naming the depth limit. + */ + @Test + public void testUnpivotCaseStackOverflow() { + ProjectionBundle bundle = buildNestedCaseProjection(393); + GandivaException ex = + assertThrows( + GandivaException.class, + () -> Projector.make(bundle.schema, Lists.newArrayList(bundle.expression))); + assertTrue(ex.getMessage().contains("depth"), ex.getMessage()); + } + + /** A shallow version of the same shape compiles fine — sanity that ordinary CASEs still work. */ + @Test + public void testUnpivotCaseShallow() throws GandivaException { + ProjectionBundle bundle = buildNestedCaseProjection(8); + Projector eval = Projector.make(bundle.schema, Lists.newArrayList(bundle.expression)); + eval.close(); + } + + /** Filter variant of the same shape: rejected by the guard for the same reason. */ + @Test + public void testUnpivotCaseStackOverflowFilter() { + FilterBundle bundle = buildNestedCaseFilter(393); + GandivaException ex = + assertThrows( + GandivaException.class, () -> Filter.make(bundle.schema, bundle.condition)); + assertTrue(ex.getMessage().contains("depth"), ex.getMessage()); + } + + /** Shallow filter sanity check matching the projector shallow test. */ + @Test + public void testUnpivotCaseShallowFilter() throws GandivaException { + FilterBundle bundle = buildNestedCaseFilter(8); + Filter filter = Filter.make(bundle.schema, bundle.condition); + filter.close(); + } + + private static final class FilterBundle { + final Schema schema; + final Condition condition; + + FilterBundle(Schema schema, Condition condition) { + this.schema = schema; + this.condition = condition; + } + } + + private static final class ProjectionBundle { + final Schema schema; + final ExpressionTree expression; + + ProjectionBundle(Schema schema, ExpressionTree expression) { + this.schema = schema; + this.expression = expression; + } + } +} diff --git a/gandiva/src/test/resources/unpivot_393_literals.txt b/gandiva/src/test/resources/unpivot_393_literals.txt new file mode 100644 index 0000000000..02d826b422 --- /dev/null +++ b/gandiva/src/test/resources/unpivot_393_literals.txt @@ -0,0 +1,393 @@ +peg-krones.ns=3;s=V:0/3/0/0/76.value +peg-krones.ns=3;s=V:0/3/1/0/24.value +peg-krones.ns=3;s=V:0/3/100/0/316.value +peg-krones.ns=3;s=V:0/3/100/0/552.value +peg-krones.ns=3;s=V:0/3/3/0/127.value +peg-krones.ns=3;s=V:0/3/2/0/65.value +peg-krones.ns=3;s=V:0/3/100/0/303.value +peg-krones.ns=3;s=V:0/3/3/0/93.value +peg-krones.ns=3;s=V:0/3/100/0/329.value +peg-krones.ns=3;s=V:0/3/100/0/447.value +peg-krones.ns=3;s=V:0/3/100/0/396.value +peg-krones.ns=3;s=V:0/3/2/0/52.value +peg-krones.ns=3;s=V:0/3/100/0/421.value +peg-krones.ns=3;s=V:0/3/4/0/363.value +peg-krones.ns=3;s=V:0/3/3/0/80.value +peg-krones.ns=3;s=V:0/3/100/0/434.value +peg-krones.ns=3;s=V:0/3/0/0/63.value +peg-krones.ns=3;s=V:0/3/1/0/11.value +peg-krones.ns=3;s=V:0/3/2/1/63.value +peg-krones.ns=3;s=V:0/3/2/1/55.value +peg-krones.ns=3;s=V:0/3/100/0/260.value +peg-krones. ns=3;s=V:0/3/103/0/0.value +peg-krones.ns=3;s=V:0/3/100/0/565.value +peg-krones.ns=3;s=V:0/3/4/0/309.value +peg-krones.ns=3;s=V:0/3/4/0/350.value +peg-krones.ns=3;s=V:0/3/100/0/278.value +peg-krones.ns=3;s=V:0/3/3/0/122.value +peg-krones.ns=3;s=V:0/3/100/0/134.value +peg-krones.ns=3;s=V:0/3/100/0/252.value +peg-krones.ns=3;s=V:0/3/3/0/26.value +peg-krones.ns=3;s=V:0/3/0/1/53.value +peg-krones.ns=3;s=V:0/3/0/1/61.value +peg-krones.ns=3;s=V:0/3/100/0/142.value +peg-krones.ns=3;s=V:0/3/3/0/114.value +peg-krones.ns=3;s=V:0/3/101/0/142.value +peg-krones.ns=3;s=V:0/3/100/0/370.value +peg-krones.ns=3;s=V:0/3/101/0/8.value +peg-krones.ns=3;s=V:0/3/100/0/603.value +peg-krones.ns=3;s=V:0/3/101/0/121.value +peg-krones.ns=3;s=V:0/3/100/0/544.value +peg-krones.ns=3;s=V:0/3/3/0/135.value +peg-krones.ns=3;s=V:0/3/2/0/60.value +peg-krones.ns=3;s=V:0/3/100/0/299.value +peg-krones.ns=3;s=V:0/3/100/0/121.value +peg-krones.ns=3;s=V:0/3/100/0/308.value +peg-krones.ns=3;s=V:0/3/101/0/168.value +peg-krones.ns=3;s=V:0/3/100/0/531.value +peg-krones.ns=3;s=V:0/3/100/0/168.value +peg-krones.ns=3;s=V:0/3/2/0/44.value +peg-krones.ns=3;s=V:0/3/100/0/219.value +peg-krones.ns=3;s=V:0/3/100/0/324.value +peg-krones.ns=3;s=V:0/3/2/0/57.value +peg-krones.ns=3;s=V:0/3/100/0/391.value +peg-krones.ns=3;s=V:0/3/1/0/16.value +peg-krones.ns=3;s=V:0/3/100/0/311.value +peg-krones.ns=3;s=V:0/3/100/0/439.value +peg-krones.ns=3;s=V:0/3/3/0/85.value +peg-krones.ns=3;s=V:0/3/4/0/304.value +peg-krones.ns=3;s=V:0/3/100/0/265.value +peg-krones.ns=3;s=V:0/3/101/0/155.value +peg-krones.ns=3;s=V:0/3/3/0/39.value +peg-krones.ns=3;s=V:0/3/3/0/72.value +peg-krones.ns=3;s=V:0/3/0/0/50.value +peg-krones.ns=3;s=V:0/3/3/0/101.value +peg-krones.ns=3;s=V:0/3/100/0/155.value +peg-krones.ns=3;s=V:0/3/100/0/139.value +peg-krones.ns=3;s=V:0/3/100/0/560.value +peg-krones.ns=3;s=V:0/3/4/0/355.value +peg-krones.ns=3;s=V:0/3/100/0/608.value +peg-krones.ns=3;s=V:0/3/0/0/97.value +peg-krones.ns=3;s=V:0/3/0/0/17.value +peg-krones.ns=3;s=V:0/3/100/0/468.value +peg-krones.ns=3;s=V:0/3/1/0/45.value +peg-krones.ns=3;s=V:0/3/2/0/73.value +peg-krones.ns=3;s=V:0/3/1/1/14.value +peg-krones.ns=3;s=V:0/3/100/0/578.value +peg-krones.ns=3;s=V:0/3/3/0/119.value +peg-krones.ns=3;s=V:0/3/101/0/83.value +peg-krones.ns=3;s=V:0/3/100/0/375.value +peg-krones.ns=3;s=V:0/3/3/0/21.value +peg-krones.ns=3;s=V:0/3/2/1/42.value +peg-krones.ns=3;s=V:0/3/100/0/624.value +peg-krones.ns=3;s=V:0/3/1/1/43.value +peg-krones.ns=3;s=V:0/3/101/0/70.value +peg-krones.ns=3;s=V:0/3/101/0/189.value +peg-krones.ns=3;s=V:0/3/100/0/536.value +peg-krones.ns=3;s=V:0/3/101/0/113.value +peg-krones.ns=3;s=V:0/3/2/1/34.value +peg-krones.ns=3;s=V:0/3/100/0/506.value +peg-krones.ns=3;s=V:0/3/101/0/163.value +peg-krones.ns=3;s=V:0/3/3/0/130.value +peg-krones.ns=3;s=V:0/3/100/0/0.value +peg-krones.ns=3;s=V:0/3/100/0/126.value +peg-krones.ns=3;s=V:0/3/1/1/30.value +peg-krones.ns=3;s=V:0/3/100/0/231.value +peg-krones.ns=3;s=V:0/3/100/0/176.value +peg-krones.ns=3;s=V:0/3/0/1/32.value +peg-krones.ns=3;s=V:0/3/100/0/400.value +peg-krones.ns=3;s=V:0/3/100/0/150.value +peg-krones.ns=3;s=V:0/3/100/0/25.value +peg-krones.ns=3;s=V:0/3/3/0/106.value +peg-krones.ns=3;s=V:0/3/100/0/498.value +peg-krones.ns=3;s=V:0/3/100/0/510.value +peg-krones.ns=3;s=V:0/3/100/0/455.value +peg-krones.ns=3;s=V:0/3/3/0/34.value +peg-krones.ns=3;s=V:0/3/0/0/47.value +peg-krones.ns=3;s=V:0/3/100/0/227.value +peg-krones.ns=3;s=V:0/3/3/0/42.value +peg-krones.ns=3;s=V:0/3/100/0/337.value +peg-krones.ns=3;s=V:0/3/100/0/388.value +peg-krones.ns=3;s=V:0/3/4/0/334.value +peg-krones.ns=3;s=V:0/3/101/0/109.value +peg-krones.ns=3;s=V:0/3/0/0/55.value +peg-krones.ns=3;s=V:0/3/2/0/31.value +peg-krones.ns=3;s=V:0/3/100/0/480.value +peg-krones.ns=3;s=V:0/3/101/0/88.value +peg-krones.ns=3;s=V:0/3/100/0/418.value +peg-krones.ns=3;s=V:0/3/100/0/590.value +peg-krones.ns=3;s=V:0/3/1/0/40.value +peg-krones.ns=3;s=V:0/3/2/0/23.value +peg-krones.ns=3;s=V:0/3/0/0/84.value +peg-krones.ns=3;s=V:0/3/4/0/342.value +peg-krones.ns=3;s=V:0/3/100/0/345.value +peg-krones.ns=3;s=V:0/3/100/0/118.value +peg-krones.ns=3;s=V:0/3/100/0/426.value +peg-krones.ns=3;s=V:0/3/1/1/51.value +peg-krones.ns=3;s=V:0/3/100/0/616.value +peg-krones.ns=3;s=V:0/3/101/0/134.value +peg-krones.ns=3;s=V:0/3/0/1/90.value +peg-krones.ns=3;s=V:0/3/0/1/40.value +peg-krones.ns=3;s=V:0/3/0/1/10.value +peg-krones.ns=3;s=V:0/3/100/0/476.value +peg-krones.ns=3;s=V:0/3/0/0/25.value +peg-krones.ns=3;s=V:0/3/100/0/198.value +peg-krones.ns=3;s=V:0/3/100/0/557.value +peg-krones.ns=3;s=V:0/3/101/0/62.value +peg-krones.ns=3;s=V:0/3/100/0/206.value +peg-krones.ns=3;s=V:0/3/100/0/286.value +peg-krones.ns=3;s=V:0/3/100/0/147.value +peg-krones.ns=3;s=V:0/3/100/0/210.value +peg-krones.ns=3;s=V:0/3/0/1/11.value +peg-krones.ns=3;s=V:0/3/100/0/257.value +peg-krones.ns=3;s=V:0/3/100/0/180.value +peg-krones.ns=3;s=V:0/3/100/0/586.value +peg-krones.ns=3;s=V:0/3/1/1/22.value +peg-krones.ns=3;s=V:0/3/100/0/367.value +peg-krones.ns=3;s=V:0/3/101/0/91.value +peg-krones.ns=3;s=V:0/3/1/0/37.value +peg-krones.ns=3;s=V:0/3/100/0/290.value +peg-krones.ns=3;s=V:0/3/3/0/64.value +peg-krones.ns=3;s=V:0/3/0/1/91.value +peg-krones.ns=3;s=V:0/3/4/0/312.value +peg-krones.ns=3;s=V:0/3/102/0/2.value +peg-krones.ns=3;s=V:0/3/3/0/43.value +peg-krones.ns=3;s=V:0/3/4/0/333.value +peg-krones.ns=3;s=V:0/3/2/0/22.value +peg-krones.ns=3;s=V:0/3/4/0/313.value +peg-krones.ns=3;s=V:0/3/0/0/46.value +peg-krones.ns=3;s=V:0/3/100/0/497.value +peg-krones.ns=3;s=V:0/3/0/0/26.value +peg-krones.ns=3;s=V:0/3/4/0/320.value +peg-krones.ns=3;s=V:0/3/3/0/63.value +peg-krones.ns=3;s=V:0/3/100/0/228.value +peg-krones.ns=3;s=V:0/3/1/0/41.value +peg-krones.ns=3;s=V:0/3/100/0/451.value +peg-krones.ns=3;s=V:0/3/100/0/477.value +peg-krones.ns=3;s=V:0/3/100/0/502.value +peg-krones.ns=3;s=V:0/3/4/0/326.value +peg-krones.ns=3;s=V:0/3/3/0/50.value +peg-krones.ns=3;s=V:0/3/0/0/33.value +peg-krones.ns=3;s=V:0/3/101/0/87.value +peg-krones.ns=3;s=V:0/3/100/0/379.value +peg-krones.ns=3;s=V:0/3/0/0/93.value +peg-krones.ns=3;s=V:0/3/100/0/484.value +peg-krones.ns=3;s=V:0/3/100/0/620.value +peg-krones.ns=3;s=V:0/3/0/1/24.value +peg-krones.ns=3;s=V:0/3/100/0/333.value +peg-krones.ns=3;s=V:0/3/101/0/61.value +peg-krones.ns=3;s=V:0/3/1/1/52.value +peg-krones.ns=3;s=V:0/3/2/0/15.value +peg-krones.ns=3;s=V:0/3/100/0/105.value +peg-krones.ns=3;s=V:0/3/100/0/215.value +peg-krones.ns=3;s=V:0/3/3/0/131.value +peg-krones.ns=3;s=V:0/3/101/0/105.value +peg-krones.ns=3;s=V:0/3/3/0/89.value +peg-krones.ns=3;s=V:0/3/100/0/117.value +peg-krones.ns=3;s=V:0/3/100/0/548.value +peg-krones.ns=3;s=V:0/3/100/0/374.value +peg-krones.ns=3;s=V:0/3/101/0/95.value +peg-krones.ns=3;s=V:0/3/0/1/83.value +peg-krones.ns=3;s=V:0/3/101/0/138.value +peg-krones.ns=3;s=V:0/3/0/1/70.value +peg-krones.ns=3;s=V:0/3/100/0/387.value +peg-krones.ns=3;s=V:0/3/3/0/118.value +peg-krones.ns=3;s=V:0/3/100/0/594.value +peg-krones.ns=3;s=V:0/3/100/0/612.value +peg-krones.ns=3;s=V:0/3/100/0/171.value +peg-krones.ns=3;s=V:0/3/4/0/321.value +peg-krones.ns=3;s=V:0/3/2/0/74.value +peg-krones.ns=3;s=V:0/3/100/0/341.value +peg-krones.ns=3;s=V:0/3/3/0/55.value +peg-krones.ns=3;s=V:0/3/100/0/230.value +peg-krones.ns=3;s=V:0/3/2/0/56.value +peg-krones.ns=3;s=V:0/3/0/1/31.value +peg-krones.ns=3;s=V:0/3/1/0/33.value +peg-krones.ns=3;s=V:0/3/100/0/104.value +peg-krones.ns=3;s=V:0/3/0/0/85.value +peg-krones.ns=3;s=V:0/3/101/0/104.value +peg-krones.ns=3;s=V:0/3/100/0/120.value +peg-krones.ns=3;s=V:0/3/2/0/61.value +peg-krones.ns=3;s=V:0/3/3/0/56.value +peg-krones.ns=3;s=V:0/3/100/0/595.value +peg-krones.ns=3;s=V:0/3/100/0/485.value +peg-krones.ns=3;s=V:0/3/100/0/282.value +peg-krones.ns=3;s=V:0/3/2/0/10.value +peg-krones.ns=3;s=V:0/3/100/0/340.value +peg-krones.ns=3;s=V:0/3/100/0/543.value +peg-krones.ns=3;s=V:0/3/0/0/34.value +peg-krones.ns=3;s=V:0/3/100/0/450.value +peg-krones.ns=3;s=V:0/3/100/0/172.value +peg-krones.ns=3;s=V:0/3/100/0/156.value +peg-krones. ns=3;s=V:0/3/103/0/4.value +peg-krones.ns=3;s=V:0/3/1/1/13.value +peg-krones.ns=3;s=V:0/3/100/0/274.value +peg-krones.ns=3;s=V:0/3/0/1/75.value +peg-krones.ns=3;s=V:0/3/100/0/599.value +peg-krones.ns=3;s=V:0/3/100/0/549.value +peg-krones.ns=3;s=V:0/3/100/0/604.value +peg-krones.ns=3;s=V:0/3/100/0/382.value +peg-krones.ns=3;s=V:0/3/0/1/15.value +peg-krones.ns=3;s=V:0/3/100/0/214.value +peg-krones.ns=3;s=V:0/3/2/1/51.value +peg-krones.ns=3;s=V:0/3/100/0/281.value +peg-krones.ns=3;s=V:0/3/100/0/159.value +peg-krones.ns=3;s=V:0/3/3/0/113.value +peg-krones.ns=3;s=V:0/3/101/0/139.value +peg-krones.ns=3;s=V:0/3/3/0/123.value +peg-krones.ns=3;s=V:0/3/100/0/163.value +peg-krones.ns=3;s=V:0/3/100/0/113.value +peg-krones.ns=3;s=V:0/3/100/0/607.value +peg-krones.ns=3;s=V:0/3/0/1/82.value +peg-krones.ns=3;s=V:0/3/1/1/10.value +peg-krones.ns=3;s=V:0/3/101/0/146.value +peg-krones.ns=3;s=V:0/3/3/0/97.value +peg-krones.ns=3;s=V:0/3/100/0/435.value +peg-krones.ns=3;s=V:0/3/2/0/66.value +peg-krones.ns=3;s=V:0/3/101/0/150.value +peg-krones.ns=3;s=V:0/3/100/0/553.value +peg-krones.ns=3;s=V:0/3/100/0/443.value +peg-krones.ns=3;s=V:0/3/1/0/23.value +peg-krones.ns=3;s=V:0/3/101/0/96.value +peg-krones.ns=3;s=V:0/3/100/0/109.value +peg-krones.ns=3;s=V:0/3/100/0/438.value +peg-krones.ns=3;s=V:0/3/100/0/611.value +peg-krones.ns=3;s=V:0/3/4/0/317.value +peg-krones.ns=3;s=V:0/3/1/0/32.value +peg-krones.ns=3;s=V:0/3/0/0/92.value +peg-krones.ns=3;s=V:0/3/3/0/51.value +peg-krones.ns=3;s=V:0/3/100/0/325.value +peg-krones.ns=3;s=V:0/3/100/0/489.value +peg-krones.ns=3;s=V:0/3/3/0/115.value +peg-krones.ns=3;s=V:0/3/100/0/384.value +peg-krones.ns=3;s=V:0/3/100/0/501.value +peg-krones.ns=3;s=V:0/3/100/0/164.value +peg-krones.ns=3;s=V:0/3/4/0/318.value +peg-krones.ns=3;s=V:0/3/100/0/328.value +peg-krones.ns=3;s=V:0/3/3/0/48.value +peg-krones.ns=3;s=V:0/3/3/0/98.value +peg-krones.ns=3;s=V:0/3/4/0/325.value +peg-krones.ns=3;s=V:0/3/101/0/148.value +peg-krones.ns=3;s=V:0/3/0/0/41.value +peg-krones.ns=3;s=V:0/3/100/0/492.value +peg-krones.ns=3;s=V:0/3/100/0/442.value +peg-krones.ns=3;s=V:0/3/100/0/223.value +peg-krones.ns=3;s=V:0/3/2/0/14.value +peg-krones.ns=3;s=V:0/3/100/0/269.value +peg-krones.ns=3;s=V:0/3/2/0/64.value +peg-krones.ns=3;s=V:0/3/101/0/151.value +peg-krones.ns=3;s=V:0/3/0/1/74.value +peg-krones.ns=3;s=V:0/3/1/0/25.value +peg-krones.ns=3;s=V:0/3/0/0/42.value +peg-krones.ns=3;s=V:0/3/100/0/493.value +peg-krones.ns=3;s=V:0/3/101/0/147.value +peg-krones.ns=3;s=V:0/3/100/0/273.value +peg-krones.ns=3;s=V:0/3/101/0/100.value +peg-krones.ns=3;s=V:0/3/3/0/47.value +peg-krones. ns=3;s=V:0/3/103/0/5.value +peg-krones.ns=3;s=V:0/3/100/0/551.value +peg-krones.ns=3;s=V:0/3/2/1/50.value +peg-krones.ns=3;s=V:0/3/100/0/383.value +peg-krones.ns=3;s=V:0/3/0/1/23.value +peg-krones.ns=3;s=V:0/3/3/0/121.value +peg-krones.ns=3;s=V:0/3/100/0/332.value +peg-krones.ns=3;s=V:0/3/100/0/222.value +peg-krones.ns=3;s=V:0/3/3/0/99.value +peg-krones.ns=3;s=V:0/3/100/0/112.value +peg-krones.ns=3;s=V:0/3/1/0/44.value +peg-krones.ns=3;s=V:0/3/4/0/343.value +peg-krones.ns=3;s=V:0/3/100/0/572.value +peg-krones.ns=3;s=V:0/3/100/0/336.value +peg-krones.ns=3;s=V:0/3/2/0/32.value +peg-krones.ns=3;s=V:0/3/0/0/56.value +peg-krones.ns=3;s=V:0/3/3/0/107.value +peg-krones.ns=3;s=V:0/3/3/0/73.value +peg-krones.ns=3;s=V:0/3/100/0/467.value +peg-krones.ns=3;s=V:0/3/4/0/316.value +peg-krones.ns=3;s=V:0/3/101/0/97.value +peg-krones.ns=3;s=V:0/3/3/0/20.value +peg-krones.ns=3;s=V:0/3/100/0/481.value +peg-krones.ns=3;s=V:0/3/100/0/376.value +peg-krones.ns=3;s=V:0/3/100/0/494.value +peg-krones.ns=3;s=V:0/3/100/0/414.value +peg-krones.ns=3;s=V:0/3/100/0/309.value +peg-krones.ns=3;s=V:0/3/100/0/389.value +peg-krones.ns=3;s=V:0/3/100/0/240.value +peg-krones.ns=3;s=V:0/3/100/0/505.value +peg-krones.ns=3;s=V:0/3/0/1/41.value +peg-krones.ns=3;s=V:0/3/2/1/75.value +peg-krones.ns=3;s=V:0/3/100/0/545.value +peg-krones.ns=3;s=V:0/3/100/0/114.value +peg-krones.ns=3;s=V:0/3/100/0/298.value +peg-krones.ns=3;s=V:0/3/4/0/329.value +peg-krones.ns=3;s=V:0/3/100/0/615.value +peg-krones.ns=3;s=V:0/3/3/0/46.value +peg-krones. ns=3;s=V:0/3/103/0/6.value +peg-krones.ns=3;s=V:0/3/101/0/122.value +peg-krones.ns=3;s=V:0/3/100/0/350.value +peg-krones.ns=3;s=V:0/3/3/0/134.value +peg-krones.ns=3;s=V:0/3/0/0/51.value +peg-krones.ns=3;s=V:0/3/3/0/78.value +peg-krones.ns=3;s=V:0/3/100/0/446.value +peg-krones.ns=3;s=V:0/3/101/0/154.value +peg-krones.ns=3;s=V:0/3/100/0/213.value +peg-krones.ns=3;s=V:0/3/100/0/154.value +peg-krones.ns=3;s=V:0/3/3/0/102.value +peg-krones.ns=3;s=V:0/3/100/0/462.value +peg-krones.ns=3;s=V:0/3/0/1/60.value +peg-krones.ns=3;s=V:0/3/0/0/91.value +peg-krones.ns=3;s=V:0/3/100/0/486.value +peg-krones.ns=3;s=V:0/3/100/0/602.value +peg-krones.ns=3;s=V:0/3/1/1/55.value +peg-krones.ns=3;s=V:0/3/0/1/14.value +peg-krones.ns=3;s=V:0/3/2/1/43.value +peg-krones.ns=3;s=V:0/3/100/0/577.value +peg-krones.ns=3;s=V:0/3/1/1/15.value +peg-krones.ns=3;s=V:0/3/100/0/226.value +peg-krones.ns=3;s=V:0/3/100/0/266.value +peg-krones.ns=3;s=V:0/3/101/0/127.value +peg-krones.ns=3;s=V:0/3/0/0/22.value +peg-krones.ns=3;s=V:0/3/1/0/10.value +peg-krones.ns=3;s=V:0/3/101/0/71.value +peg-krones.ns=3;s=V:0/3/1/0/17.value +peg-krones.ns=3;s=V:0/3/100/0/293.value +peg-krones.ns=3;s=V:0/3/0/1/94.value +peg-krones.ns=3;s=V:0/3/100/0/323.value +peg-krones.ns=3;s=V:0/3/2/1/70.value +peg-krones.ns=3;s=V:0/3/100/0/433.value +peg-krones.ns=3;s=V:0/3/100/0/167.value +peg-krones.ns=3;s=V:0/3/101/0/78.value +peg-krones.ns=3;s=V:0/3/100/0/211.value +peg-krones.ns=3;s=V:0/3/2/1/54.value +peg-krones.ns=3;s=V:0/3/0/1/65.value +peg-krones.ns=3;s=V:0/3/100/0/103.value +peg-krones.ns=3;s=V:0/3/100/0/579.value +peg-krones.ns=3;s=V:0/3/101/0/90.value +peg-krones.ns=3;s=V:0/3/1/1/53.value +peg-krones.ns=3;s=V:0/3/100/0/264.value +peg-krones.ns=3;s=V:0/3/100/0/556.value +peg-krones.ns=3;s=V:0/3/101/0/143.value +peg-krones.ns=3;s=V:0/3/100/0/169.value +peg-krones.ns=3;s=V:0/3/101/0/159.value +peg-krones.ns=3;s=V:0/3/101/0/129.value +peg-krones.ns=3;s=V:0/3/100/0/529.value +peg-krones.ns=3;s=V:0/3/100/0/183.value +peg-krones.ns=3;s=V:0/3/0/1/92.value +peg-krones.ns=3;s=V:0/3/1/1/50.value +peg-krones.ns=3;s=V:0/3/0/1/12.value +peg-krones.ns=3;s=V:0/3/100/0/116.value +peg-krones.ns=3;s=V:0/3/100/0/261.value +peg-krones.ns=3;s=V:0/3/100/0/617.value +peg-krones.ns=3;s=V:0/3/101/0/156.value +peg-krones.ns=3;s=V:0/3/0/1/62.value +peg-krones.ns=3;s=V:0/3/100/0/291.value +peg-krones.ns=3;s=V:0/3/101/0/106.value +peg-krones.ns=3;s=V:0/3/4/0/314.value +peg-krones.ns=3;s=V:0/3/1/0/15.value +peg-krones.ns=3;s=V:0/3/100/0/483.value +peg-krones.ns=3;s=V:0/3/101/0/170.value +peg-krones.ns=3;s=V:0/3/0/0/75.value +peg-krones.ns=3;s=V:0/3/3/0/57.value +peg-krones.ns=3;s=V:0/3/100/0/130.value +peg-krones.ns=3;s=V:0/3/2/0/51.value +peg-krones.ns=3;s=V:0/3/3/0/94.value