From a46b8806335793aad36de98741af80bb47edeceb Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 24 Aug 2026 09:52:53 +0800 Subject: [PATCH] [refactor](nereids) Remove the PLAN REPLAYER PLAY command ### What problem does this PR solve? Issue Number: None Related PR: #66817 Problem Summary: PLAN REPLAYER PLAY was a developer-only command for replaying a minidump on another FE and has no known users. Remove its lexer and parser alternatives, runtime loading and comparison logic, offline runner, and FE packaging. PLAN REPLAYER DUMP remains unchanged, while play becomes available as an ordinary identifier. ### Release note Remove the PLAN REPLAYER PLAY statement and the packaged minidump/nereids_ut.sh offline runner. PLAN REPLAYER DUMP is unchanged. ### Check List (For Author) - Test: Manual test (bash -n build.sh and diff/reference checks passed; ./run-fe-ut.sh --run org.apache.doris.nereids.parser.NereidsParserTest was attempted twice, but the host killed Maven with exit 137 during compilation before test execution; ANTLR generation and Maven checkstyle passed in both attempts) - Behavior changed: Yes (PLAN REPLAYER PLAY is removed and play is now an ordinary identifier; PLAN REPLAYER DUMP is unchanged) - Does this need documentation: No --- build.sh | 3 +- .../org/apache/doris/nereids/DorisLexer.g4 | 1 - .../org/apache/doris/nereids/DorisParser.g4 | 3 +- .../doris/nereids/minidump/Minidump.java | 36 ---- .../doris/nereids/minidump/MinidumpUtils.java | 108 ----------- .../nereids/parser/LogicalPlanBuilder.java | 11 +- .../trees/plans/commands/ReplayCommand.java | 57 +----- .../nereids/parser/NereidsParserTest.java | 13 +- minidump/README.md | 38 ---- minidump/minidump.conf | 73 -------- minidump/nereids_ut.sh | 176 ------------------ 11 files changed, 13 insertions(+), 506 deletions(-) delete mode 100644 minidump/README.md delete mode 100644 minidump/minidump.conf delete mode 100755 minidump/nereids_ut.sh diff --git a/build.sh b/build.sh index 428633870c83f6..76fd3599dac2d7 100755 --- a/build.sh +++ b/build.sh @@ -1020,6 +1020,8 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then cp -r -p "${DORIS_HOME}/conf/ldap.conf" "${DORIS_OUTPUT}/fe/conf"/ cp -r -p "${DORIS_HOME}/conf/mysql_ssl_default_certificate" "${DORIS_OUTPUT}/fe/"/ rm -rf "${DORIS_OUTPUT}/fe/lib"/* + # The offline minidump runner is gone; drop it from reused output trees too. + rm -rf "${DORIS_OUTPUT}/fe/minidump" cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/ cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/ for extra_module_path in "${FE_EXTRA_MODULE_PATHS[@]}"; do @@ -1044,7 +1046,6 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Third-party filesystem jars (JuiceFS, JindoFS) are packaged by post-build.sh "${DORIS_HOME}/post-build.sh" --fe --output "${DORIS_OUTPUT}" - cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/ cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/ cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/fe/webroot/static"/ diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 1b6b76a2059f72..a56811806fff8d 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -438,7 +438,6 @@ PHYSICAL: 'PHYSICAL'; PI: 'PI'; PLACEHOLDER: '?'; PLAN: 'PLAN'; -PLAY: 'PLAY'; PRIVILEGES: 'PRIVILEGES'; PROCESS: 'PROCESS'; PYTHON: 'PYTHON'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index f3c076852998db..e7305beeb02148 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1183,8 +1183,7 @@ replayCommand : PLAN REPLAYER replayType; replayType - : DUMP query - | PLAY filePath=STRING_LITERAL; + : DUMP query; mergeType : APPEND diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java index 37c7ff9a165b7c..11e1e57163d2b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java @@ -19,15 +19,10 @@ import org.apache.doris.catalog.ColocateTableIndex; import org.apache.doris.catalog.TableIf; -import org.apache.doris.nereids.StatementContext; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.Histogram; -import org.json.JSONObject; - import java.util.HashMap; import java.util.List; import java.util.Map; @@ -110,35 +105,4 @@ public Map getTotalColumnStatisticMap() { public Map getTotalHistogramMap() { return totalHistogramMap; } - - /** Nereids minidump entry, argument should be absolute address of minidump path */ - public static void main(String[] args) { - assert (args.length == 1); - Minidump minidump = null; - try { - minidump = MinidumpUtils.loadMinidumpInputs(args[0]); - } catch (Exception e) { - e.printStackTrace(); - } - - StatementContext statementContext = new StatementContext(ConnectContext.get(), - new OriginStatement(minidump.getSql(), 0)); - statementContext.setTables(minidump.getTables()); - ConnectContext.get().setStatementContext(statementContext); - JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql()); - JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson()); - - List differences = MinidumpUtils.compareJsonObjects(minidumpResult, resultPlan, ""); - - if (differences.isEmpty()) { - System.out.println(minidump.sql); - } else { - System.out.println(minidump.sql + "\n" + "Result plan are not expected. Differences:"); - System.out.println("----------------------------Result plan--------------------------"); - System.out.println(resultPlan + "\n"); - System.out.println("----------------------------Minidump plan--------------------------"); - System.out.println(minidumpResult + "\n"); - } - System.exit(0); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java index b0bdf41dafc5f3..1602b79e54d0a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java @@ -60,9 +60,7 @@ import java.lang.reflect.Type; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -232,29 +230,6 @@ public static void setConnectContext(Minidump minidump) { connectContext.getSessionVariable().enableNereidsTimeout = false; } - /** - * Loading minidump messages from file to memory - * @param minidumpPath path of minidump file - * @return minidump messages in memory - */ - public static Minidump loadMinidumpInputs(String minidumpPath) throws AnalysisException { - Minidump minidump = null; - try { - minidump = MinidumpUtils.jsonMinidumpLoad(minidumpPath); - } catch (AnalysisException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); - } - setConnectContext(minidump); - - Env env = Env.getCurrentEnv(); - ConnectContext.get().setEnv(env); - ConnectContext.get().getSessionVariable().setEnableNereidsTrace(false); - ConnectContext.get().getSessionVariable().setNereidsTraceEventMode("all"); - return minidump; - } - /** * Executing sql with minidump and return jsontype result for more use of unit test * @param sql original sql clause @@ -363,89 +338,6 @@ public static void serializeOutputToDumpFile(Plan resultPlan) { } } - /** compare two json object and print detail information about difference */ - public static List compareJsonObjects(JSONObject json1, JSONObject json2, String path) { - List differences = new ArrayList<>(); - - Iterator keys = json1.keys(); - while (keys.hasNext()) { - String key = keys.next(); - String currentPath = ""; - if (path.isEmpty()) { - if (key.equals("PlanType")) { - path = json1.getString(key); - } else { - currentPath = key; - } - } else { - if (key.equals("PlanType")) { - currentPath = path + "." + json1.getString(key); - } else { - currentPath = path + "." + key; - } - } - - if (!json2.has(key)) { - differences.add("Key '" + currentPath + "' not found in the second JSON object."); - continue; - } - - Object value1 = json1.get(key); - Object value2 = json2.get(key); - - if (!value1.toString().equals(value2.toString())) { - differences.add("Value for key '" + currentPath + "' is different: " + value1 + " != " + value2); - } - - if (value1 instanceof JSONObject && value2 instanceof JSONObject) { - List nestedDifferences = - compareJsonObjects((JSONObject) value1, (JSONObject) value2, currentPath); - differences.addAll(nestedDifferences); - } else if (value1 instanceof JSONArray && value2 instanceof JSONArray) { - List nestedDifferences = compareJsonArrays((JSONArray) value1, (JSONArray) value2, currentPath); - differences.addAll(nestedDifferences); - } - } - - keys = json2.keys(); - while (keys.hasNext()) { - String key = keys.next(); - String currentPath = (path.isEmpty()) ? key : path + "." + key; - - if (!json1.has(key)) { - differences.add("Key '" + currentPath + "' not found in the first JSON object."); - } - } - - return differences; - } - - private static List compareJsonArrays(JSONArray array1, JSONArray array2, String path) { - List differences = new ArrayList<>(); - - if (array1.length() != array2.length()) { - differences.add("Array length for key '" + path + "' is different: " - + array1.length() + " != " + array2.length()); - return differences; - } - - for (int i = 0; i < array1.length(); i++) { - Object value1 = array1.get(i); - Object value2 = array2.get(i); - String currentPath = path + "[" + i + "]"; - - if (value1 instanceof JSONObject && value2 instanceof JSONObject) { - List nestedDifferences = - compareJsonObjects((JSONObject) value1, (JSONObject) value2, currentPath); - differences.addAll(nestedDifferences); - } else if (!value1.equals(value2)) { - differences.add("Value for key '" + currentPath + "' is different: " + value1 + " != " + value2); - } - } - - return differences; - } - private static String getValue(Object obj, Field field) { try { switch (field.getType().getSimpleName()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index a33f7ba4f64b26..7d9d2f05e380aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1778,15 +1778,8 @@ public MTMVRefreshTriggerInfo visitRefreshTrigger(RefreshTriggerContext ctx) { @Override public ReplayCommand visitReplay(DorisParser.ReplayContext ctx) { - if (ctx.replayCommand().replayType().DUMP() != null) { - LogicalPlan plan = plan(ctx.replayCommand().replayType().query()); - return new ReplayCommand(PlanType.REPLAY_COMMAND, null, plan, ReplayCommand.ReplayType.DUMP); - } else if (ctx.replayCommand().replayType().PLAY() != null) { - String tmpPath = ctx.replayCommand().replayType().filePath.getText(); - String path = LogicalPlanBuilderAssistant.escapeBackSlash(tmpPath.substring(1, tmpPath.length() - 1)); - return new ReplayCommand(PlanType.REPLAY_COMMAND, path, null, ReplayCommand.ReplayType.PLAY); - } - return null; + LogicalPlan plan = plan(ctx.replayCommand().replayType().query()); + return new ReplayCommand(PlanType.REPLAY_COMMAND, plan); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java index 4eabbb9e9595d2..22edc0919069c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java @@ -19,66 +19,29 @@ import org.apache.doris.analysis.StmtType; import org.apache.doris.nereids.NereidsPlanner; -import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; -import org.apache.doris.nereids.minidump.Minidump; import org.apache.doris.nereids.minidump.MinidumpUtils; import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.StmtExecutor; -import org.json.JSONObject; - -import java.util.List; - /** - * replay command. + * replay command: PLAN REPLAYER DUMP query. */ public class ReplayCommand extends Command implements NoForward { - private final String dumpFileFullPath; - private final LogicalPlan plan; - private final ReplayType replayType; - - public ReplayCommand(PlanType type, String dumpFileFullPath, LogicalPlan plan, ReplayType replayType) { + public ReplayCommand(PlanType type, LogicalPlan plan) { super(type); - this.dumpFileFullPath = dumpFileFullPath; this.plan = plan; - this.replayType = replayType; - } - - public String getDumpFileFullPath() { - return dumpFileFullPath; - } - - public ReplayType getReplayType() { - return replayType; - } - - /** - * explain level. - */ - public enum ReplayType { - DUMP, - PLAY } @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { - if (this.replayType == ReplayType.DUMP) { - handleDump(ctx, executor); - } else if (this.replayType == ReplayType.PLAY) { - handleLoad(); - } - } - - private void handleDump(ConnectContext ctx, StmtExecutor executor) throws Exception { LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(plan, ctx.getStatementContext()); MinidumpUtils.openDump(); executor.setParsedStmt(logicalPlanAdapter); @@ -92,22 +55,6 @@ private void handleDump(ConnectContext ctx, StmtExecutor executor) throws Except executor.handleReplayStmt(MinidumpUtils.getHttpGetString()); } - private void handleLoad() throws Exception { - // 1. check fe version, if not matched throw exception - // 2. load every thing from minidump file and replace original ones - Minidump minidump = MinidumpUtils.loadMinidumpInputs(dumpFileFullPath); - // 3. run nereids planner with sql in minidump file - StatementContext statementContext = new StatementContext(ConnectContext.get(), - new OriginStatement(minidump.getSql(), 0)); - statementContext.setTables(minidump.getTables()); - ConnectContext.get().setStatementContext(statementContext); - JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql()); - JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson()); - - List differences = MinidumpUtils.compareJsonObjects(minidumpResult, resultPlan, ""); - assert (differences.isEmpty()); - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitReplayCommand(this, context); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index bc9e68dd06b98b..8dd5fbe0f49f3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -327,13 +327,12 @@ public void testPlanReplayer() { String sql = "plan replayer dump select `AD``D` from t1 where a = 1"; NereidsParser nereidsParser = new NereidsParser(); LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); - ReplayCommand replayCommand = (ReplayCommand) logicalPlan; - Assertions.assertEquals(ReplayCommand.ReplayType.DUMP, replayCommand.getReplayType()); - sql = "plan replayer play 'path'"; - logicalPlan = nereidsParser.parseSingle(sql); - replayCommand = (ReplayCommand) logicalPlan; - Assertions.assertEquals(ReplayCommand.ReplayType.PLAY, replayCommand.getReplayType()); - Assertions.assertEquals("path", replayCommand.getDumpFileFullPath()); + Assertions.assertInstanceOf(ReplayCommand.class, logicalPlan); + Assertions.assertThrows(ParseException.class, + () -> nereidsParser.parseSingle("plan replayer play 'path'")); + // PLAY is no longer a keyword, so it is an ordinary identifier in any case. + Assertions.assertDoesNotThrow(() -> nereidsParser.parseSingle("select pLaY from play")); + Assertions.assertDoesNotThrow(() -> nereidsParser.parseSingle("select play.play as play from play")); } @Test diff --git a/minidump/README.md b/minidump/README.md deleted file mode 100644 index da45769d80f785..00000000000000 --- a/minidump/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Tutorial of Nereids minidump -Minidump is a basic dump file for Nereids optimizer which arms -to replay problems and used as unit test tool for developers. -It include minimal necessary informations for Nereids optimizer -to replay a scenario in Optimizer. - -## Save minidump file -In two cases fe would save minidump file -- when query throw an exception -- when use 'set enable_minidump=true;' switch by User - -## File structure of minidump file -Minidump files would save in $DORIS_HOME/log/minidump path -and named by queryId like: c9bf82b70ce34c28-8d281e31412f3ee6.json -File structure of minidump file includes: -| key | Value | -|:----|:----| -|Sql | original sql string | -|SessionVariables | sessionVariables different than default | -|DatabaseName | database name| -|Tables | schemas in table, partition info and some table properties | -|ColocateTableIndex | table distribute informations | -|ColumnStatistics | all ColumnStatistics in table | -|Histogram | all histogram information in table| -|ParsedPlan | plan after parsed in json format| -|ResultPlan | plan after all optimized in json format| - -## Use of nereids_ut.sh -When we want to use this shell, we need to go to output/fe/minidump path -Now only one parameter is allowed for this shell which is directory -of minidump files like $DORIS_HOME/log/minidump -```shell -cd output/fe/minidump -./nereids_ut.sh --d $minidump_files_path -``` -The running reports can be found in output/fe/minidump/log -which is minidump.out - diff --git a/minidump/minidump.conf b/minidump/minidump.conf deleted file mode 100644 index 04b3159adeaa40..00000000000000 --- a/minidump/minidump.conf +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -##################################################################### -## The uppercase properties are read and exported by bin/start_fe.sh. -## To see all Frontend configurations, -## see fe/src/org/apache/doris/common/Config.java -##################################################################### - -CUR_DATE=`date +%Y%m%d-%H%M%S` - -# the output dir of stderr and stdout -LOG_DIR = ${DORIS_HOME}/minidump/log - -JAVA_OPTS="-Dorg.slf4j.simpleLogger.showLogName=false -Dorg.slf4j.simpleLogger.showShortLogName=false -Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0" - -# For jdk 9+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_9="-Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0" - -## -## the lowercase properties are read by main program. -## - -# INFO, WARN, ERROR, FATAL -sys_log_level = ERROR - -# store metadata, must be created before start FE. -# Default value is ${DORIS_HOME}/doris-meta -# meta_dir = ${DORIS_HOME}/doris-meta - -# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers -# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers - -http_port = 8030 -rpc_port = 9020 -query_port = 9030 -edit_log_port = 9010 -mysql_service_nio_enabled = true - -# Choose one if there are more than one ip except loopback address. -# Note that there should at most one ip match this list. -# If no ip match this rule, will choose one randomly. -# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1 -# Default value is empty. -# priority_networks = 10.10.10.0/24;192.168.0.0/16 - -# Advanced configurations -# log_roll_size_mb = 1024 -# sys_log_dir = ${DORIS_HOME}/log -# sys_log_roll_num = 10 -# sys_log_verbose_modules = org.apache.doris -# audit_log_dir = ${DORIS_HOME}/log -# audit_log_modules = slow_query, query -# audit_log_roll_num = 10 -# meta_delay_toleration_second = 10 -# qe_max_connection = 1024 -# max_conn_per_user = 100 -# qe_query_timeout_second = 300 -# qe_slow_log_ms = 5000 diff --git a/minidump/nereids_ut.sh b/minidump/nereids_ut.sh deleted file mode 100755 index 08191c15b1be9d..00000000000000 --- a/minidump/nereids_ut.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -cur_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - -if [[ "$(uname -s)" == 'Darwin' ]] && command -v brew &>/dev/null; then - PATH="$(brew --prefix)/opt/gnu-getopt/bin:${PATH}" - export PATH -fi - -OPTS="$(getopt \ - -n "$0" \ - -o '' \ - -l 'daemon' \ - -l 'helper:' \ - -l 'd:' \ - -l 'version' \ - -- "$@")" - -eval set -- "${OPTS}" - -DUMP_FILE_DIR='' -OPT_VERSION='' -while true; do - case "$1" in - --version) - OPT_VERSION="--version" - shift - ;; - --d) - DUMP_FILE_DIR="$2" - shift 2 - ;; - --) - shift - break - ;; - *) - echo "Internal error" - exit 1 - ;; - esac -done - -DORIS_HOME="$( - cd "${cur_dir}/.." - pwd -)" -export DORIS_HOME - -# export env variables from fe.conf -# -# JAVA_OPTS -# LOG_DIR -# PID_DIR -export JAVA_OPTS="-Xmx1024m" -export LOG_DIR="${DORIS_HOME}/minidump" - -while read -r line; do - env_line="$(echo "${line}" | - sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | - sed 's/^[[:blank:]]*//g' | - grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" || - true)" - env_line="$(eval "echo ${env_line}")" - if [[ "${env_line}" == *"="* ]]; then - eval 'export "${env_line}"' - fi -done <"${DORIS_HOME}/minidump/minidump.conf" - -if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then - # shellcheck disable=1091 - source "${DORIS_HOME}/bin/palo_env.sh" -fi - -if [[ -z "${JAVA_HOME}" ]]; then - JAVA="$(command -v java)" -else - JAVA="${JAVA_HOME}/bin/java" -fi - -if [[ ! -x "${JAVA}" ]]; then - echo "The JAVA_HOME environment variable is not defined correctly" - echo "This environment variable is needed to run this program" - echo "NB: JAVA_HOME should point to a JDK not a JRE" - exit 1 -fi - -# get jdk version, return version as an Integer. -# 1.8 => 8, 13.0 => 13 -jdk_version() { - local java_cmd="${1}" - local result - local IFS=$'\n' - - if [[ -z "${java_cmd}" ]]; then - result=no_java - return 1 - else - local version - # remove \r for Cygwin - version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" - version="${version//\"/}" - if [[ "${version}" =~ ^1\. ]]; then - result="$(echo "${version}" | awk -F '.' '{print $2}')" - else - result="$(echo "${version}" | awk -F '.' '{print $1}')" - fi - fi - echo "${result}" - return 0 -} - -# need check and create if the log directory existed before outing message to the log file. -if [[ ! -d "${LOG_DIR}" ]]; then - mkdir -p "${LOG_DIR}" -fi - -# check java version and choose correct JAVA_OPTS -java_version="$( - set -e - jdk_version "${JAVA}" -)" -final_java_opt="${JAVA_OPTS}" -if [[ "${java_version}" -gt 8 ]]; then - if [[ -z "${JAVA_OPTS_FOR_JDK_9}" ]]; then - echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >>"${LOG_DIR}/minidump.out" - exit 1 - fi - final_java_opt="${JAVA_OPTS_FOR_JDK_9}" -fi - -# add libs to CLASSPATH -DORIS_FE_JAR= -for f in "${DORIS_HOME}/lib"/*.jar; do - if [[ "${f}" == *"doris-fe.jar" ]]; then - DORIS_FE_JAR="${f}" - continue - fi - CLASSPATH="${f}:${CLASSPATH}" -done - -# make sure the doris-fe.jar is at first order, so that some classed -# with same qualified name can be loaded priority from doris-fe.jar -CLASSPATH="${DORIS_FE_JAR}:${CLASSPATH}" -export CLASSPATH="${CLASSPATH}:${DORIS_HOME}/lib:${DORIS_HOME}/conf" - -date >>"${LOG_DIR}/minidump.out" -query_id_head="|QueryId=" -sql_head="|Sql=" -# shellcheck disable=SC2045 -for path in $(ls "${DUMP_FILE_DIR}"); do - query_id=${path:0:33} - temp_file=${LOG_DIR}/temp - "${JAVA}" ${final_java_opt:+${final_java_opt}} -Xmx2g org.apache.doris.nereids.minidump.Minidump "${DUMP_FILE_DIR}/${path}" "${OPT_VERSION}" "$@" >"${temp_file}" - result=$(cat "${temp_file}") - echo "${query_id_head} ${query_id} ${sql_head} ${result}" | tee >>"${LOG_DIR}/minidump.out" -done -date >>"${LOG_DIR}/minidump.out"