diff --git a/build.sh b/build.sh index 428633870c83f6..76fd3599dac2d7 100755 --- a/build.sh +++ b/build.sh @@ -1020,6 +1020,8 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then cp -r -p "${DORIS_HOME}/conf/ldap.conf" "${DORIS_OUTPUT}/fe/conf"/ cp -r -p "${DORIS_HOME}/conf/mysql_ssl_default_certificate" "${DORIS_OUTPUT}/fe/"/ rm -rf "${DORIS_OUTPUT}/fe/lib"/* + # The offline minidump runner is gone; drop it from reused output trees too. + rm -rf "${DORIS_OUTPUT}/fe/minidump" cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/ cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/ for extra_module_path in "${FE_EXTRA_MODULE_PATHS[@]}"; do @@ -1044,7 +1046,6 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Third-party filesystem jars (JuiceFS, JindoFS) are packaged by post-build.sh "${DORIS_HOME}/post-build.sh" --fe --output "${DORIS_OUTPUT}" - cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/ cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/ cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/fe/webroot/static"/ diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 1b6b76a2059f72..a56811806fff8d 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -438,7 +438,6 @@ PHYSICAL: 'PHYSICAL'; PI: 'PI'; PLACEHOLDER: '?'; PLAN: 'PLAN'; -PLAY: 'PLAY'; PRIVILEGES: 'PRIVILEGES'; PROCESS: 'PROCESS'; PYTHON: 'PYTHON'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index f3c076852998db..e7305beeb02148 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1183,8 +1183,7 @@ replayCommand : PLAN REPLAYER replayType; replayType - : DUMP query - | PLAY filePath=STRING_LITERAL; + : DUMP query; mergeType : APPEND diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java index 37c7ff9a165b7c..11e1e57163d2b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java @@ -19,15 +19,10 @@ import org.apache.doris.catalog.ColocateTableIndex; import org.apache.doris.catalog.TableIf; -import org.apache.doris.nereids.StatementContext; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.Histogram; -import org.json.JSONObject; - import java.util.HashMap; import java.util.List; import java.util.Map; @@ -110,35 +105,4 @@ public Map getTotalColumnStatisticMap() { public Map getTotalHistogramMap() { return totalHistogramMap; } - - /** Nereids minidump entry, argument should be absolute address of minidump path */ - public static void main(String[] args) { - assert (args.length == 1); - Minidump minidump = null; - try { - minidump = MinidumpUtils.loadMinidumpInputs(args[0]); - } catch (Exception e) { - e.printStackTrace(); - } - - StatementContext statementContext = new StatementContext(ConnectContext.get(), - new OriginStatement(minidump.getSql(), 0)); - statementContext.setTables(minidump.getTables()); - ConnectContext.get().setStatementContext(statementContext); - JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql()); - JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson()); - - List differences = MinidumpUtils.compareJsonObjects(minidumpResult, resultPlan, ""); - - if (differences.isEmpty()) { - System.out.println(minidump.sql); - } else { - System.out.println(minidump.sql + "\n" + "Result plan are not expected. Differences:"); - System.out.println("----------------------------Result plan--------------------------"); - System.out.println(resultPlan + "\n"); - System.out.println("----------------------------Minidump plan--------------------------"); - System.out.println(minidumpResult + "\n"); - } - System.exit(0); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java index b0bdf41dafc5f3..1602b79e54d0a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java @@ -60,9 +60,7 @@ import java.lang.reflect.Type; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -232,29 +230,6 @@ public static void setConnectContext(Minidump minidump) { connectContext.getSessionVariable().enableNereidsTimeout = false; } - /** - * Loading minidump messages from file to memory - * @param minidumpPath path of minidump file - * @return minidump messages in memory - */ - public static Minidump loadMinidumpInputs(String minidumpPath) throws AnalysisException { - Minidump minidump = null; - try { - minidump = MinidumpUtils.jsonMinidumpLoad(minidumpPath); - } catch (AnalysisException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); - } - setConnectContext(minidump); - - Env env = Env.getCurrentEnv(); - ConnectContext.get().setEnv(env); - ConnectContext.get().getSessionVariable().setEnableNereidsTrace(false); - ConnectContext.get().getSessionVariable().setNereidsTraceEventMode("all"); - return minidump; - } - /** * Executing sql with minidump and return jsontype result for more use of unit test * @param sql original sql clause @@ -363,89 +338,6 @@ public static void serializeOutputToDumpFile(Plan resultPlan) { } } - /** compare two json object and print detail information about difference */ - public static List compareJsonObjects(JSONObject json1, JSONObject json2, String path) { - List differences = new ArrayList<>(); - - Iterator keys = json1.keys(); - while (keys.hasNext()) { - String key = keys.next(); - String currentPath = ""; - if (path.isEmpty()) { - if (key.equals("PlanType")) { - path = json1.getString(key); - } else { - currentPath = key; - } - } else { - if (key.equals("PlanType")) { - currentPath = path + "." + json1.getString(key); - } else { - currentPath = path + "." + key; - } - } - - if (!json2.has(key)) { - differences.add("Key '" + currentPath + "' not found in the second JSON object."); - continue; - } - - Object value1 = json1.get(key); - Object value2 = json2.get(key); - - if (!value1.toString().equals(value2.toString())) { - differences.add("Value for key '" + currentPath + "' is different: " + value1 + " != " + value2); - } - - if (value1 instanceof JSONObject && value2 instanceof JSONObject) { - List nestedDifferences = - compareJsonObjects((JSONObject) value1, (JSONObject) value2, currentPath); - differences.addAll(nestedDifferences); - } else if (value1 instanceof JSONArray && value2 instanceof JSONArray) { - List nestedDifferences = compareJsonArrays((JSONArray) value1, (JSONArray) value2, currentPath); - differences.addAll(nestedDifferences); - } - } - - keys = json2.keys(); - while (keys.hasNext()) { - String key = keys.next(); - String currentPath = (path.isEmpty()) ? key : path + "." + key; - - if (!json1.has(key)) { - differences.add("Key '" + currentPath + "' not found in the first JSON object."); - } - } - - return differences; - } - - private static List compareJsonArrays(JSONArray array1, JSONArray array2, String path) { - List differences = new ArrayList<>(); - - if (array1.length() != array2.length()) { - differences.add("Array length for key '" + path + "' is different: " - + array1.length() + " != " + array2.length()); - return differences; - } - - for (int i = 0; i < array1.length(); i++) { - Object value1 = array1.get(i); - Object value2 = array2.get(i); - String currentPath = path + "[" + i + "]"; - - if (value1 instanceof JSONObject && value2 instanceof JSONObject) { - List nestedDifferences = - compareJsonObjects((JSONObject) value1, (JSONObject) value2, currentPath); - differences.addAll(nestedDifferences); - } else if (!value1.equals(value2)) { - differences.add("Value for key '" + currentPath + "' is different: " + value1 + " != " + value2); - } - } - - return differences; - } - private static String getValue(Object obj, Field field) { try { switch (field.getType().getSimpleName()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index a33f7ba4f64b26..7d9d2f05e380aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1778,15 +1778,8 @@ public MTMVRefreshTriggerInfo visitRefreshTrigger(RefreshTriggerContext ctx) { @Override public ReplayCommand visitReplay(DorisParser.ReplayContext ctx) { - if (ctx.replayCommand().replayType().DUMP() != null) { - LogicalPlan plan = plan(ctx.replayCommand().replayType().query()); - return new ReplayCommand(PlanType.REPLAY_COMMAND, null, plan, ReplayCommand.ReplayType.DUMP); - } else if (ctx.replayCommand().replayType().PLAY() != null) { - String tmpPath = ctx.replayCommand().replayType().filePath.getText(); - String path = LogicalPlanBuilderAssistant.escapeBackSlash(tmpPath.substring(1, tmpPath.length() - 1)); - return new ReplayCommand(PlanType.REPLAY_COMMAND, path, null, ReplayCommand.ReplayType.PLAY); - } - return null; + LogicalPlan plan = plan(ctx.replayCommand().replayType().query()); + return new ReplayCommand(PlanType.REPLAY_COMMAND, plan); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java index 4eabbb9e9595d2..22edc0919069c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java @@ -19,66 +19,29 @@ import org.apache.doris.analysis.StmtType; import org.apache.doris.nereids.NereidsPlanner; -import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; -import org.apache.doris.nereids.minidump.Minidump; import org.apache.doris.nereids.minidump.MinidumpUtils; import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.StmtExecutor; -import org.json.JSONObject; - -import java.util.List; - /** - * replay command. + * replay command: PLAN REPLAYER DUMP query. */ public class ReplayCommand extends Command implements NoForward { - private final String dumpFileFullPath; - private final LogicalPlan plan; - private final ReplayType replayType; - - public ReplayCommand(PlanType type, String dumpFileFullPath, LogicalPlan plan, ReplayType replayType) { + public ReplayCommand(PlanType type, LogicalPlan plan) { super(type); - this.dumpFileFullPath = dumpFileFullPath; this.plan = plan; - this.replayType = replayType; - } - - public String getDumpFileFullPath() { - return dumpFileFullPath; - } - - public ReplayType getReplayType() { - return replayType; - } - - /** - * explain level. - */ - public enum ReplayType { - DUMP, - PLAY } @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { - if (this.replayType == ReplayType.DUMP) { - handleDump(ctx, executor); - } else if (this.replayType == ReplayType.PLAY) { - handleLoad(); - } - } - - private void handleDump(ConnectContext ctx, StmtExecutor executor) throws Exception { LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(plan, ctx.getStatementContext()); MinidumpUtils.openDump(); executor.setParsedStmt(logicalPlanAdapter); @@ -92,22 +55,6 @@ private void handleDump(ConnectContext ctx, StmtExecutor executor) throws Except executor.handleReplayStmt(MinidumpUtils.getHttpGetString()); } - private void handleLoad() throws Exception { - // 1. check fe version, if not matched throw exception - // 2. load every thing from minidump file and replace original ones - Minidump minidump = MinidumpUtils.loadMinidumpInputs(dumpFileFullPath); - // 3. run nereids planner with sql in minidump file - StatementContext statementContext = new StatementContext(ConnectContext.get(), - new OriginStatement(minidump.getSql(), 0)); - statementContext.setTables(minidump.getTables()); - ConnectContext.get().setStatementContext(statementContext); - JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql()); - JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson()); - - List differences = MinidumpUtils.compareJsonObjects(minidumpResult, resultPlan, ""); - assert (differences.isEmpty()); - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitReplayCommand(this, context); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index bc9e68dd06b98b..8dd5fbe0f49f3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -327,13 +327,12 @@ public void testPlanReplayer() { String sql = "plan replayer dump select `AD``D` from t1 where a = 1"; NereidsParser nereidsParser = new NereidsParser(); LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); - ReplayCommand replayCommand = (ReplayCommand) logicalPlan; - Assertions.assertEquals(ReplayCommand.ReplayType.DUMP, replayCommand.getReplayType()); - sql = "plan replayer play 'path'"; - logicalPlan = nereidsParser.parseSingle(sql); - replayCommand = (ReplayCommand) logicalPlan; - Assertions.assertEquals(ReplayCommand.ReplayType.PLAY, replayCommand.getReplayType()); - Assertions.assertEquals("path", replayCommand.getDumpFileFullPath()); + Assertions.assertInstanceOf(ReplayCommand.class, logicalPlan); + Assertions.assertThrows(ParseException.class, + () -> nereidsParser.parseSingle("plan replayer play 'path'")); + // PLAY is no longer a keyword, so it is an ordinary identifier in any case. + Assertions.assertDoesNotThrow(() -> nereidsParser.parseSingle("select pLaY from play")); + Assertions.assertDoesNotThrow(() -> nereidsParser.parseSingle("select play.play as play from play")); } @Test diff --git a/minidump/README.md b/minidump/README.md deleted file mode 100644 index da45769d80f785..00000000000000 --- a/minidump/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Tutorial of Nereids minidump -Minidump is a basic dump file for Nereids optimizer which arms -to replay problems and used as unit test tool for developers. -It include minimal necessary informations for Nereids optimizer -to replay a scenario in Optimizer. - -## Save minidump file -In two cases fe would save minidump file -- when query throw an exception -- when use 'set enable_minidump=true;' switch by User - -## File structure of minidump file -Minidump files would save in $DORIS_HOME/log/minidump path -and named by queryId like: c9bf82b70ce34c28-8d281e31412f3ee6.json -File structure of minidump file includes: -| key | Value | -|:----|:----| -|Sql | original sql string | -|SessionVariables | sessionVariables different than default | -|DatabaseName | database name| -|Tables | schemas in table, partition info and some table properties | -|ColocateTableIndex | table distribute informations | -|ColumnStatistics | all ColumnStatistics in table | -|Histogram | all histogram information in table| -|ParsedPlan | plan after parsed in json format| -|ResultPlan | plan after all optimized in json format| - -## Use of nereids_ut.sh -When we want to use this shell, we need to go to output/fe/minidump path -Now only one parameter is allowed for this shell which is directory -of minidump files like $DORIS_HOME/log/minidump -```shell -cd output/fe/minidump -./nereids_ut.sh --d $minidump_files_path -``` -The running reports can be found in output/fe/minidump/log -which is minidump.out - diff --git a/minidump/minidump.conf b/minidump/minidump.conf deleted file mode 100644 index 04b3159adeaa40..00000000000000 --- a/minidump/minidump.conf +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -##################################################################### -## The uppercase properties are read and exported by bin/start_fe.sh. -## To see all Frontend configurations, -## see fe/src/org/apache/doris/common/Config.java -##################################################################### - -CUR_DATE=`date +%Y%m%d-%H%M%S` - -# the output dir of stderr and stdout -LOG_DIR = ${DORIS_HOME}/minidump/log - -JAVA_OPTS="-Dorg.slf4j.simpleLogger.showLogName=false -Dorg.slf4j.simpleLogger.showShortLogName=false -Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0" - -# For jdk 9+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_9="-Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0" - -## -## the lowercase properties are read by main program. -## - -# INFO, WARN, ERROR, FATAL -sys_log_level = ERROR - -# store metadata, must be created before start FE. -# Default value is ${DORIS_HOME}/doris-meta -# meta_dir = ${DORIS_HOME}/doris-meta - -# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers -# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers - -http_port = 8030 -rpc_port = 9020 -query_port = 9030 -edit_log_port = 9010 -mysql_service_nio_enabled = true - -# Choose one if there are more than one ip except loopback address. -# Note that there should at most one ip match this list. -# If no ip match this rule, will choose one randomly. -# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1 -# Default value is empty. -# priority_networks = 10.10.10.0/24;192.168.0.0/16 - -# Advanced configurations -# log_roll_size_mb = 1024 -# sys_log_dir = ${DORIS_HOME}/log -# sys_log_roll_num = 10 -# sys_log_verbose_modules = org.apache.doris -# audit_log_dir = ${DORIS_HOME}/log -# audit_log_modules = slow_query, query -# audit_log_roll_num = 10 -# meta_delay_toleration_second = 10 -# qe_max_connection = 1024 -# max_conn_per_user = 100 -# qe_query_timeout_second = 300 -# qe_slow_log_ms = 5000 diff --git a/minidump/nereids_ut.sh b/minidump/nereids_ut.sh deleted file mode 100755 index 08191c15b1be9d..00000000000000 --- a/minidump/nereids_ut.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -cur_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - -if [[ "$(uname -s)" == 'Darwin' ]] && command -v brew &>/dev/null; then - PATH="$(brew --prefix)/opt/gnu-getopt/bin:${PATH}" - export PATH -fi - -OPTS="$(getopt \ - -n "$0" \ - -o '' \ - -l 'daemon' \ - -l 'helper:' \ - -l 'd:' \ - -l 'version' \ - -- "$@")" - -eval set -- "${OPTS}" - -DUMP_FILE_DIR='' -OPT_VERSION='' -while true; do - case "$1" in - --version) - OPT_VERSION="--version" - shift - ;; - --d) - DUMP_FILE_DIR="$2" - shift 2 - ;; - --) - shift - break - ;; - *) - echo "Internal error" - exit 1 - ;; - esac -done - -DORIS_HOME="$( - cd "${cur_dir}/.." - pwd -)" -export DORIS_HOME - -# export env variables from fe.conf -# -# JAVA_OPTS -# LOG_DIR -# PID_DIR -export JAVA_OPTS="-Xmx1024m" -export LOG_DIR="${DORIS_HOME}/minidump" - -while read -r line; do - env_line="$(echo "${line}" | - sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | - sed 's/^[[:blank:]]*//g' | - grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" || - true)" - env_line="$(eval "echo ${env_line}")" - if [[ "${env_line}" == *"="* ]]; then - eval 'export "${env_line}"' - fi -done <"${DORIS_HOME}/minidump/minidump.conf" - -if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then - # shellcheck disable=1091 - source "${DORIS_HOME}/bin/palo_env.sh" -fi - -if [[ -z "${JAVA_HOME}" ]]; then - JAVA="$(command -v java)" -else - JAVA="${JAVA_HOME}/bin/java" -fi - -if [[ ! -x "${JAVA}" ]]; then - echo "The JAVA_HOME environment variable is not defined correctly" - echo "This environment variable is needed to run this program" - echo "NB: JAVA_HOME should point to a JDK not a JRE" - exit 1 -fi - -# get jdk version, return version as an Integer. -# 1.8 => 8, 13.0 => 13 -jdk_version() { - local java_cmd="${1}" - local result - local IFS=$'\n' - - if [[ -z "${java_cmd}" ]]; then - result=no_java - return 1 - else - local version - # remove \r for Cygwin - version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" - version="${version//\"/}" - if [[ "${version}" =~ ^1\. ]]; then - result="$(echo "${version}" | awk -F '.' '{print $2}')" - else - result="$(echo "${version}" | awk -F '.' '{print $1}')" - fi - fi - echo "${result}" - return 0 -} - -# need check and create if the log directory existed before outing message to the log file. -if [[ ! -d "${LOG_DIR}" ]]; then - mkdir -p "${LOG_DIR}" -fi - -# check java version and choose correct JAVA_OPTS -java_version="$( - set -e - jdk_version "${JAVA}" -)" -final_java_opt="${JAVA_OPTS}" -if [[ "${java_version}" -gt 8 ]]; then - if [[ -z "${JAVA_OPTS_FOR_JDK_9}" ]]; then - echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >>"${LOG_DIR}/minidump.out" - exit 1 - fi - final_java_opt="${JAVA_OPTS_FOR_JDK_9}" -fi - -# add libs to CLASSPATH -DORIS_FE_JAR= -for f in "${DORIS_HOME}/lib"/*.jar; do - if [[ "${f}" == *"doris-fe.jar" ]]; then - DORIS_FE_JAR="${f}" - continue - fi - CLASSPATH="${f}:${CLASSPATH}" -done - -# make sure the doris-fe.jar is at first order, so that some classed -# with same qualified name can be loaded priority from doris-fe.jar -CLASSPATH="${DORIS_FE_JAR}:${CLASSPATH}" -export CLASSPATH="${CLASSPATH}:${DORIS_HOME}/lib:${DORIS_HOME}/conf" - -date >>"${LOG_DIR}/minidump.out" -query_id_head="|QueryId=" -sql_head="|Sql=" -# shellcheck disable=SC2045 -for path in $(ls "${DUMP_FILE_DIR}"); do - query_id=${path:0:33} - temp_file=${LOG_DIR}/temp - "${JAVA}" ${final_java_opt:+${final_java_opt}} -Xmx2g org.apache.doris.nereids.minidump.Minidump "${DUMP_FILE_DIR}/${path}" "${OPT_VERSION}" "$@" >"${temp_file}" - result=$(cat "${temp_file}") - echo "${query_id_head} ${query_id} ${sql_head} ${result}" | tee >>"${LOG_DIR}/minidump.out" -done -date >>"${LOG_DIR}/minidump.out"