Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1020,6 +1020,8 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
cp -r -p "${DORIS_HOME}/conf/ldap.conf" "${DORIS_OUTPUT}/fe/conf"/
cp -r -p "${DORIS_HOME}/conf/mysql_ssl_default_certificate" "${DORIS_OUTPUT}/fe/"/
rm -rf "${DORIS_OUTPUT}/fe/lib"/*
# The offline minidump runner is gone; drop it from reused output trees too.
rm -rf "${DORIS_OUTPUT}/fe/minidump"
cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/
cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/
for extra_module_path in "${FE_EXTRA_MODULE_PATHS[@]}"; do
Expand All @@ -1044,7 +1046,6 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
# Third-party filesystem jars (JuiceFS, JindoFS) are packaged by post-build.sh
"${DORIS_HOME}/post-build.sh" --fe --output "${DORIS_OUTPUT}"

cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/
cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/

cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/fe/webroot/static"/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,6 @@ PHYSICAL: 'PHYSICAL';
PI: 'PI';
PLACEHOLDER: '?';
PLAN: 'PLAN';
PLAY: 'PLAY';
PRIVILEGES: 'PRIVILEGES';
PROCESS: 'PROCESS';
PYTHON: 'PYTHON';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1183,8 +1183,7 @@ replayCommand
: PLAN REPLAYER replayType;

replayType
: DUMP query
| PLAY filePath=STRING_LITERAL;
: DUMP query;

mergeType
: APPEND
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,10 @@

import org.apache.doris.catalog.ColocateTableIndex;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Histogram;

import org.json.JSONObject;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -110,35 +105,4 @@ public Map<String, ColumnStatistic> getTotalColumnStatisticMap() {
public Map<String, Histogram> getTotalHistogramMap() {
return totalHistogramMap;
}

/** Nereids minidump entry, argument should be absolute address of minidump path */
public static void main(String[] args) {
assert (args.length == 1);
Minidump minidump = null;
try {
minidump = MinidumpUtils.loadMinidumpInputs(args[0]);
} catch (Exception e) {
e.printStackTrace();
}

StatementContext statementContext = new StatementContext(ConnectContext.get(),
new OriginStatement(minidump.getSql(), 0));
statementContext.setTables(minidump.getTables());
ConnectContext.get().setStatementContext(statementContext);
JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql());
JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson());

List<String> differences = MinidumpUtils.compareJsonObjects(minidumpResult, resultPlan, "");

if (differences.isEmpty()) {
System.out.println(minidump.sql);
} else {
System.out.println(minidump.sql + "\n" + "Result plan are not expected. Differences:");
System.out.println("----------------------------Result plan--------------------------");
System.out.println(resultPlan + "\n");
System.out.println("----------------------------Minidump plan--------------------------");
System.out.println(minidumpResult + "\n");
}
System.exit(0);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,7 @@
import java.lang.reflect.Type;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -232,29 +230,6 @@ public static void setConnectContext(Minidump minidump) {
connectContext.getSessionVariable().enableNereidsTimeout = false;
}

/**
* Loading minidump messages from file to memory
* @param minidumpPath path of minidump file
* @return minidump messages in memory
*/
public static Minidump loadMinidumpInputs(String minidumpPath) throws AnalysisException {
Minidump minidump = null;
try {
minidump = MinidumpUtils.jsonMinidumpLoad(minidumpPath);
} catch (AnalysisException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
setConnectContext(minidump);

Env env = Env.getCurrentEnv();
ConnectContext.get().setEnv(env);
ConnectContext.get().getSessionVariable().setEnableNereidsTrace(false);
ConnectContext.get().getSessionVariable().setNereidsTraceEventMode("all");
return minidump;
}

/**
* Executing sql with minidump and return jsontype result for more use of unit test
* @param sql original sql clause
Expand Down Expand Up @@ -363,89 +338,6 @@ public static void serializeOutputToDumpFile(Plan resultPlan) {
}
}

/** compare two json object and print detail information about difference */
public static List<String> compareJsonObjects(JSONObject json1, JSONObject json2, String path) {
List<String> differences = new ArrayList<>();

Iterator<String> keys = json1.keys();
while (keys.hasNext()) {
String key = keys.next();
String currentPath = "";
if (path.isEmpty()) {
if (key.equals("PlanType")) {
path = json1.getString(key);
} else {
currentPath = key;
}
} else {
if (key.equals("PlanType")) {
currentPath = path + "." + json1.getString(key);
} else {
currentPath = path + "." + key;
}
}

if (!json2.has(key)) {
differences.add("Key '" + currentPath + "' not found in the second JSON object.");
continue;
}

Object value1 = json1.get(key);
Object value2 = json2.get(key);

if (!value1.toString().equals(value2.toString())) {
differences.add("Value for key '" + currentPath + "' is different: " + value1 + " != " + value2);
}

if (value1 instanceof JSONObject && value2 instanceof JSONObject) {
List<String> nestedDifferences =
compareJsonObjects((JSONObject) value1, (JSONObject) value2, currentPath);
differences.addAll(nestedDifferences);
} else if (value1 instanceof JSONArray && value2 instanceof JSONArray) {
List<String> nestedDifferences = compareJsonArrays((JSONArray) value1, (JSONArray) value2, currentPath);
differences.addAll(nestedDifferences);
}
}

keys = json2.keys();
while (keys.hasNext()) {
String key = keys.next();
String currentPath = (path.isEmpty()) ? key : path + "." + key;

if (!json1.has(key)) {
differences.add("Key '" + currentPath + "' not found in the first JSON object.");
}
}

return differences;
}

private static List<String> compareJsonArrays(JSONArray array1, JSONArray array2, String path) {
List<String> differences = new ArrayList<>();

if (array1.length() != array2.length()) {
differences.add("Array length for key '" + path + "' is different: "
+ array1.length() + " != " + array2.length());
return differences;
}

for (int i = 0; i < array1.length(); i++) {
Object value1 = array1.get(i);
Object value2 = array2.get(i);
String currentPath = path + "[" + i + "]";

if (value1 instanceof JSONObject && value2 instanceof JSONObject) {
List<String> nestedDifferences =
compareJsonObjects((JSONObject) value1, (JSONObject) value2, currentPath);
differences.addAll(nestedDifferences);
} else if (!value1.equals(value2)) {
differences.add("Value for key '" + currentPath + "' is different: " + value1 + " != " + value2);
}
}

return differences;
}

private static String getValue(Object obj, Field field) {
try {
switch (field.getType().getSimpleName()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1778,15 +1778,8 @@ public MTMVRefreshTriggerInfo visitRefreshTrigger(RefreshTriggerContext ctx) {

@Override
public ReplayCommand visitReplay(DorisParser.ReplayContext ctx) {
if (ctx.replayCommand().replayType().DUMP() != null) {
LogicalPlan plan = plan(ctx.replayCommand().replayType().query());
return new ReplayCommand(PlanType.REPLAY_COMMAND, null, plan, ReplayCommand.ReplayType.DUMP);
} else if (ctx.replayCommand().replayType().PLAY() != null) {
String tmpPath = ctx.replayCommand().replayType().filePath.getText();
String path = LogicalPlanBuilderAssistant.escapeBackSlash(tmpPath.substring(1, tmpPath.length() - 1));
return new ReplayCommand(PlanType.REPLAY_COMMAND, path, null, ReplayCommand.ReplayType.PLAY);
}
return null;
LogicalPlan plan = plan(ctx.replayCommand().replayType().query());
return new ReplayCommand(PlanType.REPLAY_COMMAND, plan);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,66 +19,29 @@

import org.apache.doris.analysis.StmtType;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.minidump.Minidump;
import org.apache.doris.nereids.minidump.MinidumpUtils;
import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.StmtExecutor;

import org.json.JSONObject;

import java.util.List;

/**
* replay command.
* replay command: PLAN REPLAYER DUMP query.
*/
public class ReplayCommand extends Command implements NoForward {

private final String dumpFileFullPath;

private final LogicalPlan plan;

private final ReplayType replayType;

public ReplayCommand(PlanType type, String dumpFileFullPath, LogicalPlan plan, ReplayType replayType) {
public ReplayCommand(PlanType type, LogicalPlan plan) {
super(type);
this.dumpFileFullPath = dumpFileFullPath;
this.plan = plan;
this.replayType = replayType;
}

public String getDumpFileFullPath() {
return dumpFileFullPath;
}

public ReplayType getReplayType() {
return replayType;
}

/**
* explain level.
*/
public enum ReplayType {
DUMP,
PLAY
}

@Override
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
if (this.replayType == ReplayType.DUMP) {
handleDump(ctx, executor);
} else if (this.replayType == ReplayType.PLAY) {
handleLoad();
}
}

private void handleDump(ConnectContext ctx, StmtExecutor executor) throws Exception {
LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(plan, ctx.getStatementContext());
MinidumpUtils.openDump();
executor.setParsedStmt(logicalPlanAdapter);
Expand All @@ -92,22 +55,6 @@ private void handleDump(ConnectContext ctx, StmtExecutor executor) throws Except
executor.handleReplayStmt(MinidumpUtils.getHttpGetString());
}

private void handleLoad() throws Exception {
// 1. check fe version, if not matched throw exception
// 2. load every thing from minidump file and replace original ones
Minidump minidump = MinidumpUtils.loadMinidumpInputs(dumpFileFullPath);
// 3. run nereids planner with sql in minidump file
StatementContext statementContext = new StatementContext(ConnectContext.get(),
new OriginStatement(minidump.getSql(), 0));
statementContext.setTables(minidump.getTables());
ConnectContext.get().setStatementContext(statementContext);
JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql());
JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson());

List<String> differences = MinidumpUtils.compareJsonObjects(minidumpResult, resultPlan, "");
assert (differences.isEmpty());
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitReplayCommand(this, context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -327,13 +327,12 @@ public void testPlanReplayer() {
String sql = "plan replayer dump select `AD``D` from t1 where a = 1";
NereidsParser nereidsParser = new NereidsParser();
LogicalPlan logicalPlan = nereidsParser.parseSingle(sql);
ReplayCommand replayCommand = (ReplayCommand) logicalPlan;
Assertions.assertEquals(ReplayCommand.ReplayType.DUMP, replayCommand.getReplayType());
sql = "plan replayer play 'path'";
logicalPlan = nereidsParser.parseSingle(sql);
replayCommand = (ReplayCommand) logicalPlan;
Assertions.assertEquals(ReplayCommand.ReplayType.PLAY, replayCommand.getReplayType());
Assertions.assertEquals("path", replayCommand.getDumpFileFullPath());
Assertions.assertInstanceOf(ReplayCommand.class, logicalPlan);
Assertions.assertThrows(ParseException.class,
() -> nereidsParser.parseSingle("plan replayer play 'path'"));
// PLAY is no longer a keyword, so it is an ordinary identifier in any case.
Assertions.assertDoesNotThrow(() -> nereidsParser.parseSingle("select pLaY from play"));
Assertions.assertDoesNotThrow(() -> nereidsParser.parseSingle("select play.play as play from play"));
}

@Test
Expand Down
38 changes: 0 additions & 38 deletions minidump/README.md

This file was deleted.

Loading
Loading