Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@924060929
924060929 committed Jul 18, 2024
1 parent 0568401 commit d7ee26d
Show file tree
Hide file tree
Showing 693 changed files with 2,216 additions and 18 deletions.
23 changes: 5 additions & 18 deletions fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -785,21 +785,14 @@ private void sendPipelineCtx() throws TException, RpcException, UserException {
needCheckBackendState = true;
}

Map<TUniqueId, Boolean> fragmentInstancesMap = new HashMap<TUniqueId, Boolean>();
for (Map.Entry<TNetworkAddress, TPipelineFragmentParams> entry : tParams.entrySet()) {
for (TPipelineInstanceParams instanceParam : entry.getValue().local_params) {
fragmentInstancesMap.put(instanceParam.fragment_instance_id, false);
}
}

int numBackendsWithSink = 0;
// 3. group PipelineExecContext by BE.
// So that we can use one RPC to send all fragment instances of a BE.
for (Map.Entry<TNetworkAddress, TPipelineFragmentParams> entry : tParams.entrySet()) {
Long backendId = this.addressToBackendID.get(entry.getKey());
backendFragments.add(Pair.of(fragment.getFragmentId(), backendId));
PipelineExecContext pipelineExecContext = new PipelineExecContext(fragment.getFragmentId(),
entry.getValue(), backendId, fragmentInstancesMap, executionProfile);
entry.getValue(), backendId, executionProfile);
// Each tParam will set the total number of Fragments that need to be executed on the same BE,
// and the BE will determine whether all Fragments have been executed based on this information.
// Notice. load fragment has a small probability that FragmentNumOnHost is 0, for unknown reasons.
Expand Down Expand Up @@ -833,14 +826,14 @@ private void sendPipelineCtx() throws TException, RpcException, UserException {
}
++backendIdx;
}
int loadStreamPerNode = 1;
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) {
loadStreamPerNode = ConnectContext.get().getSessionVariable().getLoadStreamPerNode();
}
for (Map.Entry<TNetworkAddress, TPipelineFragmentParams> entry : tParams.entrySet()) {
if (entry.getValue().getFragment().getOutputSink() != null
&& entry.getValue().getFragment().getOutputSink().getType()
== TDataSinkType.OLAP_TABLE_SINK) {
int loadStreamPerNode = 1;
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) {
loadStreamPerNode = ConnectContext.get().getSessionVariable().getLoadStreamPerNode();
}
entry.getValue().setLoadStreamPerNode(loadStreamPerNode);
entry.getValue().setTotalLoadStreams(numBackendsWithSink * loadStreamPerNode);
entry.getValue().setNumLocalSink(entry.getValue().getLocalParams().size());
Expand Down Expand Up @@ -2754,23 +2747,17 @@ public class PipelineExecContext {
boolean initiated;
boolean done;

Map<TUniqueId, Boolean> fragmentInstancesMap;
TNetworkAddress brpcAddress;
TNetworkAddress address;
Backend backend;
long lastMissingHeartbeatTime = -1;
long profileReportProgress = 0;
long beProcessEpoch = 0;
private final int numInstances;

public PipelineExecContext(PlanFragmentId fragmentId,
TPipelineFragmentParams rpcParams, Long backendId,
Map<TUniqueId, Boolean> fragmentInstancesMap,
ExecutionProfile executionProfile) {
this.fragmentId = fragmentId;
this.rpcParams = rpcParams;
this.numInstances = rpcParams.local_params.size();
this.fragmentInstancesMap = fragmentInstancesMap;

this.initiated = false;
this.done = false;
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/load.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ suite("load") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql """
DROP TABLE IF EXISTS hits
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query1.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
*/

suite("query1") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query10.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
*/

suite("query10") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query11.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query11") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query12.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query12") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query13.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query13") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query14.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query14") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query15.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query15") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query16.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query16") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query17.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query17") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query18.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query18") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query19.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query19") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query2.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
*/

suite("query2") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query20.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query20") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query21.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query21") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query22.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query22") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query23.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query23") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query24.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query24") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query25.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query25") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query26.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query26") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query27.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query27") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query28.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query28") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query29.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query29") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query3.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
*/

suite("query3") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query30.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query30") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query31.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query31") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query32.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query32") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query33.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query33") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query34.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query34") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query35.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query35") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query36.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query36") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query37.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query37") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
3 changes: 3 additions & 0 deletions regression-test/suites/new_shapes_p0/clickbench/query38.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ suite("query38") {
if (isCloudMode()) {
return
}
if (isCloudMode()) {
return
}

sql 'set enable_nereids_planner=true'
sql 'set enable_nereids_distribute_planner=true'
Expand Down
Loading

0 comments on commit d7ee26d

Please sign in to comment.