diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java index 188d37ffae3f77..6ddd29fb13a065 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/ColocateMetaService.java @@ -77,7 +77,7 @@ private static GroupId checkAndGetGroupId(HttpServletRequest request) throws Ddl GroupId groupId = new GroupId(dbId, grpId); if (!colocateIndex.isGroupExist(groupId)) { - throw new DdlException("the group " + groupId + "isn't exist"); + throw new DdlException("the group " + groupId + " isn't exist"); } return groupId; } @@ -95,7 +95,7 @@ public Object colocate(HttpServletRequest request, HttpServletResponse response) return redirectToHttps(request); } - if (!Env.getCurrentEnv().isMaster()) { + if (!checkForwardToMaster(request)) { return forwardToMaster(request); } executeWithoutPassword(request, response); @@ -109,7 +109,7 @@ public Object group_stable(HttpServletRequest request, HttpServletResponse respo return redirectToHttps(request); } - if (!Env.getCurrentEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java index 207d35dbd99181..59c0579747a331 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/CancelLoadAction.java @@ -50,7 +50,7 @@ public Object execute(@PathVariable(value = DB_KEY) final String dbName, } executeCheckPassword(request, response); - if (!Env.getCurrentEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java index 0c2ae929ebe5a4..6a16459adb0bcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetLoadInfoAction.java @@ -55,6 +55,9 @@ public class GetLoadInfoAction extends RestBaseController { public Object execute( @PathVariable(value = DB_KEY) final String dbName, HttpServletRequest request, HttpServletResponse response) { + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } executeCheckPassword(request, response); String fullDbName = getFullDbName(dbName); @@ -67,7 +70,7 @@ public Object execute( if (Strings.isNullOrEmpty(info.label)) { return new RestBaseResult("No label selected"); } - if (!Env.getServingEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetStreamLoadState.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetStreamLoadState.java index eb03e140f04fba..688ccc413fadbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetStreamLoadState.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/GetStreamLoadState.java @@ -38,7 +38,11 @@ public Object execute(@PathVariable(value = DB_KEY) final String dbName, HttpServletRequest request, HttpServletResponse response) { executeCheckPassword(request, response); - if (!Env.getCurrentEnv().isMaster()) { + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } + + if (!checkForwardToMaster(request)) { return forwardToMaster(request); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java index 34cda7d7ef0c45..addb3865cf0708 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MultiAction.java @@ -67,7 +67,7 @@ public Object multi_desc( checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); // only Master has these load info - if (!Env.getCurrentEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } @@ -95,7 +95,7 @@ public Object multi_list( checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); // only Master has these load info - if (!Env.getCurrentEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } @@ -128,7 +128,7 @@ public Object multi_start( // Multi start request must redirect to master, because all following sub requests will be handled // on Master - if (!Env.getCurrentEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } @@ -177,7 +177,7 @@ public Object multi_unload( String fullDbName = getFullDbName(dbName); checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); - if (!Env.getCurrentEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } @@ -209,7 +209,7 @@ public Object multi_commit( checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), fullDbName, PrivPredicate.LOAD); // only Master has these load info - if (!Env.getCurrentEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java index a029b4cc71f7d1..193dc2d3bda254 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.NetUtils; import org.apache.doris.httpv2.controller.BaseController; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; @@ -65,6 +66,7 @@ public class RestBaseController extends BaseController { protected static final String TXN_ID_KEY = "txn_id"; protected static final String TXN_OPERATION_KEY = "txn_operation"; protected static final String SINGLE_REPLICA_KEY = "single_replica"; + protected static final String FORBID_FORWARD_UT_TEST = "forbid_forward_ut_test"; private static final Logger LOG = LogManager.getLogger(RestBaseController.class); public ActionAuthorizationInfo executeCheckPassword(HttpServletRequest request, @@ -220,6 +222,18 @@ public Object forwardToMaster(HttpServletRequest request) { } } + public boolean checkForwardToMaster(HttpServletRequest request) { + if (FeConstants.runningUnitTest) { + String forbidForward = request.getHeader(FORBID_FORWARD_UT_TEST); + if (forbidForward != null) { + return !"true".equals(forbidForward); + } + return true; + } + return !Env.getCurrentEnv().isMaster(); + } + + private String getRequestBody(HttpServletRequest request) throws IOException { BufferedReader reader = request.getReader(); return reader.lines().collect(Collectors.joining(System.lineSeparator())); @@ -228,8 +242,14 @@ private String getRequestBody(HttpServletRequest request) throws IOException { public Object forwardToMaster(HttpServletRequest request, @Nullable Object body) { try { Env env = Env.getCurrentEnv(); - String redirectUrl = getRedirectUrL(request, - new TNetworkAddress(env.getMasterHost(), env.getMasterHttpPort())); + String redirectUrl = null; + if (FeConstants.runningUnitTest) { + redirectUrl = + getRedirectUrL(request, new TNetworkAddress(request.getServerName(), request.getServerPort())); + } else { + redirectUrl = getRedirectUrL(request, + new TNetworkAddress(env.getMasterHost(), env.getMasterHttpPort())); + } String method = request.getMethod(); HttpHeaders headers = new HttpHeaders(); @@ -237,6 +257,11 @@ public Object forwardToMaster(HttpServletRequest request, @Nullable Object body) headers.add(headerName, request.getHeader(headerName)); } + if (FeConstants.runningUnitTest) { + //Add a header to avoid forward. + headers.add(FORBID_FORWARD_UT_TEST, "true"); + } + HttpEntity entity = new HttpEntity<>(body, headers); RestTemplate restTemplate = new RestTemplate(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java index a185add412ed70..eb1b78666383d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java @@ -115,7 +115,7 @@ public Object show_proc(HttpServletRequest request, HttpServletResponse response } // forward to master if necessary - if (!Env.getCurrentEnv().isMaster() && isForward) { + if (checkForwardToMaster(request) && isForward) { return forwardToMaster(request); } else { ProcNodeInterface procNode = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java index 53107f38f05b45..f42c09d4737c72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java @@ -144,7 +144,11 @@ public Object brokers_info(HttpServletRequest request, HttpServletResponse respo private Object fetchNodeInfo(HttpServletRequest request, HttpServletResponse response, String procPath) throws Exception { try { - if (!Env.getCurrentEnv().isMaster()) { + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } + + if (checkForwardToMaster(request)) { return forwardToMaster(request, null); } @@ -604,7 +608,11 @@ public Object setConfigBe(HttpServletRequest request, HttpServletResponse respon public Object operateBackend(HttpServletRequest request, HttpServletResponse response, @PathVariable String action, @RequestBody BackendReqInfo reqInfo) { try { - if (!Env.getCurrentEnv().isMaster()) { + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } + + if (checkForwardToMaster(request)) { return forwardToMaster(request, reqInfo); } @@ -647,7 +655,11 @@ public Object operateBackend(HttpServletRequest request, HttpServletResponse res public Object operateFrontends(HttpServletRequest request, HttpServletResponse response, @PathVariable String action, @RequestBody FrontendReqInfo reqInfo) { try { - if (!Env.getCurrentEnv().isMaster()) { + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } + + if (checkForwardToMaster(request)) { return forwardToMaster(request, reqInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/AddStoragePolicyAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/AddStoragePolicyAction.java index 9842ffccb095e2..446f9f5306b047 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/AddStoragePolicyAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/AddStoragePolicyAction.java @@ -52,7 +52,11 @@ public Object clusterOverview(@RequestBody StoragePolicyVo body, } try { - if (!Env.getCurrentEnv().isMaster()) { + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } + + if (checkForwardToMaster(request)) { return forwardToMaster(request, body); } } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ESCatalogAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ESCatalogAction.java index 174b2b5797dbae..aeb2591eced87f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ESCatalogAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ESCatalogAction.java @@ -55,7 +55,11 @@ private Object handleRequest(HttpServletRequest request, HttpServletResponse res executeCheckPassword(request, response); } - if (!Env.getCurrentEnv().isMaster()) { + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } + + if (checkForwardToMaster(request)) { return forwardToMaster(request); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/StatisticAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/StatisticAction.java index 0b4e7ad3707778..2b2a6d49b6394e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/StatisticAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/StatisticAction.java @@ -50,8 +50,11 @@ public Object clusterOverview(HttpServletRequest request, HttpServletResponse re if (Config.enable_all_http_auth) { executeCheckPassword(request, response); } + if (needRedirect(request.getScheme())) { + return redirectToHttps(request); + } - if (!Env.getCurrentEnv().isMaster()) { + if (checkForwardToMaster(request)) { return forwardToMaster(request); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/ForwardToMasterTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/ForwardToMasterTest.java new file mode 100644 index 00000000000000..52ef5bcf618696 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/http/ForwardToMasterTest.java @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.http; + +import org.apache.doris.common.FeConstants; + +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import org.junit.Assert; +import org.junit.Test; + +public class ForwardToMasterTest extends DorisHttpTestCase { + @Test + public void testAddBeDropBe() throws Exception { + FeConstants.runningUnitTest = true; + String port = "48013"; + { + //query backends + String url = "http://localhost:" + HTTP_PORT + "/rest/v2/manager/node/backends"; + Request request = new Request.Builder() + .get() + .addHeader("Authorization", rootAuth) + .url(url) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + Assert.assertNotNull(response.body()); + String respStr = response.body().string(); + JSONObject object = (JSONObject) JSONValue.parse(respStr); + + JSONObject data = (JSONObject) object.get("data"); + JSONArray columnNames = (JSONArray) ((JSONObject) data.get("columnNames")).get("columnNames"); + JSONArray rows = (JSONArray) ((JSONObject) data.get("rows")).get("rows"); + int sz = columnNames.size(); + int index = columnNames.indexOf("HeartbeatPort"); + int existsbe = 0; + for (int i = 0; i < rows.size(); i += sz) { + if (port.equals(rows.get(i + index).toString())) { + existsbe++; + } + } + Assert.assertEquals(0, existsbe); + } + + { + String jsonBody = "{" + + " \"hostPorts\": [\"localhost:" + port + "\"]," + + " \"properties\": {" + + " \"key1\": \"value1\"," + + " \"key2\": \"value2\"" + + " }" + + "}"; + + String url = "http://localhost:" + HTTP_PORT + "/rest/v2/manager/node/ADD/be"; + Request request = new Request.Builder() + .url(url) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", rootAuth) + .post(RequestBody.create(jsonBody, MediaType.parse("application/json"))) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + } + + { + //query be + String url = "http://localhost:" + HTTP_PORT + "/rest/v2/manager/node/backends"; + Request request = new Request.Builder() + .get() + .addHeader("Authorization", rootAuth) + .url(url) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + Assert.assertNotNull(response.body()); + String respStr = response.body().string(); + + JSONObject object = (JSONObject) JSONValue.parse(respStr); + JSONObject data = (JSONObject) object.get("data"); + JSONArray columnNames = (JSONArray) ((JSONObject) data.get("columnNames")).get("columnNames"); + JSONArray rows = (JSONArray) ((JSONObject) data.get("rows")).get("rows"); + int sz = columnNames.size(); + int index = columnNames.indexOf("HeartbeatPort"); + int existsbe = 0; + for (int i = 0; i < rows.size(); i += sz) { + if (port.equals(rows.get(i + index).toString())) { + existsbe++; + } + } + Assert.assertEquals(1, existsbe); + } + + { + // DROP be + String jsonBody = "{" + + " \"hostPorts\": [\"localhost:" + port + "\"]," + + " \"properties\": {" + + " \"key1\": \"value1\"," + + " \"key2\": \"value2\"" + + " }" + + "}"; + + String url = "http://localhost:" + HTTP_PORT + "/rest/v2/manager/node/DROP/be"; + Request request = new Request.Builder() + .url(url) + .addHeader("Content-Type", "application/json") + .addHeader("Authorization", rootAuth) + .post(RequestBody.create(jsonBody, MediaType.parse("application/json"))) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + } + + { + //query be + String url = "http://localhost:" + HTTP_PORT + "/rest/v2/manager/node/backends"; + Request request = new Request.Builder() + .get() + .addHeader("Authorization", rootAuth) + .url(url) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + Assert.assertNotNull(response.body()); + String respStr = response.body().string(); + JSONObject object = (JSONObject) JSONValue.parse(respStr); + + JSONObject data = (JSONObject) object.get("data"); + JSONArray columnNames = (JSONArray) ((JSONObject) data.get("columnNames")).get("columnNames"); + JSONArray rows = (JSONArray) ((JSONObject) data.get("rows")).get("rows"); + int sz = columnNames.size(); + int index = columnNames.indexOf("HeartbeatPort"); + int existsbe = 0; + for (int i = 0; i < rows.size(); i += sz) { + if (port.equals(rows.get(i + index).toString())) { + existsbe++; + } + } + Assert.assertEquals(0, existsbe); + } + } + + @Test + public void testPost1() throws Exception { + FeConstants.runningUnitTest = true; + RequestBody emptyBody = RequestBody.create(new byte[0], null); + String url = "http://localhost:" + HTTP_PORT + "/api/dbforwardmaster/_cancel?label=1"; + Request request = new Request.Builder() + .url(url) + .addHeader("Authorization", rootAuth) + .post(emptyBody) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + Assert.assertNotNull(response.body()); + String respStr = response.body().string(); + JSONObject object = (JSONObject) JSONValue.parse(respStr); + + String data = (String) object.get("data"); + Assert.assertTrue(data.contains("does not exist")); + } + + @Test + public void testPost2() throws Exception { + FeConstants.runningUnitTest = true; + String url = "http://localhost:" + HTTP_PORT + "/api/colocate/group_stable"; + RequestBody requestBody = new okhttp3.FormBody.Builder() + .add("group_id", "18888") + .add("db_id", "99999999") + .build(); + + Request request = new Request.Builder() + .url(url) + .post(requestBody) + .addHeader("Authorization", rootAuth) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + Assert.assertNotNull(response.body()); + String respStr = response.body().string(); + JSONObject object = (JSONObject) JSONValue.parse(respStr); + + String data = (String) object.get("data"); + Assert.assertTrue(data.contains("the group 99999999.18888 isn't exist")); + } + + @Test + public void testGet1() throws Exception { + FeConstants.runningUnitTest = true; + String url = "http://localhost:" + HTTP_PORT + "/rest/v2/api/cluster_overview"; + Request request = new Request.Builder() + .get() + .addHeader("Authorization", rootAuth) + .url(url) + .build(); + Response response = networkClient.newCall(request).execute(); + Assert.assertTrue(response.isSuccessful()); + Assert.assertNotNull(response.body()); + String respStr = response.body().string(); + JSONObject object = (JSONObject) JSONValue.parse(respStr); + + JSONObject data = (JSONObject) object.get("data"); + Assert.assertTrue(data.toString().contains("diskOccupancy")); + } +}