Merge branch 'dev' into dev_wenjun_fixGlobalMasterFailoverMightCauseM…

…asterDead
apache · Jan 10, 2025 · 46791b2 · 46791b2
2 parents 56555ca + 890376a
commit 46791b2
Show file tree

Hide file tree

Showing 12 changed files with 45 additions and 38 deletions.
diff --git a/docs/docs/en/guide/security/security.md b/docs/docs/en/guide/security/security.md
@@ -19,7 +19,9 @@ Administrator login, default username/password: admin/dolphinscheduler123
 - Tenant Code: **The tenant code is the user on Linux, unique and cannot be repeated**
 - The administrator enters the `Security Center->Tenant Management` page, and clicks the `Create Tenant` button to create a tenant.
 
-> Note: Currently, only admin users can modify tenant.
+> Note:
+> 1. Currently, only admin users can modify tenant.
+> 2. If you create a tenant manually in the Linux, you need to add the manually created tenant to the dolphinscheduler bootstrap user's group, so that the tenant will have enough working directory permissions.
 
 ![create-tenant](../../../../img/new_ui/dev/security/create-tenant.png)
 

diff --git a/docs/docs/zh/guide/security/security.md b/docs/docs/zh/guide/security/security.md
@@ -18,7 +18,9 @@
 - 租户编码：**租户编码是 Linux上 的用户，唯一，不能重复**
 - 管理员进入安全中心->租户管理页面，点击“创建租户”按钮，创建租户。
 
-> 注意：目前仅有 admin 用户可以修改租户。
+> 注意：
+> 1. 目前仅有 admin 用户可以修改租户；
+> 2. 如果您在 Linux 中手动创建一个租户，则需要将手动创建的租户添加到 dolphinscheduler 启动用户组，以便该租户拥有足够的工作目录权限。
 
 ![create-tenant](../../../../img/new_ui/dev/security/create-tenant.png)
 

diff --git a/...hinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/...hinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -62,7 +62,9 @@ public class FileUtils {
 
     public static final String KUBE_CONFIG_FILE = "config";
 
-    private static final Set<PosixFilePermission> PERMISSION_755 = PosixFilePermissions.fromString("rwxr-xr-x");
+    public static final Set<PosixFilePermission> PERMISSION_755 = PosixFilePermissions.fromString("rwxr-xr-x");
+
+    public static final Set<PosixFilePermission> PERMISSION_775 = PosixFilePermissions.fromString("rwxrwxr-x");
 
     /**
      * get download file absolute path and name
@@ -239,7 +241,7 @@ public static String getFileChecksum(String pathName) throws IOException {
     public static void createFileWith755(@NonNull Path path) throws IOException {
         final Path parent = path.getParent();
         if (!parent.toFile().exists()) {
-            createDirectoryWith755(parent);
+            createDirectoryWithPermission(parent, PERMISSION_755);
         }
         if (SystemUtils.IS_OS_WINDOWS) {
             Files.createFile(path);
@@ -249,29 +251,6 @@ public static void createFileWith755(@NonNull Path path) throws IOException {
         }
     }
 
-    public static void createDirectoryWith755(@NonNull Path path) throws IOException {
-        if (path.toFile().exists()) {
-            return;
-        }
-        if (OSUtils.isWindows()) {
-            Files.createDirectories(path);
-        } else {
-            Path parent = path.getParent();
-            if (parent != null && !parent.toFile().exists()) {
-                createDirectoryWith755(parent);
-            }
-
-            try {
-                Files.createDirectory(path);
-                Files.setPosixFilePermissions(path, PERMISSION_755);
-            } catch (FileAlreadyExistsException fileAlreadyExistsException) {
-                // Catch the FileAlreadyExistsException here to avoid create the same parent directory in parallel
-                log.debug("The directory: {} already exists", path);
-            }
-
-        }
-    }
-
     public static void setFileTo755(File file) throws IOException {
         if (OSUtils.isWindows()) {
             return;
@@ -289,6 +268,29 @@ public static void setFileTo755(File file) throws IOException {
         }
     }
 
+    public static void createDirectoryWithPermission(@NonNull Path path,
+                                                     @NonNull Set<PosixFilePermission> permissions) throws IOException {
+        if (path.toFile().exists()) {
+            return;
+        }
+
+        if (OSUtils.isWindows()) {
+            Files.createDirectories(path);
+        } else {
+            Path parent = path.getParent();
+            if (parent != null && !parent.toFile().exists()) {
+                createDirectoryWithPermission(parent, permissions);
+            }
+
+            try {
+                Files.createDirectory(path);
+                Files.setPosixFilePermissions(path, permissions);
+            } catch (FileAlreadyExistsException fileAlreadyExistsException) {
+                log.error("The directory: {} already exists", path);
+            }
+        }
+    }
+
     public static String concatFilePath(String... paths) {
         if (paths.length == 0) {
             throw new IllegalArgumentException("At least one path should be provided");

diff --git a/...cheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/...cheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
@@ -60,18 +60,18 @@ public void testGetProcessExecDir() {
     }
 
     @Test
-    public void createDirectoryWith755() throws IOException {
+    public void testCreateDirectoryWithPermission() throws IOException {
         Path path = Paths.get("/tmp/createWorkDirAndUserIfAbsent");
         try {
-            FileUtils.createDirectoryWith755(path);
+            FileUtils.createDirectoryWithPermission(path, FileUtils.PERMISSION_755);
             File file = path.toFile();
             Assertions.assertTrue(file.exists());
             Assertions.assertTrue(file.isDirectory());
             Assertions.assertTrue(file.canExecute());
             Assertions.assertTrue(file.canRead());
             Assertions.assertTrue(file.canWrite());
 
-            FileUtils.createDirectoryWith755(Paths.get("/"));
+            FileUtils.createDirectoryWithPermission(Paths.get("/"), FileUtils.PERMISSION_755);
         } catch (Exception e) {
             e.printStackTrace();
             Assertions.fail(e.getMessage());

diff --git a/...-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java b/...-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java
@@ -99,7 +99,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite)
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755);
         }
 
         BlobClient blobClient = blobContainerClient.getBlobClient(srcFilePath);

diff --git a/...-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java b/...-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java
@@ -148,7 +148,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite)
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755);
         }
 
         GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, cosKey);

diff --git a/...-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/...-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java
@@ -112,7 +112,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite)
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755);
         }
 
         Blob blob = gcsStorage.get(BlobId.of(bucketName, srcFilePath));

diff --git a/...-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/...-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java
@@ -111,7 +111,7 @@ public void download(String srcFilePath, String dstFilePath, boolean overwrite)
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755);
         }
         ObsObject obsObject = obsClient.getObject(bucketName, srcFilePath);
         try (

diff --git a/...-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/...-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
@@ -169,7 +169,7 @@ public void download(String srcFilePath,
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755);
         }
         OSSObject ossObject = ossClient.getObject(bucketName, srcFilePath);
         try (

diff --git a/...age-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/...age-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java
@@ -113,7 +113,7 @@ public void download(String srcFilePath,
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWithPermission(dstFile.getParentFile().toPath(), FileUtils.PERMISSION_755);
         }
         S3Object o = s3Client.getObject(bucketName, srcFilePath);
         try (

diff --git a/.../main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/.../main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
@@ -50,7 +50,8 @@ public static void createTaskInstanceWorkingDirectory(TaskExecutionContext taskE
                 log.warn("The TaskInstance WorkingDirectory: {} is exist, will recreate again",
                         taskInstanceWorkingDirectory);
             }
-            FileUtils.createDirectoryWith755(Paths.get(taskInstanceWorkingDirectory));
+
+            FileUtils.createDirectoryWithPermission(Paths.get(taskInstanceWorkingDirectory), FileUtils.PERMISSION_775);
 
             taskExecutionContext.setExecutePath(taskInstanceWorkingDirectory);
             taskExecutionContext.setAppInfoPath(FileUtils.getAppInfoPath(taskInstanceWorkingDirectory));

diff --git a/...t/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java b/...t/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java
@@ -44,7 +44,7 @@ void createTaskInstanceWorkingDirectory() throws IOException {
         try {
             // Test if the working directory is exist
             // will delete it and recreate
-            FileUtils.createDirectoryWith755(Paths.get(taskWorkingDirectory));
+            FileUtils.createDirectoryWithPermission(Paths.get(taskWorkingDirectory), FileUtils.PERMISSION_775);
             Files.createFile(Paths.get(taskWorkingDirectory, "text.txt"));
             Assertions.assertTrue(Files.exists(Paths.get(taskWorkingDirectory, "text.txt")));