From 748af92b617046afab49a50006d2aa6b623834bc Mon Sep 17 00:00:00 2001 From: Yann Ann Date: Sun, 6 Nov 2022 16:32:36 +0800 Subject: [PATCH] run spotless:apply --- docs/docs/en/architecture/configuration.md | 30 +++++------ docs/docs/zh/architecture/configuration.md | 29 ++++++----- docs/docs/zh/guide/security.md | 2 +- .../plugin/task/api/ShellCommandExecutor.java | 52 +++++++++---------- 4 files changed, 57 insertions(+), 56 deletions(-) diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 0b746eae38d80..ddb24226287ee 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -300,23 +300,23 @@ Location: `worker-server/conf/application.yaml` |worker.task-execute-threads-full-policy|REJECT|If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution| ## SSH Session相关配置 -位置：`worker-server/conf/application.yaml` -| 参数 | 默认值 | 描述 | -| ----------------------------------------------- | ------ | ------------------------------------------------------------ | -| ssh.session.pool.maxTotal | 20 | The maximum number of active ssh session instances in the pool. | -| ssh.session.pool.maxTotalPerKey | 10 | The maximum number of active ssh session of per ssh host & account in the pool. | -| ssh.session.pool.maxIdlePerKey | 4 | The maximum number of sleeping instances of per ssh host in the pool. | -| ssh.session.pool.blockWhenExhausted | true | When borrow an ssh session, whether to block waiting for a session when the pool is exhausted | -| ssh.session.pool.maxWaitDuration | 500ms | The maximum waiting time for the caller when the pool session is exhausted, it will be ignored if `blockWhenExhausted` is `false` | -| ssh.session.pool.minEvictableIdleDuration | 60s | The minimum time to allow idle session to live (or not be evicted) | -| ssh.session.pool.durationBetweenEvictionRuns | 60s | The interval at which the eviction thread pool scans the pool for free sessions | -| ssh.session.pool.removeAbandonedOnBorrow | true | Whether borrowObject performs abandoned session removal | -| ssh.session.pool.removeAbandonedTimeoutDuration | 30s | How long the object has been unused and returned after it was lent out and used for the last time is considered a leak | -| ssh.session.sftp.enableUploadMonitor | true | enable upload monitor thread when upload files to remote host | -| ssh.session.sftp.maxUploadRate | 256 | max upload rate (KB), if negative, will not limit | -| ssh.session.sftp.maxFileSize | 100 | max file size (MB), if negative, will not limit | +位置：`worker-server/conf/application.yaml` +| 参数 | 默认值 | 描述 | +|-------------------------------------------------|-------|-----------------------------------------------------------------------------------------------------------------------------------| +| ssh.session.pool.maxTotal | 20 | The maximum number of active ssh session instances in the pool. | +| ssh.session.pool.maxTotalPerKey | 10 | The maximum number of active ssh session of per ssh host & account in the pool. | +| ssh.session.pool.maxIdlePerKey | 4 | The maximum number of sleeping instances of per ssh host in the pool. | +| ssh.session.pool.blockWhenExhausted | true | When borrow an ssh session, whether to block waiting for a session when the pool is exhausted | +| ssh.session.pool.maxWaitDuration | 500ms | The maximum waiting time for the caller when the pool session is exhausted, it will be ignored if `blockWhenExhausted` is `false` | +| ssh.session.pool.minEvictableIdleDuration | 60s | The minimum time to allow idle session to live (or not be evicted) | +| ssh.session.pool.durationBetweenEvictionRuns | 60s | The interval at which the eviction thread pool scans the pool for free sessions | +| ssh.session.pool.removeAbandonedOnBorrow | true | Whether borrowObject performs abandoned session removal | +| ssh.session.pool.removeAbandonedTimeoutDuration | 30s | How long the object has been unused and returned after it was lent out and used for the last time is considered a leak | +| ssh.session.sftp.enableUploadMonitor | true | enable upload monitor thread when upload files to remote host | +| ssh.session.sftp.maxUploadRate | 256 | max upload rate (KB), if negative, will not limit | +| ssh.session.sftp.maxFileSize | 100 | max file size (MB), if negative, will not limit | ### Alert Server related configuration diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index d90c7517662c9..e9a75ff855ebb 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -294,22 +294,23 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId |worker.task-execute-threads-full-policy|REJECT|如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务| ## SSH Session相关配置 + 位置：`worker-server/conf/application.yaml` -| 参数 | 默认值 | 描述 | -| ----------------------------------------------- | ------ |--------------------------------------------------------------------------------------------------| -| ssh.session.pool.maxTotal | 20 | Worker能够同时创建SSH Session的最大数量，当对象池满，新请求将阻塞或直接拒绝 | -| ssh.session.pool.maxTotalPerKey | 10 | Worker能够对单一主机 + 用户创建SSH Session的最大数量。当该主机+用户创建的SSH Session满，则Worker对该主机+用户的SSH ession请求将阻塞或直接拒绝。 | -| ssh.session.pool.maxIdlePerKey | 4 | Worker能够对单一主机 + 用户创建SSH Session的最大空闲数量。当释放的主机+用户创建的SSH Session超过该值，则Worker将直接销毁新收回的SSH Session。 | -| ssh.session.pool.blockWhenExhausted | true | 当对象池满，是否阻塞新请求。true为阻塞，false则为直接异常返回。推荐使用阻塞策略。 | -| ssh.session.pool.maxWaitDuration | 500ms | 该值仅当`blockWhenExhausted`设置为`true`时生效。表示请求最大阻塞时长。建议该值必须设置 | -| ssh.session.pool.minEvictableIdleDuration | 60s | 允许空闲SSH Session的最大存活时间。假如空闲SSH Session超过该值未被使用，将被驱逐 | -| ssh.session.pool.durationBetweenEvictionRuns | 60s | 检测是否驱逐空闲SSH Session的周期 | -| ssh.session.pool.removeAbandonedOnBorrow | true | 当每次从池子中借取SSH Session时，去扫描可以被丢弃的SSH Session | -| ssh.session.pool.removeAbandonedTimeoutDuration | 30s | SSH Session借出去最后一次使用后，多长时间未使用和归还就认为是泄漏 | -| ssh.session.sftp.enableUploadMonitor | true | 当通过SFTP上传文件时，是否开启进度监控 | -| ssh.session.sftp.maxUploadRate | 256 | SFTP的最大速率，单位为KB，若为负数则不限制 | -| ssh.session.sftp.maxFileSize | 100 | 单任务能够SFTP的最大文件大小，单位为MB，若为负数则不限制 | +| 参数 | 默认值 | 描述 | +|-------------------------------------------------|-------|--------------------------------------------------------------------------------------------------| +| ssh.session.pool.maxTotal | 20 | Worker能够同时创建SSH Session的最大数量，当对象池满，新请求将阻塞或直接拒绝 | +| ssh.session.pool.maxTotalPerKey | 10 | Worker能够对单一主机 + 用户创建SSH Session的最大数量。当该主机+用户创建的SSH Session满，则Worker对该主机+用户的SSH ession请求将阻塞或直接拒绝。 | +| ssh.session.pool.maxIdlePerKey | 4 | Worker能够对单一主机 + 用户创建SSH Session的最大空闲数量。当释放的主机+用户创建的SSH Session超过该值，则Worker将直接销毁新收回的SSH Session。 | +| ssh.session.pool.blockWhenExhausted | true | 当对象池满，是否阻塞新请求。true为阻塞，false则为直接异常返回。推荐使用阻塞策略。 | +| ssh.session.pool.maxWaitDuration | 500ms | 该值仅当`blockWhenExhausted`设置为`true`时生效。表示请求最大阻塞时长。建议该值必须设置 | +| ssh.session.pool.minEvictableIdleDuration | 60s | 允许空闲SSH Session的最大存活时间。假如空闲SSH Session超过该值未被使用，将被驱逐 | +| ssh.session.pool.durationBetweenEvictionRuns | 60s | 检测是否驱逐空闲SSH Session的周期 | +| ssh.session.pool.removeAbandonedOnBorrow | true | 当每次从池子中借取SSH Session时，去扫描可以被丢弃的SSH Session | +| ssh.session.pool.removeAbandonedTimeoutDuration | 30s | SSH Session借出去最后一次使用后，多长时间未使用和归还就认为是泄漏 | +| ssh.session.sftp.enableUploadMonitor | true | 当通过SFTP上传文件时，是否开启进度监控 | +| ssh.session.sftp.maxUploadRate | 256 | SFTP的最大速率，单位为KB，若为负数则不限制 | +| ssh.session.sftp.maxFileSize | 100 | 单任务能够SFTP的最大文件大小，单位为MB，若为负数则不限制 | ## Alert Server相关配置 diff --git a/docs/docs/zh/guide/security.md b/docs/docs/zh/guide/security.md index 74fd82063ac7e..6e905a7b7593c 100644 --- a/docs/docs/zh/guide/security.md +++ b/docs/docs/zh/guide/security.md @@ -190,4 +190,4 @@ worker: - 配置远程主机的相关信息之后，可以通过测试按钮测试当前主机是否能够执行远程SSH。因为能够远程SSH的主机才是有效的. -![create-task-remote-host](../../../img/new_ui/dev/security/create-task-remote-host.png) \ No newline at end of file +![create-task-remote-host](../../../img/new_ui/dev/security/create-task-remote-host.png) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java index 76bf5576ca2a6..cc3206a5bd735 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java @@ -90,37 +90,37 @@ protected void createCommandFileIfNotExists(String execCommand, String commandFi return; } - StringBuilder sb = new StringBuilder(); - if (SystemUtils.IS_OS_WINDOWS) { - sb.append("@echo off\n"); - sb.append("cd /d %~dp0\n"); - if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { - sb.append(taskRequest.getEnvironmentConfig()).append("\n"); - } else { - if (taskRequest.getEnvFile() != null) { - sb.append("call ").append(taskRequest.getEnvFile()).append("\n"); - } + StringBuilder sb = new StringBuilder(); + if (SystemUtils.IS_OS_WINDOWS) { + sb.append("@echo off\n"); + sb.append("cd /d %~dp0\n"); + if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { + sb.append(taskRequest.getEnvironmentConfig()).append("\n"); + } else { + if (taskRequest.getEnvFile() != null) { + sb.append("call ").append(taskRequest.getEnvFile()).append("\n"); } + } + } else { + sb.append("#!/bin/sh\n"); + sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); + sb.append("cd $BASEDIR\n"); + if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { + sb.append(taskRequest.getEnvironmentConfig()).append("\n"); } else { - sb.append("#!/bin/sh\n"); - sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); - sb.append("cd $BASEDIR\n"); - if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { - sb.append(taskRequest.getEnvironmentConfig()).append("\n"); - } else { - if (taskRequest.getEnvFile() != null) { - if (runningOnSSH) { - File envFile = new File(taskRequest.getEnvFile()); - sb.append("source ").append(taskRequest.getExecutePath()).append(File.separator) - .append(envFile.getName()).append("\n"); - } else { - sb.append("source ").append(taskRequest.getEnvFile()).append("\n"); - } + if (taskRequest.getEnvFile() != null) { + if (runningOnSSH) { + File envFile = new File(taskRequest.getEnvFile()); + sb.append("source ").append(taskRequest.getExecutePath()).append(File.separator) + .append(envFile.getName()).append("\n"); + } else { + sb.append("source ").append(taskRequest.getEnvFile()).append("\n"); } } } - sb.append(execCommand); - String commandContent = sb.toString(); + } + sb.append(execCommand); + String commandContent = sb.toString(); FileUtils.createFileWith755(commandFilePath); Files.write(commandFilePath, commandContent.getBytes(), StandardOpenOption.APPEND);