Skip to content

Commit

Permalink
[Fix](broker)fix change afs and bos properties not working cause of h…
Browse files Browse the repository at this point in the history
…adoop filesystem cache. (apache#39117)

Due to hadoop filesystem cache changing afs and bos properties not
working,so add some cache disable config to them.
  • Loading branch information
biohazard4321 authored Aug 11, 2024
1 parent dc8bd00 commit d0793fa
Showing 1 changed file with 4 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -152,13 +152,15 @@ public class FileSystemManager {
private static final String FS_BOS_SECRET_KEY = "fs.bos.secret.access.key";
private static final String FS_BOS_ENDPOINT = "fs.bos.endpoint";
private static final String FS_BOS_IMPL = "fs.bos.impl";
private static final String FS_BOS_IMPL_DISABLE_CACHE = "fs.bos.impl.disable.cache";
private static final String FS_BOS_MULTIPART_UPLOADS_BLOCK_SIZE = "fs.bos.multipart.uploads.block.size";

// arguments for afs
private static final String HADOOP_JOB_GROUP_NAME = "hadoop.job.group.name";
private static final String HADOOP_JOB_UGI = "hadoop.job.ugi";
private static final String FS_DEFAULT_NAME = "fs.default.name";
private static final String FS_AFS_IMPL = "fs.afs.impl";
private static final String FS_AFS_IMPL_DISABLE_CACHE = "fs.afs.impl.disable.cache";
private static final String DFS_AGENT_PORT = "dfs.agent.port";
private static final String DFS_CLIENT_AUTH_METHOD = "dfs.client.auth.method";
private static final String DFS_RPC_TIMEOUT = "dfs.rpc.timeout";
Expand Down Expand Up @@ -847,6 +849,7 @@ public BrokerFileSystem getBOSFileSystem(String path, Map<String, String> proper
conf.set(FS_BOS_SECRET_KEY, secretKey);
conf.set(FS_BOS_ENDPOINT, endpoint);
conf.set(FS_BOS_IMPL, "org.apache.hadoop.fs.bos.BaiduBosFileSystem");
conf.set(FS_BOS_IMPL_DISABLE_CACHE, "true");
conf.set(FS_BOS_MULTIPART_UPLOADS_BLOCK_SIZE, multiPartUploadBlockSize);
FileSystem bosFileSystem = FileSystem.get(pathUri.getUri(), conf);
fileSystem.setFileSystem(bosFileSystem);
Expand Down Expand Up @@ -1011,6 +1014,7 @@ private BrokerFileSystem getAfsFileSystem(String path, Map<String, String> prope
conf.set(HADOOP_JOB_GROUP_NAME, group);
conf.set(FS_DEFAULT_NAME, host);
conf.set(FS_AFS_IMPL, properties.getOrDefault(FS_AFS_IMPL, "org.apache.hadoop.fs.LiteFileSystem"));
conf.set(FS_AFS_IMPL_DISABLE_CACHE, "true");
conf.set(DFS_CLIENT_AUTH_METHOD, properties.getOrDefault(DFS_CLIENT_AUTH_METHOD, "3"));
conf.set(DFS_AGENT_PORT, properties.getOrDefault(DFS_AGENT_PORT, "20001"));
conf.set(DFS_RPC_TIMEOUT, properties.getOrDefault(DFS_RPC_TIMEOUT, "300000"));
Expand Down

0 comments on commit d0793fa

Please sign in to comment.