Skip to content

Commit

Permalink
apacheGH-643: provide interfaces for caching file attributes on paths
Browse files Browse the repository at this point in the history
Remote file systems may have a need to cache file attributes on Path
instances. A typical use case is iterating over all files in a
directory: the directory listing returns paths, but the underlying
remote listing operation may also return file attributes for each entry.
This is the case in Sftp, but may also occur for other file systems, for
instance a file system wrapping Amazon S3.

It would be unfortunate and inefficient if iterating through the paths
returned and doing something with the attributes would have to re-fetch
the attributes again if they were already available.

By implementing WithFileAttributes of its Paths, a file system can
associate file attributes with a path instance, and client code can
access them. If a file system also makes its paths implement the second
interface WithFileAttributeCache, then the SftpSubsystem uses it
internally to avoid making repeated remote calls to get file attributes.
  • Loading branch information
tomaswolf committed Dec 18, 2024
1 parent 7cc9c49 commit 12a581c
Show file tree
Hide file tree
Showing 8 changed files with 128 additions and 65 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,6 @@
import org.apache.sshd.sftp.client.SftpErrorDataHandler;
import org.apache.sshd.sftp.client.SftpVersionSelector;
import org.apache.sshd.sftp.client.extensions.CopyFileExtension;
import org.apache.sshd.sftp.client.impl.SftpPathImpl;
import org.apache.sshd.sftp.client.impl.SftpRemotePathChannel;
import org.apache.sshd.sftp.common.SftpConstants;
import org.apache.sshd.sftp.common.SftpException;
Expand Down Expand Up @@ -1117,7 +1116,7 @@ public SftpClient.Attributes readRemoteAttributes(SftpPath path, LinkOption... o
// SftpPathImpl.withAttributeCache() invocation. So we ensure here that if we are already within a caching
// scope, we do use the cached attributes, but if we are not, we clear any possibly cached attributes and
// do actually read them from the remote.
return SftpPathImpl.withAttributeCache(path, p -> resolveRemoteFileAttributes(path, options));
return WithFileAttributeCache.withAttributeCache(path, p -> resolveRemoteFileAttributes(path, options));
}

protected SftpClient.Attributes resolveRemoteFileAttributes(SftpPath path, LinkOption... options) throws IOException {
Expand All @@ -1136,8 +1135,8 @@ protected SftpClient.Attributes resolveRemoteFileAttributes(SftpPath path, LinkO
if (log.isTraceEnabled()) {
log.trace("resolveRemoteFileAttributes({})[{}]: {}", fs, path, attrs);
}
if (path instanceof SftpPathImpl) {
((SftpPathImpl) path).cacheAttributes(attrs);
if (path instanceof WithFileAttributeCache) {
((WithFileAttributeCache) path).setAttributes(attrs);
}
return attrs;
} catch (SftpException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,13 @@
/**
* A {@link java.nio.file.Path} on an {@link SftpFileSystem}.
*/
public class SftpPath extends BasePath<SftpPath, SftpFileSystem> {
public class SftpPath extends BasePath<SftpPath, SftpFileSystem> implements WithFileAttributes {

public SftpPath(SftpFileSystem fileSystem, String root, List<String> names) {
super(fileSystem, root, names);
}

/**
* Retrieves the cached {@link SftpClient.Attributes} of this {@link SftpPath}, if it has any.
*
* @return the cached {@link SftpClient.Attributes} or {@code null} if there are none cached
*/
@SuppressWarnings("javadoc")
@Override
public SftpClient.Attributes getAttributes() {
// Subclasses may override
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.util.Objects;

import org.apache.sshd.sftp.client.SftpClient;
import org.apache.sshd.sftp.client.impl.SftpPathImpl;

/**
* Implements and {@link Iterator} of {@link SftpPath}-s returned by a {@link DirectoryStream#iterator()} method.
Expand Down Expand Up @@ -116,8 +115,8 @@ protected SftpPath nextEntry(SftpPath root, DirectoryStream.Filter<? super Path>
dotdotIgnored = true;
} else {
SftpPath candidate = root.resolve(entry.getFilename());
if (candidate instanceof SftpPathImpl) {
((SftpPathImpl) candidate).setAttributes(entry.getAttributes());
if (candidate instanceof WithFileAttributeCache) {
((WithFileAttributeCache) candidate).setAttributes(entry.getAttributes());
}
try {
if ((selector == null) || selector.accept(candidate)) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.sshd.sftp.client.fs;

import java.io.IOException;
import java.nio.file.Path;

import org.apache.sshd.common.util.io.functors.IOFunction;
import org.apache.sshd.sftp.client.SftpClient;

/**
* A mix-in interface for paths that can carry and cache file attributes of the referenced file.
*/
public interface WithFileAttributeCache extends WithFileAttributes {

/**
* Sets the attributes.
*
* @param attributes {@link SftpClient.Attributes} to set
*/
void setAttributes(SftpClient.Attributes attributes);

/**
* Performs the given operation with attribute caching. If {@code SftpClient.Attributes} are fetched by the
* operation, they will be cached and subsequently these cached attributes will be re-used for this {@link SftpPath}
* instance throughout the operation. Calls to {@link #withAttributeCache(IOFunction)} may be nested. The cache is
* cleared at the start and at the end of the outermost invocation.
*
* @param <T> result type of the {@code operation}
* @param operation to perform; may return {@code null} if it has no result
* @return the result of the {@code operation}
* @throws IOException if thrown by the {@code operation}
*/
<T> T withAttributeCache(IOFunction<Path, T> operation) throws IOException;

/**
* Performs the given operation with attribute caching, if the given {@link Path} implements the
* {@link WithFileAttributeCache} interface, otherwise simply executes the operation.
*
* @param <T> result type of the {@code operation}
* @param path {@link Path} to operate on
* @param operation to perform; may return {@code null} if it has no result
* @return the result of the {@code operation}
* @throws IOException if thrown by the {@code operation}
*
* @see #withAttributeCache(IOFunction)
*/
static <T> T withAttributeCache(Path path, IOFunction<Path, T> operation) throws IOException {
if (path instanceof WithFileAttributeCache) {
return ((WithFileAttributeCache) path).withAttributeCache(operation);
}
return operation.apply(path);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.sshd.sftp.client.fs;

import org.apache.sshd.sftp.client.SftpClient.Attributes;

/**
* A mix-in interface for paths that may have file attributes of the file referenced by the path.
*/
public interface WithFileAttributes {

/**
* Retrieves the {@link Attributes} of this object, if it has any.
*
* @return the {@link Attributes} or {@code null} if there are none
*/
Attributes getAttributes();

}
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@
import org.apache.sshd.sftp.client.SftpClient;
import org.apache.sshd.sftp.client.fs.SftpFileSystem;
import org.apache.sshd.sftp.client.fs.SftpPath;
import org.apache.sshd.sftp.client.fs.WithFileAttributeCache;

/**
* An {@link SftpPath} that can cache {@code SftpClient.Attributes}.
*/
public class SftpPathImpl extends SftpPath {
public class SftpPathImpl extends SftpPath implements WithFileAttributeCache {

private SftpClient.Attributes attributes;

Expand Down Expand Up @@ -77,23 +78,14 @@ protected void cacheAttributes(boolean doCache) {
}
}

/**
* Sets the cached attributes to the argument if this {@link SftpPath} instance is currently caching attributes.
* Otherwise a no-op.
*
* @param attributes the {@code SftpClient.Attributes} to cache
*/
@Override
public void cacheAttributes(SftpClient.Attributes attributes) {
if (cachingLevel > 0) {
setAttributes(attributes);
}
}

/**
* Unconditionally set the cached attributes, whether or not this instance's attribute cache is enabled.
*
* @param attributes the {@code SftpClient.Attributes} to cache
*/
@Override
public void setAttributes(SftpClient.Attributes attributes) {
this.attributes = attributes;
}
Expand All @@ -103,17 +95,7 @@ public SftpClient.Attributes getAttributes() {
return attributes;
}

/**
* Performs the given operation with attribute caching. If {@code SftpClient.Attributes} are fetched by the
* operation, they will be cached and subsequently these cached attributes will be re-used for this {@link SftpPath}
* instance throughout the operation. Calls to {@link #withAttributeCache(IOFunction)} may be nested. The cache is
* cleared at the start and at the end of the outermost invocation.
*
* @param <T> result type of the {@code operation}
* @param operation to perform; may return {@code null} if it has no result
* @return the result of the {@code operation}
* @throws IOException if thrown by the {@code operation}
*/
@Override
public <T> T withAttributeCache(IOFunction<Path, T> operation) throws IOException {
cacheAttributes(true);
try {
Expand All @@ -123,22 +105,4 @@ public <T> T withAttributeCache(IOFunction<Path, T> operation) throws IOExceptio
}
}

/**
* Performs the given operation with attribute caching, if the given {@link Path} can cache attributes, otherwise
* simply executes the operation.
*
* @param <T> result type of the {@code operation}
* @param path {@link Path} to operate on
* @param operation to perform; may return {@code null} if it has no result
* @return the result of the {@code operation}
* @throws IOException if thrown by the {@code operation}
*
* @see #withAttributeCache(IOFunction)
*/
public static <T> T withAttributeCache(Path path, IOFunction<Path, T> operation) throws IOException {
if (path instanceof SftpPathImpl) {
return ((SftpPathImpl) path).withAttributeCache(operation);
}
return operation.apply(path);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@
import org.apache.sshd.sftp.SftpModuleProperties;
import org.apache.sshd.sftp.client.SftpClient;
import org.apache.sshd.sftp.client.extensions.openssh.OpenSSHLimitsExtensionInfo;
import org.apache.sshd.sftp.client.fs.SftpPath;
import org.apache.sshd.sftp.client.impl.SftpPathImpl;
import org.apache.sshd.sftp.client.fs.WithFileAttributeCache;
import org.apache.sshd.sftp.client.fs.WithFileAttributes;
import org.apache.sshd.sftp.common.SftpConstants;
import org.apache.sshd.sftp.common.SftpException;
import org.apache.sshd.sftp.common.SftpHelper;
Expand Down Expand Up @@ -1654,7 +1654,7 @@ protected void doMakeDirectory(
LinkOption[] options = accessor.resolveFileAccessLinkOptions(
this, resolvedPath, SftpConstants.SSH_FXP_MKDIR, "", false);
final boolean followLinks = resolvePathResolutionFollowLinks(SftpConstants.SSH_FXP_MKDIR, "", resolvedPath);
SftpPathImpl.withAttributeCache(resolvedPath, p -> {
WithFileAttributeCache.withAttributeCache(resolvedPath, p -> {
Boolean symlinkCheck = validateParentExistWithNoSymlinksIfNeverFollowSymlinks(p, !followLinks);
if (!Boolean.TRUE.equals(symlinkCheck)) {
throw new AccessDeniedException(p.toString(), p.toString(),
Expand Down Expand Up @@ -1715,7 +1715,7 @@ protected void doRemoveFile(int id, String path) throws IOException {
// never resolve links in the final path to remove as we want to remove the symlink, not the target
LinkOption[] options = accessor.resolveFileAccessLinkOptions(
this, resolvedPath, SftpConstants.SSH_FXP_REMOVE, "", false);
SftpPathImpl.withAttributeCache(resolvedPath, p -> {
WithFileAttributeCache.withAttributeCache(resolvedPath, p -> {
Boolean status = checkSymlinkState(p, followLinks, options);
if (status == null) {
throw signalRemovalPreConditionFailure(id, path, p,
Expand Down Expand Up @@ -2253,8 +2253,8 @@ protected int doReadDir(
} else {
Path f = dir.next();
String shortName = getShortName(f);
if (f instanceof SftpPath) {
SftpClient.Attributes attributes = ((SftpPath) f).getAttributes();
if (f instanceof WithFileAttributes) {
SftpClient.Attributes attributes = ((WithFileAttributes) f).getAttributes();
if (attributes != null) {
entries.put(shortName, f);
writeDirEntry(session, id, buffer, nb, f, shortName, attributes);
Expand Down Expand Up @@ -2416,7 +2416,7 @@ protected String getShortName(Path f) throws IOException {
protected NavigableMap<String, Object> resolveFileAttributes(
Path path, int flags, boolean neverFollowSymLinks, LinkOption... options)
throws IOException {
return SftpPathImpl.withAttributeCache(path, file -> {
return WithFileAttributeCache.withAttributeCache(path, file -> {
Boolean status = checkSymlinkState(file, neverFollowSymLinks, options);
if (status == null) {
return handleUnknownStatusFileAttributes(file, flags, options);
Expand Down Expand Up @@ -2492,7 +2492,7 @@ protected NavigableMap<String, Object> handleUnknownStatusFileAttributes(
protected NavigableMap<String, Object> getAttributes(Path path, int flags, LinkOption... options)
throws IOException {
NavigableMap<String, Object> attrs
= SftpPathImpl.withAttributeCache(path, file -> resolveReportedFileAttributes(file, flags, options));
= WithFileAttributeCache.withAttributeCache(path, file -> resolveReportedFileAttributes(file, flags, options));
SftpFileSystemAccessor accessor = getFileSystemAccessor();
return accessor.resolveReportedFileAttributes(this, path, flags, attrs, options);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
import org.apache.sshd.server.session.ServerSession;
import org.apache.sshd.sftp.SftpModuleProperties;
import org.apache.sshd.sftp.client.fs.SftpPath;
import org.apache.sshd.sftp.client.impl.SftpPathImpl;
import org.apache.sshd.sftp.client.fs.WithFileAttributeCache;
import org.apache.sshd.sftp.common.SftpConstants;
import org.apache.sshd.sftp.common.SftpException;
import org.apache.sshd.sftp.common.SftpHelper;
Expand Down Expand Up @@ -782,7 +782,7 @@ protected void doReadDir(Buffer buffer, int id) throws IOException {

@Override
protected String doOpenDir(int id, String path, Path dir, LinkOption... options) throws IOException {
SftpPathImpl.withAttributeCache(dir, p -> {
WithFileAttributeCache.withAttributeCache(dir, p -> {
Boolean status = IoUtils.checkFileExistsAnySymlinks(p, !IoUtils.followLinks(options));
if (status == null) {
throw signalOpenFailure(id, path, p, true,
Expand Down

0 comments on commit 12a581c

Please sign in to comment.