Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pgconfig: save resource store temporary or auxiliary folders only on disk #454

Merged
merged 1 commit into from
Apr 25, 2024
Merged

pgconfig: save resource store temporary or auxiliary folders only on disk #454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ...main/java/org/geoserver/cloud/backend/pgconfig/resource/FileSystemResourceStoreCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

import com.google.common.base.Preconditions;

import lombok.Getter;
import lombok.NonNull;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;

import org.geoserver.platform.resource.FileSystemResourceStore;
import org.geoserver.platform.resource.Resource;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.util.FileSystemUtils;
Expand All @@ -25,6 +27,7 @@ public class FileSystemResourceStoreCache implements DisposableBean {

private final Path base;
private boolean disposable;
private @Getter FileSystemResourceStore localOnlyStore;

private FileSystemResourceStoreCache(@NonNull Path cacheDirectory, boolean disposable) {
this.disposable = disposable;
Expand All @@ -37,6 +40,7 @@ private FileSystemResourceStoreCache(@NonNull Path cacheDirectory, boolean dispo
"Cache directory is not writable: %s",
cacheDirectory.toAbsolutePath());
this.base = cacheDirectory;
this.localOnlyStore = new FileSystemResourceStore(new File(this.base.toUri()));
}

@SneakyThrows
Expand Down
29 changes: 12 additions & 17 deletions ...s/pgconfig/src/main/java/org/geoserver/cloud/backend/pgconfig/resource/PgsqlResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@
*/
@EqualsAndHashCode(exclude = {"store"})
class PgsqlResource implements Resource {

static final long ROOT_ID = 0L;
static final long UNDEFINED_ID = -1L;

@Getter long id;
@Getter long parentId;
Resource.Type type;
Expand All @@ -49,14 +45,15 @@ class PgsqlResource implements Resource {
this.lastmodified = lastmodified;
}

/** Undefined type constructor */
PgsqlResource(@NonNull PgsqlResourceStore store, @NonNull String path) {
this.store = store;
this.id = UNDEFINED_ID;
this.parentId = UNDEFINED_ID;
this.type = Type.UNDEFINED;
this.path = path;
this.lastmodified = 0L;
/** Undefined type factory method */
static PgsqlResource undefined(@NonNull PgsqlResourceStore store, @NonNull String path) {
return new PgsqlResource(
store,
PgsqlResourceStore.UNDEFINED_ID,
PgsqlResourceStore.UNDEFINED_ID,
Type.UNDEFINED,
path,
0L);
}

void copy(PgsqlResource other) {
Expand Down Expand Up @@ -110,8 +107,7 @@ public long lastmodified() {

@Override
public PgsqlResource parent() {
if (ROOT_ID == id) return null;
return (PgsqlResource) store.get(parentPath());
return store.getParent(this);
}

@Override
Expand Down Expand Up @@ -163,12 +159,11 @@ public String toString() {
}

public PgsqlResource mkdirs() {
store.mkdirs(this);
return this;
return store.mkdirs(this);
}

public boolean exists() {
return id != UNDEFINED_ID;
return id != PgsqlResourceStore.UNDEFINED_ID;
}

public boolean isFile() {
Expand Down
2 changes: 1 addition & 1 deletion ...g/src/main/java/org/geoserver/cloud/backend/pgconfig/resource/PgsqlResourceRowMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,6 @@ public PgsqlResource mapRow(ResultSet rs, int rowNum) throws SQLException {
}

public PgsqlResource undefined(String path) {
return new PgsqlResource(store, path);
return PgsqlResource.undefined(store, path);
}
}
126 changes: 107 additions & 19 deletions ...onfig/src/main/java/org/geoserver/cloud/backend/pgconfig/resource/PgsqlResourceStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@

import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.experimental.Delegate;
import lombok.extern.slf4j.Slf4j;

import org.geoserver.platform.resource.FilePaths;
import org.geoserver.platform.resource.Files;
import org.geoserver.platform.resource.LockProvider;
import org.geoserver.platform.resource.Paths;
import org.geoserver.platform.resource.Resource;
Expand All @@ -34,7 +34,9 @@
import java.nio.file.Path;
import java.sql.Timestamp;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Predicate;
import java.util.stream.Stream;

/**
Expand All @@ -43,6 +45,8 @@
@Slf4j
@Transactional(transactionManager = "pgconfigTransactionManager", propagation = SUPPORTS)
public class PgsqlResourceStore implements ResourceStore {
static final long ROOT_ID = 0L;
static final long UNDEFINED_ID = -1L;

private final JdbcTemplate template;
private final FileSystemResourceStoreCache cache;
Expand All @@ -51,47 +55,115 @@ public class PgsqlResourceStore implements ResourceStore {

private final PgsqlResourceRowMapper queryMapper;

private final Predicate<String> fileSystemOnlyPathMatcher;

public PgsqlResourceStore(
@NonNull Path cacheDirectory,
@NonNull JdbcTemplate template,
@NonNull PgsqlLockProvider lockProvider) {
this(FileSystemResourceStoreCache.of(cacheDirectory), template, lockProvider);
@NonNull PgsqlLockProvider lockProvider,
@NonNull Predicate<String> fileSystemOnlyPathMatcher) {
this(
FileSystemResourceStoreCache.of(cacheDirectory),
template,
lockProvider,
fileSystemOnlyPathMatcher);
}

public PgsqlResourceStore(
@NonNull FileSystemResourceStoreCache cache,
@NonNull JdbcTemplate template,
@NonNull PgsqlLockProvider lockProvider) {
@NonNull PgsqlLockProvider lockProvider,
@NonNull Predicate<String> fileSystemOnlyPathMatcher) {
this.template = template;
this.lockProvider = lockProvider;
this.queryMapper = new PgsqlResourceRowMapper(this);
this.cache = cache;
final String root = "";
Predicate<String> notRoot = path -> !root.equals(path);
this.fileSystemOnlyPathMatcher = notRoot.and(fileSystemOnlyPathMatcher);
}

public static Predicate<String> defaultIgnoredDirs() {
return PgsqlResourceStore.simplePathMatcher("temp", "tmp", "legendsamples", "data", "logs");
}

public static Predicate<String> simplePathMatcher(String... paths) {
Predicate<String> matcher = path -> false;
for (String path : paths) {
path = normalize(path);
matcher = matcher.or(path::equals);
final String dirpath = path + "/";
matcher = matcher.or(r -> r.startsWith(dirpath));
}
return matcher;
}

@Override
public Resource get(@NonNull String path) {
String validPath = normalize(path);
if (FilePaths.isAbsolute(validPath)) {
return Files.asResource(new File(validPath));
final String validPath = normalize(path);
if (fileSystemOnlyPathMatcher.test(validPath)) {
Resource fsResource = cache.getLocalOnlyStore().get(validPath);
return new FileSystemResourceAdaptor(fsResource, this);
}
return findByPath(validPath).orElseGet(() -> queryMapper.undefined(validPath));
}

@RequiredArgsConstructor
static class FileSystemResourceAdaptor implements Resource {
@Delegate @NonNull private final Resource delegate;
private final @NonNull PgsqlResourceStore store;

@Override
public Resource parent() {
String parentPath = Paths.parent(this.path());
return store.get(parentPath);
}

@Override
public boolean equals(Object obj) {
return obj instanceof FileSystemResourceAdaptor fra
&& Objects.equals(path(), fra.path())
&& Objects.equals(getType(), fra.getType());
}

@Override
public int hashCode() {
return delegate.hashCode();
}
}

@Override
@Transactional(transactionManager = "pgconfigTransactionManager", propagation = REQUIRED)
public boolean remove(@NonNull String path) {
return findByPath(normalize(path)).map(PgsqlResource::delete).orElse(false);
String validPath = normalize(path);
if (fileSystemOnlyPathMatcher.test(validPath)) {
return cache.getLocalOnlyStore().remove(validPath);
}
return findByPath(validPath).map(PgsqlResource::delete).orElse(false);
}

@Override
@Transactional(transactionManager = "pgconfigTransactionManager", propagation = REQUIRED)
public boolean move(@NonNull String path, @NonNull String target) {
normalize(path);
normalize(target);
return PgsqlResourceStore.this.move((PgsqlResource) get(path), (PgsqlResource) get(target));
Resource from = get(path);
Resource to = get(target);
if (from instanceof PgsqlResource pgFrom && to instanceof PgsqlResource pgTo) {
return PgsqlResourceStore.this.move(pgFrom, pgTo);
}
if (from instanceof PgsqlResource) {
throw new UnsupportedOperationException(
"source resource targets database but target resource matches the ignored resources predicate. Source: %s, target: %s"
.formatted(path, target));
}
if (to instanceof PgsqlResource) {
throw new UnsupportedOperationException(
"target resource targets database but source resource matches the ignored resources predicate. Source: %s, target: %s"
.formatted(path, target));
}
return cache.getLocalOnlyStore().move(path, target);
}

private String normalize(String path) {
private static String normalize(String path) {
path = Paths.valid(path);
if (path.startsWith("/")) {
path = path.substring(1);
Expand All @@ -102,7 +174,7 @@ private String normalize(String path) {
return path;
}

public Optional<PgsqlResource> findByPath(@NonNull String path) {
private Optional<PgsqlResource> findByPath(@NonNull String path) {
path = Paths.valid(path);
Preconditions.checkArgument(
!path.startsWith("/"), "Absolute paths not supported: %s", path);
Expand Down Expand Up @@ -154,6 +226,10 @@ INSERT INTO resourcestore (parentid, "type", path, content)
byte[] contents = resource.getType() == Type.DIRECTORY ? null : new byte[0];
template.update(sql, parentId, type, path, contents);
}
PgsqlResource updated = (PgsqlResource) get(resource.path);
resource.id = updated.getId();
resource.lastmodified = updated.lastmodified();
resource.parentId = updated.getParentId();
}

/**
Expand Down Expand Up @@ -329,24 +405,25 @@ public File asDir(PgsqlResource resource) {
}

@Transactional(transactionManager = "pgconfigTransactionManager", propagation = REQUIRED)
public void mkdirs(PgsqlResource resource) {
public PgsqlResource mkdirs(PgsqlResource resource) {
if (resource.exists() && resource.isDirectory()) {
return;
return resource;
}
if (resource.isFile())
throw new IllegalStateException(
"mkdirs() can only be called on DIRECTORY or UNDEFINED resources");

PgsqlResource parent = resource.parent();
if (null == parent) return;
PgsqlResource parent = getParent(resource);
if (null == parent) return resource;
if (!parent.exists()) {
parent = parent.mkdirs();
parent = (PgsqlResource) parent.mkdirs();
}
resource.parentId = parent.getId();
resource.type = Type.DIRECTORY;
PgsqlResourceStore.this.save(resource);
PgsqlResource saved = (PgsqlResource) get(resource.path());
resource.copy(saved);
return resource;
}

public OutputStream out(PgsqlResource res) {
Expand All @@ -372,4 +449,15 @@ public void close() {
}
};
}

public PgsqlResource getParent(PgsqlResource resource) {
if (ROOT_ID == resource.getId()) return null;
String parentPath = resource.parentPath();
try {
return (PgsqlResource) get(parentPath);
} catch (RuntimeException e) {
e.printStackTrace();
throw e;
}
}
}
5 changes: 4 additions & 1 deletion ...n/java/org/geoserver/cloud/config/catalog/backend/pgconfig/PgsqlBackendConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.util.StringUtils;

import java.util.function.Predicate;

import javax.sql.DataSource;

/**
Expand Down Expand Up @@ -114,7 +116,8 @@ protected ResourceStore resourceStoreImpl() {
FileSystemResourceStoreCache resourceStoreCache = pgsqlFileSystemResourceStoreCache();
JdbcTemplate template = template();
PgsqlLockProvider lockProvider = pgsqlLockProvider();
return new PgsqlResourceStore(resourceStoreCache, template, lockProvider);
Predicate<String> ignoreDirs = PgsqlResourceStore.defaultIgnoredDirs();
return new PgsqlResourceStore(resourceStoreCache, template, lockProvider, ignoreDirs);
}

@Bean
Expand Down
Loading
Loading