Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] AFS data store implementation #52

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[WIP] AFS data store implementation #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
518812d
AppStorageDataStore implementation
quinarygio Jul 1, 2020
2548a68
Add test and small fixes
quinarygio Jul 7, 2020
a6931c6
Simplify implementation
quinarygio Sep 15, 2020
0edb5a3
Build ImportedCase from DataPack
quinarygio Sep 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 44 additions & 16 deletions afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ImportedCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,28 +6,35 @@
*/
package com.powsybl.afs.ext.base;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;

import com.powsybl.afs.AfsException;
import com.powsybl.afs.ProjectFile;
import com.powsybl.afs.ProjectFileCreationContext;
import com.powsybl.afs.storage.AppStorageDataSource;
import com.powsybl.afs.storage.AppStorageDataStore;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.datastore.DataPack;
import com.powsybl.commons.datastore.NonUniqueResultException;
import com.powsybl.commons.datastore.ReadOnlyDataStore;
import com.powsybl.commons.exceptions.NetworkImportException;
import com.powsybl.iidm.import_.Importer;
import com.powsybl.iidm.import_.ImportersLoader;
import com.powsybl.iidm.network.Network;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import java.util.Properties;

/**
* A type of {@code ProjectFile} which represents a {@link Network} imported to the project,
* and provides methods to get the {@code Network} object or query it with a script.
* A type of {@code ProjectFile} which represents a {@link Network} imported to
* the project, and provides methods to get the {@code Network} object or query
* it with a script.
*
* @author Geoffroy Jamgotchian <geoffroy.jamgotchian at rte-france.com>
* @author Geoffroy Jamgotchian <geoffroy.jamgotchian at rte-france.com>
*/
public class ImportedCase extends ProjectFile implements ProjectCase {

Expand All @@ -36,6 +43,7 @@ public class ImportedCase extends ProjectFile implements ProjectCase {

static final String FORMAT = "format";
static final String PARAMETERS = "parameters";
static final String HAS_DATA_PACK = "HAS_DATA_PACK";

private final ImportersLoader importersLoader;

Expand All @@ -48,9 +56,31 @@ public ReadOnlyDataSource getDataSource() {
return new AppStorageDataSource(storage, info.getId(), info.getName());
}

private ReadOnlyDataStore getDataStore() {
return new AppStorageDataStore(storage, info.getId());
}

public Optional<DataPack> getDataPack() {
if (info.getGenericMetadata().getBoolean(HAS_DATA_PACK)) {
try {
return getImporter().getDataFormat().newDataResolver().resolve(getDataStore(), getDataPackMainEntry(), getParameters());
} catch (IOException | NonUniqueResultException e) {
throw new NetworkImportException(e);
}
} else {
return Optional.empty();
}
}

private String getDataPackMainEntry() {
return info.getGenericMetadata().getString(DataPack.MAIN_ENTRY_TAG);
}

public Properties getParameters() {
Properties parameters = new Properties();
try (Reader reader = new InputStreamReader(storage.readBinaryData(info.getId(), PARAMETERS).orElseThrow(AssertionError::new), StandardCharsets.UTF_8)) {
try (Reader reader = new InputStreamReader(
storage.readBinaryData(info.getId(), PARAMETERS).orElseThrow(AssertionError::new),
StandardCharsets.UTF_8)) {
parameters.load(reader);
} catch (IOException e) {
throw new UncheckedIOException(e);
Expand All @@ -60,11 +90,9 @@ public Properties getParameters() {

public Importer getImporter() {
String format = info.getGenericMetadata().getString(FORMAT);
return importersLoader.loadImporters()
.stream()
return importersLoader.loadImporters().stream()
.filter(importer -> importer.getFormat().equals(format))
.findFirst()
.orElseThrow(() -> new AfsException("Importer not found for format " + format));
.findFirst().orElseThrow(() -> new AfsException("Importer not found for format " + format));
}

@Override
Expand Down
66 changes: 46 additions & 20 deletions afs-ext-base/src/main/java/com/powsybl/afs/ext/base/ImportedCaseBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,30 @@
*/
package com.powsybl.afs.ext.base;

import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UncheckedIOException;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;

import com.powsybl.afs.AfsException;
import com.powsybl.afs.ProjectFileBuildContext;
import com.powsybl.afs.ProjectFileBuilder;
import com.powsybl.afs.ProjectFileCreationContext;
import com.powsybl.afs.ext.base.events.CaseImported;
import com.powsybl.afs.storage.*;
import com.powsybl.afs.storage.AppStorageDataSource;
import com.powsybl.afs.storage.AppStorageDataStore;
import com.powsybl.afs.storage.NodeGenericMetadata;
import com.powsybl.afs.storage.NodeInfo;
import com.powsybl.commons.datasource.DataSource;
import com.powsybl.commons.datasource.DataSourceUtil;
import com.powsybl.commons.datasource.MemDataSource;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.datastore.DataPack;
import com.powsybl.iidm.export.Exporters;
import com.powsybl.iidm.export.ExportersLoader;
import com.powsybl.iidm.export.ExportersServiceLoader;
Expand All @@ -25,16 +39,6 @@
import com.powsybl.iidm.import_.ImportersLoader;
import com.powsybl.iidm.network.Network;

import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UncheckedIOException;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;

/**
* Builder for the project file {@link ImportedCase}.
*
Expand All @@ -58,6 +62,8 @@ public class ImportedCaseBuilder implements ProjectFileBuilder<ImportedCase> {

private final Properties parameters = new Properties();

private DataPack dataPack;

public ImportedCaseBuilder(ProjectFileBuildContext context, ImportersLoader importersLoader, ImportConfig importConfig) {
this(context, new ExportersServiceLoader(), importersLoader, importConfig);
}
Expand Down Expand Up @@ -102,6 +108,11 @@ public ImportedCaseBuilder withDatasource(ReadOnlyDataSource dataSource) {
return this;
}

public ImportedCaseBuilder withDataPack(DataPack dataPack) {
this.dataPack = Objects.requireNonNull(dataPack);
return this;
}

public ImportedCaseBuilder withNetwork(Network network) {
Objects.requireNonNull(network);
if (name == null) {
Expand All @@ -124,7 +135,7 @@ public ImportedCaseBuilder withParameters(Map<String, String> parameters) {

@Override
public ImportedCase build() {
if (dataSource == null) {
if (dataSource == null && dataPack == null) {
throw new AfsException("Case or data source is not set");
}
if (name == null) {
Expand All @@ -134,14 +145,29 @@ public ImportedCase build() {
if (context.getStorage().getChildNode(context.getFolderInfo().getId(), name).isPresent()) {
throw new AfsException("Parent folder already contains a '" + name + "' node");
}

// create project file
NodeInfo info = context.getStorage().createNode(context.getFolderInfo().getId(), name, ImportedCase.PSEUDO_CLASS, "", ImportedCase.VERSION,
new NodeGenericMetadata().setString(ImportedCase.FORMAT, importer.getFormat()));

// store case data
importer.copy(dataSource, new AppStorageDataSource(context.getStorage(), info.getId(), info.getName()));

NodeInfo info = null;
if (dataSource != null) {
NodeGenericMetadata meta = new NodeGenericMetadata().setString(ImportedCase.FORMAT, importer.getFormat());
meta.setBoolean(ImportedCase.HAS_DATA_PACK, false);
// create project file
info = context.getStorage().createNode(context.getFolderInfo().getId(), name, ImportedCase.PSEUDO_CLASS, "", ImportedCase.VERSION,
meta);
// store case data
importer.copy(dataSource, new AppStorageDataSource(context.getStorage(), info.getId(), info.getName()));
} else {
NodeGenericMetadata meta = new NodeGenericMetadata().setString(ImportedCase.FORMAT, dataPack.getDataFormatId());
meta.setBoolean(ImportedCase.HAS_DATA_PACK, true);
dataPack.getMainEntry().ifPresent(entry -> meta.setString(DataPack.MAIN_ENTRY_TAG, entry.getName()));
// create project file
info = context.getStorage().createNode(context.getFolderInfo().getId(), name, ImportedCase.PSEUDO_CLASS, "", ImportedCase.VERSION,
meta);

try {
dataPack.copyTo(new AppStorageDataStore(context.getStorage(), info.getId()));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
// store parameters
try (Writer writer = new OutputStreamWriter(context.getStorage().writeBinaryData(info.getId(), ImportedCase.PARAMETERS), StandardCharsets.UTF_8)) {
parameters.store(writer, "");
Expand Down
18 changes: 11 additions & 7 deletions afs-ext-base/src/main/java/com/powsybl/afs/ext/base/LocalNetworkCacheService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,21 @@
*/
package com.powsybl.afs.ext.base;

import java.util.Objects;
import java.util.Properties;
import java.util.UUID;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.powsybl.afs.AfsException;
import com.powsybl.afs.ProjectFile;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.iidm.import_.Importer;
import com.powsybl.iidm.network.Network;
import groovy.json.JsonOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.powsybl.iidm.network.NetworkFactory;

import java.util.Objects;
import java.util.Properties;
import java.util.UUID;
import groovy.json.JsonOutput;

/**
* @author Geoffroy Jamgotchian <geoffroy.jamgotchian at rte-france.com>
Expand Down Expand Up @@ -50,7 +53,8 @@ private static ScriptResult<Network> loadNetworkFromImportedCase(ImportedCase im
Importer importer = importedCase.getImporter();
ReadOnlyDataSource dataSource = importedCase.getDataSource();
Properties parameters = importedCase.getParameters();
Network network = importer.importData(dataSource, parameters);
Network network = importedCase.getDataPack().map(p -> importer.importDataPack(p, NetworkFactory.findDefault(), parameters))
.orElse(importer.importData(dataSource, parameters));
return ScriptResult.of(network);
}

Expand Down
72 changes: 59 additions & 13 deletions afs-ext-base/src/test/java/com/powsybl/afs/ext/base/ImportedCaseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,46 @@
*/
package com.powsybl.afs.ext.base;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.IOException;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.jimfs.Configuration;
import com.google.common.jimfs.Jimfs;
import com.powsybl.afs.*;
import com.powsybl.afs.AbstractProjectFileTest;
import com.powsybl.afs.AfsException;
import com.powsybl.afs.FileExtension;
import com.powsybl.afs.Folder;
import com.powsybl.afs.Project;
import com.powsybl.afs.ProjectFileExtension;
import com.powsybl.afs.ProjectFolder;
import com.powsybl.afs.ProjectNode;
import com.powsybl.afs.ServiceExtension;
import com.powsybl.afs.mapdb.storage.MapDbAppStorage;
import com.powsybl.afs.storage.AppStorage;
import com.powsybl.afs.storage.InMemoryEventsBus;
import com.powsybl.afs.storage.NodeGenericMetadata;
import com.powsybl.afs.storage.NodeInfo;
import com.powsybl.commons.datastore.DataPack;
import com.powsybl.commons.datastore.DataStore;
import com.powsybl.commons.datastore.DataStores;
import com.powsybl.commons.datastore.NonUniqueResultException;
import com.powsybl.iidm.export.ExportersLoader;
import com.powsybl.iidm.export.ExportersLoaderList;
import com.powsybl.iidm.import_.ImportConfig;
Expand All @@ -24,18 +54,6 @@
import com.powsybl.iidm.network.Network;
import com.powsybl.iidm.xml.XMLExporter;
import com.powsybl.iidm.xml.XMLImporter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import static org.junit.Assert.*;

/**
* @author Geoffroy Jamgotchian <geoffroy.jamgotchian at rte-france.com>
Expand Down Expand Up @@ -207,4 +225,32 @@ public void testNetwork() {
assertNotNull(importedCase2);
assertEquals("NetworkID", importedCase2.getName());
}

@Test
public void testDataStore() throws IOException, NonUniqueResultException {
Folder root = afs.getRootFolder();

// create project
Project project = root.createProject("project");
assertNotNull(project);

// create project folder
ProjectFolder folder = project.getRootFolder().createFolder("folder");
assertTrue(folder.getChildren().isEmpty());

DataStore ds = DataStores.createDataStore(fileSystem.getPath("/work"));
TestDataFormat form = new TestDataFormat("TEST");
Optional<DataPack> op = form.newDataResolver().resolve(ds, "network.tst", null);
assertTrue(op.isPresent());

ImportedCase importedCase = folder.fileBuilder(ImportedCaseBuilder.class)
.withDataPack(op.get())
.withName("test")
.build();
assertNotNull(importedCase);
assertEquals("test", importedCase.getName());
assertNotNull(importedCase.getNetwork());
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should add additional test for the "reading" part, ie the getDataStore method.

We can check that it contains the single entry "network.tst".

assertTrue(importedCase.getDataPack().isPresent());
assertEquals("network.tst", importedCase.getDataPack().get().getMainEntry().get().getName());
}
}
Loading