Skip to content

Commit

Permalink
feat: download wfs files to tmp folder for bucket service
Browse files Browse the repository at this point in the history 
ING-3609
  • Loading branch information
@kapil-agnihotri
kapil-agnihotri committed Jun 27, 2023
1 parent 9a62628 commit a970f6e
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 6 deletions.
3 changes: 1 addition & 2 deletions ...esdihumboldt.hale.io.gml.ui/src/eu/esdihumboldt/hale/io/gml/ui/GmlReaderSettingsPage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,7 @@ protected void createContent(Composite page) {

Group wfsGroup = new Group(page, SWT.NONE);
wfsGroup.setText("WFS requests");
GridLayoutFactory.swtDefaults().numColumns(2).equalWidth(false)
.applyTo(wfsGroup);
GridLayoutFactory.swtDefaults().numColumns(2).equalWidth(false).applyTo(wfsGroup);
GridDataFactory.fillDefaults().grab(true, false).applyTo(wfsGroup);

requestPaginationEnabled = new Button(wfsGroup, SWT.CHECK);
Expand Down
1 change: 1 addition & 0 deletions io/plugins/eu.esdihumboldt.hale.io.gml/META-INF/MANIFEST.MF
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ Import-Package: com.google.common.base,
eu.esdihumboldt.util.http.client,
eu.esdihumboldt.util.io,
javax.annotation;version="[1.2.0,1.2.0]",
org.apache.commons.io;version="2.11.0",
org.apache.http;version="4.3.3",
org.apache.http.client.utils;version="4.3.6",
org.joda.time.format;version="2.3.0",
Expand Down
32 changes: 28 additions & 4 deletions ...humboldt.hale.io.gml/src/eu/esdihumboldt/hale/io/gml/reader/internal/StreamGmlReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ public class StreamGmlReader extends AbstractInstanceReader {
*/
public static final String PARAM_FEATURES_PER_WFS_REQUEST = "featuresPerWfsRequest";

/**
* The parameter with temporary downlaod path passed from bucket-service to
* download wfs files directly to it instead of processing wfs as instances
*/
public static final String PARAM_TMP_DIR_PATH = "tmpDirPath";

/**
* The name of the parameter specifying if hale should ignore the total
* number of features reported by the WFS.
Expand Down Expand Up @@ -115,6 +121,7 @@ public StreamGmlReader(boolean restrictToFeatures) {
addSupportedParameter(PARAM_IGNORE_NAMESPACES);
addSupportedParameter(PARAM_PAGINATE_REQUEST);
addSupportedParameter(PARAM_FEATURES_PER_WFS_REQUEST);
addSupportedParameter(PARAM_TMP_DIR_PATH);
}

/**
Expand All @@ -135,6 +142,7 @@ protected IOReport execute(ProgressIndicator progress, IOReporter reporter)
.as(Boolean.class, false);

int featuresPerRequest;
String tmpDownloadDirPath = null;
if (paginateRequest) {
featuresPerRequest = getParameter(PARAM_FEATURES_PER_WFS_REQUEST).as(Integer.class,
1000);
Expand All @@ -143,6 +151,8 @@ protected IOReport execute(ProgressIndicator progress, IOReporter reporter)
featuresPerRequest = WfsBackedGmlInstanceCollection.UNLIMITED;
}

tmpDownloadDirPath = getParameter(PARAM_TMP_DIR_PATH).getStringRepresentation();

LocatableInputSupplier<? extends InputStream> source = getSource();
String scheme = null;
String query = null;
Expand All @@ -155,11 +165,25 @@ protected IOReport execute(ProgressIndicator progress, IOReporter reporter)
&& query.toLowerCase().contains("request=getfeature")
&& (scheme.equalsIgnoreCase("http") || scheme.equalsIgnoreCase("https"))) {

// check if WFS is reachable and responds?
// for testing
// if (tmpDownloadDirPath == null) {
// tmpDownloadDirPath = Files.createTempDirectory("wfsDownloads").toFile()
// .getAbsolutePath();
// System.out.println("++++++ tmpDownloadDirPath: " + tmpDownloadDirPath);
// }

instances = new WfsBackedGmlInstanceCollection(getSource(), getSourceSchema(),
restrictToFeatures, ignoreRoot, strict, ignoreNamespaces, getCrsProvider(),
this, featuresPerRequest, ignoreNumberMatched);
// check if WFS is reachable and responds?
if (tmpDownloadDirPath != null) {
instances = new WfsBackedGmlInstanceCollection(getSource(), getSourceSchema(),
restrictToFeatures, ignoreRoot, strict, ignoreNamespaces,
getCrsProvider(), this, featuresPerRequest, ignoreNumberMatched,
tmpDownloadDirPath);
}
else {
instances = new WfsBackedGmlInstanceCollection(getSource(), getSourceSchema(),
restrictToFeatures, ignoreRoot, strict, ignoreNamespaces,
getCrsProvider(), this, featuresPerRequest, ignoreNumberMatched);
}
}
else {
instances = new GmlInstanceCollection(getSource(), getSourceSchema(),
Expand Down
59 changes: 59 additions & 0 deletions ...l/src/eu/esdihumboldt/hale/io/gml/reader/internal/wfs/WfsBackedGmlInstanceCollection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,19 @@

package eu.esdihumboldt.hale.io.gml.reader.internal.wfs;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;

import org.apache.commons.io.FileUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URIBuilder;

Expand Down Expand Up @@ -108,6 +111,7 @@ public class WfsBackedGmlInstanceCollection implements InstanceCollection {
private final CRSProvider crsProvider;
private final boolean ignoreNamespaces;
private final IOProvider ioProvider;
private final String tmpdir;

/**
* Create a GML instance collection based on the given WFS source.
Expand Down Expand Up @@ -140,6 +144,42 @@ public WfsBackedGmlInstanceCollection(LocatableInputSupplier<? extends InputStre
boolean ignoreNamespaces, CRSProvider crsProvider, IOProvider provider,
int featuresPerRequest, boolean ignoreNumberMatched) throws URISyntaxException {

this(source, sourceSchema, restrictToFeatures, ignoreRoot, strict, ignoreNamespaces,
crsProvider, provider, featuresPerRequest, ignoreNumberMatched, null);
}

/**
* Create a GML instance collection based on the given WFS source.
*
* @param source the source
* @param sourceSchema the source schema
* @param restrictToFeatures if only instances that are GML features shall
* be loaded
* @param ignoreRoot if the root element should be ignored for creating
* instances even if it is recognized as an allowed instance type
* @param strict if associating elements with properties should be done
* strictly according to the schema, otherwise a fall-back is
* used trying to populate values also on invalid property paths
* @param ignoreNamespaces if parsing of the XML instances should allow
* types and properties with namespaces that differ from those
* defined in the schema
* @param crsProvider CRS provider in case no CRS is specified, may be
* <code>null</code>
* @param provider the I/O provider to get values
* @param featuresPerRequest Number of features to retrieve at most with one
* WFS GetFeature request, or {@value #UNLIMITED} to disable
* pagination
* @param ignoreNumberMatched If featuresPerRequest is set, ignore the
* number of matches reported by the WFS
* @param tmpDirPath tmp dir path to download all wfs files
* @throws URISyntaxException thrown if the WFS request URL cannot be
* generated from the source location URI
*/
public WfsBackedGmlInstanceCollection(LocatableInputSupplier<? extends InputStream> source,
TypeIndex sourceSchema, boolean restrictToFeatures, boolean ignoreRoot, boolean strict,
boolean ignoreNamespaces, CRSProvider crsProvider, IOProvider provider,
int featuresPerRequest, boolean ignoreNumberMatched, String tmpDirPath)
throws URISyntaxException {
this.sourceSchema = sourceSchema;
this.restrictToFeatures = restrictToFeatures;
this.ignoreRoot = ignoreRoot;
Expand All @@ -150,6 +190,7 @@ public WfsBackedGmlInstanceCollection(LocatableInputSupplier<? extends InputStre

this.primordialUri = source.getLocation();

this.tmpdir = tmpDirPath;
// Build base URI from original location by removing STARTINDEX and
// MAXFEATURES/COUNT parameters if present
URIBuilder builder = new URIBuilder(primordialUri);
Expand Down Expand Up @@ -412,6 +453,24 @@ private void createNextIterator() {
throw new IllegalStateException(e.getMessage(), e);
}

// Download all the files into a folder for Bucket service
if (tmpdir != null) {
try {
URL url = nextUri.toURL();
StringBuilder fileName = new StringBuilder(tmpdir);
fileName.append("/").append(url.getPath().replaceAll("/", "."))
.append(totalFeaturesProcessed).append(".gml");

// TODO: may be download files in a thread
File downloadFile = new File(fileName.toString());
nextUri = downloadFile.toURI();
FileUtils.copyURLToFile(url, downloadFile);
} catch (IOException e) {
log.error(MessageFormat.format("Unable to download WFS request: {0}",
e.getMessage()), e);
}

}
log.debug(MessageFormat.format("Creating new iterator for URL \"{0}\"",
nextUri.toString()));

Expand Down

0 comments on commit a970f6e

Please sign in to comment.