Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add support for structured queries (opensearch only) #815

Merged
merged 3 commits into from
Jul 9, 2024
Merged

add support for structured queries (opensearch only) #815

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d9bc19f
add support for structured queries (opensearch only)
tobiass-sdl Jun 13, 2024
6e48f88
review comments / bug fixes
tobiass-sdl Jun 24, 2024
bc230d2
stricter filter for houses without house number and objects that are …
tobiass-sdl Jul 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions app/es_embedded/src/main/java/de/komoot/photon/Server.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import de.komoot.photon.DatabaseProperties;
import de.komoot.photon.Importer;
import de.komoot.photon.Updater;
import de.komoot.photon.searcher.StructuredSearchHandler;
import de.komoot.photon.searcher.ReverseHandler;
import de.komoot.photon.searcher.SearchHandler;
import de.komoot.photon.elasticsearch.*;
Expand Down Expand Up @@ -168,12 +169,12 @@ private void setupDirectories(URL directoryName) throws IOException, URISyntaxEx

}

public DatabaseProperties recreateIndex(String[] languages, Date importDate) throws IOException {
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException {
deleteIndex();

loadIndexSettings().createIndex(esClient, PhotonIndex.NAME);

new IndexMapping().addLanguages(languages).putMapping(esClient, PhotonIndex.NAME, PhotonIndex.TYPE);
createAndPutIndexMapping(languages, supportStructuredQueries);

DatabaseProperties dbProperties = new DatabaseProperties()
.setLanguages(languages)
Expand All @@ -183,6 +184,16 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate) thr
return dbProperties;
}

private void createAndPutIndexMapping(String[] languages, boolean supportStructuredQueries)
{
if (supportStructuredQueries) {
throw new UnsupportedOperationException("Structured queries are not supported for elasticsearch-based Photon. Consider to use OpenSearch.");
}

new IndexMapping().addLanguages(languages)
.putMapping(esClient, PhotonIndex.NAME, PhotonIndex.TYPE);
}

public void updateIndexSettings(String synonymFile) throws IOException {
// Load the settings from the database to make sure it is at the right
// version. If the version is wrong, we should not be messing with the
Expand All @@ -195,9 +206,7 @@ public void updateIndexSettings(String synonymFile) throws IOException {
// Sanity check: legacy databases don't save the languages, so there is no way to update
// the mappings consistently.
if (dbProperties.getLanguages() != null) {
new IndexMapping()
.addLanguages(dbProperties.getLanguages())
.putMapping(esClient, PhotonIndex.NAME, PhotonIndex.TYPE);
this.createAndPutIndexMapping(dbProperties.getLanguages(), false);
}
}

Expand Down Expand Up @@ -279,6 +288,10 @@ public SearchHandler createSearchHandler(String[] languages, int queryTimeoutSec
return new ElasticsearchSearchHandler(esClient, languages, queryTimeoutSec);
}

public StructuredSearchHandler createStructuredSearchHandler(String[] languages, int queryTimeoutSec) {
throw new UnsupportedOperationException("Structured queries are not supported for elasticsearch-based Photon. Consider to use OpenSearch.");
}

public ReverseHandler createReverseHandler(int queryTimeoutSec) {
return new ElasticsearchReverseHandler(esClient, queryTimeoutSec);
}
Expand Down
2 changes: 1 addition & 1 deletion app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public void setUpES() throws IOException {
public void setUpES(Path test_directory, String... languages) throws IOException {
server = new ElasticTestServer(test_directory.toString());
server.start(TEST_CLUSTER_NAME, new String[]{});
server.recreateIndex(languages, new Date());
server.recreateIndex(languages, new Date(), false);
refresh();
}

Expand Down
13 changes: 10 additions & 3 deletions app/opensearch/src/main/java/de/komoot/photon/Server.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import de.komoot.photon.opensearch.*;
import de.komoot.photon.searcher.ReverseHandler;
import de.komoot.photon.searcher.SearchHandler;
import de.komoot.photon.searcher.StructuredSearchHandler;
import org.apache.hc.core5.http.HttpHost;
import org.codelibs.opensearch.runner.OpenSearchRunner;
import org.opensearch.client.json.jackson.JacksonJsonpMapper;
Expand Down Expand Up @@ -97,18 +98,19 @@ public void shutdown() {
}
}

public DatabaseProperties recreateIndex(String[] languages, Date importDate) throws IOException {
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException {
// delete any existing data
if (client.indices().exists(e -> e.index(PhotonIndex.NAME)).value()) {
client.indices().delete(d -> d.index(PhotonIndex.NAME));
}

(new IndexSettingBuilder()).setShards(5).createIndex(client, PhotonIndex.NAME);

(new IndexMapping()).addLanguages(languages).putMapping(client, PhotonIndex.NAME);
(new IndexMapping(supportStructuredQueries)).addLanguages(languages).putMapping(client, PhotonIndex.NAME);

var dbProperties = new DatabaseProperties()
.setLanguages(languages)
.setSupportStructuredQueries(supportStructuredQueries)
.setImportDate(importDate);
saveToDatabase(dbProperties);

Expand All @@ -122,7 +124,7 @@ public void updateIndexSettings(String synonymFile) throws IOException {
(new IndexSettingBuilder()).setSynonymFile(synonymFile).updateIndex(client, PhotonIndex.NAME);

if (dbProperties.getLanguages() != null) {
(new IndexMapping())
(new IndexMapping(dbProperties.getSupportStructuredQueries()))
.addLanguages(dbProperties.getLanguages())
.putMapping(client, PhotonIndex.NAME);
}
Expand Down Expand Up @@ -154,6 +156,7 @@ public void loadFromDatabase(DatabaseProperties dbProperties) throws IOException

dbProperties.setLanguages(dbEntry.source().languages);
dbProperties.setImportDate(dbEntry.source().importDate);
dbProperties.setSupportStructuredQueries(dbEntry.source().supportStructuredQueries);
}

public Importer createImporter(String[] languages, String[] extraTags) {
Expand All @@ -170,6 +173,10 @@ public SearchHandler createSearchHandler(String[] languages, int queryTimeoutSec
return new OpenSearchSearchHandler(client, languages, queryTimeoutSec);
}

public StructuredSearchHandler createStructuredSearchHandler(String[] languages, int queryTimeoutSec) {
return new OpenSearchStructuredSearchHandler(client, languages, queryTimeoutSec);
}

public ReverseHandler createReverseHandler(int queryTimeoutSec) {
return new OpenSearchReverseHandler(client, queryTimeoutSec);
}
Expand Down
Loading