Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Hot fixes #1360

Merged
merged 5 commits into from
Sep 19, 2024
Merged

Hot fixes #1360

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f7cf95f
Case-insensitive check of NaN values.
mchrza Sep 19, 2024
821a0e2
Temporary deactivate reducing precision.
mchrza Sep 19, 2024
d740969
Add missing "END"
mchrza Sep 19, 2024
a4056b8
Rollback tests related to deactivation of precision reduction.
mchrza Sep 19, 2024
e3f9dea
Rollback tests related to deactivation of precision reduction.
mchrza Sep 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion xyz-hub-test/src/test/java/com/here/xyz/hub/rest/ReadFeatureApiIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,8 @@ public void testReadingFeatureByHereTileIdFalse() {
get(getSpacesPath() + "/x-psql-test/tile/here/5148795631.geojson").
then().
statusCode(OK.code()).
body("features.size()", equalTo(0));
// Fix if we enable back precision method which removes duplicate points
body("features.size()", equalTo(1));
}


Expand Down
6 changes: 3 additions & 3 deletions xyz-hub-test/src/test/java/com/here/xyz/hub/rest/jobs/JobApiCompositeExportIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public class JobApiCompositeExportIT extends JobApiIT{
add("id5");
add("id6");
add("id7");
add("01010000A0E61000000B462575422C5CC03140C3177D5249400000000000000000");
add("01010000A0E6100000B0CD1A681EE23C403411B4C4CEE939400000000000000000");
}};

private static List<String> l1ChangesContentTILEID_FC_B64 = new ArrayList<>(){
Expand All @@ -85,8 +85,8 @@ public class JobApiCompositeExportIT extends JobApiIT{
add("idX");
add("id3");
add("id7");
add("01010000A0E61000000B462575422C54C03140C3177DD246400000000000000000");
add("01010000A0E6100000E4F17A5137D34440D6C58E42E97B43400000000000000000");
add("01010000A0E61000007DF82175422C54C0D056CC177DD246400000000000000000");
add("01010000A0E610000038D07F5137D34440F0968E42E97B43400000000000000000");
}};

private static List<String> l1ChangesContentPARTITIONED_JSON_WKB = new ArrayList<>(){
Expand Down
38 changes: 19 additions & 19 deletions xyz-hub-test/src/test/java/com/here/xyz/hub/rest/jobs/JobApiExportIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ public void testFullWKBExport() throws Exception {
List<URL> urls = performExport(job, getScopedSpaceId(testSpaceId2, scope), finalized, failed);

List<String> mustContain = Arrays.asList(
"01020000A0E61000000300000042C17BFDFF662140E08442041C14494000000000000000006BDE27FD732F21406C5CFFAECF0A49400000000000000000A9B7ABFCD7162140A96A82A8FB0E49400000000000000000",
"01030000A0E6100000010000000500000083E4F8FC8B172140876D8B321B18494000000000000000004DF564FED1A32140876D8B321B18494000000000000000004DF564FED1A3214074F04C6892024940000000000000000083E4F8FC8B17214074F04C6892024940000000000000000083E4F8FC8B172140876D8B321B1849400000000000000000",
"01020000A0E61000000300000041C17BFDFF662140E08442041C14494000000000000000006BDE27FD732F21406C5CFFAECF0A49400000000000000000A9B7ABFCD7162140A96A82A8FB0E49400000000000000000",
"01030000A0E6100000010000000500000083E4F8FC8B17214074F04C689202494000000000000000004DF564FED1A3214074F04C689202494000000000000000004DF564FED1A32140876D8B321B184940000000000000000083E4F8FC8B172140876D8B321B184940000000000000000083E4F8FC8B17214074F04C68920249400000000000000000",
"foo_polygon"
);

Expand Down Expand Up @@ -114,7 +114,7 @@ public void testFilteredWKBExportWithSpatialFilterUnclipped() throws Exception {

List<String> mustContain = Arrays.asList(
"foo_polygon",
"01020000A0E610000004000000901FE0FF7D70214087A4164A260B4940000000000000000041237DFF8B4421402A7288B8390549400000000000000000407E80FFF70521400264E8D8410749400000000000000000F60E12FD350421401AA9F7544E0349400000000000000000"
"01020000A0E610000004000000901FE0FF7D70214087A4164A260B4940000000000000000041237DFF8B4421402A7288B8390549400000000000000000407E80FFF70521400264E8D8410749400000000000000000F60E12FD3504214019A9F7544E0349400000000000000000"
);

downloadAndCheck(urls, 2467, 5, mustContain);
Expand All @@ -133,7 +133,7 @@ public void testFilteredWKBExportWithPropFilter() throws Exception {

List<String> mustContain = Arrays.asList(
"onPropertyLevel",
"01010000A0E6100000ABBEA9FEF7322140C9563C9A8F1049400000000000000000"
"01010000A0E610000037E38FFD875721402A55A2EC2D0D49400000000000000000"
);

downloadAndCheck(urls, 890, 3, mustContain);
Expand All @@ -157,7 +157,7 @@ public void testFilteredWKBExportWithPropAndSpatialFilter() throws Exception {

List<String> mustContain = Arrays.asList(
"Point",
"01010000A0E610000038E38FFD875721402A55A2EC2D0D49400000000000000000"
"01010000A0E610000037E38FFD875721402A55A2EC2D0D49400000000000000000"
);

downloadAndCheck(urls, 328, 1, mustContain);
Expand All @@ -182,7 +182,7 @@ public void testFullGEOJSONExport() throws Exception {
"Polygon with hole"
);

downloadAndCheck(urls, 3971, 11, mustContain);
downloadAndCheck(urls, 3973, 11, mustContain);
}

/**
Expand All @@ -207,7 +207,7 @@ public void testFullGEOJSONCompositeL1Export() throws Exception {
);

// 252 + 11 = 263
downloadAndCheck(urls, 96743, 263, mustContain);
downloadAndCheck(urls, 96744, 263, mustContain);
}

/**
Expand All @@ -229,7 +229,7 @@ public void testFullGEOJSONCompositeL1SuperExport() throws Exception {
);

// 252 + 11 = 263
downloadAndCheck(urls, 3971, 11, mustContain);
downloadAndCheck(urls, 3973, 11, mustContain);
}

/**
Expand Down Expand Up @@ -269,7 +269,7 @@ public void testFilteredGEOJSONCompositeL1ExportWithSpatialFilter() throws Excep
"foo_new" //Got overridden in composite space
);

downloadAndCheck(urls, 10917, 30, mustContain);
downloadAndCheck(urls, 10916, 30, mustContain);
}

@Test
Expand Down Expand Up @@ -315,7 +315,7 @@ public void testFilteredGEOJSONCompositeL2WithPropAndSpatialFilter() throws Exce
"2LPoint" //Got added in composite space L2
);

downloadAndCheck(urls, 11150, 31, mustContain);
downloadAndCheck(urls, 11149, 31, mustContain);
}

@Test
Expand All @@ -335,7 +335,7 @@ public void testFullGEOJSONCompositeL2Export() throws Exception {
"2LPoint" //Got added in composite space L2
);
// 252 + 11 + 1 = 264
downloadAndCheck(urls, 96976, 264, mustContain);
downloadAndCheck(urls, 96977, 264, mustContain);
}

@Test
Expand All @@ -354,7 +354,7 @@ public void testFullGEOJSONCompositeL2SuperExport() throws Exception {
);

// 252 + 11 = 263
downloadAndCheck(urls, 96743, 263, mustContain);
downloadAndCheck(urls, 96744, 263, mustContain);
}

/**
Expand All @@ -378,7 +378,7 @@ public void testFullVMLExport() throws Exception {
List<String> mustContain = Arrays.asList("360", "eyJ0eXBlIjogIkZl");

/** Expect 10 features instead of 11 because feature without geometry is not included in result */
downloadAndCheckFC(urls, 5069, 10, mustContain, 1);
downloadAndCheckFC(urls, 5073, 10, mustContain, 1);
}

/**
Expand All @@ -402,7 +402,7 @@ public void testFullVMLExportMaxTilesPerFile2() throws Exception {
List<String> mustContain = Arrays.asList("23600780", "23600794", "W9uIiwgImZlYXR1cmV");

/** Expect 39 features because the geometry of some features is intersecting multiple tiles */
downloadAndCheckFC(urls, 23046, 39, mustContain, 17);
downloadAndCheckFC(urls, 23098, 39, mustContain, 17);
}

/**
Expand Down Expand Up @@ -541,7 +541,7 @@ public void testFullVMLCompositeL2Export() throws Exception {
List<String> mustContain = Arrays.asList("374", "352", "HVyZSIsICJnZW9", "374");

// 252 + 10 (-feature w/o geometry) + 1 = 263
downloadAndCheckFC(urls, 131365, 263, mustContain, 37);
downloadAndCheckFC(urls, 131369, 263, mustContain, 37);
}

@Test
Expand All @@ -563,7 +563,7 @@ public void testFilteredVMLCompositeL1ExportWithSpatialFilter() throws Exception

List<String> mustContain = Arrays.asList("360", "RyeSI6IHsidHlwZSI6I");

downloadAndCheckFC(urls, 14621, 30, mustContain, 1 );
downloadAndCheckFC(urls, 14617, 30, mustContain, 1 );
}

@Test
Expand Down Expand Up @@ -613,8 +613,8 @@ public void testFilteredVMLCompositeL2WithPropAndSpatialFilter() throws Exceptio
List<URL> urls = performExport(job, getScopedSpaceId(testSpaceId2ExtExt, scope), finalized, failed);

List<String> mustContain = Arrays.asList(
"360",
"2ludCIsICJjb29yZGluYXRlcyI6IFs4LjQ4M"
"360",
"2ludCIsICJjb29yZGluYXRlcyI6IFs4LjQ4M"
);

downloadAndCheckFC(urls, 14929, 31, mustContain, 1);
Expand Down Expand Up @@ -816,7 +816,7 @@ public void testParallelFullWKBExport() throws Exception {
System.out.println(urls.size());
}

// @Test
// @Test
public void testParallelFullVMLExport() throws Exception {
/** Create job */
String spaceId = "test-space";
Expand Down
2 changes: 1 addition & 1 deletion xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseMaintainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public class DatabaseMaintainer {
private static final Logger logger = LogManager.getLogger();

/** Is used to check against xyz_ext_version() */
public static final int XYZ_EXT_VERSION = 199;
public static final int XYZ_EXT_VERSION = 200;

public static final int H3_CORE_VERSION = 108;

Expand Down
2 changes: 1 addition & 1 deletion xyz-psql-connector/src/main/java/com/here/xyz/psql/query/GetFeatures.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ protected SQLQuery buildGeoFragment(E event) {
}

protected SQLQuery buildGeoJsonExpression(E event) {
return new SQLQuery("replace(ST_AsGeojson(${{rawGeoExpression}}, ${{precision}}), 'nan', '0')")
return new SQLQuery("REGEXP_REPLACE(ST_AsGeojson(${{rawGeoExpression}}, ${{precision}}), '(?i)\\bnan\\b', '0', 'g')")
.withQueryFragment("rawGeoExpression", buildRawGeoExpression(event))
.withQueryFragment("precision", "" + GetFeatures.GEOMETRY_DECIMAL_DIGITS);
}
Expand Down
7 changes: 3 additions & 4 deletions xyz-psql-connector/src/main/java/com/here/xyz/psql/query/GetFeaturesByBBoxTweaked.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ private SQLQuery buildSamplingTweaksQuery(GetFeaturesByBBoxEvent event, Map twea
String tweaksGeoSql = clipProjGeom(bbox,"geo");
tweaksGeoSql = map2MvtGeom( event, bbox, tweaksGeoSql );
//convert to geojson
tweaksGeoSql = ( bConvertGeo2Geojson ? DhString.format("replace(ST_AsGeojson(" + getForceMode(event.isForce2D()) + "( %s ),%d),'nan','0')",tweaksGeoSql,GEOMETRY_DECIMAL_DIGITS)
tweaksGeoSql = ( bConvertGeo2Geojson ? DhString.format("REGEXP_REPLACE(ST_AsGeojson(" + getForceMode(event.isForce2D()) + "( %s ),%d), '(?i)\\bnan\\b', '0', 'g')",tweaksGeoSql,GEOMETRY_DECIMAL_DIGITS)
: DhString.format( getForceMode(event.isForce2D()) + "( %s )",tweaksGeoSql ) );

return generateCombinedQueryTweaks(event, tweakQuery , tweaksGeoSql, bTestTweaksGeoIfNull, tblSampleRatio, bSortByHashedValue );
Expand Down Expand Up @@ -421,9 +421,8 @@ private SQLQuery buildSimplificationTweaksQuery(GetFeaturesByBBoxEvent event, Ma

default: break;
}

//convert to geojson
tweaksGeoSql = ( convertGeo2Geojson ? DhString.format("replace(ST_AsGeojson(" + getForceMode(event.isForce2D()) + "( %s ),%d),'nan','0')",tweaksGeoSql,GEOMETRY_DECIMAL_DIGITS)
tweaksGeoSql = ( convertGeo2Geojson ? DhString.format("REGEXP_REPLACE(ST_AsGeojson(" + getForceMode(event.isForce2D()) + "( %s ),%d),'(?i)\\bnan\\b', '0', 'g')",tweaksGeoSql,GEOMETRY_DECIMAL_DIGITS)
: DhString.format( getForceMode(event.isForce2D()) + "( %s )",tweaksGeoSql) );
}

Expand All @@ -443,7 +442,7 @@ private SQLQuery buildSimplificationTweaksQuery(GetFeaturesByBBoxEvent event, Ma
else if ( strength <= 80 ) { minGeoHashLenForLineMerge = 4; } //medhigh

if( "geo".equals(tweaksGeoSql) ) // formal, just in case
tweaksGeoSql = ( convertGeo2Geojson ? DhString.format("replace(ST_AsGeojson(" + getForceMode(event.isForce2D()) + "( %s ),%d),'nan','0')",tweaksGeoSql,GEOMETRY_DECIMAL_DIGITS)
tweaksGeoSql = ( convertGeo2Geojson ? DhString.format("REGEXP_REPLACE(ST_AsGeojson(" + getForceMode(event.isForce2D()) + "( %s ),%d),'(?i)\\bnan\\b', '0', 'g')",tweaksGeoSql,GEOMETRY_DECIMAL_DIGITS)
: DhString.format(getForceMode(event.isForce2D()) + "( %s )",tweaksGeoSql) );

if( convertGeo2Geojson )
Expand Down
3 changes: 1 addition & 2 deletions xyz-psql-connector/src/main/java/com/here/xyz/psql/query/IterateChangesets.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,7 @@ public XyzResponse handle(ResultSet rs) throws SQLException {
}

public SQLQuery buildIterateChangesets(IterateChangesetsEvent event){

String geo = "replace(ST_AsGeojson(geo, " + GetFeatures.GEOMETRY_DECIMAL_DIGITS + "), 'nan', '0')::jsonb";
String geo = "REGEXP_REPLACE(ST_AsGeojson(geo, " + GetFeatures.GEOMETRY_DECIMAL_DIGITS + "), '(?i)\\bnan\\b', '0', 'g')::jsonb";

SQLQuery query = new SQLQuery(
"SELECT " +
Expand Down
17 changes: 10 additions & 7 deletions xyz-psql-connector/src/main/resources/xyz_ext.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@
CREATE OR REPLACE FUNCTION xyz_ext_version()
RETURNS integer AS
$BODY$
select 199
select 200
$BODY$
LANGUAGE sql IMMUTABLE;

Expand All @@ -123,12 +123,15 @@ CREATE OR REPLACE FUNCTION xyz_reduce_precision(geo GEOMETRY, enable_logging boo
RETURNS GEOMETRY AS
$BODY$
BEGIN
RETURN ST_ReducePrecision(geo, 0.00000001);
EXCEPTION WHEN OTHERS THEN
IF enable_logging THEN
RAISE WARNING 'Invalid geometry detected: %',ST_AsGeoJson(geo);
END IF;
RETURN geo;
RETURN geo;
-- Temporary deactivated, till we found a proper solution.
--
-- RETURN ST_ReducePrecision(geo, 0.00000001);
-- EXCEPTION WHEN OTHERS THEN
-- IF enable_logging THEN
-- RAISE WARNING 'Invalid geometry detected: %',ST_AsGeoJson(geo);
-- END IF;
-- RETURN geo;
END
$BODY$
LANGUAGE plpgsql VOLATILE;
Expand Down
Loading