fix: added cql_filter to table search endpoint (#4231)

* fix: added cql_filter to table search endpoint.

* fix: add dataset link to the response of table api

* refactor: add parseCqlFilter to index.ts

.changeset/wild-houses-buy.md

@@ -0,0 +1,5 @@
+---
+"geohub": patch
+---
+
+fix: added cql_filter to table search endpoint.

sites/geohub/src/lib/server/helpers/index.ts

@@ -32,3 +32,4 @@ export * from './isGeoHubBlobStorage';
 export * from './recolorPngDataUrl';
 export * from './clipSprite';
 export * from './loadStorymapById';
+export * from './parseCqlFilter';

sites/geohub/src/lib/server/helpers/parseCqlFilter.test.ts

@@ -0,0 +1,170 @@
+import { describe, it, expect } from 'vitest';
+import { parseCqlFilter } from './parseCqlFilter';
+import type { FeatureCollection } from 'geojson';
+
+// テスト用のGeoJSONデータ
+const geoJsonData: FeatureCollection = {
+	type: 'FeatureCollection',
+	features: [
+		{
+			type: 'Feature',
+			id: 1,
+			geometry: { type: 'MultiPoint', coordinates: [[30.750211060218042, -2.30839626064316]] },
+			properties: {
+				id: 34,
+				facility_name: 'EP Rumuri',
+				type_of_facility: 'Primary School',
+				student_number: 714,
+				staff_number: '25',
+				connect_or_not: 'Connect',
+				wss_id: 50530,
+				condition: 'Normal',
+				availability: 'Not Regularly',
+				district: 'KIREHE',
+				sector: 'NYAMUGARI',
+				cell: 'NYAMUGARI',
+				latitude: -2.3083963,
+				longitude: 30.7502111
+			}
+		},
+		{
+			type: 'Feature',
+			id: 2,
+			geometry: { type: 'MultiPoint', coordinates: [[30.75756957103567, -2.2820901415712784]] },
+			properties: {
+				id: 32,
+				facility_name: 'Lychee de Rusumo',
+				type_of_facility: 'Secondary School',
+				student_number: 701,
+				staff_number: '42',
+				connect_or_not: 'Connect',
+				wss_id: 50532,
+				condition: 'Normal',
+				availability: 'Not Regularly',
+				district: 'KIREHE',
+				sector: 'NYAMUGARI',
+				cell: 'NYAMUGARI',
+				latitude: -2.2820901,
+				longitude: 30.7575696
+			}
+		},
+		{
+			type: 'Feature',
+			id: 3,
+			geometry: { type: 'MultiPoint', coordinates: [[30.757090489837932, -2.2844122973492267]] },
+			properties: {
+				id: 23,
+				facility_name: 'GS Rusumo',
+				type_of_facility: 'Secondary School',
+				student_number: 2184,
+				staff_number: '93',
+				connect_or_not: 'Connect',
+				wss_id: 50519,
+				condition: 'Normal',
+				availability: 'Regularly',
+				district: 'KIREHE',
+				sector: 'NYAMUGARI',
+				cell: 'NYAMUGARI',
+				latitude: -2.2844123,
+				longitude: 30.7570905
+			}
+		},
+		{
+			type: 'Feature',
+			id: 4,
+			geometry: { type: 'MultiPoint', coordinates: [[30.783671603104686, -2.26796484425878]] },
+			properties: {
+				id: 24,
+				facility_name: 'EP Kazizi',
+				type_of_facility: 'Primary School',
+				student_number: 782,
+				staff_number: '25',
+				connect_or_not: 'Connect',
+				wss_id: 50525,
+				condition: undefined,
+				availability: 'Regularly',
+				district: 'KIREHE',
+				sector: 'NYAMUGARI',
+				cell: 'KAZIZI',
+				latitude: -2.2679648,
+				longitude: 30.7836716
+			}
+		}
+	]
+};
+
+describe('parseCqlFilter', () => {
+	it('should filter by EQUAL TO operator', () => {
+		const result = parseCqlFilter("type_of_facility = 'Primary School'", geoJsonData.features);
+		expect(result).toHaveLength(2);
+	});
+
+	it('should filter by LESS THAN operator', () => {
+		const result = parseCqlFilter('student_number < 714', geoJsonData.features);
+		expect(result).toHaveLength(1);
+	});
+
+	it('should filter by LESS THAN OR EQUAL TO operator', () => {
+		const result = parseCqlFilter('student_number <= 714', geoJsonData.features);
+		expect(result).toHaveLength(2);
+	});
+
+	it('should filter by GREATER THAN operator', () => {
+		const result = parseCqlFilter('student_number > 782', geoJsonData.features);
+		expect(result).toHaveLength(1);
+	});
+
+	it('should filter by GREATER THAN OR EQUAL TO operator', () => {
+		const result = parseCqlFilter('student_number >= 782', geoJsonData.features);
+		expect(result).toHaveLength(2);
+	});
+
+	it('should filter by IS NULL operator', () => {
+		const result = parseCqlFilter('condition IS NULL', geoJsonData.features);
+		expect(result).toHaveLength(1);
+	});
+
+	it('should filter by LIKE operator', () => {
+		const result = parseCqlFilter('facility_name LIKE EP', geoJsonData.features);
+		expect(result).toHaveLength(2);
+	});
+
+	it('should filter by IN operator with single condition', () => {
+		const result = parseCqlFilter("type_of_facility IN ('Primary School')", geoJsonData.features);
+		expect(result).toHaveLength(2);
+	});
+
+	it('should filter by IN operator with multiple conditions', () => {
+		const result = parseCqlFilter(
+			"type_of_facility IN ('Primary School', 'Secondary School')",
+			geoJsonData.features
+		);
+		expect(result).toHaveLength(4);
+	});
+
+	it('should filter by BETWEEN operator', () => {
+		const result = parseCqlFilter('student_number BETWEEN 600 AND 800', geoJsonData.features);
+		expect(result).toHaveLength(3);
+	});
+
+	it('should filter by BETWEEN operator and EQUAL TO', () => {
+		const result = parseCqlFilter(
+			'student_number BETWEEN 600 AND 800 AND type_of_facility = Secondary school',
+			geoJsonData.features
+		);
+		expect(result).toHaveLength(1);
+	});
+
+	it('should filter by NOT EQUAL TO operator', () => {
+		const result = parseCqlFilter("availability <> 'Regularly'", geoJsonData.features);
+		expect(result).toHaveLength(2);
+	});
+
+	it('should return an empty array when no matches', () => {
+		const result = parseCqlFilter(
+			"type_of_facility = 'Non-existent Facility'",
+			geoJsonData.features
+		);
+		expect(result).toHaveLength(0);
+	});
+});

sites/geohub/src/lib/server/helpers/parseCqlFilter.ts

@@ -0,0 +1,111 @@
+import type { Feature } from 'geojson';
+
+/**
+ * parse the given CQL filter string to return only matched objects
+ *
+ * Only the following operators are currently supported
+ * EQUAL TO [ = ], LESS THAN [ < ], LESS THAN OR EQUAL TO [ <= ], GREATER THAN [ > ], GREATER THAN OR EQUAL TO [ >= ], IS NULL, LIKE, IN, BETWEEN
+ * AND, OR, NOT [ <> ]
+ *
+ * @param cqlFilter CQL filter string
+ * @param data an array of GeoJSON feature object
+ * @returns filtered data
+ */
+export const parseCqlFilter = (cqlFilter: string, data: Feature[]): Feature[] => {
+	const conditions = cqlFilter
+		.replace(/(\w+)\s+BETWEEN\s+\S+\s+AND\s+\S+/g, (match) => {
+			return match.replace(/\s+AND\s+/g, ' BETWEEN_AND');
+		})
+		.split(/\s+(AND|OR)\s+/)
+		.map((condition) => condition.replace('BETWEEN_AND', 'AND '))
+		.filter(Boolean);
+
+	const filteredData: Feature[] = [];
+
+	for (const item of data) {
+		let result = false;
+		for (let i = 0; i < conditions.length; i++) {
+			const cond = conditions[i].trim();
+
+			if (cond === 'AND') {
+				continue;
+			} else if (cond === 'OR') {
+				continue;
+			} else {
+				const isMatched = evaluateCondition(item, cond);
+
+				const previous = conditions[i - 1];
+				if (previous === 'AND') {
+					result = result && isMatched;
+				} else if (previous === 'OR') {
+					result = result || isMatched;
+				} else {
+					result = isMatched;
+				}
+			}
+		}
+		if (result === true) {
+			filteredData.push(item);
+		}
+	}
+
+	return filteredData;
+};
+
+const evaluateCondition = (item: Feature, condition: string): boolean => {
+	const regex = /(\w+)\s*(<=|>=|<>|=|<|>|LIKE|IN|IS NULL|BETWEEN)\s*(.*)/;
+	const match = condition.trim().match(regex);
+	if (!match) return false;
+
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	const [_, field, operator, value] = match;
+
+	if (!item.properties) return false;
+	const targetField = Object.keys(item.properties).find(
+		(key) => key.toLowerCase() === field.toLowerCase()
+	);
+	if (!targetField) return false;
+	let targetProp = item.properties[targetField];
+	if (typeof targetProp === 'string') {
+		targetProp = targetProp.toLowerCase();
+	}
+
+	if (operator === 'IN') {
+		const values = value
+			.replace(/[()']/g, '')
+			.split(',')
+			.map((v) => v.trim().toLowerCase());
+		return values.includes(targetProp?.toString());
+	} else if (operator === 'BETWEEN') {
+		const [minValue, maxValue] = value.split('AND').map((v) => v.trim());
+		if (isNaN(Number(minValue)) || isNaN(Number(maxValue))) return false;
+		const min = Number(minValue);
+		const max = Number(maxValue);
+		return typeof targetProp === 'number' && targetProp >= min && targetProp <= max;
+	}
+
+	const processedValue: string | number = isNaN(Number(value))
+		? value.replace(/'/g, '').toLowerCase()
+		: Number(value);
+
+	switch (operator) {
+		case '=':
+			return targetProp === processedValue;
+		case '<':
+			return targetProp < processedValue;
+		case '<=':
+			return targetProp <= processedValue;
+		case '>':
+			return targetProp > processedValue;
+		case '>=':
+			return targetProp >= processedValue;
+		case '<>':
+			return targetProp !== processedValue;
+		case 'IS NULL':
+			return targetProp === null || targetProp === undefined;
+		case 'LIKE':
+			return targetProp?.toString().indexOf(processedValue) !== -1;
+		default:
+			return false;
+	}
+};

sites/geohub/src/routes/api/datasets/[id]/table/layers/[layer].[format]/+server.ts

@@ -1,5 +1,11 @@
 import type { RequestHandler } from './$types';
-import { createDatasetLinks, getDatasetById, isSuperuser, pageNumber } from '$lib/server/helpers';
+import {
+	createDatasetLinks,
+	getDatasetById,
+	isSuperuser,
+	pageNumber,
+	parseCqlFilter
+} from '$lib/server/helpers';
 import { env } from '$env/dynamic/private';
 import { AccessLevel, Permission } from '$lib/config/AppConfig';
 import { getDomainFromEmail } from '$lib/helper';
@@ -163,6 +169,11 @@ export const GET: RequestHandler = async ({ params, locals, url, fetch }) => {
 		fc.features.push({ ...feature, id: fc.features.length + 1 });
 	}
 
+	const cqlFilter = url.searchParams.get('cql_filter');
+	if (cqlFilter) {
+		fc.features = parseCqlFilter(cqlFilter, fc.features);
+	}
+
 	// sort by target column
 	const sortby = url.searchParams.get('sortby');
 	let sortByColumn = '';
@@ -235,6 +246,11 @@ export const GET: RequestHandler = async ({ params, locals, url, fetch }) => {
 			rel: 'self',
 			type: 'application/json',
 			href: url.toString()
+		},
+		{
+			rel: 'dataset',
+			type: 'application/json',
+			href: `${url.origin}/api/datasets/${dataset.properties.id}`
 		}
 	];
 

sites/geohub/static/api/swagger/spec.yaml

@@ -499,7 +499,29 @@ paths:
             - If no flatgeobuf exists to the dataset
           headers: {}
       operationId: get-datasets-id-attributes-layer
-      description: 'This endpoint is to provide a capability to query attribute table for a vector layer. Currently, it is not available for raster layer.'
+      description: |-
+        This endpoint is to provide a capability to query attribute table for a vector layer. Currently, it is not available for raster layer.
+
+        `query` param is to scan all properties to return any matched features. If you want to filter by specific columns, please use `cql_filter` instead.
+
+        Use `cql_filter` to filter by advanced search. `cql_filter` only supports the following operators:
+
+        `EQUAL TO [ = ]`, `LESS THAN [ < ]`, `LESS THAN OR EQUAL TO [ <= ]`, `GREATER THAN [ > ]`, `GREATER THAN OR EQUAL TO [ >= ]`, `IS NULL`, `LIKE`, `IN`, `BETWEEN aaa AND bbb`, `NOT [ <> ]`
+
+        each condition can be concatenated by either `AND`, `OR`. operators should be capital letters.
+
+        The below are examples of cql_filter for each operator.
+
+        - `type_of_facility = 'Primary School'`
+        - `student_number < 714`
+        - `student_number <= 714`
+        - `student_number > 782`
+        - `student_number >= 782`
+        - `condition IS NULL`
+        - `facility_name LIKE EP`
+        - `type_of_facility IN ('Primary School')`, `type_of_facility IN ('Primary School', 'Secondary School')`
+        - `student_number BETWEEN 600 AND 800`, `student_number BETWEEN 600 AND 800 AND type_of_facility = Secondary school`
+        - `availability <> 'Regularly'`
       security:
         - Azure AD authentication: []
         - API access token: []
@@ -531,6 +553,11 @@ paths:
           in: query
           name: sortby
           description: 'optional. sorting column. format should be `{field name},{asc|desc}`'
+        - schema:
+            type: string
+          in: query
+          name: cql_filter
+          description: optional. CQL filter to search data
   '/datasets/{id}/permission':
     parameters:
       - schema: