diff --git a/.changeset/wild-houses-buy.md b/.changeset/wild-houses-buy.md new file mode 100644 index 000000000..5c8b52586 --- /dev/null +++ b/.changeset/wild-houses-buy.md @@ -0,0 +1,5 @@ +--- +"geohub": patch +--- + +fix: added cql_filter to table search endpoint. diff --git a/sites/geohub/src/lib/server/helpers/index.ts b/sites/geohub/src/lib/server/helpers/index.ts index 656f28290..98a6d8d18 100644 --- a/sites/geohub/src/lib/server/helpers/index.ts +++ b/sites/geohub/src/lib/server/helpers/index.ts @@ -32,3 +32,4 @@ export * from './isGeoHubBlobStorage'; export * from './recolorPngDataUrl'; export * from './clipSprite'; export * from './loadStorymapById'; +export * from './parseCqlFilter'; diff --git a/sites/geohub/src/lib/server/helpers/parseCqlFilter.test.ts b/sites/geohub/src/lib/server/helpers/parseCqlFilter.test.ts new file mode 100644 index 000000000..1b50c6bcc --- /dev/null +++ b/sites/geohub/src/lib/server/helpers/parseCqlFilter.test.ts @@ -0,0 +1,170 @@ +import { describe, it, expect } from 'vitest'; +import { parseCqlFilter } from './parseCqlFilter'; +import type { FeatureCollection } from 'geojson'; + +// テスト用のGeoJSONデータ +const geoJsonData: FeatureCollection = { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + id: 1, + geometry: { type: 'MultiPoint', coordinates: [[30.750211060218042, -2.30839626064316]] }, + properties: { + id: 34, + facility_name: 'EP Rumuri', + type_of_facility: 'Primary School', + student_number: 714, + staff_number: '25', + connect_or_not: 'Connect', + wss_id: 50530, + condition: 'Normal', + availability: 'Not Regularly', + district: 'KIREHE', + sector: 'NYAMUGARI', + cell: 'NYAMUGARI', + latitude: -2.3083963, + longitude: 30.7502111 + } + }, + { + type: 'Feature', + id: 2, + geometry: { type: 'MultiPoint', coordinates: [[30.75756957103567, -2.2820901415712784]] }, + properties: { + id: 32, + facility_name: 'Lychee de Rusumo', + type_of_facility: 'Secondary School', + student_number: 701, + staff_number: '42', + connect_or_not: 'Connect', + wss_id: 50532, + condition: 'Normal', + availability: 'Not Regularly', + district: 'KIREHE', + sector: 'NYAMUGARI', + cell: 'NYAMUGARI', + latitude: -2.2820901, + longitude: 30.7575696 + } + }, + { + type: 'Feature', + id: 3, + geometry: { type: 'MultiPoint', coordinates: [[30.757090489837932, -2.2844122973492267]] }, + properties: { + id: 23, + facility_name: 'GS Rusumo', + type_of_facility: 'Secondary School', + student_number: 2184, + staff_number: '93', + connect_or_not: 'Connect', + wss_id: 50519, + condition: 'Normal', + availability: 'Regularly', + district: 'KIREHE', + sector: 'NYAMUGARI', + cell: 'NYAMUGARI', + latitude: -2.2844123, + longitude: 30.7570905 + } + }, + { + type: 'Feature', + id: 4, + geometry: { type: 'MultiPoint', coordinates: [[30.783671603104686, -2.26796484425878]] }, + properties: { + id: 24, + facility_name: 'EP Kazizi', + type_of_facility: 'Primary School', + student_number: 782, + staff_number: '25', + connect_or_not: 'Connect', + wss_id: 50525, + condition: undefined, + availability: 'Regularly', + district: 'KIREHE', + sector: 'NYAMUGARI', + cell: 'KAZIZI', + latitude: -2.2679648, + longitude: 30.7836716 + } + } + ] +}; + +describe('parseCqlFilter', () => { + it('should filter by EQUAL TO operator', () => { + const result = parseCqlFilter("type_of_facility = 'Primary School'", geoJsonData.features); + expect(result).toHaveLength(2); + }); + + it('should filter by LESS THAN operator', () => { + const result = parseCqlFilter('student_number < 714', geoJsonData.features); + expect(result).toHaveLength(1); + }); + + it('should filter by LESS THAN OR EQUAL TO operator', () => { + const result = parseCqlFilter('student_number <= 714', geoJsonData.features); + expect(result).toHaveLength(2); + }); + + it('should filter by GREATER THAN operator', () => { + const result = parseCqlFilter('student_number > 782', geoJsonData.features); + expect(result).toHaveLength(1); + }); + + it('should filter by GREATER THAN OR EQUAL TO operator', () => { + const result = parseCqlFilter('student_number >= 782', geoJsonData.features); + expect(result).toHaveLength(2); + }); + + it('should filter by IS NULL operator', () => { + const result = parseCqlFilter('condition IS NULL', geoJsonData.features); + expect(result).toHaveLength(1); + }); + + it('should filter by LIKE operator', () => { + const result = parseCqlFilter('facility_name LIKE EP', geoJsonData.features); + expect(result).toHaveLength(2); + }); + + it('should filter by IN operator with single condition', () => { + const result = parseCqlFilter("type_of_facility IN ('Primary School')", geoJsonData.features); + expect(result).toHaveLength(2); + }); + + it('should filter by IN operator with multiple conditions', () => { + const result = parseCqlFilter( + "type_of_facility IN ('Primary School', 'Secondary School')", + geoJsonData.features + ); + expect(result).toHaveLength(4); + }); + + it('should filter by BETWEEN operator', () => { + const result = parseCqlFilter('student_number BETWEEN 600 AND 800', geoJsonData.features); + expect(result).toHaveLength(3); + }); + + it('should filter by BETWEEN operator and EQUAL TO', () => { + const result = parseCqlFilter( + 'student_number BETWEEN 600 AND 800 AND type_of_facility = Secondary school', + geoJsonData.features + ); + expect(result).toHaveLength(1); + }); + + it('should filter by NOT EQUAL TO operator', () => { + const result = parseCqlFilter("availability <> 'Regularly'", geoJsonData.features); + expect(result).toHaveLength(2); + }); + + it('should return an empty array when no matches', () => { + const result = parseCqlFilter( + "type_of_facility = 'Non-existent Facility'", + geoJsonData.features + ); + expect(result).toHaveLength(0); + }); +}); diff --git a/sites/geohub/src/lib/server/helpers/parseCqlFilter.ts b/sites/geohub/src/lib/server/helpers/parseCqlFilter.ts new file mode 100644 index 000000000..d3ae3ae45 --- /dev/null +++ b/sites/geohub/src/lib/server/helpers/parseCqlFilter.ts @@ -0,0 +1,111 @@ +import type { Feature } from 'geojson'; + +/** + * parse the given CQL filter string to return only matched objects + * + * Only the following operators are currently supported + * EQUAL TO [ = ], LESS THAN [ < ], LESS THAN OR EQUAL TO [ <= ], GREATER THAN [ > ], GREATER THAN OR EQUAL TO [ >= ], IS NULL, LIKE, IN, BETWEEN + * AND, OR, NOT [ <> ] + * + * @param cqlFilter CQL filter string + * @param data an array of GeoJSON feature object + * @returns filtered data + */ +export const parseCqlFilter = (cqlFilter: string, data: Feature[]): Feature[] => { + const conditions = cqlFilter + .replace(/(\w+)\s+BETWEEN\s+\S+\s+AND\s+\S+/g, (match) => { + return match.replace(/\s+AND\s+/g, ' BETWEEN_AND'); + }) + .split(/\s+(AND|OR)\s+/) + .map((condition) => condition.replace('BETWEEN_AND', 'AND ')) + .filter(Boolean); + + const filteredData: Feature[] = []; + + for (const item of data) { + let result = false; + for (let i = 0; i < conditions.length; i++) { + const cond = conditions[i].trim(); + + if (cond === 'AND') { + continue; + } else if (cond === 'OR') { + continue; + } else { + const isMatched = evaluateCondition(item, cond); + + const previous = conditions[i - 1]; + if (previous === 'AND') { + result = result && isMatched; + } else if (previous === 'OR') { + result = result || isMatched; + } else { + result = isMatched; + } + } + } + if (result === true) { + filteredData.push(item); + } + } + + return filteredData; +}; + +const evaluateCondition = (item: Feature, condition: string): boolean => { + const regex = /(\w+)\s*(<=|>=|<>|=|<|>|LIKE|IN|IS NULL|BETWEEN)\s*(.*)/; + const match = condition.trim().match(regex); + if (!match) return false; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const [_, field, operator, value] = match; + + if (!item.properties) return false; + const targetField = Object.keys(item.properties).find( + (key) => key.toLowerCase() === field.toLowerCase() + ); + if (!targetField) return false; + let targetProp = item.properties[targetField]; + if (typeof targetProp === 'string') { + targetProp = targetProp.toLowerCase(); + } + + if (operator === 'IN') { + const values = value + .replace(/[()']/g, '') + .split(',') + .map((v) => v.trim().toLowerCase()); + return values.includes(targetProp?.toString()); + } else if (operator === 'BETWEEN') { + const [minValue, maxValue] = value.split('AND').map((v) => v.trim()); + if (isNaN(Number(minValue)) || isNaN(Number(maxValue))) return false; + const min = Number(minValue); + const max = Number(maxValue); + return typeof targetProp === 'number' && targetProp >= min && targetProp <= max; + } + + const processedValue: string | number = isNaN(Number(value)) + ? value.replace(/'/g, '').toLowerCase() + : Number(value); + + switch (operator) { + case '=': + return targetProp === processedValue; + case '<': + return targetProp < processedValue; + case '<=': + return targetProp <= processedValue; + case '>': + return targetProp > processedValue; + case '>=': + return targetProp >= processedValue; + case '<>': + return targetProp !== processedValue; + case 'IS NULL': + return targetProp === null || targetProp === undefined; + case 'LIKE': + return targetProp?.toString().indexOf(processedValue) !== -1; + default: + return false; + } +}; diff --git a/sites/geohub/src/routes/api/datasets/[id]/table/layers/[layer].[format]/+server.ts b/sites/geohub/src/routes/api/datasets/[id]/table/layers/[layer].[format]/+server.ts index 840c9b335..8424aa2f5 100644 --- a/sites/geohub/src/routes/api/datasets/[id]/table/layers/[layer].[format]/+server.ts +++ b/sites/geohub/src/routes/api/datasets/[id]/table/layers/[layer].[format]/+server.ts @@ -1,5 +1,11 @@ import type { RequestHandler } from './$types'; -import { createDatasetLinks, getDatasetById, isSuperuser, pageNumber } from '$lib/server/helpers'; +import { + createDatasetLinks, + getDatasetById, + isSuperuser, + pageNumber, + parseCqlFilter +} from '$lib/server/helpers'; import { env } from '$env/dynamic/private'; import { AccessLevel, Permission } from '$lib/config/AppConfig'; import { getDomainFromEmail } from '$lib/helper'; @@ -163,6 +169,11 @@ export const GET: RequestHandler = async ({ params, locals, url, fetch }) => { fc.features.push({ ...feature, id: fc.features.length + 1 }); } + const cqlFilter = url.searchParams.get('cql_filter'); + if (cqlFilter) { + fc.features = parseCqlFilter(cqlFilter, fc.features); + } + // sort by target column const sortby = url.searchParams.get('sortby'); let sortByColumn = ''; @@ -235,6 +246,11 @@ export const GET: RequestHandler = async ({ params, locals, url, fetch }) => { rel: 'self', type: 'application/json', href: url.toString() + }, + { + rel: 'dataset', + type: 'application/json', + href: `${url.origin}/api/datasets/${dataset.properties.id}` } ]; diff --git a/sites/geohub/static/api/swagger/spec.yaml b/sites/geohub/static/api/swagger/spec.yaml index ad296cc94..1e974438b 100644 --- a/sites/geohub/static/api/swagger/spec.yaml +++ b/sites/geohub/static/api/swagger/spec.yaml @@ -499,7 +499,29 @@ paths: - If no flatgeobuf exists to the dataset headers: {} operationId: get-datasets-id-attributes-layer - description: 'This endpoint is to provide a capability to query attribute table for a vector layer. Currently, it is not available for raster layer.' + description: |- + This endpoint is to provide a capability to query attribute table for a vector layer. Currently, it is not available for raster layer. + + `query` param is to scan all properties to return any matched features. If you want to filter by specific columns, please use `cql_filter` instead. + + Use `cql_filter` to filter by advanced search. `cql_filter` only supports the following operators: + + `EQUAL TO [ = ]`, `LESS THAN [ < ]`, `LESS THAN OR EQUAL TO [ <= ]`, `GREATER THAN [ > ]`, `GREATER THAN OR EQUAL TO [ >= ]`, `IS NULL`, `LIKE`, `IN`, `BETWEEN aaa AND bbb`, `NOT [ <> ]` + + each condition can be concatenated by either `AND`, `OR`. operators should be capital letters. + + The below are examples of cql_filter for each operator. + + - `type_of_facility = 'Primary School'` + - `student_number < 714` + - `student_number <= 714` + - `student_number > 782` + - `student_number >= 782` + - `condition IS NULL` + - `facility_name LIKE EP` + - `type_of_facility IN ('Primary School')`, `type_of_facility IN ('Primary School', 'Secondary School')` + - `student_number BETWEEN 600 AND 800`, `student_number BETWEEN 600 AND 800 AND type_of_facility = Secondary school` + - `availability <> 'Regularly'` security: - Azure AD authentication: [] - API access token: [] @@ -531,6 +553,11 @@ paths: in: query name: sortby description: 'optional. sorting column. format should be `{field name},{asc|desc}`' + - schema: + type: string + in: query + name: cql_filter + description: optional. CQL filter to search data '/datasets/{id}/permission': parameters: - schema: