From 2637d09016a43559f4d99fba82a6b8fa8b75bee7 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 10 Jul 2023 20:31:32 +0200 Subject: [PATCH 01/35] feat: add @iot.id and @iot.selfLink --- database/istsos_schema.sql | 8 ++++++++ docker-compose.yml | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/database/istsos_schema.sql b/database/istsos_schema.sql index d8318b2..a22d48f 100644 --- a/database/istsos_schema.sql +++ b/database/istsos_schema.sql @@ -81,6 +81,14 @@ CREATE TABLE IF NOT EXISTS sensorthings."Observation" ( UNIQUE ("datastream_id", "phenomenonTime") ); +CREATE OR REPLACE FUNCTION "@iot.id"(anyelement) RETURNS text AS $$ + SELECT $1.id; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION "@iot.selfLink"(anyelement) RETURNS text AS $$ + SELECT concat(current_setting('custom.hostname'), '/v1.1/',substring(pg_typeof($1)::text from 2 for length(pg_typeof($1)::text) - 2),'(' || $1.id || ')'); +$$ LANGUAGE SQL; + --- ======================= --- SYSTEM_TIME extension --- ======================= diff --git a/docker-compose.yml b/docker-compose.yml index 1a6918e..0336047 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: POSTGRES_PASSWORD: admin DATADIR: /var/lib/postgresql/data POSTGRES_MULTIPLE_EXTENSIONS: postgis,hstore,postgis_topology,postgis_raster,pgrouting,ltree,pg_cron,uuid-ossp + command: postgres -c custom.hostname="http://localhost:3000" volumes: # - v-istsos-miu-postgis-sql:/docker-entrypoint-initdb.d - v-istsos-miu-database-data:/var/lib/postgresql/data @@ -32,8 +33,6 @@ services: depends_on: - database - - # POSTGRESQL # database_test: # build: From 2ed3d9d473e2d3354b0ca4eac5c4e85333f02b2b Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 10 Jul 2023 20:52:25 +0200 Subject: [PATCH 02/35] feat: create @iot.navigationLink functions --- database/istsos_schema.sql | 80 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/database/istsos_schema.sql b/database/istsos_schema.sql index a22d48f..67f7ed1 100644 --- a/database/istsos_schema.sql +++ b/database/istsos_schema.sql @@ -89,6 +89,86 @@ CREATE OR REPLACE FUNCTION "@iot.selfLink"(anyelement) RETURNS text AS $$ SELECT concat(current_setting('custom.hostname'), '/v1.1/',substring(pg_typeof($1)::text from 2 for length(pg_typeof($1)::text) - 2),'(' || $1.id || ')'); $$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Thing") RETURNS table( + "Locations@iot.navigationLink" text, + "Datastreams@iot.navigationLink" text, + "HistoricalLocations@iot.navigationLink" text +) AS +$$ + SELECT 'Things(' || $1.id || ')/Locations', + 'Things(' || $1.id || ')/Datastreams', + 'Things(' || $1.id || ')/HistoricalLocations'; +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Location") RETURNS table( + "Things@iot.navigationLink" text, + "HistoricalLocations@iot.navigationLink" text +) AS +$$ + SELECT 'Locations(' || $1.id || ')/Things', + 'Locations(' || $1.id || ')/HistoricalLocations'; +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."HistoricalLocation") RETURNS table( + "Locations@iot.navigationLink" text, + "Thing@iot.navigationLink" text +) AS +$$ + SELECT 'HistoricalLocations(' || $1.id || ')/Locations', + 'HistoricalLocations(' || $1.id || ')/Thing'; +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Datastream") RETURNS table( + "Thing@iot.navigationLink" text, + "Sensor@iot.navigationLink" text, + "ObservedProperty@iot.navigationLink" text, + "Observations@iot.navigationLink" text +) AS +$$ + SELECT 'Datastreams(' || $1.id || ')/Thing', + 'Datastreams(' || $1.id || ')/Sensor', + 'Datastreams(' || $1.id || ')/ObservedProperty', + 'Datastreams(' || $1.id || ')/Observations'; +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Sensor") RETURNS table( + "Datastreams@iot.navigationLink" text +) AS +$$ + SELECT 'Sensors(' || $1.id || ')/Datastreams'; +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."ObservedProperty") RETURNS table( + "Datastreams@iot.navigationLink" text +) AS +$$ + SELECT 'ObservedProperties(' || $1.id || ')/Datastreams'; +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Observation") RETURNS table( + "FeatureOfInterest@iot.navigationLink" text, + "Datastream@iot.navigationLink" text +) AS +$$ + SELECT 'Observations(' || $1.id || ')/FeatureOfInterest', + 'Observations(' || $1.id || ')/Datastream'; +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."FeaturesOfInterest") RETURNS table( + "Observations@iot.navigationLink" text +) AS +$$ + SELECT 'FeaturesOfInterest(' || $1.id || ')/Observations'; +$$ +LANGUAGE SQL; + --- ======================= --- SYSTEM_TIME extension --- ======================= From 335e417699f770fb8ca79c733131c8c234bd6549 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 10 Jul 2023 21:09:14 +0200 Subject: [PATCH 03/35] feat: add default parameters --- fastapi/app/sta2rest/sta2rest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py index 9025d99..1fe91cc 100644 --- a/fastapi/app/sta2rest/sta2rest.py +++ b/fastapi/app/sta2rest/sta2rest.py @@ -216,7 +216,7 @@ def visit_ExpandNode(self, node: ast.ExpandNode, parent=None): if not expand_identifier.subquery or not expand_identifier.subquery.select: if not select: select = ast.SelectNode([]) - select.identifiers.append(ast.IdentifierNode(f'{expand_identifier.identifier}(*)')) + select.identifiers.append(ast.IdentifierNode(f'{expand_identifier.identifier}("@iot.id", "@iot.selfLink", *)')) # Return the converted expand node return { @@ -263,6 +263,16 @@ def visit_QueryNode(self, node: ast.QueryNode): if result['filter']: query_parts.append(result['filter']) + if not node.select: + node.select = ast.SelectNode([]) + # Add "@iot.id", "@iot.selfLink" and "*" to the select node + node.select.identifiers.extend(( + ast.IdentifierNode('"@iot.id"'), + ast.IdentifierNode('"@iot.selfLink"'), + ast.IdentifierNode('"@iot.navigationLink"'), + ast.IdentifierNode('*') + )) + # Check if we have a select, filter, orderby, skip, top or count in the query if node.select: query_parts.append(self.visit(node.select)) From ae3b3e11f033839dd1eb40a6057e9e3392e98004 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 10 Jul 2023 21:09:21 +0200 Subject: [PATCH 04/35] feat: flatten navigationLink --- fastapi/app/v1/endpoints/general.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index bcfcb83..e997f52 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -4,6 +4,12 @@ v1 = APIRouter() +def __flatten_navigation_links(row): + if "@iot.navigationLink" in row: + # merge all the keys from the navigationLink + row.update(row["@iot.navigationLink"]) + del row["@iot.navigationLink"] + @v1.api_route("/{path_name:path}", methods=["GET"]) async def catch_all(request: Request, path_name: str): try: @@ -25,11 +31,16 @@ async def catch_all(request: Request, path_name: str): r = await client.get(url) data = r.json() + print(result) + if result['single_result']: data = data[0] if result['value']: # get the value of the first key data = data[list(data.keys())[0]] + elif "@iot.navigationLink" in data: + __flatten_navigation_links(data) + elif result['ref']: # Get the value of the first key key_name = list(data[0].keys())[0] @@ -47,6 +58,10 @@ async def catch_all(request: Request, path_name: str): data["value"].append({ "@iot.selfLink": f"{host}/v1.1/{table_name}({row['id']})" }) + else: + # Flatten the @iot.navigationLink for each row + for row in data: + __flatten_navigation_links(row) return data except Exception as e: From 69d4ce20d71c9b55aa2e8015e4eb191356a7450c Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 10 Jul 2023 21:10:59 +0200 Subject: [PATCH 05/35] feat: change value --- fastapi/app/v1/endpoints/general.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index e997f52..5900116 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -63,6 +63,10 @@ async def catch_all(request: Request, path_name: str): for row in data: __flatten_navigation_links(row) + data = { + "value": data + } + return data except Exception as e: return {"error": str(e)} From d60af3f3b8407375bae81a914cc5e0ba5361c5bb Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 10 Jul 2023 21:14:28 +0200 Subject: [PATCH 06/35] fix: use existing selfLink --- fastapi/app/v1/endpoints/general.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 5900116..173380f 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -31,8 +31,6 @@ async def catch_all(request: Request, path_name: str): r = await client.get(url) data = r.json() - print(result) - if result['single_result']: data = data[0] if result['value']: @@ -45,18 +43,12 @@ async def catch_all(request: Request, path_name: str): # Get the value of the first key key_name = list(data[0].keys())[0] rows = data[0][key_name] - data = { "value": [] } - - table_name = key_name + "s" - - host = request.url.scheme + "://" + request.url.netloc - for row in rows: data["value"].append({ - "@iot.selfLink": f"{host}/v1.1/{table_name}({row['id']})" + "@iot.selfLink": row["@iot.selfLink"] }) else: # Flatten the @iot.navigationLink for each row From 8a2b384120cf09ef069d5a68854e324e4398c9a2 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 10 Jul 2023 21:18:05 +0200 Subject: [PATCH 07/35] fix: handle root path --- fastapi/app/v1/endpoints/general.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 173380f..e2b4e52 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -13,6 +13,11 @@ def __flatten_navigation_links(row): @v1.api_route("/{path_name:path}", methods=["GET"]) async def catch_all(request: Request, path_name: str): try: + if not path_name: + # Handle the root path + # TODO(@filippofinke): handle the root path + return + # get full path from request full_path = request.url.path if request.url.query: From 7b40e5b6d448d2adf412ca71215893f40daf8979 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 10 Jul 2023 21:19:56 +0200 Subject: [PATCH 08/35] fix: default hostname --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0336047..de4b3d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: POSTGRES_PASSWORD: admin DATADIR: /var/lib/postgresql/data POSTGRES_MULTIPLE_EXTENSIONS: postgis,hstore,postgis_topology,postgis_raster,pgrouting,ltree,pg_cron,uuid-ossp - command: postgres -c custom.hostname="http://localhost:3000" + command: postgres -c custom.hostname="http://localhost:8018" volumes: # - v-istsos-miu-postgis-sql:/docker-entrypoint-initdb.d - v-istsos-miu-database-data:/var/lib/postgresql/data From 4f0718afeae1de0f9512c5d9cfca5a080dc61c01 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Tue, 11 Jul 2023 10:53:56 +0200 Subject: [PATCH 09/35] fix: add quotes for postgREST --- fastapi/app/sta2rest/sta2rest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py index 1fe91cc..63e6a26 100644 --- a/fastapi/app/sta2rest/sta2rest.py +++ b/fastapi/app/sta2rest/sta2rest.py @@ -34,6 +34,11 @@ def visit_IdentifierNode(self, node: ast.IdentifierNode): Returns: str: The converted identifier. """ + + # if the identifier starts with @ add a double quote + if node.name.startswith('@'): + return f'"{node.name}"' + return node.name def visit_SelectNode(self, node: ast.SelectNode): @@ -61,6 +66,9 @@ def visit_FilterNode(self, node: ast.FilterNode): str: The converted filter node. """ + # replace @iot.id with id + node.filter = node.filter.replace("@iot.id", "id") + # Parse the filter using the OData lexer and parser ast = odata_filter_parser.parse(odata_filter_lexer.tokenize(node.filter)) # Visit the tree to convert the filter From 2a7cd7432a121e935741c771b28ae764a538fb69 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Fri, 14 Jul 2023 14:56:53 +0200 Subject: [PATCH 10/35] feat: add root path --- fastapi/app/v1/endpoints/general.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index e2b4e52..c697052 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -4,6 +4,14 @@ v1 = APIRouter() +tables = ["Datastreams", "FeaturesOfInterest", "HistoricalLocations", "Locations", "Observations", "ObservedProperties", "Sensors", "Things"] +serverSettings = { + "conformance": [ + "http://www.opengis.net/spec/iot_sensing/1.1/req/request-data", + ], +} + + def __flatten_navigation_links(row): if "@iot.navigationLink" in row: # merge all the keys from the navigationLink @@ -15,8 +23,22 @@ async def catch_all(request: Request, path_name: str): try: if not path_name: # Handle the root path - # TODO(@filippofinke): handle the root path - return + value = [] + # append the domain to the path for each table + for table in tables: + value.append( + { + "name": table, + "url": + request.url._url + table, + } + ) + + response = { + "value": value, + "serverSettings": serverSettings, + } + return response # get full path from request full_path = request.url.path From f86336e88ae905b082e795b70c284a282d319f4c Mon Sep 17 00:00:00 2001 From: filippofinke Date: Fri, 14 Jul 2023 15:07:22 +0200 Subject: [PATCH 11/35] fix: format response --- fastapi/app/v1/endpoints/general.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index c697052..35cae39 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -18,6 +18,15 @@ def __flatten_navigation_links(row): row.update(row["@iot.navigationLink"]) del row["@iot.navigationLink"] +def __flatten_expand_entity(data): + # Check if there is only one key and it is in an ENTITY_MAPPING from the sta2rest module + if len(data[0].keys()) == 1 and list(data[0].keys())[0] in sta2rest.STA2REST.ENTITY_MAPPING: + # Get the value of the first key + key_name = list(data[0].keys())[0] + data = data[0][key_name] + + return data + @v1.api_route("/{path_name:path}", methods=["GET"]) async def catch_all(request: Request, path_name: str): try: @@ -59,7 +68,7 @@ async def catch_all(request: Request, path_name: str): data = r.json() if result['single_result']: - data = data[0] + data = __flatten_expand_entity(data)[0] if result['value']: # get the value of the first key data = data[list(data.keys())[0]] @@ -78,7 +87,8 @@ async def catch_all(request: Request, path_name: str): "@iot.selfLink": row["@iot.selfLink"] }) else: - # Flatten the @iot.navigationLink for each row + data = __flatten_expand_entity(data) + for row in data: __flatten_navigation_links(row) From 528cebe50a862940a1e3c5be7cfa02a5e5c952e6 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Fri, 14 Jul 2023 15:11:31 +0200 Subject: [PATCH 12/35] feat: throw error on fail --- fastapi/app/v1/endpoints/general.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 35cae39..f095200 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -1,6 +1,7 @@ +import httpx +import traceback from fastapi import APIRouter, Request from app.sta2rest import sta2rest -import httpx v1 = APIRouter() @@ -19,6 +20,11 @@ def __flatten_navigation_links(row): del row["@iot.navigationLink"] def __flatten_expand_entity(data): + # Check if it is an array + if not isinstance(data, list): + # throw an error + raise Exception(data) + # Check if there is only one key and it is in an ENTITY_MAPPING from the sta2rest module if len(data[0].keys()) == 1 and list(data[0].keys())[0] in sta2rest.STA2REST.ENTITY_MAPPING: # Get the value of the first key @@ -98,4 +104,7 @@ async def catch_all(request: Request, path_name: str): return data except Exception as e: + # print stack trace + traceback.print_exc() return {"error": str(e)} + From 04a8fd555b2524ba9df918f15d487d09ae7bc178 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 17 Jul 2023 16:58:02 +0200 Subject: [PATCH 13/35] fix: FeaturesOfInterest table name --- fastapi/app/sta2rest/sta2rest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py index 63e6a26..ce2c7a4 100644 --- a/fastapi/app/sta2rest/sta2rest.py +++ b/fastapi/app/sta2rest/sta2rest.py @@ -313,7 +313,7 @@ class STA2REST: "ObservedProperties": "ObservedProperty", "Datastreams": "Datastream", "Observations": "Observation", - "FeaturesOfInterest": "FeatureOfInterest", + "FeaturesOfInterest": "FeaturesOfInterest", "HistoricalLocations": "HistoricalLocation", "Thing": "Thing", @@ -322,7 +322,7 @@ class STA2REST: "ObservedProperty": "ObservedProperty", "Datastream": "Datastream", "Observation": "Observation", - "FeatureOfInterest": "FeatureOfInterest", + "FeatureOfInterest": "FeaturesOfInterest", "HistoricalLocation": "HistoricalLocation", } From 67ae6f51137e1ab32dda69280489c10dae7ad4a4 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 17 Jul 2023 17:16:05 +0200 Subject: [PATCH 14/35] fix: navigation link for single results --- database/istsos_schema.sql | 31 ++++++++++++++++------------- fastapi/app/v1/endpoints/general.py | 5 +++++ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/database/istsos_schema.sql b/database/istsos_schema.sql index 67f7ed1..5b5b749 100644 --- a/database/istsos_schema.sql +++ b/database/istsos_schema.sql @@ -135,37 +135,40 @@ $$ $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Sensor") RETURNS table( - "Datastreams@iot.navigationLink" text +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Observation") RETURNS table( + "FeatureOfInterest@iot.navigationLink" text, + "Datastream@iot.navigationLink" text ) AS $$ - SELECT 'Sensors(' || $1.id || ')/Datastreams'; + SELECT 'Observations(' || $1.id || ')/FeatureOfInterest', + 'Observations(' || $1.id || ')/Datastream'; $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."ObservedProperty") RETURNS table( - "Datastreams@iot.navigationLink" text +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."FeaturesOfInterest") RETURNS table( + "Observations@iot.navigationLink" text, + "skip@iot.navigationLink" text ) AS $$ - SELECT 'ObservedProperties(' || $1.id || ')/Datastreams'; + SELECT 'FeaturesOfInterest(' || $1.id || ')/Observations', 'skip'; $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Observation") RETURNS table( - "FeatureOfInterest@iot.navigationLink" text, - "Datastream@iot.navigationLink" text +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Sensor") RETURNS table( + "Datastreams@iot.navigationLink" text, + "skip@iot.navigationLink" text ) AS $$ - SELECT 'Observations(' || $1.id || ')/FeatureOfInterest', - 'Observations(' || $1.id || ')/Datastream'; + SELECT 'Sensors(' || $1.id || ')/Datastreams', 'skip'; $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."FeaturesOfInterest") RETURNS table( - "Observations@iot.navigationLink" text +CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."ObservedProperty") RETURNS table( + "Datastreams@iot.navigationLink" text, + "skip@iot.navigationLink" text ) AS $$ - SELECT 'FeaturesOfInterest(' || $1.id || ')/Observations'; + SELECT 'ObservedProperties(' || $1.id || ')/Datastreams', 'skip'; $$ LANGUAGE SQL; diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index f095200..777e7c4 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -14,11 +14,16 @@ def __flatten_navigation_links(row): + print(row) if "@iot.navigationLink" in row: # merge all the keys from the navigationLink row.update(row["@iot.navigationLink"]) del row["@iot.navigationLink"] + # check if skip@iot.navigationLink is present and remove it + if "skip@iot.navigationLink" in row: + del row["skip@iot.navigationLink"] + def __flatten_expand_entity(data): # Check if it is an array if not isinstance(data, list): From 2dbe110e1e7ef1606cbe193b1686a233bfd1cb2c Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 17 Jul 2023 17:16:23 +0200 Subject: [PATCH 15/35] fix: remove debug print --- fastapi/app/v1/endpoints/general.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 777e7c4..e2f0b17 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -14,7 +14,6 @@ def __flatten_navigation_links(row): - print(row) if "@iot.navigationLink" in row: # merge all the keys from the navigationLink row.update(row["@iot.navigationLink"]) From ce770fbe47526398e0416440c90118edb4ca348b Mon Sep 17 00:00:00 2001 From: filippofinke Date: Tue, 18 Jul 2023 17:28:27 +0200 Subject: [PATCH 16/35] fix: use env variable also for navlinks --- database/istsos_schema.sql | 34 +++++++++++++++++----------------- docker-compose.yml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/database/istsos_schema.sql b/database/istsos_schema.sql index 5b5b749..79617bd 100644 --- a/database/istsos_schema.sql +++ b/database/istsos_schema.sql @@ -86,7 +86,7 @@ CREATE OR REPLACE FUNCTION "@iot.id"(anyelement) RETURNS text AS $$ $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION "@iot.selfLink"(anyelement) RETURNS text AS $$ - SELECT concat(current_setting('custom.hostname'), '/v1.1/',substring(pg_typeof($1)::text from 2 for length(pg_typeof($1)::text) - 2),'(' || $1.id || ')'); + SELECT concat(current_setting('custom.hostname'),substring(pg_typeof($1)::text from 2 for length(pg_typeof($1)::text) - 2),'(' || $1.id || ')'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Thing") RETURNS table( @@ -95,9 +95,9 @@ CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Thing") RETURNS t "HistoricalLocations@iot.navigationLink" text ) AS $$ - SELECT 'Things(' || $1.id || ')/Locations', - 'Things(' || $1.id || ')/Datastreams', - 'Things(' || $1.id || ')/HistoricalLocations'; + SELECT concat(current_setting('custom.hostname'),'Things(' || $1.id || ')/Locations'), + concat(current_setting('custom.hostname'),'Things(' || $1.id || ')/Datastreams'), + concat(current_setting('custom.hostname'),'Things(' || $1.id || ')/HistoricalLocations'); $$ LANGUAGE SQL; @@ -106,8 +106,8 @@ CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Location") RETURN "HistoricalLocations@iot.navigationLink" text ) AS $$ - SELECT 'Locations(' || $1.id || ')/Things', - 'Locations(' || $1.id || ')/HistoricalLocations'; + SELECT concat(current_setting('custom.hostname'),'Locations(' || $1.id || ')/Things'), + concat(current_setting('custom.hostname'),'Locations(' || $1.id || ')/HistoricalLocations'); $$ LANGUAGE SQL; @@ -116,8 +116,8 @@ CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."HistoricalLocatio "Thing@iot.navigationLink" text ) AS $$ - SELECT 'HistoricalLocations(' || $1.id || ')/Locations', - 'HistoricalLocations(' || $1.id || ')/Thing'; + SELECT concat(current_setting('custom.hostname'),'HistoricalLocations(' || $1.id || ')/Locations'), + concat(current_setting('custom.hostname'),'HistoricalLocations(' || $1.id || ')/Thing'); $$ LANGUAGE SQL; @@ -128,10 +128,10 @@ CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Datastream") RETU "Observations@iot.navigationLink" text ) AS $$ - SELECT 'Datastreams(' || $1.id || ')/Thing', - 'Datastreams(' || $1.id || ')/Sensor', - 'Datastreams(' || $1.id || ')/ObservedProperty', - 'Datastreams(' || $1.id || ')/Observations'; + SELECT concat(current_setting('custom.hostname'),'Datastreams(' || $1.id || ')/Thing'), + concat(current_setting('custom.hostname'),'Datastreams(' || $1.id || ')/Sensor'), + concat(current_setting('custom.hostname'),'Datastreams(' || $1.id || ')/ObservedProperty'), + concat(current_setting('custom.hostname'),'Datastreams(' || $1.id || ')/Observations'); $$ LANGUAGE SQL; @@ -140,8 +140,8 @@ CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Observation") RET "Datastream@iot.navigationLink" text ) AS $$ - SELECT 'Observations(' || $1.id || ')/FeatureOfInterest', - 'Observations(' || $1.id || ')/Datastream'; + SELECT concat(current_setting('custom.hostname'),'Observations(' || $1.id || ')/FeatureOfInterest'), + concat(current_setting('custom.hostname'),'Observations(' || $1.id || ')/Datastream'); $$ LANGUAGE SQL; @@ -150,7 +150,7 @@ CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."FeaturesOfInteres "skip@iot.navigationLink" text ) AS $$ - SELECT 'FeaturesOfInterest(' || $1.id || ')/Observations', 'skip'; + SELECT concat(current_setting('custom.hostname'),'FeaturesOfInterest(' || $1.id || ')/Observations'), 'skip'; $$ LANGUAGE SQL; @@ -159,7 +159,7 @@ CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."Sensor") RETURNS "skip@iot.navigationLink" text ) AS $$ - SELECT 'Sensors(' || $1.id || ')/Datastreams', 'skip'; + SELECT concat(current_setting('custom.hostname'),'Sensors(' || $1.id || ')/Datastreams'), 'skip'; $$ LANGUAGE SQL; @@ -168,7 +168,7 @@ CREATE OR REPLACE FUNCTION "@iot.navigationLink"(sensorthings."ObservedProperty" "skip@iot.navigationLink" text ) AS $$ - SELECT 'ObservedProperties(' || $1.id || ')/Datastreams', 'skip'; + SELECT concat(current_setting('custom.hostname'),'ObservedProperties(' || $1.id || ')/Datastreams'), 'skip'; $$ LANGUAGE SQL; diff --git a/docker-compose.yml b/docker-compose.yml index de4b3d6..69c473c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: POSTGRES_PASSWORD: admin DATADIR: /var/lib/postgresql/data POSTGRES_MULTIPLE_EXTENSIONS: postgis,hstore,postgis_topology,postgis_raster,pgrouting,ltree,pg_cron,uuid-ossp - command: postgres -c custom.hostname="http://localhost:8018" + command: postgres -c custom.hostname="http://localhost:8018/v1.1/" volumes: # - v-istsos-miu-postgis-sql:/docker-entrypoint-initdb.d - v-istsos-miu-database-data:/var/lib/postgresql/data From 17403b4cff6db0dfa8d1d3b0bae9262e5a04498d Mon Sep 17 00:00:00 2001 From: filippofinke Date: Tue, 18 Jul 2023 17:36:05 +0200 Subject: [PATCH 17/35] feat: load .env file --- .env.sample | 1 + .gitignore | 3 ++- docker-compose.yml | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .env.sample diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..32467b0 --- /dev/null +++ b/.env.sample @@ -0,0 +1 @@ +HOSTNAME=http://localhost:8018/v1.1/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 127fd6a..7543aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Python stuff __pycache__/ *.py[cod] -.cache/ \ No newline at end of file +.cache/ +.env \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 69c473c..bd944ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,8 @@ version: "3.7" services: # POSTGRESQL database: + env_file: + - .env build: context: ./database dockerfile: Dockerfile @@ -14,7 +16,7 @@ services: POSTGRES_PASSWORD: admin DATADIR: /var/lib/postgresql/data POSTGRES_MULTIPLE_EXTENSIONS: postgis,hstore,postgis_topology,postgis_raster,pgrouting,ltree,pg_cron,uuid-ossp - command: postgres -c custom.hostname="http://localhost:8018/v1.1/" + command: postgres -c custom.hostname="${HOSTNAME}" volumes: # - v-istsos-miu-postgis-sql:/docker-entrypoint-initdb.d - v-istsos-miu-database-data:/var/lib/postgresql/data From 8ac3de7f8b62472bf44c606f6de85f3910c1e679 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Tue, 18 Jul 2023 20:03:00 +0200 Subject: [PATCH 18/35] feat: select only default columns --- fastapi/app/sta2rest/sta2rest.py | 122 +++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 8 deletions(-) diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py index ce2c7a4..f21bf8e 100644 --- a/fastapi/app/sta2rest/sta2rest.py +++ b/fastapi/app/sta2rest/sta2rest.py @@ -20,6 +20,16 @@ odata_filter_parser = ODataParser() class NodeVisitor(Visitor): + + main_entity = None + + """ + Constructor for the NodeVisitor class that accepts the main entity name + """ + def __init__(self, main_entity=None): + super().__init__() + self.main_entity = main_entity + """ This class provides a visitor to convert a STA query to a PostgREST query. """ @@ -224,7 +234,10 @@ def visit_ExpandNode(self, node: ast.ExpandNode, parent=None): if not expand_identifier.subquery or not expand_identifier.subquery.select: if not select: select = ast.SelectNode([]) - select.identifiers.append(ast.IdentifierNode(f'{expand_identifier.identifier}("@iot.id", "@iot.selfLink", *)')) + default_columns = STA2REST.get_default_column_names(expand_identifier.identifier) + # join default columns as single string + default_columns = ','.join(default_columns) + select.identifiers.append(ast.IdentifierNode(f'{expand_identifier.identifier}({default_columns})')) # Return the converted expand node return { @@ -274,12 +287,11 @@ def visit_QueryNode(self, node: ast.QueryNode): if not node.select: node.select = ast.SelectNode([]) # Add "@iot.id", "@iot.selfLink" and "*" to the select node - node.select.identifiers.extend(( - ast.IdentifierNode('"@iot.id"'), - ast.IdentifierNode('"@iot.selfLink"'), - ast.IdentifierNode('"@iot.navigationLink"'), - ast.IdentifierNode('*') - )) + + # get default columns for main entity + default_columns = STA2REST.get_default_column_names(self.main_entity) + for column in default_columns: + node.select.identifiers.append(ast.IdentifierNode(column)) # Check if we have a select, filter, orderby, skip, top or count in the query if node.select: @@ -326,6 +338,100 @@ class STA2REST: "HistoricalLocation": "HistoricalLocation", } + # Default columns for each entity + DEFAULT_SELECT = { + "Thing": [ + '"@iot.id"', + '"@iot.selfLink"', + '"@iot.navigationLink"', + 'name', + 'description', + 'properties', + ], + "Location": [ + '"@iot.id"', + '"@iot.selfLink"', + '"@iot.navigationLink"', + 'name', + 'description', + 'encodingType', + 'location', + 'properties', + ], + "Sensor": [ + '"@iot.id"', + '"@iot.selfLink"', + '"@iot.navigationLink"', + 'name', + 'description', + 'encodingType', + 'metadata', + 'properties', + ], + "ObservedProperty": [ + '"@iot.id"', + '"@iot.selfLink"', + '"@iot.navigationLink"', + 'name', + 'description', + 'definition', + 'properties', + ], + "Datastream": [ + '"@iot.id"', + '"@iot.selfLink"', + '"@iot.navigationLink"', + 'name', + 'description', + 'unitOfMeasurement', + 'observationType', + 'observedArea', + 'phenomenonTime', + 'resultTime', + 'properties', + ], + "Observation": [ + '"@iot.id"', + '"@iot.selfLink"', + '"@iot.navigationLink"', + 'phenomenonTime', + 'resultTime', + 'result', + 'resultQuality', + 'validTime', + 'parameters', + ], + "FeaturesOfInterest": [ + '"@iot.id"', + '"@iot.selfLink"', + '"@iot.navigationLink"', + 'name', + 'description', + 'encodingType', + 'feature', + 'properties', + ], + "HistoricalLocation": [ + '"@iot.id"', + '"@iot.selfLink"', + '"@iot.navigationLink"', + 'time', + ], + } + + @staticmethod + def get_default_column_names(entity: str) -> list: + """ + Get the default column names for a given entity. + + Args: + entity (str): The entity name. + + Returns: + list: The default column names. + """ + return STA2REST.DEFAULT_SELECT.get(entity, ["*"]) + @staticmethod def convert_entity(entity: str) -> str: """ @@ -441,7 +547,7 @@ def convert_query(full_path: str) -> str: single_result = True # Visit the query ast to convert it - visitor = NodeVisitor() + visitor = NodeVisitor(main_entity) query_converted = visitor.visit(query_ast) return { From a1fdd52153a8e8384caad3c25a81d8ca62f12725 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Tue, 18 Jul 2023 20:14:40 +0200 Subject: [PATCH 19/35] fix: return empty data --- fastapi/app/v1/endpoints/general.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index e2f0b17..84994fd 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -28,6 +28,10 @@ def __flatten_expand_entity(data): if not isinstance(data, list): # throw an error raise Exception(data) + + # check if data is empty + if not data: + return data # Check if there is only one key and it is in an ENTITY_MAPPING from the sta2rest module if len(data[0].keys()) == 1 and list(data[0].keys())[0] in sta2rest.STA2REST.ENTITY_MAPPING: From e30acf558279161f22132f3345498af0bf8c0fd5 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Tue, 18 Jul 2023 20:22:51 +0200 Subject: [PATCH 20/35] feat: add missing columns --- database/istsos_schema.sql | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/database/istsos_schema.sql b/database/istsos_schema.sql index 79617bd..87d973b 100644 --- a/database/istsos_schema.sql +++ b/database/istsos_schema.sql @@ -12,14 +12,15 @@ CREATE TABLE IF NOT EXISTS sensorthings."Location" ( "name" VARCHAR(255) UNIQUE NOT NULL, "description" TEXT NOT NULL, "encodingType" VARCHAR(100) NOT NULL, - "location" geometry(geometry, 4326) NOT NULL + "location" geometry(geometry, 4326) NOT NULL, + "properties" jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS sensorthings."Thing" ( "id" BIGSERIAL NOT NULL PRIMARY KEY, "name" VARCHAR(255) UNIQUE NOT NULL, "description" TEXT NOT NULL, - "properties" jsonb, + "properties" jsonb NOT NULL, "location_id" BIGINT REFERENCES sensorthings."Location" (id) ); @@ -34,16 +35,18 @@ CREATE TABLE IF NOT EXISTS sensorthings."HistoricalLocation" ( CREATE TABLE IF NOT EXISTS sensorthings."ObservedProperty" ( "id" BIGSERIAL PRIMARY KEY, "name" VARCHAR(255) UNIQUE NOT NULL, - --"definition" URI NOT NULL, "definition" VARCHAR(255) NOT NULL, - "description" VARCHAR(255) NOT NULL + "description" VARCHAR(255) NOT NULL, + "properties" jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS sensorthings."Sensor" ( "id" BIGSERIAL NOT NULL PRIMARY KEY, "name" VARCHAR(255) UNIQUE NOT NULL, + "description" VARCHAR(255) NOT NULL, "encodingType" VARCHAR(100) NOT NULL, - "metadata" jsonb NOT NULL + "metadata" jsonb NOT NULL, + "properties" jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS sensorthings."Datastream" ( @@ -55,6 +58,7 @@ CREATE TABLE IF NOT EXISTS sensorthings."Datastream" ( "observedArea" geometry(Polygon, 4326), "phenomenonTime" tstzrange, "resultTime" tstzrange, + "properties" jsonb, "thing_id" BIGINT REFERENCES sensorthings."Thing"(id) NOT NULL, "sensor_id" BIGINT REFERENCES sensorthings."Sensor"(id) NOT NULL, "observedproperty_id" BIGINT REFERENCES sensorthings."ObservedProperty"(id) @@ -64,8 +68,10 @@ CREATE TABLE IF NOT EXISTS sensorthings."Datastream" ( CREATE TABLE IF NOT EXISTS sensorthings."FeaturesOfInterest" ( "id" BIGSERIAL NOT NULL PRIMARY KEY, "name" VARCHAR(255) NOT NULL, + "description" VARCHAR(255) NOT NULL, "encodingType" VARCHAR(100) NOT NULL, - "feature" geometry(geometry, 4326) NOT NULL + "feature" geometry(geometry, 4326) NOT NULL, + "properties" jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS sensorthings."Observation" ( From 2997a7752ce2d550ed8477fe471ed53b1dc8e193 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Tue, 18 Jul 2023 20:23:03 +0200 Subject: [PATCH 21/35] feat: update example data to meet new schema --- database/istsos_example_data.sql | 371 +++++-------------------------- 1 file changed, 61 insertions(+), 310 deletions(-) diff --git a/database/istsos_example_data.sql b/database/istsos_example_data.sql index 197dc89..89796ba 100644 --- a/database/istsos_example_data.sql +++ b/database/istsos_example_data.sql @@ -1,340 +1,91 @@ -- location -insert - into - sensorthings."Location" ("name", - "description", - "encodingType", - "location") -values ('Room 101', -'The first room in the building', -'application/vnd.geo+json', -ST_SetSRID(ST_MakePoint(-73.987, -40.766), -4326)); - -insert - into - sensorthings."Location" ("name", - "description", - "encodingType", - "location") -values ('Room 102', -'The second room in the building', -'application/vnd.geo+json', -ST_SetSRID(ST_MakePoint(-73.987, -40.766), -4326)); - -insert - into - sensorthings."Location" ("name", - "description", - "encodingType", - "location") -values ('SIC 102', -'Lab at IIT Bombay', -'application/vnd.geo+json', -ST_SetSRID(ST_MakePoint(-19.131004980831737, 72.91701812621127), -4326)); - +INSERT INTO sensorthings."Location" ("name", "description", "encodingType", "location", "properties") +VALUES ('Room 101', 'The first room in the building', 'application/vnd.geo+json', ST_SetSRID(ST_MakePoint(-73.987, 40.766), 4326), '{}'); +INSERT INTO sensorthings."Location" ("name", "description", "encodingType", "location", "properties") +VALUES ('Room 102', 'The second room in the building', 'application/vnd.geo+json', ST_SetSRID(ST_MakePoint(-73.987, 40.766), 4326), '{}'); +INSERT INTO sensorthings."Location" ("name", "description", "encodingType", "location", "properties") +VALUES ('SIC 102', 'Lab at IIT Bombay', 'application/vnd.geo+json', ST_SetSRID(ST_MakePoint(-19.131004980831737, 72.91701812621127), 4326), '{}'); -- thing -insert - into - sensorthings."Thing" ("name", - "description", - "properties", - "location_id") -values ('Temperature Sensor', -'A sensor that measures the temperature in a room', -'{"manufacturer": "ACME Inc.", "model": "TS-100"}', -1); - -insert - into - sensorthings."Thing" ("name", - "description", - "properties", - "location_id") -values ('Humidity Sensor', -'A sensor that measures the humidity in a room', -'{"manufacturer": "ACME Inc.", "model": "TS-100"}', -2); - -insert - into - sensorthings."Thing" ("name", - "description", - "properties", - "location_id") -values ('Pressure Sensor', -'A sensor that measures the humidity in a room', -'{"manufacturer": "ACME Inc.", "model": "TS-100"}', -3); +INSERT INTO sensorthings."Thing" ("name", "description", "properties", "location_id") +VALUES ('Temperature Sensor', 'A sensor that measures the temperature in a room', '{"manufacturer": "ACME Inc.", "model": "TS-100"}', 1); + +INSERT INTO sensorthings."Thing" ("name", "description", "properties", "location_id") +VALUES ('Humidity Sensor', 'A sensor that measures the humidity in a room', '{"manufacturer": "ACME Inc.", "model": "TS-100"}', 2); + +INSERT INTO sensorthings."Thing" ("name", "description", "properties", "location_id") +VALUES ('Pressure Sensor', 'A sensor that measures the humidity in a room', '{"manufacturer": "ACME Inc.", "model": "TS-100"}', 3); -- historical location -insert - into - sensorthings."HistoricalLocation" ("time", - "thing_id", - "location_id") -values ('2023-03-25 10:00:00-04', -1, -1); - - -insert - into - sensorthings."HistoricalLocation" ("time", - "thing_id", - "location_id") -values ('2023-03-25 10:00:00-04', -2, -2); - -insert - into - sensorthings."HistoricalLocation" ("time", - "thing_id", - "location_id") -values ('2023-03-25 10:00:00-04', -3, -3); +INSERT INTO sensorthings."HistoricalLocation" ("time", "thing_id", "location_id") +VALUES ('2023-03-25 10:00:00-04', 1, 1); +INSERT INTO sensorthings."HistoricalLocation" ("time", "thing_id", "location_id") +VALUES ('2023-03-25 10:00:00-04', 2, 2); --- observed property +INSERT INTO sensorthings."HistoricalLocation" ("time", "thing_id", "location_id") +VALUES ('2023-03-25 10:00:00-04', 3, 3); -insert - into - sensorthings."ObservedProperty" ("name", - "definition", - "description") -values ('Temperature', -'http://www.qudt.org/qudt/owl/1.0.0/quantity/Instances.html#Temperature', -'The degree or intensity of heat present in a substance or object'); -insert - into - sensorthings."ObservedProperty" ("name", - "definition", - "description") -values ('Humidity', -'http://www.qudt.org/qudt/owl/1.0.0/quantity/Instances.html#Humidity', -'The percentage of humidity present in a substance or object'); +-- observed property + +INSERT INTO sensorthings."ObservedProperty" ("name", "definition", "description", "properties") +VALUES ('Temperature', 'http://www.qudt.org/qudt/owl/1.0.0/quantity/Instances.html#Temperature', 'The degree or intensity of heat present in a substance or object', '{}'); -insert - into - sensorthings."ObservedProperty" ("name", - "definition", - "description") -values ('Pressure', -'http://www.qudt.org/qudt/owl/1.0.0/quantity/Instances.html#Humidity', -'The percentage of humidity present in a substance or object'); +INSERT INTO sensorthings."ObservedProperty" ("name", "definition", "description", "properties") +VALUES ('Humidity', 'http://www.qudt.org/qudt/owl/1.0.0/quantity/Instances.html#Humidity', 'The percentage of humidity present in a substance or object', '{}'); +INSERT INTO sensorthings."ObservedProperty" ("name", "definition", "description", "properties") +VALUES ('Pressure', 'http://www.qudt.org/qudt/owl/1.0.0/quantity/Instances.html#Pressure', 'The pressure of a substance or object', '{}'); -- sensor -insert - into - sensorthings."Sensor" ("name", - "encodingType", - "metadata") -values ('Temperature Sensor', -'application/pdf', -'{"specification": "https://example.com/temperature-sensor-specs.pdf"}'); - -insert - into - sensorthings."Sensor" ("name", - "encodingType", - "metadata") -values ('Humidity Sensor', -'application/pdf', -'{"specification": "https://example.com/humidity-sensor-specs.pdf"}'); - -insert - into - sensorthings."Sensor" ("name", - "encodingType", - "metadata") -values ('Pressure Sensor', -'application/pdf', -'{"specification": "https://example.com/humidity-sensor-specs.pdf"}'); +INSERT INTO sensorthings."Sensor" ("name", "description", "encodingType", "metadata", "properties") +VALUES ('Temperature Sensor', 'A temperature sensor', 'application/pdf', '{"specification": "https://example.com/temperature-sensor-specs.pdf"}', '{}'); + +INSERT INTO sensorthings."Sensor" ("name", "description", "encodingType", "metadata", "properties") +VALUES ('Humidity Sensor', 'A humidity sensor', 'application/pdf', '{"specification": "https://example.com/humidity-sensor-specs.pdf"}', '{}'); + +INSERT INTO sensorthings."Sensor" ("name", "description", "encodingType", "metadata", "properties") +VALUES ('Pressure Sensor', 'A pressure sensor', 'application/pdf', '{"specification": "https://example.com/pressure-sensor-specs.pdf"}', '{}'); -- datastream -insert - into - sensorthings."Datastream" ("name", - "description", - "unitOfMeasurement", - "observationType", - "observedArea", - "phenomenonTime", - "resultTime", - "thing_id", - "sensor_id", - "observedproperty_id") -values ('Temperature Datastream', -'A datastream that provides the temperature measurements from a temperature sensor', -'{"name": "degree Celsius", "symbol": "degC", "definition": "http://www.qudt.org/qudt/owl/1.0.0/unit/Instances.html#DegreeCelsius"}', -'http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement', -ST_MakePolygon(ST_GeomFromText('LINESTRING(-73.987 40.766, -73.987 40.768, -73.983 40.768, -73.983 40.766, -73.987 40.766)')), -tstzrange('2023-03-25 10:00:00-04', -'2023-03-25 11:00:00-04'), -tstzrange('2023-03-25 10:00:00-04', -'2023-03-25 11:00:00-04'), -1, -1, -1); - - -insert - into - sensorthings."Datastream" ("name", - "description", - "unitOfMeasurement", - "observationType", - "observedArea", - "phenomenonTime", - "resultTime", - "thing_id", - "sensor_id", - "observedproperty_id") -values ('Humidity Datastream', -'A datastream that provides the temperature measurements from a temperature sensor', -'{"name": "degree Celsius", "symbol": "degC", "definition": "http://www.qudt.org/qudt/owl/1.0.0/unit/Instances.html#DegreeCelsius"}', -'http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement', -ST_MakePolygon(ST_GeomFromText('LINESTRING(-73.987 40.766, -73.987 40.768, -73.983 40.768, -73.983 40.766, -73.987 40.766)')), -tstzrange('2023-03-25 10:00:00-04', -'2023-03-25 11:00:00-04'), -tstzrange('2023-03-25 10:00:00-04', -'2023-03-25 11:00:00-04'), -2, -2, -2); - -insert - into - sensorthings."Datastream" ("name", - "description", - "unitOfMeasurement", - "observationType", - "observedArea", - "phenomenonTime", - "resultTime", - "thing_id", - "sensor_id", - "observedproperty_id") -values ('Pressure Datastream', -'A datastream that provides the temperature measurements from a temperature sensor', -'{"name": "degree Celsius", "symbol": "degC", "definition": "http://www.qudt.org/qudt/owl/1.0.0/unit/Instances.html#DegreeCelsius"}', -'http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement', -ST_MakePolygon(ST_GeomFromText('LINESTRING(-73.987 40.766, -73.987 40.768, -73.983 40.768, -73.983 40.766, -73.987 40.766)')), -tstzrange('2023-03-25 10:00:00-04', -'2023-03-25 11:00:00-04'), -tstzrange('2023-03-25 10:00:00-04', -'2023-03-25 11:00:00-04'), -3, -3, -3); +INSERT INTO sensorthings."Datastream" ("name", "description", "unitOfMeasurement", "observationType", "observedArea", "phenomenonTime", "resultTime", "properties", "thing_id", "sensor_id", "observedproperty_id") +VALUES ('Temperature Datastream', 'A datastream for temperature measurements', '{"name": "degree Celsius", "symbol": "degC", "definition": "http://www.qudt.org/qudt/owl/1.0.0/unit/Instances.html#DegreeCelsius"}', 'http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement', ST_MakePolygon(ST_GeomFromText('LINESTRING(-73.987 40.766, -73.987 40.768, -73.983 40.768, -73.983 40.766, -73.987 40.766)')), tstzrange('2023-03-25 10:00:00-04', '2023-03-25 11:00:00-04'), tstzrange('2023-03-25 10:00:00-04', '2023-03-25 11:00:00-04'), '{}', 1, 1, 1); + +INSERT INTO sensorthings."Datastream" ("name", "description", "unitOfMeasurement", "observationType", "observedArea", "phenomenonTime", "resultTime", "properties", "thing_id", "sensor_id", "observedproperty_id") +VALUES ('Humidity Datastream', 'A datastream for humidity measurements', '{"name": "percent", "symbol": "%", "definition": "http://www.qudt.org/qudt/owl/1.0.0/unit/Instances.html#Percent"}', 'http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement', ST_MakePolygon(ST_GeomFromText('LINESTRING(-73.987 40.766, -73.987 40.768, -73.983 40.768, -73.983 40.766, -73.987 40.766)')), tstzrange('2023-03-25 10:00:00-04', '2023-03-25 11:00:00-04'), tstzrange('2023-03-25 10:00:00-04', '2023-03-25 11:00:00-04'), '{}', 2, 2, 2); + +INSERT INTO sensorthings."Datastream" ("name", "description", "unitOfMeasurement", "observationType", "observedArea", "phenomenonTime", "resultTime", "properties", "thing_id", "sensor_id", "observedproperty_id") +VALUES ('Pressure Datastream', 'A datastream for pressure measurements', '{"name": "pascal", "symbol": "Pa", "definition": "http://www.qudt.org/qudt/owl/1.0.0/unit/Instances.html#Pascal"}', 'http://www.opengis.net/def/observationType/OGC-OM/2.0/OM_Measurement', ST_MakePolygon(ST_GeomFromText('LINESTRING(-73.987 40.766, -73.987 40.768, -73.983 40.768, -73.983 40.766, -73.987 40.766)')), tstzrange('2023-03-25 10:00:00-04', '2023-03-25 11:00:00-04'), tstzrange('2023-03-25 10:00:00-04', '2023-03-25 11:00:00-04'), '{}', 3, 3, 3); -- features of interest -insert - into - sensorthings."FeaturesOfInterest" ("name", - "encodingType", - "feature") -values ('Room 101', -'application/vnd.geo+json', -ST_SetSRID(ST_MakePoint(-73.987, -40.766), -4326)); - -insert - into - sensorthings."FeaturesOfInterest" ("name", - "encodingType", - "feature") -values ('SIC 102', -'application/vnd.geo+json', -ST_SetSRID(ST_MakePoint(-19.131004980831737, 72.91701812621127), -4326)); - -insert - into - sensorthings."FeaturesOfInterest" ("name", - "encodingType", - "feature") -values ('Room 102', -'application/vnd.geo+json', -ST_SetSRID(ST_MakePoint(-73.987, -40.766), -4326)); + +INSERT INTO sensorthings."FeaturesOfInterest" ("name", "description", "encodingType", "feature", "properties") +VALUES ('Room 101', 'Feature of interest for Room 101', 'application/vnd.geo+json', ST_SetSRID(ST_MakePoint(-73.987, 40.766), 4326), '{}'); + +INSERT INTO sensorthings."FeaturesOfInterest" ("name", "description", "encodingType", "feature", "properties") +VALUES ('SIC 102', 'Feature of interest for SIC 102', 'application/vnd.geo+json', ST_SetSRID(ST_MakePoint(-19.131004980831737, 72.91701812621127), 4326), '{}'); + +INSERT INTO sensorthings."FeaturesOfInterest" ("name", "description", "encodingType", "feature", "properties") +VALUES ('Room 102', 'Feature of interest for Room 102', 'application/vnd.geo+json', ST_SetSRID(ST_MakePoint(-73.987, 40.766), 4326), '{}'); + -- observation +INSERT INTO sensorthings."Observation" ("phenomenonTime", "resultTime", "result", "resultQuality", "validTime", "parameters", "datastream_id", "feature_of_interest_id") +VALUES ('2023-03-25 10:30:00-04', '2023-03-25 10:30:00-04', 23.5, NULL, NULL, NULL, 1, 1); + +INSERT INTO sensorthings."Observation" ("phenomenonTime", "resultTime", "result", "resultQuality", "validTime", "parameters", "datastream_id", "feature_of_interest_id") +VALUES ('2023-03-25 10:30:00-04', '2023-03-25 10:30:00-04', 23.5, NULL, NULL, NULL, 2, 2); -insert - into - sensorthings."Observation" ("phenomenonTime", - "resultTime", - "result", - "resultQuality", - "validTime", - "parameters", - "datastream_id", - "feature_of_interest_id") -values ('2023-03-25 10:30:00-04', -'2023-03-25 10:30:00-04', -23.5, -null, -null, -null, -1, -1); - -insert - into - sensorthings."Observation" ("phenomenonTime", - "resultTime", - "result", - "resultQuality", - "validTime", - "parameters", - "datastream_id", - "feature_of_interest_id") -values ('2023-03-25 10:30:00-04', -'2023-03-25 10:30:00-04', -23.5, -null, -null, -null, -2, -2); - -insert - into - sensorthings."Observation" ("phenomenonTime", - "resultTime", - "result", - "resultQuality", - "validTime", - "parameters", - "datastream_id", - "feature_of_interest_id") -values ('2023-03-25 10:30:00-04', -'2023-03-25 10:30:00-04', -23.5, -null, -null, -null, -3, -3); +INSERT INTO sensorthings."Observation" ("phenomenonTime", "resultTime", "result", "resultQuality", "validTime", "parameters", "datastream_id", "feature_of_interest_id") +VALUES ('2023-03-25 10:30:00-04', '2023-03-25 10:30:00-04', 23.5, NULL, NULL, NULL, 3, 3); From 35d0d0e71312761c27727d3677feede2c0b291ee Mon Sep 17 00:00:00 2001 From: filippofinke Date: Wed, 19 Jul 2023 10:19:02 +0200 Subject: [PATCH 22/35] fix: nested query --- fastapi/app/sta2rest/sta2rest.py | 18 +++++++++++++++--- fastapi/app/v1/endpoints/general.py | 9 +++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py index f21bf8e..30bc09b 100644 --- a/fastapi/app/sta2rest/sta2rest.py +++ b/fastapi/app/sta2rest/sta2rest.py @@ -470,9 +470,7 @@ def convert_query(full_path: str) -> str: if not uri: raise Exception("Error parsing uri") - - main_entity, main_entity_id = uri['entity'] - url = f"/{main_entity}" + # Check if we have a query query_ast = ast.QueryNode(None, None, None, None, None, None, None, False) @@ -482,7 +480,21 @@ def convert_query(full_path: str) -> str: parser = Parser(tokens) query_ast = parser.parse() + + main_entity, main_entity_id = uri['entity'] entities = uri['entities'] + + # Check if size of entities is bigger than 2 + if len(entities) > 1: + # Replace the main entity with the first entity + m = entities.pop(0) + main_entity = m[0] + main_entity_id = m[1] + single_result = True + + url = f"/{main_entity}" + + if entities: if not query_ast.expand: query_ast.expand = ast.ExpandNode([]) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 84994fd..087fdd5 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -32,7 +32,7 @@ def __flatten_expand_entity(data): # check if data is empty if not data: return data - + # Check if there is only one key and it is in an ENTITY_MAPPING from the sta2rest module if len(data[0].keys()) == 1 and list(data[0].keys())[0] in sta2rest.STA2REST.ENTITY_MAPPING: # Get the value of the first key @@ -82,7 +82,12 @@ async def catch_all(request: Request, path_name: str): data = r.json() if result['single_result']: - data = __flatten_expand_entity(data)[0] + data = __flatten_expand_entity(data) + + # check if the result is an array + if isinstance(data, list): + data = data[0] + if result['value']: # get the value of the first key data = data[list(data.keys())[0]] From 2561868fe1ca88f78db9e35ec747efebcd3625b2 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Wed, 19 Jul 2023 10:37:00 +0200 Subject: [PATCH 23/35] fix: $ref with multiple values --- fastapi/app/v1/endpoints/general.py | 45 ++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 087fdd5..00e9d67 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -41,6 +41,32 @@ def __flatten_expand_entity(data): return data +def __create_ref_format(data): + + rows = [data] + + # Check if it is an array + if isinstance(data, list): + key_name = list(data[0].keys())[0] + # Check if the key is in an ENTITY_MAPPING from the sta2rest module + if key_name in sta2rest.STA2REST.ENTITY_MAPPING: + rows = data[0][key_name] + if not isinstance(rows, list): + rows = [rows] + else: + rows = data + + data = { + "value": [] + } + + for row in rows: + data["value"].append({ + "@iot.selfLink": row["@iot.selfLink"] + }) + + return data + @v1.api_route("/{path_name:path}", methods=["GET"]) async def catch_all(request: Request, path_name: str): try: @@ -81,6 +107,9 @@ async def catch_all(request: Request, path_name: str): r = await client.get(url) data = r.json() + print("data:\t\t", data) + print("result:\t\t", result) + if result['single_result']: data = __flatten_expand_entity(data) @@ -91,22 +120,18 @@ async def catch_all(request: Request, path_name: str): if result['value']: # get the value of the first key data = data[list(data.keys())[0]] + elif result['ref']: + data = __create_ref_format(data) elif "@iot.navigationLink" in data: __flatten_navigation_links(data) elif result['ref']: - # Get the value of the first key - key_name = list(data[0].keys())[0] - rows = data[0][key_name] - data = { - "value": [] - } - for row in rows: - data["value"].append({ - "@iot.selfLink": row["@iot.selfLink"] - }) + data = __create_ref_format(data) else: data = __flatten_expand_entity(data) + # check if the result is an array + if not isinstance(data, list): + data = [data] for row in data: __flatten_navigation_links(row) From 3d596b34eb58dad29cde7b96689a81b7e7c16efa Mon Sep 17 00:00:00 2001 From: filippofinke Date: Wed, 19 Jul 2023 10:41:34 +0200 Subject: [PATCH 24/35] refactor: remove debug messages --- fastapi/app/v1/endpoints/general.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 00e9d67..e3d5ad0 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -106,10 +106,6 @@ async def catch_all(request: Request, path_name: str): async with httpx.AsyncClient() as client: r = await client.get(url) data = r.json() - - print("data:\t\t", data) - print("result:\t\t", result) - if result['single_result']: data = __flatten_expand_entity(data) From 3835bc25b6751d7f666fcb1b955c1c91f8443d8d Mon Sep 17 00:00:00 2001 From: filippofinke Date: Thu, 20 Jul 2023 10:54:11 +0200 Subject: [PATCH 25/35] fix: support json columns --- fastapi/app/sta2rest/sta2rest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py index 30bc09b..da0fcb3 100644 --- a/fastapi/app/sta2rest/sta2rest.py +++ b/fastapi/app/sta2rest/sta2rest.py @@ -48,6 +48,9 @@ def visit_IdentifierNode(self, node: ast.IdentifierNode): # if the identifier starts with @ add a double quote if node.name.startswith('@'): return f'"{node.name}"' + + # Replace / with -> for json columns + node.name = node.name.replace('/', '->>') return node.name From 791d4217ed3c3c79c115f6b2aa15680ec8cdfb10 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Thu, 20 Jul 2023 10:58:28 +0200 Subject: [PATCH 26/35] refactor: root path handler --- fastapi/app/v1/endpoints/general.py | 42 ++++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index e3d5ad0..faf049b 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -67,28 +67,32 @@ def __create_ref_format(data): return data +def __handle_root(request: Request): + # Handle the root path + value = [] + # append the domain to the path for each table + for table in tables: + value.append( + { + "name": table, + "url": + request.url._url + table, + } + ) + + response = { + "value": value, + "serverSettings": serverSettings, + } + return response + @v1.api_route("/{path_name:path}", methods=["GET"]) async def catch_all(request: Request, path_name: str): - try: - if not path_name: - # Handle the root path - value = [] - # append the domain to the path for each table - for table in tables: - value.append( - { - "name": table, - "url": - request.url._url + table, - } - ) - - response = { - "value": value, - "serverSettings": serverSettings, - } - return response + if not path_name: + # Handle the root path + return __handle_root(request) + try: # get full path from request full_path = request.url.path if request.url.query: From 3f52edee6096edc8d0f4379b32b2426e2050e236 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Thu, 20 Jul 2023 11:02:51 +0200 Subject: [PATCH 27/35] feat: better error messages --- fastapi/app/v1/endpoints/general.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index faf049b..79cdcf3 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -28,7 +28,7 @@ def __flatten_expand_entity(data): if not isinstance(data, list): # throw an error raise Exception(data) - + # check if data is empty if not data: return data @@ -110,6 +110,11 @@ async def catch_all(request: Request, path_name: str): async with httpx.AsyncClient() as client: r = await client.get(url) data = r.json() + + # print r status + if r.status_code != 200: + raise Exception(data["message"]) + if result['single_result']: data = __flatten_expand_entity(data) From 714e92440f16f95f9f9e0f4bf1d97bacd90047e9 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Thu, 20 Jul 2023 11:05:06 +0200 Subject: [PATCH 28/35] refactor: error message result --- fastapi/app/v1/endpoints/general.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 79cdcf3..248dc95 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -149,5 +149,9 @@ async def catch_all(request: Request, path_name: str): except Exception as e: # print stack trace traceback.print_exc() - return {"error": str(e)} + return { + "code": 404, + "type": "error", + "message": str(e) + } From e7b6afca45d3b1d1dcb2095a6e0375bb3341a431 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Thu, 20 Jul 2023 11:06:56 +0200 Subject: [PATCH 29/35] fix: return 404 status code on error --- fastapi/app/v1/endpoints/general.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 248dc95..03c3751 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -1,6 +1,8 @@ import httpx import traceback from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse +from fastapi import status from app.sta2rest import sta2rest v1 = APIRouter() @@ -149,9 +151,12 @@ async def catch_all(request: Request, path_name: str): except Exception as e: # print stack trace traceback.print_exc() - return { - "code": 404, - "type": "error", - "message": str(e) - } + return JSONResponse( + status_code=status.HTTP_404_NOT_FOUND, + content={ + "code": 404, + "type": "error", + "message": str(e) + } + ) From 368bd2a7ef2f06cbc1061f91cd2afecfa9ac4e82 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Thu, 20 Jul 2023 11:11:06 +0200 Subject: [PATCH 30/35] fix: distinguish between syntax and postgrest error --- fastapi/app/v1/endpoints/general.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 03c3751..78bdc3a 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -14,6 +14,9 @@ ], } +class PostgRESTError(Exception): + pass + def __flatten_navigation_links(row): if "@iot.navigationLink" in row: @@ -29,7 +32,7 @@ def __flatten_expand_entity(data): # Check if it is an array if not isinstance(data, list): # throw an error - raise Exception(data) + raise PostgRESTError(data) # check if data is empty if not data: @@ -115,7 +118,7 @@ async def catch_all(request: Request, path_name: str): # print r status if r.status_code != 200: - raise Exception(data["message"]) + raise PostgRESTError(data["message"]) if result['single_result']: data = __flatten_expand_entity(data) @@ -148,14 +151,24 @@ async def catch_all(request: Request, path_name: str): } return data - except Exception as e: - # print stack trace + except PostgRESTError as pge: traceback.print_exc() return JSONResponse( status_code=status.HTTP_404_NOT_FOUND, content={ "code": 404, "type": "error", + "message": str(pge) + } + ) + except Exception as e: + # print stack trace + traceback.print_exc() + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content={ + "code": 400, + "type": "error", "message": str(e) } ) From f210869a92b4ee9ec3672755b408ba3a61784991 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Fri, 21 Jul 2023 22:44:55 +0200 Subject: [PATCH 31/35] feat: basic POST handler --- fastapi/app/v1/endpoints/general.py | 35 ++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 78bdc3a..8ecd523 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -92,7 +92,7 @@ def __handle_root(request: Request): return response @v1.api_route("/{path_name:path}", methods=["GET"]) -async def catch_all(request: Request, path_name: str): +async def catch_all_get(request: Request, path_name: str): if not path_name: # Handle the root path return __handle_root(request) @@ -173,3 +173,36 @@ async def catch_all(request: Request, path_name: str): } ) +# Handle POST requests +@v1.api_route("/{path_name:path}", methods=["POST"]) +async def catch_all_post(request: Request, path_name: str): + try: + full_path = request.url.path + + # parse uri + result = sta2rest.STA2REST.parse_uri(full_path) + + + print("original:\t", full_path) + print("result:\t\t", result) + + # Return okay + return JSONResponse( + status_code=status.HTTP_200_OK, + content={ + "code": 200, + "type": "success", + "message": result + } + ) + except Exception as e: + # print stack trace + traceback.print_exc() + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content={ + "code": 400, + "type": "error", + "message": str(e) + } + ) From 297cc0abb18b5c66baf6e96c6acb129f7ecf6aef Mon Sep 17 00:00:00 2001 From: filippofinke Date: Fri, 21 Jul 2023 22:53:41 +0200 Subject: [PATCH 32/35] feat: basic entities creation --- fastapi/app/v1/endpoints/general.py | 50 +++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 8ecd523..149e30b 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -176,25 +176,55 @@ async def catch_all_get(request: Request, path_name: str): # Handle POST requests @v1.api_route("/{path_name:path}", methods=["POST"]) async def catch_all_post(request: Request, path_name: str): + # Accept only content-type application/json + if not "content-type" in request.headers or request.headers["content-type"] != "application/json": + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content={ + "code": 400, + "type": "error", + "message": "Only content-type application/json is supported." + } + ) + try: full_path = request.url.path # parse uri result = sta2rest.STA2REST.parse_uri(full_path) + # get json body + body = await request.json() print("original:\t", full_path) - print("result:\t\t", result) + print("parsed:\t", result) + print("body:\t", body) - # Return okay - return JSONResponse( - status_code=status.HTTP_200_OK, - content={ - "code": 200, - "type": "success", - "message": result - } - ) + main_table = result["entity"][0] + + url = "http://postgrest:3000/" + main_table + + async with httpx.AsyncClient() as client: + # post to postgrest + r = await client.post(url, json=body, headers={"Prefer": "return=representation"}) + + # get response + result = r.json() + + # print r status + if r.status_code != 201: + raise PostgRESTError(result["message"]) + + # Return okay + return JSONResponse( + status_code=status.HTTP_200_OK, + content={ + "code": 200, + "type": "success", + "message": result + } + ) + except Exception as e: # print stack trace traceback.print_exc() From 76a8ba0a090e335725066ee8215fb027e15cc231 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Fri, 21 Jul 2023 23:00:24 +0200 Subject: [PATCH 33/35] fix: empty data --- fastapi/app/v1/endpoints/general.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 149e30b..5e2917c 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -19,7 +19,7 @@ class PostgRESTError(Exception): def __flatten_navigation_links(row): - if "@iot.navigationLink" in row: + if row and "@iot.navigationLink" in row: # merge all the keys from the navigationLink row.update(row["@iot.navigationLink"]) del row["@iot.navigationLink"] @@ -141,7 +141,7 @@ async def catch_all_get(request: Request, path_name: str): data = __flatten_expand_entity(data) # check if the result is an array if not isinstance(data, list): - data = [data] + data = [data] if data else [] for row in data: __flatten_navigation_links(row) @@ -149,7 +149,6 @@ async def catch_all_get(request: Request, path_name: str): data = { "value": data } - return data except PostgRESTError as pge: traceback.print_exc() From 2d495a4171907bba414604d0d52a254b6601e5e5 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Fri, 21 Jul 2023 23:03:34 +0200 Subject: [PATCH 34/35] refactor: remove unused scripts --- fastapi/app/v1/fetch_postgrest.py | 35 --------------- fastapi/app/v1/get_datastream.py | 75 ------------------------------- 2 files changed, 110 deletions(-) delete mode 100644 fastapi/app/v1/fetch_postgrest.py delete mode 100644 fastapi/app/v1/get_datastream.py diff --git a/fastapi/app/v1/fetch_postgrest.py b/fastapi/app/v1/fetch_postgrest.py deleted file mode 100644 index 8bd2a15..0000000 --- a/fastapi/app/v1/fetch_postgrest.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -sys.path.append('../../../src/sta2rest') - -import requests -import urllib.parse -from fastapi import FastAPI, Request -from sta2rest import STA2REST - -""" -Example query: http://127.0.0.1:8000/Datastream?$filter=thing_id eq 1 or sensor_id eq 2 -""" - -#fetching database data from postgrest -app = FastAPI() - -#sending response to client -# print(p) -@app.get("/Datastream") -async def root(request: Request): - try: - # Get the original query parameters from the request - query_params = urllib.parse.unquote_plus(str(request.query_params)) - print("Original query: ", query_params) - # Convert the STA query to a PostgREST query - converted_query = STA2REST.convert_query(query_params) - print("Converted query: ", converted_query) - - # Send the converted query to PostgREST - r = requests.get('http://localhost:3000/Datastream?' + converted_query) - - # Return the response from PostgREST - return r.json() - except Exception as e: - # send the error message to the client - return {"error": str(e)} \ No newline at end of file diff --git a/fastapi/app/v1/get_datastream.py b/fastapi/app/v1/get_datastream.py deleted file mode 100644 index d786d42..0000000 --- a/fastapi/app/v1/get_datastream.py +++ /dev/null @@ -1,75 +0,0 @@ -from fastapi import FastAPI, HTTPException -from typing import List -from pydantic import BaseModel -import psycopg2 - -# Define a Pydantic model for the Datastream data -class Datastream(BaseModel): - id: int - name: str - description: str - unitOfMeasurement: dict - observationType: str - observedArea: str - # phenomenonTime:str - thing_id: int - sensor_id: int - observedproperty_id: int - -# Create a FastAPI instance -app = FastAPI() - -# Endpoint to fetch Datastream data by ID from the database -@app.get("/datastreams/{datastream_id}", response_model=Datastream) -async def get_datastream_by_id(datastream_id: int): - # Connect to the PostgreSQL database - conn = psycopg2.connect( - host="172.17.0.1", - port="45432", - database="istsos", - user="admin", - - password="admin" - - - ) - - # Create a cursor object to execute SQL queries - cursor = conn.cursor() - - # Execute the SQL query to fetch the Datastream data by ID from the database - cursor.execute(""" - SELECT - id, name, description, "unitOfMeasurement","observationType","observedArea",thing_id,sensor_id,observedproperty_id - FROM - sensorthings."Datastream" - WHERE - id = %s - """, (datastream_id,)) - - # Fetch the row returned by the query - row = cursor.fetchone() - - # Close the cursor and database connection - cursor.close() - conn.close() - - # If no datastream is found for the provided ID, raise an HTTPException - if row is None: - raise HTTPException(status_code=404, detail="Datastream not found") - - # Create a Datastream object from the retrieved row - datastream = Datastream( - id=row[0], - name=row[1], - description=row[2], - unitOfMeasurement=row[3], - observationType=row[4], - observedArea=row[5], - thing_id= row[6], - sensor_id= row[7], - observedproperty_id=row[8] - - ) - - return datastream From f4e8fc2dd1268286d5245fc439a6b0cbc9c2b909 Mon Sep 17 00:00:00 2001 From: filippofinke Date: Mon, 24 Jul 2023 12:01:43 +0200 Subject: [PATCH 35/35] feat: convert id to database mapping --- fastapi/app/sta2rest/sta2rest.py | 8 +++++++- fastapi/app/v1/endpoints/general.py | 23 ++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py index da0fcb3..4d14f3c 100644 --- a/fastapi/app/sta2rest/sta2rest.py +++ b/fastapi/app/sta2rest/sta2rest.py @@ -447,7 +447,13 @@ def convert_entity(entity: str) -> str: str: The converted entity name in REST format. """ return STA2REST.ENTITY_MAPPING.get(entity, entity) - + + @staticmethod + def convert_to_database_id(entity: str) -> str: + # First we convert the entity to lower case + entity = STA2REST.convert_entity(entity).lower() + return entity + "_id" + @staticmethod def convert_query(full_path: str) -> str: """ diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py index 5e2917c..dc67a69 100644 --- a/fastapi/app/v1/endpoints/general.py +++ b/fastapi/app/v1/endpoints/general.py @@ -203,9 +203,30 @@ async def catch_all_post(request: Request, path_name: str): url = "http://postgrest:3000/" + main_table + + formatted_body = {} + + # Loop trought all the keys in the body + for key in body: + if key in sta2rest.STA2REST.ENTITY_MAPPING: + value = body[key] + if "@iot.id" in value: + # Convert the id + new_key = sta2rest.STA2REST.convert_to_database_id(key) + formatted_body[new_key] = value["@iot.id"] + else: + # TODO(@filippofinke): Create nested entities + pass + else: + formatted_body[key] = body[key] + + print("ORIGINAL BODY:", body) + + print("FORMATTED BODY:", formatted_body) + async with httpx.AsyncClient() as client: # post to postgrest - r = await client.post(url, json=body, headers={"Prefer": "return=representation"}) + r = await client.post(url, json=formatted_body, headers={"Prefer": "return=representation"}) # get response result = r.json()