From 00933844ff816a176daeed746e1bc4274066488b Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 28 Jun 2023 11:27:42 +0200
Subject: [PATCH 01/23] docs: add comments

---
 src/sta2rest/sta2rest.py | 124 ++++++++++++++++++++++++++++++++++++---
 src/sta2rest/test.py     |  18 ++++++
 2 files changed, 135 insertions(+), 7 deletions(-)

diff --git a/src/sta2rest/sta2rest.py b/src/sta2rest/sta2rest.py
index 7c426e9..5a56e99 100644
--- a/src/sta2rest/sta2rest.py
+++ b/src/sta2rest/sta2rest.py
@@ -16,6 +16,12 @@
 odata_filter_parser = ODataParser()
 
 class STA2REST:
+    """
+    This class provides utility functions to convert various elements used in SensorThings queries to their corresponding
+    representations in a compatible PostgREST REST API.
+    """
+
+    # Mapping from SensorThings entities to their corresponding database table names
     ENTITY_MAPPING = {
         "Things": "Thing",
         "Locations": "Location",
@@ -27,6 +33,7 @@ class STA2REST:
         "HistoricalLocations": "HistoricalLocation",
     }
 
+    # Mapping from SensorThings properties to their corresponding database column names
     PROPERTY_MAPPING = {
         "name": "name",
         "description": "description",
@@ -35,6 +42,7 @@ class STA2REST:
         "definition": "definition",
     }
 
+    # Mapping from SensorThings query parameters to their corresponding PostgRESTR query parameters
     QUERY_PARAM_MAPPING = {
         "$filter": "filter",
         "$orderby": "order",
@@ -47,27 +55,70 @@ class STA2REST:
 
     @staticmethod
     def convert_entity(entity: str) -> str:
+        """
+        Converts an entity name from STA format to REST format.
+
+        Args:
+            entity (str): The entity name in STA format.
+
+        Returns:
+            str: The converted entity name in REST format.
+        """
         return STA2REST.ENTITY_MAPPING.get(entity, entity)
 
     @staticmethod
     def convert_property(property_name: str) -> str:
+        """
+        Converts a property name from STA format to REST format.
+
+        Args:
+            property_name (str): The property name in STA format.
+
+        Returns:
+            str: The converted property name in REST format.
+        """
         return STA2REST.PROPERTY_MAPPING.get(property_name, property_name)
 
     @staticmethod
     def convert_query_param(query_param: str) -> str:
+        """
+        Converts a query parameter name from STA format to REST format.
+
+        Args:
+            query_param (str): The query parameter name in STA format.
+
+        Returns:
+            str: The converted query parameter name in REST format.
+        """
         return STA2REST.QUERY_PARAM_MAPPING.get(query_param, query_param)
 
     @staticmethod
     def split_but_not_between(query: str, split: str, delim1: str, delim2: str) -> list:
+        """
+        Splits a string by a given character, but only if the character is not between two other given characters.
+
+        Args:
+            query (str): The string to split.
+            split (str): The character to split by.
+            delim1 (str): The first delimiter character.
+            delim2 (str): The second delimiter character.
+
+        Returns:
+            list: The list of substrings.
+        """
+
         result = []
         stack = []
         current = ''
         for char in query:
+            # Check if the current character is a delimiter
             if char == delim1:
                 current += char
                 stack.append(delim1)
+            # Check if the current character is a delimiter
             elif char == delim2:
                 current += char
+                # Check if the current delimiter is the same as the last one in the stack
                 if stack and stack[-1] == delim1:
                     stack.pop()
             elif char == split:
@@ -82,6 +133,7 @@ def split_but_not_between(query: str, split: str, delim1: str, delim2: str) -> l
                 # Regular character, add it to the current substring
                 current += char
         
+        # Add the last substring
         if current:
             result.append(current.strip())
 
@@ -89,6 +141,16 @@ def split_but_not_between(query: str, split: str, delim1: str, delim2: str) -> l
 
     @staticmethod
     def convert_order_by_value(value: str) -> str:
+        """
+        Converts an order by value from STA format to REST format.
+
+        Args:
+            value (str): The order by value in STA format.
+
+        Returns:
+            str: The converted order by value in REST format.
+        """
+
         # strip all the spaces after commas ",   " -> ","
         value = re.sub(r",\s+", ",", value)
         # replace space and slash with dot
@@ -97,15 +159,38 @@ def convert_order_by_value(value: str) -> str:
 
     @staticmethod
     def convert_filter_by_value(value: str) -> str:
-        # see https://docs.ogc.org/is/18-088/18-088.html#_built_in_filter_operations
-        # see https://postgrest.org/en/stable/references/api/tables_views.html#logical-operators
-        
+        """
+        Converts a filter by value from STA format to REST format.
+
+        see https://docs.ogc.org/is/18-088/18-088.html#_built_in_filter_operations
+        see https://postgrest.org/en/stable/references/api/tables_views.html#logical-operators
+
+        Args:
+            value (str): The filter by value in STA format.
+
+        Returns:
+            str: The converted filter by value in REST format.
+        """
+        # Get the AST tree from the filter
         ast = odata_filter_parser.parse(odata_filter_lexer.tokenize(value))
+        # Visit the tree to convert the filter
         res = FilterVisitor().visit(ast)
         return res
 
     @staticmethod
     def convert_expand(expand_query: str, previous_entity: str = None) -> str:
+        """
+        Converts an expand query from STA format to REST format.
+
+        Args:
+            expand_query (str): The expand query in STA format.
+            previous_entity (str, optional): The previous entity name. Defaults to None.
+
+        Returns:
+            str: The converted expand query in REST format.
+        """
+
+        # split by comma but not between parentheses
         result = STA2REST.split_but_not_between(expand_query, ",", "(", ")")
     
         entities = []
@@ -113,16 +198,19 @@ def convert_expand(expand_query: str, previous_entity: str = None) -> str:
         for entity in result:
             converted_entity = ""
             has_select = False
+            # Check if the entity has a subquery
             if "(" in entity:
                 entity, subquery = entity.split("(",1)
                 c_entity = STA2REST.convert_entity(entity)
                 converted_entity = c_entity + "("
                 subquery = subquery[:-1]
+                # Get all the actions in the subquery
                 actions = STA2REST.split_but_not_between(subquery, ";", "(", ")")
                 for action in actions:
                     param, value = action.split("=",1)
                     converted_param = STA2REST.convert_query_param(param)
                     if param == "$expand":
+                        # Recursively convert the subquery
                         result = STA2REST.convert_expand(value, c_entity)
                         entities.extend(result["select"].split(","))
                         additionals += result["additionals"]
@@ -136,8 +224,10 @@ def convert_expand(expand_query: str, previous_entity: str = None) -> str:
                         elif param == "$filter":
                             value = STA2REST.convert_filter_by_value(value)
 
+                        # Add the previous entity if present
                         p_entity = previous_entity + "." if previous_entity != None else ""
 
+                        # Add the action to the additionals
                         if param != "$filter":
                             additionals.append(f"{p_entity}{c_entity}.{converted_param}={value}")
                         else:
@@ -157,11 +247,24 @@ def convert_expand(expand_query: str, previous_entity: str = None) -> str:
 
     @staticmethod
     def convert_query(sta_query: str) -> str:
-        # url decode the query
+        """
+        Converts a query from STA format to REST format.
+
+        Args:
+            sta_query (str): The query in STA format.
+
+        Returns:
+            str: The converted query in REST format.
+        """
+
+        # remove unwanted characters from the query
         sta_query = urllib.parse.unquote(sta_query)
 
+        # get the query parameters
         sta_query = sta_query.split("&")
         converted_query_params = {}
+
+        # convert the query parameters
         for query_param in sta_query:
             key, value = query_param.split("=", 1)
             if key == "$expand":
@@ -182,29 +285,36 @@ def convert_query(sta_query: str) -> str:
             else:
                 converted_query_params["select"] = converted_query_params["expand"]["select"]
         
-        # Get additionals and remove them from the query
+        # Get additionals parameters and remove them from the query
         additionals = []
         if "expand" in converted_query_params and "additionals" in converted_query_params["expand"]:
             additionals = converted_query_params["expand"]["additionals"]
             del converted_query_params["expand"]
 
+        # Check if the filter is present and move it to the additionals
         if "filter" in converted_query_params:
             additionals.append(converted_query_params["filter"])
             del converted_query_params["filter"]
 
-        # merge in format key=value&key=value 
+        # Merge in format key=value&key=value 
         converted_query = "&".join([f"{key}={value}" for key, value in converted_query_params.items()])
         
+        # Add additionals
         for additional in additionals:
             converted_query += "&" + additional
 
-        # remove the first & if present
+        # Remove the first & if present
         if converted_query.startswith("&"):
             converted_query = converted_query[1:]
         
         return converted_query
 
 if __name__ == "__main__":
+    """
+    Example usage of the STA2REST module.
+
+    This example converts a STA query to a REST query.
+    """
     query = "$filter=result gt 20 or result le 3.5"
     print("QUERY", query)
     print("CONVERTED", STA2REST.convert_query(query))
\ No newline at end of file
diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
index 3818482..daa54be 100644
--- a/src/sta2rest/test.py
+++ b/src/sta2rest/test.py
@@ -9,7 +9,13 @@
 from sta2rest import STA2REST
 
 class STA2RESTTestCase(unittest.TestCase):
+    """
+    Test case for STA2REST module.
+    """
     def test_convert_entity(self):
+        """
+        Test the conversion of entities.
+        """
         entity_mappings = {
             "Things": 'Thing',
             "Locations": 'Location',
@@ -26,6 +32,10 @@ def test_convert_entity(self):
             self.assertEqual(STA2REST.convert_entity(entity), expected)
 
     def test_convert_property(self):
+        """
+        Test the conversion of properties.
+        """
+
         property_mappings = {
             "name": 'name',
             "description": 'description',
@@ -38,6 +48,10 @@ def test_convert_property(self):
             self.assertEqual(STA2REST.convert_property(prop), expected)
 
     def test_convert_query_param(self):
+        """
+        Test the conversion of query parameters.
+        """
+
         query_param_mappings = {
             "$orderby": "order",
             "$top": "limit",
@@ -49,6 +63,9 @@ def test_convert_query_param(self):
             self.assertEqual(STA2REST.convert_query_param(param), expected)
 
     def test_convert_sensor_things_query(self):
+        """
+        Test the conversion of sensor things queries.
+        """
         query_mappings = {
             "$filter=type eq 'temperature'&$orderby=timestamp desc&$top=10&$skip=5":
                 "order=timestamp.desc&limit=10&offset=5&type=eq.temperature",
@@ -85,4 +102,5 @@ def test_convert_sensor_things_query(self):
             self.assertEqual(STA2REST.convert_query(query), expected)
 
 if __name__ == '__main__':
+    # Run all tests
     unittest.main()
\ No newline at end of file

From 7043f021bb21d3e14989f54f63fd3626954adc20 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 28 Jun 2023 11:28:00 +0200
Subject: [PATCH 02/23] refactor: remove unused operators (for now)

---
 src/sta2rest/filter_visitor.py | 92 +++++-----------------------------
 1 file changed, 12 insertions(+), 80 deletions(-)

diff --git a/src/sta2rest/filter_visitor.py b/src/sta2rest/filter_visitor.py
index 266faa0..7bca763 100644
--- a/src/sta2rest/filter_visitor.py
+++ b/src/sta2rest/filter_visitor.py
@@ -1,11 +1,16 @@
+"""
+Module: STA2REST filter visitor
+
+Author: Filippo Finke
+
+This module provides a visitor for the filter AST.
+"""
 from odata_query import ast, visitor
 
 class FilterVisitor(visitor.NodeVisitor):
-     
-     def visit_Add(self, node: ast.Add) -> str:
-        print("Add")
-        print(node)
-        return node
+     """
+       Visitor for the filter AST.  
+     """
      
      def visit_All(self, node: ast.All) -> str:
         return "all"
@@ -17,13 +22,9 @@ def visit_Any(self, node: ast.Any) -> str:
         return "any"
      
      def visit_Attribute(self, node: ast.Attribute) -> str:
-        print("Attribute")
-        print(node)
         return node
      
      def visit_BinOp(self, node: ast.BinOp) -> str:
-        print("BinOp")
-        print(node)
         return node
      
      def visit_BoolOp(self, node: ast.BoolOp) -> str:
@@ -38,21 +39,6 @@ def visit_BoolOp(self, node: ast.BoolOp) -> str:
          right = right.replace("=", ".")
          return f"{operator}=({left},{right})"
      
-     def visit_Boolean(self, node: ast.Boolean) -> str:
-        print("Boolean")
-        print(node)
-        return node
-     
-     def visit_Call(self, node: ast.Call) -> str:
-        print("Call")
-        print(node)
-        return node
-     
-     def visit_CollectionLambda(self, node: ast.CollectionLambda) -> str:
-        print("CollectionLambda")
-        print(node)
-        return node
-     
      def visit_Compare(self, node: ast.Compare) -> str:
         
         print(node.left, node.comparator, node.right)
@@ -73,35 +59,16 @@ def visit_Compare(self, node: ast.Compare) -> str:
 
         return f"{left}={comparator}.{right}"
      
-     def visit_Date(self, node: ast.Date) -> str:
-        print("Date")
-        print(node)
-        return node
-     
+
      def visit_DateTime(self, node: ast.DateTime) -> str:
         return node.val
-     
-     def visit_Div(self, node: ast.Div) -> str:
-        print("Div")
-        print(node)
-        return node
-     
-     def visit_Duration(self, node: ast.Duration) -> str:
-        print("Duration")
-        print(node)
-        return node
-     
+          
      def visit_Eq(self, node: ast.Eq) -> str:
         return "eq"
      
      def visit_Float(self, node: ast.Float) -> str:
         return node.val
      
-     def visit_GUID(self, node: ast.GUID) -> str:
-        print("GUID")
-        print(node)
-        return node
-     
      def visit_Gt(self, node: ast.Gt) -> str:
         return "gt"
      
@@ -114,16 +81,6 @@ def visit_In(self, node: ast.In) -> str:
      def visit_Integer(self, node: ast.Integer) -> str:
         return node.val
      
-     def visit_Lambda(self, node: ast.Lambda) -> str:
-        print("Lambda")
-        print(node)
-        return node
-     
-     def visit_List(self, node: ast.List) -> str:
-        print("List")
-        print(node)
-        return node
-     
      def visit_Lt(self, node: ast.Lt) -> str:
         return "lt"
      
@@ -136,36 +93,11 @@ def visit_Not(self, node: ast.Not) -> str:
      def visit_NotEq(self, node: ast.NotEq) -> str:
         return "neq"
      
-     def visit_Null(self, node: ast.Null) -> str:
-        print("Null")
-        print(node)
-        return node
-     
      def visit_Or(self, node: ast.Or) -> str:
         return "or"
      
      def visit_String(self, node: ast.String) -> str:
         return node.val
-     
-     def visit_Sub(self, node: ast.Sub) -> str:
-        print("Sub")
-        print(node)
-        return node
-     
-     def visit_Time(self, node: ast.Time) -> str:
-        print("Time")
-        print(node)
-        return node
-     
-     def visit_USub(self, node: ast.USub) -> str:
-        print("USub")
-        print(node)
-        return node
-     
-     def visit_UnaryOp(self, node: ast.UnaryOp) -> str:
-        print("UnaryOp")
-        print(node)
-        return node
 
      def visit_Identifier(self, node: ast.Identifier) -> str:
         return node
\ No newline at end of file

From c6d1dfaf57549fdb493f727d56298cfe355b656c Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 28 Jun 2023 11:29:26 +0200
Subject: [PATCH 03/23] docs: add comments

---
 src/sta2rest/filter_visitor.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/sta2rest/filter_visitor.py b/src/sta2rest/filter_visitor.py
index 7bca763..e9acc86 100644
--- a/src/sta2rest/filter_visitor.py
+++ b/src/sta2rest/filter_visitor.py
@@ -32,28 +32,32 @@ def visit_BoolOp(self, node: ast.BoolOp) -> str:
         left = self.visit(node.left)
         right = self.visit(node.right)
 
+        # Check if the is AND, because it is the default operator
         if(isinstance(node.op, ast.And)):
            return f"{left}&{right}"
+        # Otherwise it will be OR
         else:
          left = left.replace("=", ".")
          right = right.replace("=", ".")
+         # Or syntax is different from the other operators
          return f"{operator}=({left},{right})"
      
      def visit_Compare(self, node: ast.Compare) -> str:
         
-        print(node.left, node.comparator, node.right)
-
         left = super().visit(node.left)
         comparator = super().visit(node.comparator)
         right = super().visit(node.right)
 
+         # Check if the left is an attribute
         if isinstance(left, (ast.Attribute)):
            owner = left.owner.name
            attr = left.attr
            left = f"{owner}->>{attr}"
+         # Otherwise it is an identifier
         elif isinstance(left, (ast.Identifier)):
             left = left.name
          
+         # Check if the right is an attribute
         if isinstance(right, (ast.Identifier)):
             right = right.name
 

From 7f41438a9231dbd82f15782354e5e1ee09f9b7a1 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 29 Jun 2023 11:43:43 +0200
Subject: [PATCH 04/23] fix: allow multiple or operators

---
 src/sta2rest/filter_visitor.py | 8 +++++++-
 src/sta2rest/sta2rest.py       | 2 +-
 src/sta2rest/test.py           | 2 ++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/sta2rest/filter_visitor.py b/src/sta2rest/filter_visitor.py
index e9acc86..c733572 100644
--- a/src/sta2rest/filter_visitor.py
+++ b/src/sta2rest/filter_visitor.py
@@ -37,7 +37,13 @@ def visit_BoolOp(self, node: ast.BoolOp) -> str:
            return f"{left}&{right}"
         # Otherwise it will be OR
         else:
-         left = left.replace("=", ".")
+         # check if another or is present by checking the first or=
+         if "or=" in left:
+            # remove the parenthesis
+            left = left[4:-1]
+         else: 
+            left = left.replace("=", ".")
+         
          right = right.replace("=", ".")
          # Or syntax is different from the other operators
          return f"{operator}=({left},{right})"
diff --git a/src/sta2rest/sta2rest.py b/src/sta2rest/sta2rest.py
index 5a56e99..52e08fd 100644
--- a/src/sta2rest/sta2rest.py
+++ b/src/sta2rest/sta2rest.py
@@ -315,6 +315,6 @@ def convert_query(sta_query: str) -> str:
 
     This example converts a STA query to a REST query.
     """
-    query = "$filter=result gt 20 or result le 3.5"
+    query = "$filter=id eq 1 or id eq 2 or id eq 3 or id eq 4"
     print("QUERY", query)
     print("CONVERTED", STA2REST.convert_query(query))
\ No newline at end of file
diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
index daa54be..6e69491 100644
--- a/src/sta2rest/test.py
+++ b/src/sta2rest/test.py
@@ -96,6 +96,8 @@ def test_convert_sensor_things_query(self):
             "$filter=result le 100": "result=lte.100",
             "$filter=result le 3.5 and FeatureOfInterest/id eq 1": "result=lte.3.5&FeatureOfInterest->>id=eq.1",
             "$filter=result gt 20 or result le 3.5": "or=(result.gt.20,result.lte.3.5)",
+            "$filter=id eq 1 or id eq 2 or id eq 3": "or=(id.eq.1,id.eq.2,id.eq.3)",
+            "$filter=id eq 1 or id eq 2 or id eq 3 or id eq 4": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)",
         }
 
         for query, expected in query_mappings.items():

From 0779992608b3e33359725da9c13cbcb95664e085 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 29 Jun 2023 11:46:08 +0200
Subject: [PATCH 05/23] feat: add test with or and and

---
 src/sta2rest/sta2rest.py | 2 +-
 src/sta2rest/test.py     | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/sta2rest/sta2rest.py b/src/sta2rest/sta2rest.py
index 52e08fd..12b6b66 100644
--- a/src/sta2rest/sta2rest.py
+++ b/src/sta2rest/sta2rest.py
@@ -315,6 +315,6 @@ def convert_query(sta_query: str) -> str:
 
     This example converts a STA query to a REST query.
     """
-    query = "$filter=id eq 1 or id eq 2 or id eq 3 or id eq 4"
+    query = "$filter=(id eq 1 or id eq 2 or id eq 3 or id eq 4) and location_id eq 2"
     print("QUERY", query)
     print("CONVERTED", STA2REST.convert_query(query))
\ No newline at end of file
diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
index 6e69491..ba4c7fd 100644
--- a/src/sta2rest/test.py
+++ b/src/sta2rest/test.py
@@ -98,6 +98,7 @@ def test_convert_sensor_things_query(self):
             "$filter=result gt 20 or result le 3.5": "or=(result.gt.20,result.lte.3.5)",
             "$filter=id eq 1 or id eq 2 or id eq 3": "or=(id.eq.1,id.eq.2,id.eq.3)",
             "$filter=id eq 1 or id eq 2 or id eq 3 or id eq 4": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)",
+            "$filter=(id eq 1 or id eq 2 or id eq 3 or id eq 4) and location_id eq 2": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)&location_id=eq.2"
         }
 
         for query, expected in query_mappings.items():

From fbc4a220b78b0b827cd2a060fc039901913d0c2e Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 29 Jun 2023 12:59:10 +0200
Subject: [PATCH 06/23] feat: add multiple and test case

---
 src/sta2rest/sta2rest.py | 2 +-
 src/sta2rest/test.py     | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/sta2rest/sta2rest.py b/src/sta2rest/sta2rest.py
index 12b6b66..67375d5 100644
--- a/src/sta2rest/sta2rest.py
+++ b/src/sta2rest/sta2rest.py
@@ -315,6 +315,6 @@ def convert_query(sta_query: str) -> str:
 
     This example converts a STA query to a REST query.
     """
-    query = "$filter=(id eq 1 or id eq 2 or id eq 3 or id eq 4) and location_id eq 2"
+    query = "$filter=location_id eq 2 and id eq 2"
     print("QUERY", query)
     print("CONVERTED", STA2REST.convert_query(query))
\ No newline at end of file
diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
index ba4c7fd..5d2b010 100644
--- a/src/sta2rest/test.py
+++ b/src/sta2rest/test.py
@@ -98,7 +98,8 @@ def test_convert_sensor_things_query(self):
             "$filter=result gt 20 or result le 3.5": "or=(result.gt.20,result.lte.3.5)",
             "$filter=id eq 1 or id eq 2 or id eq 3": "or=(id.eq.1,id.eq.2,id.eq.3)",
             "$filter=id eq 1 or id eq 2 or id eq 3 or id eq 4": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)",
-            "$filter=(id eq 1 or id eq 2 or id eq 3 or id eq 4) and location_id eq 2": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)&location_id=eq.2"
+            "$filter=(id eq 1 or id eq 2 or id eq 3 or id eq 4) and location_id eq 2": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)&location_id=eq.2",
+            "$filter=liocation_id eq 2 and id eq 2": "location_id=eq.2&id=eq.2",
         }
 
         for query, expected in query_mappings.items():

From 382ae3724d5a2fdbd1b9f6300fc98be8f8e62a36 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 29 Jun 2023 13:40:17 +0200
Subject: [PATCH 07/23] feat: add PoC from Saail and adapt it to use sta2rest

---
 fastapi/app/v1/fetch_postgrest.py | 32 +++++++++++++++++++++++++++++++
 src/__init__.py                   |  0
 2 files changed, 32 insertions(+)
 create mode 100644 fastapi/app/v1/fetch_postgrest.py
 create mode 100644 src/__init__.py

diff --git a/fastapi/app/v1/fetch_postgrest.py b/fastapi/app/v1/fetch_postgrest.py
new file mode 100644
index 0000000..72bf440
--- /dev/null
+++ b/fastapi/app/v1/fetch_postgrest.py
@@ -0,0 +1,32 @@
+import sys
+sys.path.append('../../../src/sta2rest')
+
+import requests
+import urllib.parse
+from fastapi import FastAPI, Request
+from sta2rest import STA2REST
+
+"""
+Example query: http://127.0.0.1:8000/Datastream?$filter=thing_id eq 1 or sensor_id eq 2
+"""
+
+#fetching database data from postgrest 
+app = FastAPI()
+
+#sending response to client
+# print(p)
+@app.get("/Datastream")
+async def root(request: Request):
+
+    # Get the original query parameters from the request
+    query_params = urllib.parse.unquote_plus(str(request.query_params))
+    print("Original query: ", query_params)
+    # Convert the STA query to a PostgREST query
+    converted_query = STA2REST.convert_query(query_params)
+    print("Converted query: ", converted_query)
+
+    # Send the converted query to PostgREST
+    r = requests.get('http://localhost:3000/Datastream?' + converted_query)
+
+    # Return the response from PostgREST
+    return r.json()
\ No newline at end of file
diff --git a/src/__init__.py b/src/__init__.py
new file mode 100644
index 0000000..e69de29

From 21e0b9a0347a12bb893b6b3c13f951d930728abd Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Tue, 4 Jul 2023 13:36:29 +0200
Subject: [PATCH 08/23] fix: test case

---
 src/sta2rest/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
index 5d2b010..eb7258b 100644
--- a/src/sta2rest/test.py
+++ b/src/sta2rest/test.py
@@ -99,7 +99,7 @@ def test_convert_sensor_things_query(self):
             "$filter=id eq 1 or id eq 2 or id eq 3": "or=(id.eq.1,id.eq.2,id.eq.3)",
             "$filter=id eq 1 or id eq 2 or id eq 3 or id eq 4": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)",
             "$filter=(id eq 1 or id eq 2 or id eq 3 or id eq 4) and location_id eq 2": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)&location_id=eq.2",
-            "$filter=liocation_id eq 2 and id eq 2": "location_id=eq.2&id=eq.2",
+            "$filter=location_id eq 2 and id eq 2": "location_id=eq.2&id=eq.2",
         }
 
         for query, expected in query_mappings.items():

From cfc16e598bd36e9c194be0601107606d0a4f7808 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Tue, 4 Jul 2023 15:09:45 +0200
Subject: [PATCH 09/23] feat: use new converter library

---
 src/sta2rest/sta2rest.py | 419 ++++++++++++++-------------------------
 src/sta2rest/test.py     |  50 +----
 2 files changed, 161 insertions(+), 308 deletions(-)

diff --git a/src/sta2rest/sta2rest.py b/src/sta2rest/sta2rest.py
index 67375d5..70194db 100644
--- a/src/sta2rest/sta2rest.py
+++ b/src/sta2rest/sta2rest.py
@@ -6,20 +6,157 @@
 This module provides utility functions to convert various elements used in SensorThings queries to their corresponding
 representations in a REST API.
 """
-import re 
-import urllib.parse
 from odata_query.grammar import ODataLexer
 from odata_query.grammar import ODataParser
 from filter_visitor import FilterVisitor
+from sta_parser.lexer import Lexer
+from sta_parser.visitor import Visitor
+from sta_parser.parser import Parser
+from sta_parser import ast
 
 odata_filter_lexer = ODataLexer()
 odata_filter_parser = ODataParser()
 
+class NodeVisitor(Visitor):
+    def visit_IdentifierNode(self, node: ast.IdentifierNode):
+        return node.name
+
+    def visit_SelectNode(self, node: ast.SelectNode):
+        identifiers = ','.join([self.visit(identifier) for identifier in node.identifiers])
+        return f'select={identifiers}'
+
+    def visit_FilterNode(self, node: ast.FilterNode):
+        ast = odata_filter_parser.parse(odata_filter_lexer.tokenize(node.filter))
+        # Visit the tree to convert the filter
+        res = FilterVisitor().visit(ast)
+        return res
+
+    def visit_OrderByNodeIdentifier(self, node: ast.OrderByNodeIdentifier):
+        return f'{node.identifier}.{node.order}'
+
+    def visit_OrderByNode(self, node: ast.OrderByNode):
+        identifiers = ','.join([self.visit(identifier) for identifier in node.identifiers])
+        return f'order={identifiers}'
+
+    def visit_SkipNode(self, node: ast.SkipNode):
+        return f'offset={node.count}'
+
+    def visit_TopNode(self, node: ast.TopNode):
+        return f'limit={node.count}'
+
+    def visit_CountNode(self, node: ast.CountNode):
+        return f'count={node.value}'
+    
+    def visit_ExpandNode(self, node: ast.ExpandNode, parent=None):
+
+        select = None
+        filter = ""
+        orderby = ""
+        skip = ""
+        top = ""
+        count = ""
+
+        for expand_identifier in node.identifiers:
+                expand_identifier.identifier = STA2REST.convert_entity(expand_identifier.identifier)
+                prefix = ""
+                if parent:
+                    prefix = parent
+                prefix += expand_identifier.identifier + "."
+        
+                if expand_identifier.subquery:
+                    if expand_identifier.subquery.select:
+                        if not select:
+                            select = ast.SelectNode([])
+                        identifiers = ','.join([self.visit(identifier) for identifier in expand_identifier.subquery.select.identifiers])
+                        select.identifiers.append(ast.IdentifierNode(f'{expand_identifier.identifier}({identifiers})'))
+                    if expand_identifier.subquery.filter:
+                        result = self.visit_FilterNode(expand_identifier.subquery.filter)
+                        filter = prefix + result
+                    if expand_identifier.subquery.orderby:
+                        orderby = prefix + "order=" + ','.join([self.visit(identifier) for identifier in expand_identifier.subquery.orderby.identifiers])
+                    if expand_identifier.subquery.skip:
+                        skip = prefix + "offset=" + str(expand_identifier.subquery.skip.count)
+                    if expand_identifier.subquery.top:
+                        top = prefix + "limit=" + str(expand_identifier.subquery.top.count)
+                    if expand_identifier.subquery.count:
+                        count = prefix + "count=" + str(expand_identifier.subquery.count.value).lower()
+
+                    if expand_identifier.subquery.expand:
+                        result = self.visit_ExpandNode(expand_identifier.subquery.expand, prefix)
+                        if result['select']:
+                            if not select:
+                                select = ast.SelectNode([])
+                            select.identifiers.extend(result['select'].identifiers)
+                        if result['orderby']:
+                            if orderby:
+                                orderby += "&"
+                            orderby += result['orderby']
+                        if result['skip']:
+                            if skip:
+                                skip += "&"
+                            skip += result['skip']
+                        if result['top']:
+                            if top:
+                                top += "&"
+                            top += result['top']
+                        if result['count']:
+                            if count:
+                                count += "&"
+                            count += result['count']
+                        if result['filter']:
+                            if filter:
+                                filter += "&"
+                            filter += result['filter']
+                
+                if not expand_identifier.subquery or not expand_identifier.subquery.select:
+                    if not select:
+                        select = ast.SelectNode([])
+                    select.identifiers.append(ast.IdentifierNode(f'{expand_identifier.identifier}(*)'))
+        return {
+            'select': select,
+            'filter': filter,
+            'orderby': orderby,
+            'skip': skip,
+            'top': top,
+            'count': count
+        }
+
+    def visit_QueryNode(self, node: ast.QueryNode):
+        query_parts = []
+
+        if node.expand:
+            result = self.visit(node.expand)
+            if result['select']:
+                if not node.select:
+                    node.select = ast.SelectNode([])
+                node.select.identifiers.extend(result['select'].identifiers)
+            if result['orderby']:
+                query_parts.append(result['orderby'])
+            if result['skip']:
+                query_parts.append(result['skip'])
+            if result['top']:
+                query_parts.append(result['top'])
+            if result['count']:
+                query_parts.append(result['count'])
+            if result['filter']:
+                query_parts.append(result['filter'])
+
+        if node.select:
+            query_parts.append(self.visit(node.select))
+        if node.filter:
+            query_parts.append(self.visit(node.filter))
+        if node.orderby:
+            query_parts.append(self.visit(node.orderby))
+        if node.skip:
+            query_parts.append(self.visit(node.skip))
+        if node.top:
+            query_parts.append(self.visit(node.top))
+        if node.count:
+            query_parts.append(self.visit(node.count).lower())
+        
+        return '&'.join(query_parts)
+
 class STA2REST:
-    """
-    This class provides utility functions to convert various elements used in SensorThings queries to their corresponding
-    representations in a compatible PostgREST REST API.
-    """
 
     # Mapping from SensorThings entities to their corresponding database table names
     ENTITY_MAPPING = {
@@ -32,27 +169,6 @@ class STA2REST:
         "FeaturesOfInterest": "FeatureOfInterest",
         "HistoricalLocations": "HistoricalLocation",
     }
-
-    # Mapping from SensorThings properties to their corresponding database column names
-    PROPERTY_MAPPING = {
-        "name": "name",
-        "description": "description",
-        "encodingType": "encodingType",
-        "metadata": "metadata",
-        "definition": "definition",
-    }
-
-    # Mapping from SensorThings query parameters to their corresponding PostgRESTR query parameters
-    QUERY_PARAM_MAPPING = {
-        "$filter": "filter",
-        "$orderby": "order",
-        "$top": "limit",
-        "$skip": "offset",
-        "$select": "select",
-        "$count": "count",
-        "$expand": "expand",
-    }
-
     @staticmethod
     def convert_entity(entity: str) -> str:
         """
@@ -65,249 +181,16 @@ def convert_entity(entity: str) -> str:
             str: The converted entity name in REST format.
         """
         return STA2REST.ENTITY_MAPPING.get(entity, entity)
-
-    @staticmethod
-    def convert_property(property_name: str) -> str:
-        """
-        Converts a property name from STA format to REST format.
-
-        Args:
-            property_name (str): The property name in STA format.
-
-        Returns:
-            str: The converted property name in REST format.
-        """
-        return STA2REST.PROPERTY_MAPPING.get(property_name, property_name)
-
-    @staticmethod
-    def convert_query_param(query_param: str) -> str:
-        """
-        Converts a query parameter name from STA format to REST format.
-
-        Args:
-            query_param (str): The query parameter name in STA format.
-
-        Returns:
-            str: The converted query parameter name in REST format.
-        """
-        return STA2REST.QUERY_PARAM_MAPPING.get(query_param, query_param)
-
-    @staticmethod
-    def split_but_not_between(query: str, split: str, delim1: str, delim2: str) -> list:
-        """
-        Splits a string by a given character, but only if the character is not between two other given characters.
-
-        Args:
-            query (str): The string to split.
-            split (str): The character to split by.
-            delim1 (str): The first delimiter character.
-            delim2 (str): The second delimiter character.
-
-        Returns:
-            list: The list of substrings.
-        """
-
-        result = []
-        stack = []
-        current = ''
-        for char in query:
-            # Check if the current character is a delimiter
-            if char == delim1:
-                current += char
-                stack.append(delim1)
-            # Check if the current character is a delimiter
-            elif char == delim2:
-                current += char
-                # Check if the current delimiter is the same as the last one in the stack
-                if stack and stack[-1] == delim1:
-                    stack.pop()
-            elif char == split:
-                if stack:
-                    # Inside parentheses, treat comma as part of the current substring
-                    current += char
-                else:
-                    # Outside parentheses, split the string and reset the current substring
-                    result.append(current.strip())
-                    current = ''
-            else:
-                # Regular character, add it to the current substring
-                current += char
-        
-        # Add the last substring
-        if current:
-            result.append(current.strip())
-
-        return result
-
-    @staticmethod
-    def convert_order_by_value(value: str) -> str:
-        """
-        Converts an order by value from STA format to REST format.
-
-        Args:
-            value (str): The order by value in STA format.
-
-        Returns:
-            str: The converted order by value in REST format.
-        """
-
-        # strip all the spaces after commas ",   " -> ","
-        value = re.sub(r",\s+", ",", value)
-        # replace space and slash with dot
-        value = value.replace(" ", ".").replace("/", ".")
-        return value
-
-    @staticmethod
-    def convert_filter_by_value(value: str) -> str:
-        """
-        Converts a filter by value from STA format to REST format.
-
-        see https://docs.ogc.org/is/18-088/18-088.html#_built_in_filter_operations
-        see https://postgrest.org/en/stable/references/api/tables_views.html#logical-operators
-
-        Args:
-            value (str): The filter by value in STA format.
-
-        Returns:
-            str: The converted filter by value in REST format.
-        """
-        # Get the AST tree from the filter
-        ast = odata_filter_parser.parse(odata_filter_lexer.tokenize(value))
-        # Visit the tree to convert the filter
-        res = FilterVisitor().visit(ast)
-        return res
-
-    @staticmethod
-    def convert_expand(expand_query: str, previous_entity: str = None) -> str:
-        """
-        Converts an expand query from STA format to REST format.
-
-        Args:
-            expand_query (str): The expand query in STA format.
-            previous_entity (str, optional): The previous entity name. Defaults to None.
-
-        Returns:
-            str: The converted expand query in REST format.
-        """
-
-        # split by comma but not between parentheses
-        result = STA2REST.split_but_not_between(expand_query, ",", "(", ")")
-    
-        entities = []
-        additionals = []
-        for entity in result:
-            converted_entity = ""
-            has_select = False
-            # Check if the entity has a subquery
-            if "(" in entity:
-                entity, subquery = entity.split("(",1)
-                c_entity = STA2REST.convert_entity(entity)
-                converted_entity = c_entity + "("
-                subquery = subquery[:-1]
-                # Get all the actions in the subquery
-                actions = STA2REST.split_but_not_between(subquery, ";", "(", ")")
-                for action in actions:
-                    param, value = action.split("=",1)
-                    converted_param = STA2REST.convert_query_param(param)
-                    if param == "$expand":
-                        # Recursively convert the subquery
-                        result = STA2REST.convert_expand(value, c_entity)
-                        entities.extend(result["select"].split(","))
-                        additionals += result["additionals"]
-                    elif param == "$select":
-                        has_select = True
-                        converted_entity += value
-                    else:
-                        # Adjust values
-                        if param == "$orderby":
-                            value = STA2REST.convert_order_by_value(value)
-                        elif param == "$filter":
-                            value = STA2REST.convert_filter_by_value(value)
-
-                        # Add the previous entity if present
-                        p_entity = previous_entity + "." if previous_entity != None else ""
-
-                        # Add the action to the additionals
-                        if param != "$filter":
-                            additionals.append(f"{p_entity}{c_entity}.{converted_param}={value}")
-                        else:
-                            additionals.append(f"{p_entity}{c_entity}.{value}")
-            
-            if not has_select:
-                converted_entity = STA2REST.convert_entity(entity) + "(*"
-            
-            converted_entity += ")"
-            
-            entities.append(converted_entity)
-
-        return {
-            "select": ",".join(entities),
-            "additionals": additionals
-        }
-
+  
     @staticmethod
     def convert_query(sta_query: str) -> str:
-        """
-        Converts a query from STA format to REST format.
-
-        Args:
-            sta_query (str): The query in STA format.
+        lexer = Lexer(sta_query)
+        tokens = lexer.tokenize()
+        parser = Parser(tokens)
+        ast = parser.parse()
+        visitor = NodeVisitor()
+        return visitor.visit(ast)
 
-        Returns:
-            str: The converted query in REST format.
-        """
-
-        # remove unwanted characters from the query
-        sta_query = urllib.parse.unquote(sta_query)
-
-        # get the query parameters
-        sta_query = sta_query.split("&")
-        converted_query_params = {}
-
-        # convert the query parameters
-        for query_param in sta_query:
-            key, value = query_param.split("=", 1)
-            if key == "$expand":
-                converted_value = STA2REST.convert_expand(value)
-            else:
-                if key == "$orderby":
-                    value = STA2REST.convert_order_by_value(value)
-                elif key == "$filter":
-                    value = STA2REST.convert_filter_by_value(value)
-                converted_value = value
-            converted_key = STA2REST.convert_query_param(key)
-            converted_query_params[converted_key] = converted_value
-
-        # Check for expand and if present merge the select
-        if "expand" in converted_query_params and "select" in converted_query_params["expand"]:
-            if "select" in converted_query_params:
-                converted_query_params["select"] += "," + converted_query_params["expand"]["select"]
-            else:
-                converted_query_params["select"] = converted_query_params["expand"]["select"]
-        
-        # Get additionals parameters and remove them from the query
-        additionals = []
-        if "expand" in converted_query_params and "additionals" in converted_query_params["expand"]:
-            additionals = converted_query_params["expand"]["additionals"]
-            del converted_query_params["expand"]
-
-        # Check if the filter is present and move it to the additionals
-        if "filter" in converted_query_params:
-            additionals.append(converted_query_params["filter"])
-            del converted_query_params["filter"]
-
-        # Merge in format key=value&key=value 
-        converted_query = "&".join([f"{key}={value}" for key, value in converted_query_params.items()])
-        
-        # Add additionals
-        for additional in additionals:
-            converted_query += "&" + additional
-
-        # Remove the first & if present
-        if converted_query.startswith("&"):
-            converted_query = converted_query[1:]
-        
-        return converted_query
 
 if __name__ == "__main__":
     """
@@ -315,6 +198,6 @@ def convert_query(sta_query: str) -> str:
 
     This example converts a STA query to a REST query.
     """
-    query = "$filter=location_id eq 2 and id eq 2"
+    query = "$expand=Observations($filter=result eq 1)"
     print("QUERY", query)
     print("CONVERTED", STA2REST.convert_query(query))
\ No newline at end of file
diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
index eb7258b..1df72bf 100644
--- a/src/sta2rest/test.py
+++ b/src/sta2rest/test.py
@@ -31,60 +31,29 @@ def test_convert_entity(self):
         for entity, expected in entity_mappings.items():
             self.assertEqual(STA2REST.convert_entity(entity), expected)
 
-    def test_convert_property(self):
-        """
-        Test the conversion of properties.
-        """
-
-        property_mappings = {
-            "name": 'name',
-            "description": 'description',
-            "encodingType": 'encodingType',
-            "metadata": 'metadata'
-            # Add more property mappings as needed
-        }
-
-        for prop, expected in property_mappings.items():
-            self.assertEqual(STA2REST.convert_property(prop), expected)
-
-    def test_convert_query_param(self):
-        """
-        Test the conversion of query parameters.
-        """
-
-        query_param_mappings = {
-            "$orderby": "order",
-            "$top": "limit",
-            "$skip": "offset"
-            # Add more query parameter mappings as needed
-        }
-
-        for param, expected in query_param_mappings.items():
-            self.assertEqual(STA2REST.convert_query_param(param), expected)
-
     def test_convert_sensor_things_query(self):
         """
         Test the conversion of sensor things queries.
         """
         query_mappings = {
             "$filter=type eq 'temperature'&$orderby=timestamp desc&$top=10&$skip=5":
-                "order=timestamp.desc&limit=10&offset=5&type=eq.temperature",
+                "type=eq.temperature&order=timestamp.desc&offset=5&limit=10",
             "$filter=type eq 'humidity'&$top=5":
-                "limit=5&type=eq.humidity",
+                "type=eq.humidity&limit=5",
             "$orderby=timestamp asc&$skip=2":
                 "order=timestamp.asc&offset=2",
             "$select=id,name,description,properties&$top=1000&$filter=properties/type eq 'station'&$expand=Locations,Datastreams($select=id,name,unitOfMeasurement;$expand=ObservedProperty($select=name),Observations($select=result,phenomenonTime;$orderby=phenomenonTime desc;$top=1))":
-            "select=id,name,description,properties,Location(*),ObservedProperty(name),Observation(result,phenomenonTime),Datastream(id,name,unitOfMeasurement)&limit=1000&Datastream.Observation.order=phenomenonTime.desc&Datastream.Observation.limit=1&properties->>type=eq.station",
-            "$select=@iot.id,description&$expand=Datastreams($select=@iot.id,description)": "select=@iot.id,description,Datastream(@iot.id,description)",
+            "Datastream.Observation.order=phenomenonTime.desc&Datastream.Observation.limit=1&select=id,name,description,properties,Location(*),Datastream(id,name,unitOfMeasurement),ObservedProperty(name),Observation(result,phenomenonTime)&properties->>type=eq.station&limit=1000",
+            "$select=id,description&$expand=Datastreams($select=id,description)": "select=id,description,Datastream(id,description)",
             "$expand=Datastreams": "select=Datastream(*)",
             "$expand=Observations,ObservedProperty": "select=Observation(*),ObservedProperty(*)",
-            "$expand=Observations($filter=result eq 1)": "select=Observation(*)&Observation.result=eq.1",
+            "$expand=Observations($filter=result eq 1)": "Observation.result=eq.1&select=Observation(*)",
             "$expand=Observations($select=result)": "select=Observation(result)",
-            "$select=result,resultTime": "select=result,resultTime",
-            "$orderby=result": "order=result",
-            "$expand=Datastream&$orderby=Datastreams/id desc,phenomenonTime": "order=Datastreams.id.desc,phenomenonTime&select=Datastream(*)",
+            "$select=result": "select=result",
+            "$orderby=result": "order=result.asc",
+            "$expand=Datastream&$orderby=Datastreams/id desc,phenomenonTime": "select=Datastream(*)&order=Datastreams/id.desc,phenomenonTime.asc",
             "$top=5": "limit=5",
-            "$top=5&$orderby=phenomenonTime%20desc": "limit=5&order=phenomenonTime.desc",
+            "$top=5&$orderby=phenomenonTime%20desc": "order=phenomenonTime.desc&limit=5",
             "$skip=5": "offset=5",
             "$count=true": "count=true",
             "$filter=result lt 10.00": "result=lt.10.00",
@@ -103,6 +72,7 @@ def test_convert_sensor_things_query(self):
         }
 
         for query, expected in query_mappings.items():
+            print("QUERY", query)
             self.assertEqual(STA2REST.convert_query(query), expected)
 
 if __name__ == '__main__':

From 3c13fd38cc278055abef7021b6508c5fcad3acaf Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Tue, 4 Jul 2023 19:51:59 +0200
Subject: [PATCH 10/23] feat: add try and catch

---
 fastapi/app/v1/fetch_postgrest.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/fastapi/app/v1/fetch_postgrest.py b/fastapi/app/v1/fetch_postgrest.py
index 72bf440..8bd2a15 100644
--- a/fastapi/app/v1/fetch_postgrest.py
+++ b/fastapi/app/v1/fetch_postgrest.py
@@ -17,16 +17,19 @@
 # print(p)
 @app.get("/Datastream")
 async def root(request: Request):
+    try:
+        # Get the original query parameters from the request
+        query_params = urllib.parse.unquote_plus(str(request.query_params))
+        print("Original query: ", query_params)
+        # Convert the STA query to a PostgREST query
+        converted_query = STA2REST.convert_query(query_params)
+        print("Converted query: ", converted_query)
 
-    # Get the original query parameters from the request
-    query_params = urllib.parse.unquote_plus(str(request.query_params))
-    print("Original query: ", query_params)
-    # Convert the STA query to a PostgREST query
-    converted_query = STA2REST.convert_query(query_params)
-    print("Converted query: ", converted_query)
+        # Send the converted query to PostgREST
+        r = requests.get('http://localhost:3000/Datastream?' + converted_query)
 
-    # Send the converted query to PostgREST
-    r = requests.get('http://localhost:3000/Datastream?' + converted_query)
-
-    # Return the response from PostgREST
-    return r.json()
\ No newline at end of file
+        # Return the response from PostgREST
+        return r.json()
+    except Exception as e:
+        # send the error message to the client
+        return {"error": str(e)}
\ No newline at end of file

From ca9c1f03979901afb2f3e2ee3d4419571daa5a20 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Tue, 4 Jul 2023 20:06:51 +0200
Subject: [PATCH 11/23] docs: add comments

---
 src/sta2rest/sta2rest.py | 138 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 138 insertions(+)

diff --git a/src/sta2rest/sta2rest.py b/src/sta2rest/sta2rest.py
index 70194db..bbc2fbb 100644
--- a/src/sta2rest/sta2rest.py
+++ b/src/sta2rest/sta2rest.py
@@ -14,41 +14,134 @@
 from sta_parser.parser import Parser
 from sta_parser import ast
 
+# Create the OData lexer and parser
 odata_filter_lexer = ODataLexer()
 odata_filter_parser = ODataParser()
 
 class NodeVisitor(Visitor):
+    """
+    This class provides a visitor to convert a STA query to a PostgREST query.
+    """
+    
     def visit_IdentifierNode(self, node: ast.IdentifierNode):
+        """
+        Visit an identifier node.
+
+        Args:
+            node (ast.IdentifierNode): The identifier node to visit.
+
+        Returns:
+            str: The converted identifier.
+        """
         return node.name
 
     def visit_SelectNode(self, node: ast.SelectNode):
+        """
+        Visit a select node.
+
+        Args:
+            node (ast.SelectNode): The select node to visit.
+
+        Returns:
+            str: The converted select node.
+        """
+
         identifiers = ','.join([self.visit(identifier) for identifier in node.identifiers])
         return f'select={identifiers}'
 
     def visit_FilterNode(self, node: ast.FilterNode):
+        """
+        Visit a filter node.
+
+        Args:
+            node (ast.FilterNode): The filter node to visit.
+        
+        Returns:
+            str: The converted filter node.
+        """
+
+        # Parse the filter using the OData lexer and parser
         ast = odata_filter_parser.parse(odata_filter_lexer.tokenize(node.filter))
         # Visit the tree to convert the filter
         res = FilterVisitor().visit(ast)
         return res
 
     def visit_OrderByNodeIdentifier(self, node: ast.OrderByNodeIdentifier):
+        """
+        Visit an orderby node identifier.
+
+        Args:
+            node (ast.OrderByNodeIdentifier): The orderby node identifier to visit.
+
+        Returns:
+            str: The converted orderby node identifier.
+        """
+
+        # Convert the identifier to the format name.order
         return f'{node.identifier}.{node.order}'
 
     def visit_OrderByNode(self, node: ast.OrderByNode):
+        """
+        Visit an orderby node.
+
+        Args:
+            node (ast.OrderByNode): The orderby node to visit.
+        
+        Returns:
+            str: The converted orderby node.
+        """
         identifiers = ','.join([self.visit(identifier) for identifier in node.identifiers])
         return f'order={identifiers}'
 
     def visit_SkipNode(self, node: ast.SkipNode):
+        """
+        Visit a skip node.
+
+        Args:
+            node (ast.SkipNode): The skip node to visit.
+        
+        Returns:
+            str: The converted skip node.
+        """
         return f'offset={node.count}'
 
     def visit_TopNode(self, node: ast.TopNode):
+        """
+        Visit a top node.
+
+        Args:
+            node (ast.TopNode): The top node to visit.
+        
+        Returns:
+            str: The converted top node.
+        """
         return f'limit={node.count}'
 
     def visit_CountNode(self, node: ast.CountNode):
+        """
+        Visit a count node.
+
+        Args:
+            node (ast.CountNode): The count node to visit.
+        
+        Returns:
+            str: The converted count node.
+        """
         return f'count={node.value}'
     
     def visit_ExpandNode(self, node: ast.ExpandNode, parent=None):
+        """
+        Visit an expand node.
+        
+        Args:
+            node (ast.ExpandNode): The expand node to visit.
+            parent (str): The parent entity name.
+        
+        Returns:
+            dict: The converted expand node.
+        """
 
+        # dict to store the converted parts of the expand node
         select = None
         filter = ""
         orderby = ""
@@ -56,14 +149,21 @@ def visit_ExpandNode(self, node: ast.ExpandNode, parent=None):
         top = ""
         count = ""
 
+        # Visit the identifiers in the expand node
         for expand_identifier in node.identifiers:
+                # Convert the table name
                 expand_identifier.identifier = STA2REST.convert_entity(expand_identifier.identifier)
+                
+                # Check if we had a parent entity
                 prefix = ""
                 if parent:
                     prefix = parent
                 prefix += expand_identifier.identifier + "."
         
+                # Check if we have a subquery
                 if expand_identifier.subquery:
+
+                    # check if we have a select, filter, orderby, skip, top or count in the subquery
                     if expand_identifier.subquery.select:
                         if not select:
                             select = ast.SelectNode([])
@@ -81,8 +181,11 @@ def visit_ExpandNode(self, node: ast.ExpandNode, parent=None):
                     if expand_identifier.subquery.count:
                         count = prefix + "count=" + str(expand_identifier.subquery.count.value).lower()
 
+                    # check if we have a subquery in the subquery
                     if expand_identifier.subquery.expand:
                         result = self.visit_ExpandNode(expand_identifier.subquery.expand, prefix)
+
+                        # merge the results
                         if result['select']:
                             if not select:
                                 select = ast.SelectNode([])
@@ -108,10 +211,13 @@ def visit_ExpandNode(self, node: ast.ExpandNode, parent=None):
                                 filter += "&"
                             filter += result['filter']
                 
+                # If we don't have a subquery, we add the identifier to the select node
                 if not expand_identifier.subquery or not expand_identifier.subquery.select:
                     if not select:
                         select = ast.SelectNode([])
                     select.identifiers.append(ast.IdentifierNode(f'{expand_identifier.identifier}(*)'))
+        
+        # Return the converted expand node
         return {
             'select': select,
             'filter': filter,
@@ -122,10 +228,25 @@ def visit_ExpandNode(self, node: ast.ExpandNode, parent=None):
         }
 
     def visit_QueryNode(self, node: ast.QueryNode):
+        """
+        Visit a query node.
+
+        Args:
+            node (ast.QueryNode): The query node to visit.
+        
+        Returns:
+            str: The converted query node.
+        """
+
+        # list to store the converted parts of the query node
         query_parts = []
 
+        # Check if we have an expand node before the other parts of the query
         if node.expand:
+            # Visit the expand node
             result = self.visit(node.expand)
+
+            # Merge the results with the other parts of the query
             if result['select']:
                 if not node.select:
                     node.select = ast.SelectNode([])
@@ -141,6 +262,7 @@ def visit_QueryNode(self, node: ast.QueryNode):
             if result['filter']:
                 query_parts.append(result['filter'])
 
+        # Check if we have a select, filter, orderby, skip, top or count in the query
         if node.select:
             query_parts.append(self.visit(node.select))
         if node.filter:
@@ -153,10 +275,16 @@ def visit_QueryNode(self, node: ast.QueryNode):
             query_parts.append(self.visit(node.top))
         if node.count:
             query_parts.append(self.visit(node.count).lower())
+
         
+        # Join the converted parts of the query
         return '&'.join(query_parts)
 
 class STA2REST:
+    """
+    This class provides utility functions to convert various elements used in SensorThings queries to their corresponding
+    representations in a REST API.
+    """
 
     # Mapping from SensorThings entities to their corresponding database table names
     ENTITY_MAPPING = {
@@ -169,6 +297,7 @@ class STA2REST:
         "FeaturesOfInterest": "FeatureOfInterest",
         "HistoricalLocations": "HistoricalLocation",
     }
+
     @staticmethod
     def convert_entity(entity: str) -> str:
         """
@@ -184,6 +313,15 @@ def convert_entity(entity: str) -> str:
   
     @staticmethod
     def convert_query(sta_query: str) -> str:
+        """
+        Converts a STA query to a PostgREST query.
+
+        Args:
+            sta_query (str): The STA query.
+        
+        Returns:
+            str: The converted PostgREST query.
+        """
         lexer = Lexer(sta_query)
         tokens = lexer.tokenize()
         parser = Parser(tokens)

From e1bbffa125cfc0fc167a6b70abc51f062aa2d5ea Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 5 Jul 2023 16:13:37 +0200
Subject: [PATCH 12/23] feat: initial uri parsing implementation

---
 src/sta2rest/sta2rest.py | 72 ++++++++++++++++++++++++++++++++++++++++
 src/sta2rest/test.py     | 23 +++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/src/sta2rest/sta2rest.py b/src/sta2rest/sta2rest.py
index bbc2fbb..9ff1157 100644
--- a/src/sta2rest/sta2rest.py
+++ b/src/sta2rest/sta2rest.py
@@ -6,6 +6,7 @@
 This module provides utility functions to convert various elements used in SensorThings queries to their corresponding
 representations in a REST API.
 """
+import re
 from odata_query.grammar import ODataLexer
 from odata_query.grammar import ODataParser
 from filter_visitor import FilterVisitor
@@ -296,6 +297,15 @@ class STA2REST:
         "Observations": "Observation",
         "FeaturesOfInterest": "FeatureOfInterest",
         "HistoricalLocations": "HistoricalLocation",
+
+        "Thing": "Thing",
+        "Location": "Location",
+        "Sensor": "Sensor",
+        "ObservedProperty": "ObservedProperty",
+        "Datastream": "Datastream",
+        "Observation": "Observation",  
+        "FeatureOfInterest": "FeatureOfInterest",
+        "HistoricalLocation": "HistoricalLocation",
     }
 
     @staticmethod
@@ -329,6 +339,68 @@ def convert_query(sta_query: str) -> str:
         visitor = NodeVisitor()
         return visitor.visit(ast)
 
+    @staticmethod
+    def parse_entity(entity: str):
+        # Check if we have an id in the entity and match only the number
+        match = re.search(r'\(\d+\)', entity)
+        id = None
+        if match:
+            # Get the id from the match without the brackets
+            id = match.group(0)[1:-1]
+            # Remove the id from the entity
+            entity = entity.replace(match.group(0), "")
+
+        # Check if the entity is in the ENTITY_MAPPING
+        if entity in STA2REST.ENTITY_MAPPING:
+            entity = STA2REST.ENTITY_MAPPING[entity]
+        else:
+            return None
+
+        return (entity, id)
+    
+    @staticmethod
+    def parse_uri(uri: str) -> str:
+        # Split the uri by the '/' character
+        parts = uri.split('/')
+        # Remove the first part
+        parts.pop(0)
+
+        # Check if we have a version number
+        version = parts.pop(0)
+
+        # Parse first entity
+        entity = STA2REST.parse_entity(parts.pop(0))
+        if not entity:
+            raise Exception("Error parsing uri: invalid entity")
+
+        # Check all the entities in the uri
+        entities = []
+        property_name = None
+        ref = False
+        value = False
+        for entity in parts:
+            # Parse the entity
+            result = STA2REST.parse_entity(entity)
+            if result:
+                entities.append(result)
+            elif entity == "$ref":
+                ref = True
+            elif entity == "$value":
+                if property_name:
+                    value = True
+                else:
+                    raise Exception("Error parsing uri: $value without property name")
+            else:
+                property_name = entity
+
+        return {
+            'version': version,
+            'entity': entity,
+            'entities': entities,
+            'property_name': property_name,
+            'ref': ref,
+            'value': value
+        }
 
 if __name__ == "__main__":
     """
diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
index 1df72bf..ab2d221 100644
--- a/src/sta2rest/test.py
+++ b/src/sta2rest/test.py
@@ -31,6 +31,29 @@ def test_convert_entity(self):
         for entity, expected in entity_mappings.items():
             self.assertEqual(STA2REST.convert_entity(entity), expected)
 
+    def test_parse_uri(self):
+        """
+        Test the parsing of URIs.
+        """
+
+        # Test the parsing of URIs
+        tests = [
+            "/v1.1/ObservedProperties"
+            "/v1.1/Things(1)",
+            "/v1.1/Observations(1)/resultTime",
+            "/v1.1/Observations(1)/resultTime/$value",
+            "/v1.1/Datastreams(1)/Observations",
+            "/v1.1/Datastreams(1)/Observations/$ref",
+            "/v1.1/Datastreams(1)/Observations(1)",
+            "/v1.1/Datastreams(1)/Observations(1)/resultTime",
+            "/v1.1/Datastreams(1)/Observations(1)/FeatureOfInterest",
+        ]
+        
+        # Test the parsing of URIs with query
+        for test in tests:
+            print(test)
+            print(STA2REST.parse_uri(test))
+
     def test_convert_sensor_things_query(self):
         """
         Test the conversion of sensor things queries.

From 2ea7928e478f8afbb203f1f691199f82e97d1f5d Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 5 Jul 2023 16:51:12 +0200
Subject: [PATCH 13/23] feat: convert also the path

---
 src/sta2rest/sta2rest.py | 85 ++++++++++++++++++++++++++++++++++------
 src/sta2rest/test.py     |  3 ++
 2 files changed, 77 insertions(+), 11 deletions(-)

diff --git a/src/sta2rest/sta2rest.py b/src/sta2rest/sta2rest.py
index 9ff1157..cff7226 100644
--- a/src/sta2rest/sta2rest.py
+++ b/src/sta2rest/sta2rest.py
@@ -7,13 +7,13 @@
 representations in a REST API.
 """
 import re
+import sta_parser.ast as ast
 from odata_query.grammar import ODataLexer
 from odata_query.grammar import ODataParser
 from filter_visitor import FilterVisitor
 from sta_parser.lexer import Lexer
 from sta_parser.visitor import Visitor
 from sta_parser.parser import Parser
-from sta_parser import ast
 
 # Create the OData lexer and parser
 odata_filter_lexer = ODataLexer()
@@ -322,7 +322,7 @@ def convert_entity(entity: str) -> str:
         return STA2REST.ENTITY_MAPPING.get(entity, entity)
   
     @staticmethod
-    def convert_query(sta_query: str) -> str:
+    def convert_query(full_path: str) -> str:
         """
         Converts a STA query to a PostgREST query.
 
@@ -332,12 +332,72 @@ def convert_query(sta_query: str) -> str:
         Returns:
             str: The converted PostgREST query.
         """
-        lexer = Lexer(sta_query)
-        tokens = lexer.tokenize()
-        parser = Parser(tokens)
-        ast = parser.parse()
+
+        # check if we have a query
+        path = full_path
+        query = None
+        if '?' in full_path:
+            # Split the query from the path
+            path, query = full_path.split('?')
+
+        # Parse the uri
+        uri = STA2REST.parse_uri(path)
+        
+        if not uri:
+            raise Exception("Error parsing uri")
+        
+        main_entity, main_entity_id = uri['entity']
+        url = f"/{main_entity}"
+
+        # Check if we have a query
+        query_ast = ast.QueryNode(None, None, None, None, None, None, None, False)
+        if query:
+            lexer = Lexer(query)
+            tokens = lexer.tokenize()
+            parser = Parser(tokens)
+            query_ast = parser.parse()
+
+        # Check if we have a filter
+        if main_entity_id:
+            query_ast.filter = ast.FilterNode(query_ast.filter.filter + f" and id eq {main_entity_id}" if query_ast.filter else f"id eq {main_entity_id}")
+
+        entities = uri['entities']
+        if entities:
+            if not query_ast.expand:
+                query_ast.expand = ast.ExpandNode([])
+            
+            index = 0
+
+            # Merge the entities with the query
+            for entity in entities:
+                entity_name = entity[0]
+                sub_query = ast.QueryNode(None, None, None, None, None, None, None, True)
+                if entity[1]:
+                    sub_query.filter = ast.FilterNode(f"id eq {entity[1]}")
+
+                # Check if we are the last entity
+                if index == len(entities) - 1:
+                    # Check if we have a property name
+                    if uri['property_name']:
+                        # Add the property name to the select node
+                        if not sub_query.select:
+                            sub_query.select = ast.SelectNode([])
+                        sub_query.select.identifiers.append(ast.IdentifierNode(uri['property_name']))
+
+                query_ast.expand.identifiers.append(ast.ExpandNodeIdentifier(entity_name, sub_query))
+                index += 1
+
+            
+
+        # Visit the query ast to convert it
         visitor = NodeVisitor()
-        return visitor.visit(ast)
+        query_converted = visitor.visit(query_ast)
+
+        return {
+            'url': url + "?" + query_converted if query_converted else url,
+            'ref': uri['ref'],
+            'value': uri['value'],
+        }
 
     @staticmethod
     def parse_entity(entity: str):
@@ -369,8 +429,8 @@ def parse_uri(uri: str) -> str:
         version = parts.pop(0)
 
         # Parse first entity
-        entity = STA2REST.parse_entity(parts.pop(0))
-        if not entity:
+        main_entity = STA2REST.parse_entity(parts.pop(0))
+        if not main_entity:
             raise Exception("Error parsing uri: invalid entity")
 
         # Check all the entities in the uri
@@ -384,6 +444,8 @@ def parse_uri(uri: str) -> str:
             if result:
                 entities.append(result)
             elif entity == "$ref":
+                if property_name:
+                    raise Exception("Error parsing uri: $ref after property name")
                 ref = True
             elif entity == "$value":
                 if property_name:
@@ -395,19 +457,20 @@ def parse_uri(uri: str) -> str:
 
         return {
             'version': version,
-            'entity': entity,
+            'entity': main_entity,
             'entities': entities,
             'property_name': property_name,
             'ref': ref,
             'value': value
         }
 
+
 if __name__ == "__main__":
     """
     Example usage of the STA2REST module.
 
     This example converts a STA query to a REST query.
     """
-    query = "$expand=Observations($filter=result eq 1)"
+    query = "/v1.1/Datastreams(1)/Observations(1)/resultTime"
     print("QUERY", query)
     print("CONVERTED", STA2REST.convert_query(query))
\ No newline at end of file
diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
index ab2d221..ed04aed 100644
--- a/src/sta2rest/test.py
+++ b/src/sta2rest/test.py
@@ -58,6 +58,9 @@ def test_convert_sensor_things_query(self):
         """
         Test the conversion of sensor things queries.
         """
+
+        # TODO(@filippofinke): fix test cases
+
         query_mappings = {
             "$filter=type eq 'temperature'&$orderby=timestamp desc&$top=10&$skip=5":
                 "type=eq.temperature&order=timestamp.desc&offset=5&limit=10",

From f6e6597503dff3d3cda0c5191f661f16e12077c0 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 5 Jul 2023 17:25:47 +0200
Subject: [PATCH 14/23] feat: move sta2rest to fastapi folder

---
 {src => fastapi/app}/sta2rest/__init__.py     |   0
 .../app}/sta2rest/filter_visitor.py           |   0
 {src => fastapi/app}/sta2rest/sta2rest.py     |   3 +-
 src/sta2rest/test.py                          | 106 ------------------
 4 files changed, 2 insertions(+), 107 deletions(-)
 rename {src => fastapi/app}/sta2rest/__init__.py (100%)
 rename {src => fastapi/app}/sta2rest/filter_visitor.py (100%)
 rename {src => fastapi/app}/sta2rest/sta2rest.py (99%)
 delete mode 100644 src/sta2rest/test.py

diff --git a/src/sta2rest/__init__.py b/fastapi/app/sta2rest/__init__.py
similarity index 100%
rename from src/sta2rest/__init__.py
rename to fastapi/app/sta2rest/__init__.py
diff --git a/src/sta2rest/filter_visitor.py b/fastapi/app/sta2rest/filter_visitor.py
similarity index 100%
rename from src/sta2rest/filter_visitor.py
rename to fastapi/app/sta2rest/filter_visitor.py
diff --git a/src/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py
similarity index 99%
rename from src/sta2rest/sta2rest.py
rename to fastapi/app/sta2rest/sta2rest.py
index cff7226..13b0150 100644
--- a/src/sta2rest/sta2rest.py
+++ b/fastapi/app/sta2rest/sta2rest.py
@@ -10,7 +10,7 @@
 import sta_parser.ast as ast
 from odata_query.grammar import ODataLexer
 from odata_query.grammar import ODataParser
-from filter_visitor import FilterVisitor
+from .filter_visitor import FilterVisitor
 from sta_parser.lexer import Lexer
 from sta_parser.visitor import Visitor
 from sta_parser.parser import Parser
@@ -361,6 +361,7 @@ def convert_query(full_path: str) -> str:
         if main_entity_id:
             query_ast.filter = ast.FilterNode(query_ast.filter.filter + f" and id eq {main_entity_id}" if query_ast.filter else f"id eq {main_entity_id}")
 
+        # TODO(@filippofinke): Move the query to the last entitiy otherwise we can't expand the last entity
         entities = uri['entities']
         if entities:
             if not query_ast.expand:
diff --git a/src/sta2rest/test.py b/src/sta2rest/test.py
deleted file mode 100644
index ed04aed..0000000
--- a/src/sta2rest/test.py
+++ /dev/null
@@ -1,106 +0,0 @@
-"""
-Module: STA2REST Test
-
-Author: Filippo Finke
-
-This module provides unit tests for the STA2REST module.
-"""
-import unittest
-from sta2rest import STA2REST
-
-class STA2RESTTestCase(unittest.TestCase):
-    """
-    Test case for STA2REST module.
-    """
-    def test_convert_entity(self):
-        """
-        Test the conversion of entities.
-        """
-        entity_mappings = {
-            "Things": 'Thing',
-            "Locations": 'Location',
-            "Sensors": 'Sensor',
-            "ObservedProperties": 'ObservedProperty',
-            "Datastreams": 'Datastream',
-            "Observations": 'Observation',
-            "FeaturesOfInterest": 'FeatureOfInterest',
-            "HistoricalLocations": 'HistoricalLocation'
-            # Add more entity mappings as needed
-        }
-
-        for entity, expected in entity_mappings.items():
-            self.assertEqual(STA2REST.convert_entity(entity), expected)
-
-    def test_parse_uri(self):
-        """
-        Test the parsing of URIs.
-        """
-
-        # Test the parsing of URIs
-        tests = [
-            "/v1.1/ObservedProperties"
-            "/v1.1/Things(1)",
-            "/v1.1/Observations(1)/resultTime",
-            "/v1.1/Observations(1)/resultTime/$value",
-            "/v1.1/Datastreams(1)/Observations",
-            "/v1.1/Datastreams(1)/Observations/$ref",
-            "/v1.1/Datastreams(1)/Observations(1)",
-            "/v1.1/Datastreams(1)/Observations(1)/resultTime",
-            "/v1.1/Datastreams(1)/Observations(1)/FeatureOfInterest",
-        ]
-        
-        # Test the parsing of URIs with query
-        for test in tests:
-            print(test)
-            print(STA2REST.parse_uri(test))
-
-    def test_convert_sensor_things_query(self):
-        """
-        Test the conversion of sensor things queries.
-        """
-
-        # TODO(@filippofinke): fix test cases
-
-        query_mappings = {
-            "$filter=type eq 'temperature'&$orderby=timestamp desc&$top=10&$skip=5":
-                "type=eq.temperature&order=timestamp.desc&offset=5&limit=10",
-            "$filter=type eq 'humidity'&$top=5":
-                "type=eq.humidity&limit=5",
-            "$orderby=timestamp asc&$skip=2":
-                "order=timestamp.asc&offset=2",
-            "$select=id,name,description,properties&$top=1000&$filter=properties/type eq 'station'&$expand=Locations,Datastreams($select=id,name,unitOfMeasurement;$expand=ObservedProperty($select=name),Observations($select=result,phenomenonTime;$orderby=phenomenonTime desc;$top=1))":
-            "Datastream.Observation.order=phenomenonTime.desc&Datastream.Observation.limit=1&select=id,name,description,properties,Location(*),Datastream(id,name,unitOfMeasurement),ObservedProperty(name),Observation(result,phenomenonTime)&properties->>type=eq.station&limit=1000",
-            "$select=id,description&$expand=Datastreams($select=id,description)": "select=id,description,Datastream(id,description)",
-            "$expand=Datastreams": "select=Datastream(*)",
-            "$expand=Observations,ObservedProperty": "select=Observation(*),ObservedProperty(*)",
-            "$expand=Observations($filter=result eq 1)": "Observation.result=eq.1&select=Observation(*)",
-            "$expand=Observations($select=result)": "select=Observation(result)",
-            "$select=result": "select=result",
-            "$orderby=result": "order=result.asc",
-            "$expand=Datastream&$orderby=Datastreams/id desc,phenomenonTime": "select=Datastream(*)&order=Datastreams/id.desc,phenomenonTime.asc",
-            "$top=5": "limit=5",
-            "$top=5&$orderby=phenomenonTime%20desc": "order=phenomenonTime.desc&limit=5",
-            "$skip=5": "offset=5",
-            "$count=true": "count=true",
-            "$filter=result lt 10.00": "result=lt.10.00",
-            "$filter=unitOfMeasurement/name eq 'degree Celsius'": "unitOfMeasurement->>name=eq.degree Celsius",
-            "$filter=unitOfMeasurement/name ne 'degree Celsius'": "unitOfMeasurement->>name=neq.degree Celsius",
-            "$filter=result gt 20.0": "result=gt.20.0",
-            "$filter=result ge 20.0": "result=gte.20.0",
-            "$filter=result lt 100": "result=lt.100",
-            "$filter=result le 100": "result=lte.100",
-            "$filter=result le 3.5 and FeatureOfInterest/id eq 1": "result=lte.3.5&FeatureOfInterest->>id=eq.1",
-            "$filter=result gt 20 or result le 3.5": "or=(result.gt.20,result.lte.3.5)",
-            "$filter=id eq 1 or id eq 2 or id eq 3": "or=(id.eq.1,id.eq.2,id.eq.3)",
-            "$filter=id eq 1 or id eq 2 or id eq 3 or id eq 4": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)",
-            "$filter=(id eq 1 or id eq 2 or id eq 3 or id eq 4) and location_id eq 2": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)&location_id=eq.2",
-            "$filter=location_id eq 2 and id eq 2": "location_id=eq.2&id=eq.2",
-        }
-
-        for query, expected in query_mappings.items():
-            print("QUERY", query)
-            self.assertEqual(STA2REST.convert_query(query), expected)
-
-if __name__ == '__main__':
-    # Run all tests
-    unittest.main()
\ No newline at end of file

From 7baa913c454f7058c33427a847d7b2ae781aaf88 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 5 Jul 2023 17:26:05 +0200
Subject: [PATCH 15/23] deps: add sta_parser and odata-query

---
 fastapi/requirements.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fastapi/requirements.txt b/fastapi/requirements.txt
index c721eda..05b8cff 100644
--- a/fastapi/requirements.txt
+++ b/fastapi/requirements.txt
@@ -6,4 +6,6 @@ ujson
 pypika
 httpx
 asyncio
-postgrest
\ No newline at end of file
+postgrest
+sta_parser @ git+https://github.com/filippofinke/sta-parser@main
+odata-query
\ No newline at end of file

From 2c8659bf23597c1492f1370f6bdb9213b5dff8ce Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 5 Jul 2023 17:26:19 +0200
Subject: [PATCH 16/23] feat: move tests

---
 fastapi/app/sta2rest/test.py | 106 +++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)
 create mode 100644 fastapi/app/sta2rest/test.py

diff --git a/fastapi/app/sta2rest/test.py b/fastapi/app/sta2rest/test.py
new file mode 100644
index 0000000..ed04aed
--- /dev/null
+++ b/fastapi/app/sta2rest/test.py
@@ -0,0 +1,106 @@
+"""
+Module: STA2REST Test
+
+Author: Filippo Finke
+
+This module provides unit tests for the STA2REST module.
+"""
+import unittest
+from sta2rest import STA2REST
+
+class STA2RESTTestCase(unittest.TestCase):
+    """
+    Test case for STA2REST module.
+    """
+    def test_convert_entity(self):
+        """
+        Test the conversion of entities.
+        """
+        entity_mappings = {
+            "Things": 'Thing',
+            "Locations": 'Location',
+            "Sensors": 'Sensor',
+            "ObservedProperties": 'ObservedProperty',
+            "Datastreams": 'Datastream',
+            "Observations": 'Observation',
+            "FeaturesOfInterest": 'FeatureOfInterest',
+            "HistoricalLocations": 'HistoricalLocation'
+            # Add more entity mappings as needed
+        }
+
+        for entity, expected in entity_mappings.items():
+            self.assertEqual(STA2REST.convert_entity(entity), expected)
+
+    def test_parse_uri(self):
+        """
+        Test the parsing of URIs.
+        """
+
+        # Test the parsing of URIs
+        tests = [
+            "/v1.1/ObservedProperties"
+            "/v1.1/Things(1)",
+            "/v1.1/Observations(1)/resultTime",
+            "/v1.1/Observations(1)/resultTime/$value",
+            "/v1.1/Datastreams(1)/Observations",
+            "/v1.1/Datastreams(1)/Observations/$ref",
+            "/v1.1/Datastreams(1)/Observations(1)",
+            "/v1.1/Datastreams(1)/Observations(1)/resultTime",
+            "/v1.1/Datastreams(1)/Observations(1)/FeatureOfInterest",
+        ]
+        
+        # Test the parsing of URIs with query
+        for test in tests:
+            print(test)
+            print(STA2REST.parse_uri(test))
+
+    def test_convert_sensor_things_query(self):
+        """
+        Test the conversion of sensor things queries.
+        """
+
+        # TODO(@filippofinke): fix test cases
+
+        query_mappings = {
+            "$filter=type eq 'temperature'&$orderby=timestamp desc&$top=10&$skip=5":
+                "type=eq.temperature&order=timestamp.desc&offset=5&limit=10",
+            "$filter=type eq 'humidity'&$top=5":
+                "type=eq.humidity&limit=5",
+            "$orderby=timestamp asc&$skip=2":
+                "order=timestamp.asc&offset=2",
+            "$select=id,name,description,properties&$top=1000&$filter=properties/type eq 'station'&$expand=Locations,Datastreams($select=id,name,unitOfMeasurement;$expand=ObservedProperty($select=name),Observations($select=result,phenomenonTime;$orderby=phenomenonTime desc;$top=1))":
+            "Datastream.Observation.order=phenomenonTime.desc&Datastream.Observation.limit=1&select=id,name,description,properties,Location(*),Datastream(id,name,unitOfMeasurement),ObservedProperty(name),Observation(result,phenomenonTime)&properties->>type=eq.station&limit=1000",
+            "$select=id,description&$expand=Datastreams($select=id,description)": "select=id,description,Datastream(id,description)",
+            "$expand=Datastreams": "select=Datastream(*)",
+            "$expand=Observations,ObservedProperty": "select=Observation(*),ObservedProperty(*)",
+            "$expand=Observations($filter=result eq 1)": "Observation.result=eq.1&select=Observation(*)",
+            "$expand=Observations($select=result)": "select=Observation(result)",
+            "$select=result": "select=result",
+            "$orderby=result": "order=result.asc",
+            "$expand=Datastream&$orderby=Datastreams/id desc,phenomenonTime": "select=Datastream(*)&order=Datastreams/id.desc,phenomenonTime.asc",
+            "$top=5": "limit=5",
+            "$top=5&$orderby=phenomenonTime%20desc": "order=phenomenonTime.desc&limit=5",
+            "$skip=5": "offset=5",
+            "$count=true": "count=true",
+            "$filter=result lt 10.00": "result=lt.10.00",
+            "$filter=unitOfMeasurement/name eq 'degree Celsius'": "unitOfMeasurement->>name=eq.degree Celsius",
+            "$filter=unitOfMeasurement/name ne 'degree Celsius'": "unitOfMeasurement->>name=neq.degree Celsius",
+            "$filter=result gt 20.0": "result=gt.20.0",
+            "$filter=result ge 20.0": "result=gte.20.0",
+            "$filter=result lt 100": "result=lt.100",
+            "$filter=result le 100": "result=lte.100",
+            "$filter=result le 3.5 and FeatureOfInterest/id eq 1": "result=lte.3.5&FeatureOfInterest->>id=eq.1",
+            "$filter=result gt 20 or result le 3.5": "or=(result.gt.20,result.lte.3.5)",
+            "$filter=id eq 1 or id eq 2 or id eq 3": "or=(id.eq.1,id.eq.2,id.eq.3)",
+            "$filter=id eq 1 or id eq 2 or id eq 3 or id eq 4": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)",
+            "$filter=(id eq 1 or id eq 2 or id eq 3 or id eq 4) and location_id eq 2": "or=(id.eq.1,id.eq.2,id.eq.3,id.eq.4)&location_id=eq.2",
+            "$filter=location_id eq 2 and id eq 2": "location_id=eq.2&id=eq.2",
+        }
+
+        for query, expected in query_mappings.items():
+            print("QUERY", query)
+            self.assertEqual(STA2REST.convert_query(query), expected)
+
+if __name__ == '__main__':
+    # Run all tests
+    unittest.main()
\ No newline at end of file

From bccf81930a6474778f1ecbaff5fe0c9cf192ad5b Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 5 Jul 2023 17:26:26 +0200
Subject: [PATCH 17/23] feat: include the general router

---
 fastapi/app/v1/api.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fastapi/app/v1/api.py b/fastapi/app/v1/api.py
index 5968488..2761ae3 100644
--- a/fastapi/app/v1/api.py
+++ b/fastapi/app/v1/api.py
@@ -2,11 +2,14 @@
 from app.v1.endpoints import sensors
 from app.v1.endpoints import contacts
 from app.v1.endpoints import observations
+from app.v1.endpoints import general
 from fastapi import FastAPI
 
 v1 = FastAPI()
 
-v1.include_router(op.v1)
+v1.include_router(general.v1)
+
+#v1.include_router(op.v1)
 # v1.include_router(sensors.v1)
 # v1.include_router(contacts.v1)
-v1.include_router(observations.v1)
\ No newline at end of file
+#v1.include_router(observations.v1)
\ No newline at end of file

From af5c50b0c5d5aa2c706ef936dd7b54869e03d07c Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Wed, 5 Jul 2023 17:26:33 +0200
Subject: [PATCH 18/23] feat: add poc general router

---
 fastapi/app/v1/endpoints/general.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 fastapi/app/v1/endpoints/general.py

diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py
new file mode 100644
index 0000000..da76ac3
--- /dev/null
+++ b/fastapi/app/v1/endpoints/general.py
@@ -0,0 +1,27 @@
+from fastapi import APIRouter, Request
+from app.sta2rest import sta2rest
+import httpx
+
+v1 = APIRouter()
+
+@v1.api_route("/{path_name:path}", methods=["GET"])
+async def catch_all(request: Request, path_name: str):
+    try:
+        # get full path from request
+        full_path = request.url.path + "?" + request.url.query
+        result = sta2rest.STA2REST.convert_query(full_path)
+        
+        path = result["url"]
+
+        print("original:\t", full_path)
+        print("url:\t\t", path)
+
+        url = "http://postgrest:3000" + path
+
+        async with httpx.AsyncClient() as client:
+            r = await client.get(url)
+            return r.json()
+        
+
+    except Exception as e:
+        return {"error": str(e)}

From 71b076cf4f185ad0aa50b39c80611c18e8a4d99d Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 6 Jul 2023 14:08:47 +0200
Subject: [PATCH 19/23] fix: merge sub query into expand

---
 fastapi/app/sta2rest/sta2rest.py | 36 +++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py
index 13b0150..93960e8 100644
--- a/fastapi/app/sta2rest/sta2rest.py
+++ b/fastapi/app/sta2rest/sta2rest.py
@@ -8,9 +8,9 @@
 """
 import re
 import sta_parser.ast as ast
+from .filter_visitor import FilterVisitor
 from odata_query.grammar import ODataLexer
 from odata_query.grammar import ODataParser
-from .filter_visitor import FilterVisitor
 from sta_parser.lexer import Lexer
 from sta_parser.visitor import Visitor
 from sta_parser.parser import Parser
@@ -357,11 +357,6 @@ def convert_query(full_path: str) -> str:
             parser = Parser(tokens)
             query_ast = parser.parse()
 
-        # Check if we have a filter
-        if main_entity_id:
-            query_ast.filter = ast.FilterNode(query_ast.filter.filter + f" and id eq {main_entity_id}" if query_ast.filter else f"id eq {main_entity_id}")
-
-        # TODO(@filippofinke): Move the query to the last entitiy otherwise we can't expand the last entity
         entities = uri['entities']
         if entities:
             if not query_ast.expand:
@@ -385,10 +380,37 @@ def convert_query(full_path: str) -> str:
                             sub_query.select = ast.SelectNode([])
                         sub_query.select.identifiers.append(ast.IdentifierNode(uri['property_name']))
 
+                    # Merge the query with the subquery
+                    if query_ast.select:
+                        sub_query.select = query_ast.select
+                        query_ast.select = None
+
+                    if query_ast.filter:
+                        sub_query.filter = query_ast.filter
+                        query_ast.filter = None
+
+                    if query_ast.orderby:
+                        sub_query.orderby = query_ast.orderby
+                        query_ast.orderby = None
+
+                    if query_ast.skip:
+                        sub_query.skip = query_ast.skip
+                        query_ast.skip = None
+
+                    if query_ast.top:
+                        sub_query.top = query_ast.top
+                        query_ast.top = None
+
+                    if query_ast.count:
+                        sub_query.count = query_ast.count
+                        query_ast.count = None
+
                 query_ast.expand.identifiers.append(ast.ExpandNodeIdentifier(entity_name, sub_query))
                 index += 1
 
-            
+        # Check if we have a filter in the query
+        if main_entity_id:
+            query_ast.filter = ast.FilterNode(query_ast.filter.filter + f" and id eq {main_entity_id}" if query_ast.filter else f"id eq {main_entity_id}")
 
         # Visit the query ast to convert it
         visitor = NodeVisitor()

From bdb48375688d19cab9f619c73e0ab85b080093e8 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 6 Jul 2023 14:09:13 +0200
Subject: [PATCH 20/23] fix: check if there is a query

---
 fastapi/app/v1/endpoints/general.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py
index da76ac3..184fc64 100644
--- a/fastapi/app/v1/endpoints/general.py
+++ b/fastapi/app/v1/endpoints/general.py
@@ -8,7 +8,10 @@
 async def catch_all(request: Request, path_name: str):
     try:
         # get full path from request
-        full_path = request.url.path + "?" + request.url.query
+        full_path = request.url.path
+        if request.url.query:
+            full_path += "?" + request.url.query
+            
         result = sta2rest.STA2REST.convert_query(full_path)
         
         path = result["url"]

From f8f624c24db53d6c4ab0bfef4c4e2e46c12ea374 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 6 Jul 2023 14:16:42 +0200
Subject: [PATCH 21/23] fix: select property

---
 fastapi/app/sta2rest/sta2rest.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py
index 93960e8..f6553be 100644
--- a/fastapi/app/sta2rest/sta2rest.py
+++ b/fastapi/app/sta2rest/sta2rest.py
@@ -407,6 +407,11 @@ def convert_query(full_path: str) -> str:
 
                 query_ast.expand.identifiers.append(ast.ExpandNodeIdentifier(entity_name, sub_query))
                 index += 1
+        else:
+            if uri['property_name']:
+                if not query_ast.select:
+                    query_ast.select = ast.SelectNode([])
+                query_ast.select.identifiers.append(ast.IdentifierNode(uri['property_name']))
 
         # Check if we have a filter in the query
         if main_entity_id:

From 705384b25f649b963a574fa620749e4da5da2ad8 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 6 Jul 2023 14:20:25 +0200
Subject: [PATCH 22/23] fix: return single result

---
 fastapi/app/sta2rest/sta2rest.py    |  6 ++++++
 fastapi/app/v1/endpoints/general.py | 12 +++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/fastapi/app/sta2rest/sta2rest.py b/fastapi/app/sta2rest/sta2rest.py
index f6553be..9025d99 100644
--- a/fastapi/app/sta2rest/sta2rest.py
+++ b/fastapi/app/sta2rest/sta2rest.py
@@ -336,6 +336,7 @@ def convert_query(full_path: str) -> str:
         # check if we have a query
         path = full_path
         query = None
+        single_result = False
         if '?' in full_path:
             # Split the query from the path
             path, query = full_path.split('?')
@@ -369,6 +370,7 @@ def convert_query(full_path: str) -> str:
                 entity_name = entity[0]
                 sub_query = ast.QueryNode(None, None, None, None, None, None, None, True)
                 if entity[1]:
+                    single_result = True
                     sub_query.filter = ast.FilterNode(f"id eq {entity[1]}")
 
                 # Check if we are the last entity
@@ -417,6 +419,9 @@ def convert_query(full_path: str) -> str:
         if main_entity_id:
             query_ast.filter = ast.FilterNode(query_ast.filter.filter + f" and id eq {main_entity_id}" if query_ast.filter else f"id eq {main_entity_id}")
 
+            if not entities:
+                single_result = True
+
         # Visit the query ast to convert it
         visitor = NodeVisitor()
         query_converted = visitor.visit(query_ast)
@@ -425,6 +430,7 @@ def convert_query(full_path: str) -> str:
             'url': url + "?" + query_converted if query_converted else url,
             'ref': uri['ref'],
             'value': uri['value'],
+            'single_result': single_result
         }
 
     @staticmethod
diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py
index 184fc64..6664fac 100644
--- a/fastapi/app/v1/endpoints/general.py
+++ b/fastapi/app/v1/endpoints/general.py
@@ -11,7 +11,7 @@ async def catch_all(request: Request, path_name: str):
         full_path = request.url.path
         if request.url.query:
             full_path += "?" + request.url.query
-            
+
         result = sta2rest.STA2REST.convert_query(full_path)
         
         path = result["url"]
@@ -23,8 +23,14 @@ async def catch_all(request: Request, path_name: str):
 
         async with httpx.AsyncClient() as client:
             r = await client.get(url)
-            return r.json()
-        
+            data = r.json()
+
+            if result['single_result']:
+                data = data[0]
+                if result['value']:
+                    # get the value of the first key
+                    data = data[list(data.keys())[0]]
 
+            return data
     except Exception as e:
         return {"error": str(e)}

From 4226cc89ef987667fbdb3c08da614734f5eacca7 Mon Sep 17 00:00:00 2001
From: filippofinke <filippofinke@protonmail.ch>
Date: Thu, 6 Jul 2023 14:29:58 +0200
Subject: [PATCH 23/23] fix: implement $ref

---
 fastapi/app/v1/endpoints/general.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/fastapi/app/v1/endpoints/general.py b/fastapi/app/v1/endpoints/general.py
index 6664fac..bcfcb83 100644
--- a/fastapi/app/v1/endpoints/general.py
+++ b/fastapi/app/v1/endpoints/general.py
@@ -29,7 +29,24 @@ async def catch_all(request: Request, path_name: str):
                 data = data[0]
                 if result['value']:
                     # get the value of the first key
-                    data = data[list(data.keys())[0]]
+                    data = data[list(data.keys())[0]]  
+            elif result['ref']:
+                # Get the value of the first key
+                key_name = list(data[0].keys())[0]
+                rows = data[0][key_name]
+                
+                data = {
+                    "value": []
+                }
+
+                table_name = key_name + "s"
+
+                host = request.url.scheme + "://" + request.url.netloc
+
+                for row in rows:
+                    data["value"].append({
+                        "@iot.selfLink": f"{host}/v1.1/{table_name}({row['id']})"
+                    })
 
             return data
     except Exception as e: