Merge pull request #193 from ClibMouse/Kusto-p3-range-op

Kusto-phase3: Add kql range operator

src/Parsers/Kusto/KQL_ReleaseNote.md

@@ -63,6 +63,9 @@ print range(endofday(datetime(2017-01-01 10:10:17)), endofday(datetime(2017-01-0
    ```
 - [take_anyif()](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/take-anyif-aggfunction)
 ## Operator
+- [range](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/rangeoperator)  
+   `range LastWeek from ago(7d) to now() step 1d`  
+   `range Steps from 1 to 8 step 3`  
 - [top-nested](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/topnestedoperator)
 
    ```

src/Parsers/Kusto/ParserKQLQuery.cpp

@@ -29,6 +29,7 @@
 #include <Parsers/ParserSelectWithUnionQuery.h>
 #include <Parsers/ParserTablesInSelectQuery.h>
 #include <Parsers/Kusto/ParserKQLTopNested.h>
+#include <Parsers/Kusto/ParserKQLRange.h>
 
 #include <format>
 
@@ -62,6 +63,7 @@ std::unordered_map<std::string, ParserKQLQuery::KQLOperatorDataFlowState> kql_pa
     {"lookup", {"lookup", true, true, false, 3}},
     {"join", {"join", true, true, false, 3}},
     {"top-nested", {"top-nested", true, true, true, 5}},
+    {"range", {"range", false, true, false, 3}},
 };
 
 bool ParserKQLBase::parseByString(const String expr, ASTPtr & node, const uint32_t max_depth)
@@ -401,6 +403,8 @@ std::unique_ptr<ParserKQLBase> ParserKQLQuery::getOperator(String & op_name)
         return std::make_unique<ParserKQLJoin>();
     else if (op_name == "top-nested")
         return std::make_unique<ParserKQLTopNested>();
+    else if (op_name == "range")
+        return std::make_unique<ParserKQLRange>();
     else
         return nullptr;
 }
@@ -409,7 +413,7 @@ bool ParserKQLQuery::getOperations(Pos & pos, Expected & expected, OperationsPos
 {
     String table_name(pos->begin, pos->end);
 
-    if (table_name == "print")
+    if (table_name == "print" || table_name == "range")
         operation_pos.push_back(std::make_pair(table_name, pos));
     else
         operation_pos.push_back(std::make_pair("table", pos));
@@ -570,6 +574,12 @@ bool ParserKQLQuery::executeImpl(Pos & pos, ASTPtr & node, Expected & expected)
             if (!ParserKQLPrint().parse(npos, node, expected))
                 return false;
         }
+        else if (kql_operator_str == "range")
+        {
+            ++npos;
+            if (!ParserKQLRange().parse(npos, node, expected))
+                return false;
+        }
         else if (kql_operator_str == "table")
         {
             if (!kql_operator_p->parse(npos, node, expected))

src/Parsers/Kusto/ParserKQLRange.cpp

@@ -0,0 +1,87 @@
+#include <format>
+#include <Parsers/ExpressionListParsers.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+#include <Parsers/Kusto/ParserKQLRange.h>
+#include <Parsers/ParserSelectQuery.h>
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int SYNTAX_ERROR;
+}
+
+bool ParserKQLRange::parseImpl(Pos & pos, ASTPtr & node, Expected & /*expected*/)
+{
+    ASTPtr select_node;
+    String columnName, start, stop, step;
+    auto start_pos = pos;
+    auto end_pos = pos;
+    while (!pos->isEnd() && pos->type != TokenType::PipeMark && pos->type != TokenType::Semicolon)
+    {
+        if (String(pos->begin, pos->end) == "from")
+        {
+            end_pos = pos;
+            --end_pos;
+            if (end_pos < start_pos)
+                throw Exception("Missing columnName for range operator", ErrorCodes::SYNTAX_ERROR);
+
+            columnName = String(start_pos->begin, end_pos->end);
+            start_pos = pos;
+            ++start_pos;
+        }
+        if (String(pos->begin, pos->end) == "to")
+        {
+            if (columnName.empty())
+                throw Exception("Missing `from` for range operator", ErrorCodes::SYNTAX_ERROR);
+            end_pos = pos;
+            --end_pos;
+            if (end_pos < start_pos)
+                throw Exception("Missing start expression for range operator", ErrorCodes::SYNTAX_ERROR);
+            start = String(start_pos->begin, end_pos->end);
+            start_pos = pos;
+            ++start_pos;
+        }
+        if (String(pos->begin, pos->end) == "step")
+        {
+            if (columnName.empty())
+                throw Exception("Missing `from` for range operator", ErrorCodes::SYNTAX_ERROR);
+            if (start.empty())
+                throw Exception("Missing 'to' for range operator", ErrorCodes::SYNTAX_ERROR);
+
+            end_pos = pos;
+            --end_pos;
+            if (end_pos < start_pos)
+                throw Exception("Missing stop expression for range operator", ErrorCodes::SYNTAX_ERROR);
+
+            stop = String(start_pos->begin, end_pos->end);
+            start_pos = pos;
+            ++start_pos;
+        }
+        ++pos;
+    }
+
+    if (columnName.empty() || start.empty() || stop.empty())
+        throw Exception("Missing required expression for range operator", ErrorCodes::SYNTAX_ERROR);
+
+    end_pos = pos;
+    --end_pos;
+    if (end_pos < start_pos)
+        throw Exception("Missing step expression for range operator", ErrorCodes::SYNTAX_ERROR);
+
+    step = String(start_pos->begin, end_pos->end);
+
+    columnName = getExprFromToken(columnName, pos.max_depth);
+    start = getExprFromToken(start, pos.max_depth);
+    stop = getExprFromToken(stop, pos.max_depth);
+    step = getExprFromToken(step, pos.max_depth);
+    String query = std::format("SELECT * FROM (SELECT kql_range({0}, {1},{2}) AS {3}) ARRAY JOIN {3}", start, stop, step, columnName);
+
+    if (!parseSQLQueryByString(std::make_unique<ParserSelectQuery>(), query, select_node, pos.max_depth))
+        return false;
+    node = std::move(select_node);
+
+    return true;
+}
+
+}

src/Parsers/Kusto/ParserKQLRange.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+#include <Parsers/Kusto/ParserKQLQuery.h>
+
+namespace DB
+{
+
+class ParserKQLRange : public ParserKQLBase
+{
+protected:
+    const char * getName() const override { return "KQL range"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+}

src/Parsers/tests/KQL/gtest_KQL_range.cpp

@@ -65,5 +65,17 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery_Range, ParserTest,
         {
             "print range(endofday(datetime(2017-01-01 10:10:17)), endofday(datetime(2017-01-03 10:10:17)), 1d)",
             "SELECT kql_range(kql_todatetime(addDays(toStartOfDay(kql_datetime('2017-01-01 10:10:17')), 0 + 1)) - toIntervalNanosecond(100), kql_todatetime(addDays(toStartOfDay(kql_datetime('2017-01-03 10:10:17')), 0 + 1)) - toIntervalNanosecond(100), toIntervalNanosecond(86400000000000))"
+        },
+        {
+            "range Age from 20 to 25 step 1",
+            "SELECT *\nFROM\n(\n    SELECT kql_range(20, 25, 1) AS Age\n)\nARRAY JOIN Age"
+        },
+        {
+            "range LastWeek from ago(7d) to now() step 1d",
+            "SELECT *\nFROM\n(\n    SELECT kql_range(now64(9, 'UTC') + (-1 * toIntervalNanosecond(604800000000000)), now64(9, 'UTC'), toIntervalNanosecond(86400000000000)) AS LastWeek\n)\nARRAY JOIN LastWeek"
+        },
+        {
+            "range FirstWeek from datetime('2023-01-01') to datetime('2023-01-07') step 1d",
+            "SELECT *\nFROM\n(\n    SELECT kql_range(kql_datetime('2023-01-01'), kql_datetime('2023-01-07'), toIntervalNanosecond(86400000000000)) AS FirstWeek\n)\nARRAY JOIN FirstWeek"
         }
 })));

tests/queries/0_stateless/02366_kql_range.reference

@@ -34,3 +34,23 @@
 [5400000000000,12600000000000]
 -- range function endofday, endofday, timespan --
 ['2017-01-01 23:59:59.999999900','2017-01-02 23:59:59.999999900','2017-01-03 23:59:59.999999900']
+-- range orerator int, int, int --
+20
+21
+22
+23
+24
+25
+-- range orerator float, float, float --
+20.5
+22
+23.5
+25
+-- range orerator datetime, datetime, timespan --
+2023-01-01 00:00:00.000000000
+2023-01-02 00:00:00.000000000
+2023-01-03 00:00:00.000000000
+2023-01-04 00:00:00.000000000
+2023-01-05 00:00:00.000000000
+2023-01-06 00:00:00.000000000
+2023-01-07 00:00:00.000000000

tests/queries/0_stateless/02366_kql_range.sql

@@ -52,4 +52,13 @@ print '-- range function float timespan, timespan, timespan --';
 print range(1.5h, 5h, 2h);
 
 print '-- range function endofday, endofday, timespan --';
-print range(endofday(datetime(2017-01-01 10:10:17)), endofday(datetime(2017-01-03 10:10:17)), 1d);
+print range(endofday(datetime(2017-01-01 10:10:17)), endofday(datetime(2017-01-03 10:10:17)), 1d);
+
+print '-- range orerator int, int, int --';
+range Age from 20 to 25 step 1;
+
+print '-- range orerator float, float, float --';
+range temp from 20.5 to 25.5 step 1.5;
+
+print '-- range orerator datetime, datetime, timespan --';
+range FirstWeek from datetime('2023-01-01') to datetime('2023-01-07') step 1d;