Add --env-admin-role CLI option; Add a lot of tests for new object types

CHANGELOG.md

@@ -6,8 +6,8 @@
 - Added `EXTERNAL_ACCESS_INTEGRATIOS` and `SECRETS` parameters for functions and procedures.
 - Added ability to set `default` for function and procedure arguments.
 - Fixed issue with event tables being dropped while processing normal tables.
-- Implemented `LIKE` check for `NETWORK POLICY` object type. Previously it was not available in Snowflake.
 - Implemented "owner" check via `SHOW GRANTS` for `NETWORK POLICY` and `EXTERNAL ACCESS INTEGRATION`. "Owner" column is normally not available for these objects types.
+- Added `--env-admin-role` CLI option.
 
 ## [0.21.0] - 2023-11-01
 

snowddl/app/base.py

@@ -100,6 +100,11 @@ def init_arguments_parser(self):
             help="Env prefix added to global object names, used to separate environments (e.g. DEV, PROD)",
             default=environ.get("SNOWFLAKE_ENV_PREFIX"),
         )
+        parser.add_argument(
+            "--env-admin-role",
+            help="Super administration role which should inherit env prefixed SnowDDL role",
+            default=environ.get("SNOWFLAKE_ENV_ADMIN_ROLE"),
+        )
         parser.add_argument(
             "--max-workers", help="Maximum number of workers to resolve objects in parallel", default=None, type=int
         )
@@ -350,6 +355,9 @@ def init_settings(self):
 
             settings.clone_table = True
 
+        if self.args.get("env_admin_role"):
+            settings.env_admin_role = Ident(self.args.get("env_admin_role"))
+
         if self.args.get("exclude_object_types"):
             try:
                 settings.exclude_object_types = [

snowddl/context.py

@@ -92,12 +92,19 @@ def activate_role_with_prefix(self):
                 },
             )
 
-            if not self.is_account_admin:
+            if self.engine.settings.env_admin_role:
+                self.engine.execute_context_ddl(
+                    "GRANT ROLE {role_with_prefix:i} TO ROLE {env_admin_role:i}",
+                    {
+                        "role_with_prefix": role_with_prefix,
+                        "env_admin_role": self.engine.settings.env_admin_role,
+                    },
+                )
+            elif not self.is_account_admin:
                 self.engine.execute_context_ddl(
                     "GRANT ROLE {role_with_prefix:i} TO ROLE ACCOUNTADMIN",
                     {
                         "role_with_prefix": role_with_prefix,
-                        "current_role": self.current_role,
                     },
                 )
 

snowddl/resolver/network_policy.py

@@ -11,14 +11,13 @@ def get_object_type(self) -> ObjectType:
     def get_existing_objects(self):
         existing_objects = {}
 
-        cur = self.engine.execute_meta(
-            "SHOW NETWORK POLICIES LIKE {env_prefix:ls}",
-            {
-                "env_prefix": self.config.env_prefix,
-            }
-        )
+        cur = self.engine.execute_meta("SHOW NETWORK POLICIES")
 
         for r in cur:
+            # SHOW NETWORK POLICIES does not support LIKE filter, so it has to be applied manually in the code
+            if self.config.env_prefix and not str(r["name"]).startswith(self.config.env_prefix):
+                continue
+
             existing_objects[r["name"]] = {
                 "name": r["name"],
                 "entries_in_allowed_ip_list": r["entries_in_allowed_ip_list"],

snowddl/settings.py

@@ -1,10 +1,11 @@
-from typing import List
+from typing import Optional, List
 
-from snowddl.blueprint import DatabaseIdent, ObjectType
+from snowddl.blueprint import DatabaseIdent, ObjectType, Ident
 from snowddl.model import BaseModelWithConfig
 
 
 class SnowDDLSettings(BaseModelWithConfig):
+    env_admin_role: Optional[Ident] = None
     execute_safe_ddl: bool = False
     execute_unsafe_ddl: bool = False
     execute_replace_table: bool = False

test/_config/step1/db1/sc1/function/fn008_fn1(number,number,float,binary,varchar,date,time,timestamp_ltz,timestamp_ntz,timestamp_tz,variant,object,array,geography,geometry).yaml

@@ -0,0 +1,77 @@
+arguments:
+  num1:
+    type: NUMBER(10,0)
+  num2:
+    type: NUMBER(10,5)
+    default: 123.456
+  dbl:
+    type: FLOAT
+    default: 123.456
+  bin1:
+    type: BINARY(10)
+    default: "TO_BINARY('SUN', 'utf-8')"
+  var1:
+    type: VARCHAR(10)
+    default: "'abc'"
+  dt1:
+    type: DATE
+    default: "'2023-01-01'::date"
+  tm1:
+    type: TIME(9)
+    default: "'01:02:03.123'::time"
+  ltz1:
+    type: TIMESTAMP_LTZ(9)
+    default: "'2023-01-02 16:00:00'::timestamp_ltz"
+  ntz1:
+    type: TIMESTAMP_NTZ(9)
+    default: "'2023-01-02 16:00:00'::timestamp_ntz"
+  tz1:
+    type: TIMESTAMP_TZ(9)
+    default: "'2023-01-02 16:00:00 +01:00'::timestamp_tz"
+  var:
+    type: VARIANT
+    default: "PARSE_JSON('{\"foo\": \"bar\"}')::variant"
+  obj:
+    type: OBJECT
+    default: "PARSE_JSON('{\"foo\": \"bar\"}')::object"
+  arr:
+    type: ARRAY
+    default: "PARSE_JSON('[1, 2, 3]')::array"
+  geo1:
+    type: GEOGRAPHY
+    default: "TO_GEOGRAPHY('POINT(0 0)')"
+  geo2:
+    type: GEOMETRY
+    default: "TO_GEOMETRY('POINT(0 0)')"
+
+
+returns:
+  num1: NUMBER(10,0)
+  num2: NUMBER(10,5)
+  dbl: FLOAT
+  bin1: BINARY(10)
+  var1: VARCHAR(10)
+  dt1: DATE
+  tm1: TIME(9)
+  ltz1: TIMESTAMP_LTZ(9)
+  ntz1: TIMESTAMP_NTZ(9)
+  tz1: TIMESTAMP_TZ(9)
+  var: VARIANT
+  obj: OBJECT
+  arr: ARRAY
+  geo1: GEOGRAPHY
+  geo2: GEOMETRY
+
+
+body: |-
+  SELECT NUM1, NUM2
+      , DBL
+      , BIN1
+      , VAR1
+      , DT1
+      , TM1
+      , LTZ1
+      , NTZ1
+      , TZ1
+      , VAR, OBJ, ARR
+      , GEO1, GEO2

test/_config/step1/db1/sc1/function/fn009_fn1().yaml

@@ -0,0 +1,27 @@
+language: python
+runtime_version: "3.10"
+handler: get_fact
+packages:
+  - requests
+
+external_access_integrations:
+  - fn009_eai1
+secrets:
+  foo: fn009_se1
+  bar: fn009_se2
+
+returns: VARIANT
+
+body: |-
+  import _snowflake
+  import requests
+
+  def get_fact():
+      response = requests.get("https://catfact.ninja/fact")
+
+      return {
+          "foo": _snowflake.get_generic_secret_string("foo"),
+          "bar": _snowflake.get_generic_secret_string("bar"),
+          "fact": response.json()["fact"],
+          "length": response.json()["length"],
+      }

test/_config/step1/db1/sc1/network_rule/eai001_nr1.yaml

@@ -0,0 +1,5 @@
+type: HOST_PORT
+mode: EGRESS
+value_list:
+  - example.com
+  - company.com:443

test/_config/step1/db1/sc1/network_rule/eai001_nr2.yaml

@@ -0,0 +1,4 @@
+type: HOST_PORT
+mode: EGRESS
+value_list:
+  - catfact.ninja

test/_config/step1/db1/sc1/network_rule/fn009_nr1.yaml

@@ -0,0 +1,4 @@
+type: HOST_PORT
+mode: EGRESS
+value_list:
+  - catfact.ninja

test/_config/step1/db1/sc1/network_rule/nr001_nr1.yaml

@@ -0,0 +1,7 @@
+type: IPV4
+mode: INGRESS
+value_list:
+  - 192.168.2.0/24
+  - 192.168.1.99
+
+comment: abc

test/_config/step1/db1/sc1/network_rule/nr002_nr1.yaml

@@ -0,0 +1,5 @@
+type: HOST_PORT
+mode: EGRESS
+value_list:
+  - example.com
+  - company.com:443

test/_config/step1/db1/sc1/procedure/pr004_pr1(number,number,float,binary,varchar,date,time,timestamp_ltz,timestamp_ntz,timestamp_tz,variant,object,array).yaml

@@ -0,0 +1,76 @@
+arguments:
+  num1:
+    type: NUMBER(10,0)
+    default: 0
+  num2:
+    type: NUMBER(10,5)
+    default: 10.15
+  dbl:
+    type: FLOAT
+    default: 10.15
+  bin1:
+    type: BINARY(10)
+    default: TO_BINARY('SUN', 'utf-8')
+  var1:
+    type: VARCHAR(10)
+    default: "'abc'"
+  dt1:
+    type: DATE
+    default: CURRENT_DATE()
+  tm1:
+    type: TIME(9)
+    default: "'01:02:03'::time"
+  ltz1:
+    type: TIMESTAMP_LTZ(9)
+    default: CURRENT_TIMESTAMP()
+  ntz1:
+    type: TIMESTAMP_NTZ(9)
+    default: CURRENT_TIMESTAMP()
+  tz1:
+    type: TIMESTAMP_TZ(9)
+    default: CURRENT_TIMESTAMP()
+  var:
+    type: VARIANT
+    default: "PARSE_JSON('{\"foo\": \"bar\"}')::variant"
+  obj:
+    type: OBJECT
+    default: "PARSE_JSON('{\"foo\": \"bar\"}')::object"
+  arr:
+    type: ARRAY
+    default: "PARSE_JSON('[1, 2, 3]')::array"
+
+
+returns:
+  num1: NUMBER(10,0)
+  num2: NUMBER(10,5)
+  dbl: FLOAT
+  bin1: BINARY(10)
+  var1: VARCHAR(10)
+  dt1: DATE
+  tm1: TIME(0)
+  ltz1: TIMESTAMP_LTZ(9)
+  ntz1: TIMESTAMP_NTZ(9)
+  tz1: TIMESTAMP_TZ(9)
+  var: VARIANT
+  obj: OBJECT
+  arr: ARRAY
+
+body: |-
+  DECLARE
+    res resultset default (
+        SELECT :NUM1, :NUM2
+            , :DBL
+            , :BIN1
+            , :VAR1
+            , :DT1
+            , :TM1
+            , :LTZ1
+            , :NTZ1
+            , :TZ1
+            , :VAR, :OBJ, :ARR
+    );
+  BEGIN
+    RETURN table(res);
+  END;
+
+comment: abc

test/_config/step1/db1/sc1/secret/eai001_se1.yaml

@@ -0,0 +1,5 @@
+type: oauth2
+api_authentication: TEST_API_SECURITY_INTEGRATION
+oauth_scopes:
+  - photo
+  - offline_access

test/_config/step1/db1/sc1/secret/eai001_se2.yaml

@@ -0,0 +1,4 @@
+type: oauth2
+api_authentication: TEST_API_SECURITY_INTEGRATION
+oauth_refresh_token: RjY2NjM5NzA2OWJjuE7c
+oauth_refresh_token_expiry_time: "2030-01-01 00:00:00"

test/_config/step1/db1/sc1/secret/fn009_se1.yaml

@@ -0,0 +1,3 @@
+type: generic_string
+secret_string: this is foo!
+

test/_config/step1/db1/sc1/secret/fn009_se2.yaml

@@ -0,0 +1,3 @@
+type: generic_string
+secret_string: this is bar!
+

test/_config/step1/db1/sc1/secret/se001_se1.yaml

@@ -0,0 +1,3 @@
+type: generic_string
+secret_string: very secret string!
+comment: abc

test/_config/step1/db1/sc1/secret/se002_se1.yaml

@@ -0,0 +1,3 @@
+type: password
+username: john_doe
+password: very secret password

test/_config/step1/db1/sc1/secret/se003_se1.yaml

@@ -0,0 +1,5 @@
+type: oauth2
+api_authentication: TEST_API_SECURITY_INTEGRATION
+oauth_scopes:
+  - photo
+  - offline_access

test/_config/step1/db1/sc1/secret/se004_se1.yaml

@@ -0,0 +1,4 @@
+type: oauth2
+api_authentication: TEST_API_SECURITY_INTEGRATION
+oauth_refresh_token: RjY2NjM5NzA2OWJjuE7c
+oauth_refresh_token_expiry_time: "2030-01-01 00:00:00"

test/_config/step1/external_access_integration.yaml

@@ -0,0 +1,17 @@
+eai001_eai1:
+  allowed_network_rules:
+    - db1.sc1.eai001_nr1
+    - db1.sc1.eai001_nr2
+  allowed_api_authentication_integrations:
+    - TEST_API_SECURITY_INTEGRATION
+  allowed_authentication_secrets:
+    - db1.sc1.eai001_se1
+    - db1.sc1.eai001_se2
+  comment: abc
+
+fn009_eai1:
+  allowed_network_rules:
+    - db1.sc1.fn009_nr1
+  allowed_authentication_secrets:
+    - db1.sc1.fn009_se1
+    - db1.sc1.fn009_se2