Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Small fixes to materialized views support #539

Closed
wants to merge 1 commit into from
Closed

Small fixes to materialized views support #539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# 2024-01-12 (5.23.1)

* Some fixes to handle field based authentication for materialized views

# 2024-01-12 (5.23.0)

* Modified create_views functions to support materialized views
Expand Down
2 changes: 1 addition & 1 deletion setup.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[metadata]
name = amsterdam-schema-tools
version = 5.23.0
version = 5.23.1
url = https://github.com/amsterdam/schema-tools
license = Mozilla Public 2.0
author = Team Data Diensten, van het Dataplatform onder de Directie Digitale Voorzieningen (Gemeente Amsterdam)
Expand Down
144 changes: 75 additions & 69 deletions src/schematools/contrib/django/management/commands/create_views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ def _get_required_permissions(
for relation in derived_from:
datasetname, tablename = relation.split(":")
all_scopes |= _get_scopes(datasetname, tablename)
if len(all_scopes) > 1:
frozenset(set(all_scopes) - {"OPENBAAR"})
return all_scopes


Expand Down Expand Up @@ -136,95 +138,99 @@ def create_views(
# Check if the view sql is valid
# If not skip this view and proceed with next view
view_sql = _clean_sql(dataset.schema.get_view_sql())
view_type = "materialized" if "materialized" in view_sql.lower() else "view"
if not _is_valid_sql(view_sql, table.db_name, write_role_name):
command.stderr.write(f" Invalid SQL for view {table.db_name}")
continue

required_permissions = _get_required_permissions(table)
view_dataset_auth = dataset.schema.auth
if _check_required_permissions_exist(view_dataset_auth, required_permissions):
try:
with connection.cursor() as cursor:

# Check if write role exists and create if it does not
_create_role_if_not_exists(cursor, write_role_name)

# Grant usage and create on schema public to write role
cursor.execute(
sql.SQL(
"GRANT usage,create on schema public TO {write_role_name}"
).format(write_role_name=sql.Identifier(write_role_name))
if view_type != "materialized":
if not _check_required_permissions_exist(
view_dataset_auth, required_permissions
):
command.stderr.write(
f" Required permissions for view {table.db_name} are not in the view dataset auth"
)

try:
with connection.cursor() as cursor:

# Check if write role exists and create if it does not
_create_role_if_not_exists(cursor, write_role_name)

# Grant usage and create on schema public to write role
cursor.execute(
sql.SQL(
"GRANT usage,create on schema public TO {write_role_name}"
).format(write_role_name=sql.Identifier(write_role_name))
)

# We create one `view_owner` role that owns all views
_create_role_if_not_exists(cursor, "view_owner")

cursor.execute("GRANT view_owner TO current_user")

cursor.execute(
sql.SQL("GRANT {write_role_name} TO view_owner").format(
write_role_name=sql.Identifier(write_role_name)
)
)

# We create one `view_owner` role that owns all views
_create_role_if_not_exists(cursor, "view_owner")

cursor.execute("GRANT view_owner TO current_user")

cursor.execute(
sql.SQL("GRANT {write_role_name} TO view_owner").format(
write_role_name=sql.Identifier(write_role_name)
# Loop though all required permissions and and grant them to the
# write user
for scope in required_permissions:
scope = f'scope_{scope.replace("/", "_").lower()}'
if scope:
cursor.execute(
sql.SQL("GRANT {scope} TO {write_role_name}").format(
scope=sql.Identifier(scope),
write_role_name=sql.Identifier(write_role_name),
)
)
)

# Loop though all required permissions and and grant them to the
# write user
for scope in required_permissions:
scope = f'scope_{scope.replace("/", "_").lower()}'
if scope:
cursor.execute(
sql.SQL("GRANT {scope} TO {write_role_name}").format(
scope=sql.Identifier(scope),
write_role_name=sql.Identifier(write_role_name),
)
)
cursor.execute(
sql.SQL("GRANT scope_openbaar TO {write_role_name}").format(
write_role_name=sql.Identifier(write_role_name)
)
)

cursor.execute(
sql.SQL("GRANT scope_openbaar TO {write_role_name}").format(
write_role_name=sql.Identifier(write_role_name)
)
# Set the role before creating the view because the view is created
# with the permissions of the role
cursor.execute(
sql.SQL("SET ROLE {write_role_name}").format(
write_role_name=sql.Identifier(write_role_name)
)
)

# Set the role before creating the view because the view is created
# with the permissions of the role
# Remove the view if it exists
# Due to the large costs of recreating materialized views, we only create
# and not drop them. When changes are made to the materialized view the view
# must be droped manually.
if view_type != "materialized":
cursor.execute(
sql.SQL("SET ROLE {write_role_name}").format(
write_role_name=sql.Identifier(write_role_name)
sql.SQL("DROP VIEW IF EXISTS {view_name} CASCADE").format(
view_name=sql.Identifier(table.db_name)
)
)

# Remove the view if it exists
# Due to the large costs of recreating materialized views, we only create
# and not drop them. When changes are made to the materialized view the view
# must be droped manually.
if "materialized" not in view_sql.lower():
cursor.execute(
sql.SQL("DROP VIEW IF EXISTS {view_name} CASCADE").format(
view_name=sql.Identifier(table.db_name)
)
)

# Create the view
cursor.execute(view_sql)
# Create the view
cursor.execute(view_sql)

# Reset the role to the default role
cursor.execute("RESET ROLE")
# Reset the role to the default role
cursor.execute("RESET ROLE")

# Remove create and usage from write role
cursor.execute(
sql.SQL(
"REVOKE usage,create on schema public FROM {write_role_name}"
).format(write_role_name=sql.Identifier(write_role_name))
)
# Remove create and usage from write role
cursor.execute(
sql.SQL(
"REVOKE usage,create on schema public FROM {write_role_name}"
).format(write_role_name=sql.Identifier(write_role_name))
)

cursor.close()
except (DatabaseError, ValueError) as e:
command.stderr.write(f" View not created: {e}")
errors += 1
else:
command.stderr.write(
f" Required permissions {required_permissions} not found in view auth"
)
cursor.close()
except (DatabaseError, ValueError) as e:
command.stderr.write(f" View not created: {e}")
errors += 1

if errors:
raise CommandError("Not all views could be created")