From 49ee221a5d4df86a2f3b5cacd25439a7004bd698 Mon Sep 17 00:00:00 2001 From: vdelacruzb Date: Tue, 9 Jan 2024 18:04:25 +0100 Subject: [PATCH] remove only udfs --- clouds/bigquery/common/run-script.js | 15 ++++++++++++++- clouds/bigquery/modules/Makefile | 4 +++- clouds/postgres/common/run_script.py | 12 +++++++++++- clouds/postgres/modules/Makefile | 3 ++- clouds/redshift/common/run_script.py | 12 +++++++++++- clouds/redshift/modules/Makefile | 3 ++- clouds/snowflake/common/run-script.js | 15 ++++++++++++++- clouds/snowflake/modules/Makefile | 3 ++- 8 files changed, 59 insertions(+), 8 deletions(-) diff --git a/clouds/bigquery/common/run-script.js b/clouds/bigquery/common/run-script.js index bc6744030..6f5c04e23 100755 --- a/clouds/bigquery/common/run-script.js +++ b/clouds/bigquery/common/run-script.js @@ -16,11 +16,24 @@ const bar = new cliProgress.SingleBar(options, cliProgress.Presets.shades_classi const client = new BigQuery({ projectId: `${BQ_PROJECT}` }); +function apply_replacements (text) { + if (process.env.REPLACEMENTS) { + const replacements = process.env.REPLACEMENTS.split(' '); + for (let replacement of replacements) { + if (replacement) { + const pattern = new RegExp(`@@${replacement}@@`, 'g'); + text = text.replace(pattern, process.env[replacement]); + } + } + } + return text; +} + async function runQueries (queries) { const n = queries.length; bar.start(n, 0); for (let i = 0; i < n; i++) { - let query = queries[i]; + let query = apply_replacements(queries[i]); const pattern = /(FUNCTION|PROCEDURE)\s+(.*?)[(\n]/g; const results = pattern.exec(query); diff --git a/clouds/bigquery/modules/Makefile b/clouds/bigquery/modules/Makefile index b982cc265..ca2f9abfe 100644 --- a/clouds/bigquery/modules/Makefile +++ b/clouds/bigquery/modules/Makefile @@ -97,7 +97,9 @@ test: check $(NODE_MODULES_DEV) remove: check echo "Removing modules..." - $(BQ) rm -r -f -d $(BQ_DEPLOY_DATASET) + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ + $(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql clean: echo "Cleaning modules..." diff --git a/clouds/postgres/common/run_script.py b/clouds/postgres/common/run_script.py index 4d89ce134..1bcd7f264 100644 --- a/clouds/postgres/common/run_script.py +++ b/clouds/postgres/common/run_script.py @@ -9,6 +9,16 @@ function = '' +def apply_replacements(text): + if os.environ.get('REPLACEMENTS'): + replacements = os.environ.get('REPLACEMENTS').split(' ') + for replacement in replacements: + if replacement: + pattern = re.compile(f'@@{replacement}@@', re.MULTILINE) + text = pattern.sub(os.environ.get(replacement, ''), text) + return text + + def run_queries(queries): global function with connect( @@ -20,7 +30,7 @@ def run_queries(queries): conn.autocommit = True with conn.cursor() as cursor: for i in trange(len(queries), ncols=97): - query = queries[i] + query = apply_replacements(queries[i]) pattern = os.environ['PG_SCHEMA'] + '.(.*?)[(|\n]' result = re.search(pattern, query) if result: diff --git a/clouds/postgres/modules/Makefile b/clouds/postgres/modules/Makefile index 44f26d760..ba3603b36 100644 --- a/clouds/postgres/modules/Makefile +++ b/clouds/postgres/modules/Makefile @@ -61,7 +61,8 @@ test: check venv3 $(NODE_MODULES_DEV) remove: venv3 echo "Removing modules..." - $(VENV3_BIN)/python $(COMMON_DIR)/run_query.py "DROP SCHEMA IF EXISTS $(PG_SCHEMA) CASCADE;" + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + $(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql clean: echo "Cleaning modules..." diff --git a/clouds/redshift/common/run_script.py b/clouds/redshift/common/run_script.py index 1d0a3e53f..167b42730 100644 --- a/clouds/redshift/common/run_script.py +++ b/clouds/redshift/common/run_script.py @@ -10,6 +10,16 @@ function = '' +def apply_replacements(text): + if os.environ.get('REPLACEMENTS'): + replacements = os.environ.get('REPLACEMENTS').split(' ') + for replacement in replacements: + if replacement: + pattern = re.compile(f'@@{replacement}@@', re.MULTILINE) + text = pattern.sub(os.environ.get(replacement, ''), text) + return text + + def run_queries(queries): global function with connect( @@ -22,7 +32,7 @@ def run_queries(queries): filter = os.environ.get('FILTER') with conn.cursor() as cursor: for i in trange(len(queries) if not filter else 1, ncols=97): - query = queries[i] + query = apply_replacements(queries[i]) if (not filter) or (filter in query): pattern = os.environ['RS_SCHEMA'] + '.(.*?)[(|\n]' result = re.search(pattern, str(query)) diff --git a/clouds/redshift/modules/Makefile b/clouds/redshift/modules/Makefile index bc16e6576..d56d2c3c4 100644 --- a/clouds/redshift/modules/Makefile +++ b/clouds/redshift/modules/Makefile @@ -61,7 +61,8 @@ test: check venv3 $(NODE_MODULES_DEV) remove: venv3 echo "Removing modules..." - $(VENV3_BIN)/python $(COMMON_DIR)/run_query.py "DROP SCHEMA IF EXISTS $(RS_SCHEMA) CASCADE;" + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + $(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql clean: echo "Cleaning modules..." diff --git a/clouds/snowflake/common/run-script.js b/clouds/snowflake/common/run-script.js index 0d04aa737..902808742 100755 --- a/clouds/snowflake/common/run-script.js +++ b/clouds/snowflake/common/run-script.js @@ -42,11 +42,24 @@ async function runQuery (query) { }); } +function apply_replacements (text) { + if (process.env.REPLACEMENTS) { + const replacements = process.env.REPLACEMENTS.split(' '); + for (let replacement of replacements) { + if (replacement) { + const pattern = new RegExp(`@@${replacement}@@`, 'g'); + text = text.replace(pattern, process.env[replacement]); + } + } + } + return text; +} + async function runQueries (queries) { const n = queries.length; bar.start(n, 0); for (let i = 0; i < n; i++) { - let query = `BEGIN\n${queries[i]}\nEND;`; + let query = apply_replacements (`BEGIN\n${queries[i]}\nEND;`); const pattern = /(FUNCTION|PROCEDURE)\s+(.*?)[(\n]/g; const results = pattern.exec(query); diff --git a/clouds/snowflake/modules/Makefile b/clouds/snowflake/modules/Makefile index dc1df1988..0ec3c5edb 100644 --- a/clouds/snowflake/modules/Makefile +++ b/clouds/snowflake/modules/Makefile @@ -106,7 +106,8 @@ test: check $(NODE_MODULES_DEV) remove: $(NODE_MODULES_DEV) echo "Removing modules..." - $(COMMON_DIR)/run-query.js "DROP SCHEMA IF EXISTS $(SF_SCHEMA) CASCADE;" + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + $(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql remove-share: $(NODE_MODULES_DEV) echo "Removing share..."