From e905b060abc895f4f81b1d3c257c563bfdc0c978 Mon Sep 17 00:00:00 2001 From: Yoshani Date: Thu, 9 Jan 2025 11:18:50 +0530 Subject: [PATCH] add saml idp metadata cleanup scripts --- .../db2-saml-idp-metadata-cleanup-restore.sql | 56 ++++++ .../db2-saml-idp-metadata-cleanup.sql | 176 +++++++++++++++++ ...ssql-saml-idp-metadata-cleanup-restore.sql | 59 ++++++ .../mssql-saml-idp-metadata-cleanup.sql | 182 ++++++++++++++++++ ...ysql-saml-idp-metadata-cleanup-restore.sql | 50 +++++ .../mysql-saml-idp-metadata-cleanup.sql | 175 +++++++++++++++++ 6 files changed, 698 insertions(+) create mode 100644 features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/db2/saml-idp-metadata-cleanup/db2-saml-idp-metadata-cleanup-restore.sql create mode 100644 features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/db2/saml-idp-metadata-cleanup/db2-saml-idp-metadata-cleanup.sql create mode 100644 features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mssql/saml-idp-metadata-cleanup/mssql-saml-idp-metadata-cleanup-restore.sql create mode 100644 features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mssql/saml-idp-metadata-cleanup/mssql-saml-idp-metadata-cleanup.sql create mode 100644 features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mysql/saml-idp-metadata-cleanup/mysql-saml-idp-metadata-cleanup-restore.sql create mode 100644 features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mysql/saml-idp-metadata-cleanup/mysql-saml-idp-metadata-cleanup.sql diff --git a/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/db2/saml-idp-metadata-cleanup/db2-saml-idp-metadata-cleanup-restore.sql b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/db2/saml-idp-metadata-cleanup/db2-saml-idp-metadata-cleanup-restore.sql new file mode 100644 index 000000000000..0e85cbeb2bac --- /dev/null +++ b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/db2/saml-idp-metadata-cleanup/db2-saml-idp-metadata-cleanup-restore.sql @@ -0,0 +1,56 @@ +CREATE OR REPLACE PROCEDURE WSO2_SAML_IDP_METADATA_CLEANUP_RESTORE AS +BEGIN + -- ------------------------------------------ + -- DECLARE VARIABLES + -- ------------------------------------------ + DECLARE rowCount INT; + DECLARE enableLog SMALLINT; + + -- ------------------------------------------ + -- CONFIGURABLE ATTRIBUTES + -- ------------------------------------------ + SET enableLog = 1; -- ENABLE LOGGING [DEFAULT : TRUE] + + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION + BEGIN + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('ERROR OCCURRED: ' || SQLERRM); + END IF; + ROLLBACK; + END; + + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('SAML IDP METADATA CLEANUP DATA RESTORATION STARTED .... !'); + END IF; + + SELECT COUNT(*) INTO rowCount FROM SYSIBM.SYSTABLES WHERE CREATOR = CURRENT SCHEMA AND NAME = 'REG_RESOURCE'; + IF (rowCount = 1) + THEN + SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; + BEGIN TRANSACTION; + + INSERT INTO REG_RESOURCE (REG_PATH_ID, REG_NAME, REG_VERSION, REG_MEDIA_TYPE, REG_CREATOR, REG_CREATED_TIME, + REG_LAST_UPDATOR, REG_LAST_UPDATED_TIME, REG_DESCRIPTION, REG_CONTENT_ID, REG_TENANT_ID, REG_UUID) SELECT + A.REG_PATH_ID, A.REG_NAME, A.REG_VERSION, A.REG_MEDIA_TYPE, A.REG_CREATOR, A.REG_CREATED_TIME, A.REG_LAST_UPDATOR, + A.REG_LAST_UPDATED_TIME, A.REG_DESCRIPTION, A.REG_CONTENT_ID, A.REG_TENANT_ID, A.REG_UUID FROM BAK_REG_RESOURCE A LEFT + JOIN REG_RESOURCE B ON A.REG_VERSION = B.REG_VERSION AND A.REG_TENANT_ID = B.REG_TENANT_ID WHERE B.REG_VERSION IS NULL; + GET DIAGNOSTICS rowCount = ROW_COUNT; + + INSERT INTO REG_CONTENT (REG_CONTENT_ID, REG_CONTENT_DATA, REG_TENANT_ID) SELECT A.REG_CONTENT_ID, + A.REG_CONTENT_DATA, A.REG_TENANT_ID FROM BAK_REG_CONTENT A LEFT JOIN REG_CONTENT B ON + A.REG_CONTENT_ID = B.REG_CONTENT_ID WHERE B.REG_CONTENT_ID IS NULL; + + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('CLEANUP DATA RESTORATION COMPLETED ON REG_RESOURCE WITH ' || rowCount); + END IF; + COMMIT; + END IF; + + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('CLEANUP DATA RESTORATION COMPLETED .... !'); + END IF; +END; diff --git a/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/db2/saml-idp-metadata-cleanup/db2-saml-idp-metadata-cleanup.sql b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/db2/saml-idp-metadata-cleanup/db2-saml-idp-metadata-cleanup.sql new file mode 100644 index 000000000000..ce87b447ad50 --- /dev/null +++ b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/db2/saml-idp-metadata-cleanup/db2-saml-idp-metadata-cleanup.sql @@ -0,0 +1,176 @@ +-- NOTE: This procedure assumes that the SAML IDP metadata is stored under the path +-- '/_system/governance/repository/identity/provider/saml' in the registry and only two corresponding RESOURCE entries +-- (one for the collection and one for the resource object) and one CONTENT entry exist for each SAML IDP metadata file. + +CREATE OR REPLACE PROCEDURE WSO2_SAML_IDP_METADATA_CLEANUP AS +BEGIN + + -- ------------------------------------------ + -- DECLARE VARIABLES + -- ------------------------------------------ + DECLARE batchSize INT; + DECLARE chunkSize INT; + DECLARE batchCount INT; + DECLARE chunkCount INT; + DECLARE rowCount INT; + DECLARE enableLog SMALLINT; + DECLARE backupTables SMALLINT; + + -- ------------------------------------------ + -- CONFIGURABLE VARIABLES + -- ------------------------------------------ + SET batchSize = 10000; -- SET BATCH SIZE TO AVOID TABLE LOCKS [DEFAULT : 10000] + SET chunkSize = 500000; -- CHUNK WISE DELETE FOR LARGE TABLES [DEFAULT : 500000] + SET enableLog = 1; -- ENABLE LOGGING [DEFAULT : 1] + SET backupTables = 1; -- SET IF REGISTRY TABLES NEEDS TO BE BACKED-UP BEFORE DELETE [DEFAULT : 1]. + + SET rowCount = 0; + SET batchCount = 1; + SET chunkCount = 1; + + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION + BEGIN + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('ERROR OCCURRED: ' || SQLERRM); + END IF; + ROLLBACK; + END; + + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('WSO2_SAML_IDP_METADATA_CLEANUP() STARTED...!'); + END IF; + + -- ------------------------------------------ + -- GET PATH ID LIST TO DELETE + -- ------------------------------------------ + DECLARE GLOBAL TEMPORARY TABLE RootPathIdList (ROOT_PATH_ID INT) ON COMMIT DELETE ROWS; + DECLARE GLOBAL TEMPORARY TABLE PathIdList (REG_PATH_ID INT) ON COMMIT DELETE ROWS; + INSERT INTO RootPathIdList (ROOT_PATH_ID) SELECT REG_PATH_ID FROM REG_PATH WHERE + REG_PATH_VALUE = '/_system/governance/repository/identity/provider/saml'; + INSERT INTO PathIdList (REG_PATH_ID) SELECT REG_PATH_ID FROM REG_PATH WHERE REG_PATH_PARENT_ID IN + (SELECT ROOT_PATH_ID FROM RootPathIdList); + + -- ------------------------------------------ + -- BACKUP DATA + -- ------------------------------------------ + IF (backupTables = 1) + THEN + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('TABLE BACKUP STARTED...!'); + END IF; + + IF (EXISTS (SELECT NAME FROM SYSIBM.SYSTABLES WHERE NAME = 'BAK_REG_RESOURCE')) + THEN + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('DELETING OLD BACKUP...'); + END IF; + DROP TABLE BAK_REG_RESOURCE; + DROP TABLE BAK_REG_CONTENT; + END IF; + + -- BACKUP REG_RESOURCE TABLE + CREATE TABLE BAK_REG_RESOURCE AS (SELECT * FROM REG_RESOURCE WHERE REG_PATH_ID IN + (SELECT REG_PATH_ID FROM PathIdList)) WITH DATA; + + -- BACKUP REG_CONTENT TABLE + DECLARE GLOBAL TEMPORARY TABLE ContentIdList (REG_CONTENT_ID INT) ON COMMIT DELETE ROWS; + INSERT INTO ContentIdList (REG_CONTENT_ID) SELECT DISTINCT REG_CONTENT_ID FROM BAK_REG_RESOURCE WHERE + REG_CONTENT_ID IS NOT NULL; + CREATE TABLE BAK_REG_CONTENT AS (SELECT * FROM REG_CONTENT WHERE REG_CONTENT_ID IN (SELECT REG_CONTENT_ID + FROM ContentIdList)) WITH DATA; + END IF; + + -- ------------------------------------------ + -- CLEANUP DATA + -- ------------------------------------------ + + BEGIN TRANSACTION + + WHILE (chunkCount > 0) + DO + -- CREATE CHUNK TABLE + DROP TABLE IF EXISTS REG_RESOURCE_CHUNK_TMP; + DROP TABLE IF EXISTS REG_CONTENT_CHUNK_TMP; + + CREATE TABLE REG_RESOURCE_CHUNK_TMP(REG_VERSION INT, REG_TENANT_ID INT, REG_CONTENT_ID INT); + CREATE TABLE REG_CONTENT_CHUNK_TMP(REG_CONTENT_ID INT); + + INSERT INTO REG_RESOURCE_CHUNK_TMP SELECT REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID FROM REG_RESOURCE WHERE + REG_PATH_ID IN (SELECT REG_PATH_ID FROM PathIdList) LIMIT chunkSize; + GET DIAGNOSTICS chunkCount = ROW_COUNT; + INSERT INTO REG_CONTENT_CHUNK_TMP SELECT REG_CONTENT_ID FROM REG_RESOURCE_CHUNK_TMP WHERE REG_CONTENT_ID + IS NOT NULL; + + CREATE INDEX REG_RESOURCE_CHUNK_TMP on REG_RESOURCE_CHUNK_TMP (REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID) + CREATE INDEX REG_CONTENT_CHUNK_TMP on REG_CONTENT_CHUNK_TMP (REG_CONTENT_ID) + + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('CREATED REG_RESOURCE_CHUNK_TMP...'); + END IF; + + -- BATCH LOOP + SET batchCount = 1; + WHILE (batchCount > 0) + DO + -- CREATE BATCH TABLE + DROP TABLE IF EXISTS REG_RESOURCE_BATCH_TMP; + DROP TABLE IF EXISTS REG_CONTENT_BATCH_TMP; + + CREATE TABLE REG_RESOURCE_BATCH_TMP(REG_VERSION INT, REG_TENANT_ID INT, REG_CONTENT_ID INT); + CREATE TABLE REG_CONTENT_BATCH_TMP(REG_CONTENT_ID INT); + + INSERT INTO REG_RESOURCE_BATCH_TMP(REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID) SELECT REG_VERSION, + REG_TENANT_ID, REG_CONTENT_ID FROM REG_RESOURCE_CHUNK_TMP LIMIT batchSize; + GET DIAGNOSTICS batchCount = ROW_COUNT; + INSERT INTO REG_CONTENT_BATCH_TMP(REG_CONTENT_ID) SELECT REG_CONTENT_ID FROM REG_RESOURCE_BATCH_TMP + WHERE REG_CONTENT_ID IS NOT NULL; + + CREATE INDEX REG_RESOURCE_BATCH_TMP on REG_RESOURCE_BATCH_TMP (REG_VERSION, REG_TENANT_ID); + CREATE INDEX REG_CONTENT_BATCH_TMP on REG_CONTENT_BATCH_TMP (REG_CONTENT_ID); + + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('CREATED REG_RESOURCE_BATCH_TMP...'); + END IF; + + -- BATCH DELETION + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('BATCH DELETE STARTED ON REG_RESOURCE...'); + END IF; + + DELETE r FROM REG_RESOURCE r INNER JOIN REG_RESOURCE_BATCH_TMP tmp ON r.REG_VERSION = tmp.REG_VERSION + AND r.REG_TENANT_ID = tmp.REG_TENANT_ID; + GET DIAGNOSTICS rowCount = ROW_COUNT; + DELETE FROM REG_CONTENT WHERE REG_CONTENT_ID IN (SELECT REG_CONTENT_ID FROM REG_CONTENT_BATCH_TMP); + + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('BATCH DELETE FINISHED ON REG_RESOURCE : ' || rowCount); + END IF; + + -- DELETE FROM CHUNK + DELETE r FROM REG_RESOURCE_CHUNK_TMP r INNER JOIN REG_RESOURCE_BATCH_TMP tmp ON + r.REG_VERSION = tmp.REG_VERSION AND r.REG_TENANT_ID = tmp.REG_TENANT_ID; + END WHILE; + END WHILE; + + -- DELETE TEMP TABLES + DROP TABLE IF EXISTS REG_RESOURCE_BATCH_TMP; + DROP TABLE IF EXISTS REG_CONTENT_BATCH_TMP; + DROP TABLE IF EXISTS REG_RESOURCE_CHUNK_TMP; + DROP TABLE IF EXISTS REG_CONTENT_CHUNK_TMP; + + COMMIT; + + IF (enableLog = 1) + THEN + CALL DBMS_OUTPUT.PUT_LINE('CLEANUP COMPLETED...!'); + END IF; + +END; diff --git a/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mssql/saml-idp-metadata-cleanup/mssql-saml-idp-metadata-cleanup-restore.sql b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mssql/saml-idp-metadata-cleanup/mssql-saml-idp-metadata-cleanup-restore.sql new file mode 100644 index 000000000000..cd611bb86825 --- /dev/null +++ b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mssql/saml-idp-metadata-cleanup/mssql-saml-idp-metadata-cleanup-restore.sql @@ -0,0 +1,59 @@ +CREATE OR ALTER PROCEDURE WSO2_SAML_IDP_METADATA_CLEANUP_RESTORE AS +BEGIN TRY + -- ------------------------------------------ + -- DECLARE VARIABLES + -- ------------------------------------------ + DECLARE @rowCount INT + DECLARE @enableLog BIT + + -- ------------------------------------------ + -- CONFIGURABLE ATTRIBUTES + -- ------------------------------------------ + SET @enableLog = 1 -- ENABLE LOGGING [DEFAULT : TRUE] + + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] SAML IDP METADATA CLEANUP DATA RESTORATION STARTED .... !' AS 'INFO_LOG' + END + + SELECT @rowCount = COUNT(1) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME IN ('REG_RESOURCE'); + IF (@rowCount = 1) + BEGIN + SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; + BEGIN TRANSACTION; + + SET IDENTITY_INSERT REG_RESOURCE ON; + INSERT INTO REG_RESOURCE (REG_PATH_ID, REG_NAME, REG_VERSION, REG_MEDIA_TYPE, REG_CREATOR, REG_CREATED_TIME, + REG_LAST_UPDATOR, REG_LAST_UPDATED_TIME, REG_DESCRIPTION, REG_CONTENT_ID, REG_TENANT_ID, REG_UUID) SELECT + A.REG_PATH_ID, A.REG_NAME, A.REG_VERSION, A.REG_MEDIA_TYPE, A.REG_CREATOR, A.REG_CREATED_TIME, A.REG_LAST_UPDATOR, + A.REG_LAST_UPDATED_TIME, A.REG_DESCRIPTION, A.REG_CONTENT_ID, A.REG_TENANT_ID, A.REG_UUID FROM BAK_REG_RESOURCE A LEFT + JOIN REG_RESOURCE B ON A.REG_VERSION = B.REG_VERSION AND A.REG_TENANT_ID = B.REG_TENANT_ID WHERE B.REG_VERSION IS NULL; + SET @rowCount = @@ROWCOUNT; + SET IDENTITY_INSERT REG_RESOURCE OFF; + + SET IDENTITY_INSERT REG_CONTENT ON; + INSERT INTO REG_CONTENT (REG_CONTENT_ID, REG_CONTENT_DATA, REG_TENANT_ID) SELECT A.REG_CONTENT_ID, + A.REG_CONTENT_DATA, A.REG_TENANT_ID FROM BAK_REG_CONTENT A LEFT JOIN REG_CONTENT B ON + A.REG_CONTENT_ID = B.REG_CONTENT_ID WHERE B.REG_CONTENT_ID IS NULL; + SET IDENTITY_INSERT REG_CONTENT OFF; + IF (@enableLog = 1) + BEGIN + SELECT CONCAT('CLEANUP DATA RESTORATION COMPLETED ON REG_RESOURCE WITH ', @rowCount) AS 'INFO_LOG' + END + COMMIT TRANSACTION; + END + + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] CLEANUP DATA RESTORATION COMPLETED .... !' AS 'INFO_LOG' + END +END TRY +BEGIN CATCH + + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] ERROR OCCURRED: ' + ERROR_MESSAGE() AS 'ERROR LOG' + END + + ROLLBACK TRANSACTION +END CATCH diff --git a/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mssql/saml-idp-metadata-cleanup/mssql-saml-idp-metadata-cleanup.sql b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mssql/saml-idp-metadata-cleanup/mssql-saml-idp-metadata-cleanup.sql new file mode 100644 index 000000000000..9c6ffceae5ab --- /dev/null +++ b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mssql/saml-idp-metadata-cleanup/mssql-saml-idp-metadata-cleanup.sql @@ -0,0 +1,182 @@ +-- NOTE: This procedure assumes that the SAML IDP metadata is stored under the path +-- '/_system/governance/repository/identity/provider/saml' in the registry and only two corresponding RESOURCE entries +-- (one for the collection and one for the resource object) and one CONTENT entry exist for each SAML IDP metadata file. + +CREATE OR ALTER PROCEDURE WSO2_SAML_IDP_METADATA_CLEANUP AS +BEGIN TRY + + -- ------------------------------------------ + -- DECLARE VARIABLES + -- ------------------------------------------ + DECLARE @batchSize INT + DECLARE @chunkSize INT + DECLARE @batchCount INT + DECLARE @chunkCount INT + DECLARE @rowCount INT + DECLARE @enableLog BIT + DECLARE @backupTables BIT + + -- ------------------------------------------ + -- CONFIGURABLE VARIABLES + -- ------------------------------------------ + SET @batchSize = 10000 -- SET BATCH SIZE TO AVOID TABLE LOCKS [DEFAULT : 10000] + SET @chunkSize = 500000 -- CHUNK WISE DELETE FOR LARGE TABLES [DEFAULT : 500000] + SET @enableLog = 1 -- ENABLE LOGGING [DEFAULT : 1] + SET @backupTables = 1 -- SET IF REGISTRY TABLES NEEDS TO BE BACKED-UP BEFORE DELETE [DEFAULT : 1]. + + SET @rowCount = 0 + SET @batchCount = 1 + SET @chunkCount = 1 + + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] WSO2_SAML_IDP_METADATA_CLEANUP() STARTED...!' AS 'INFO LOG' + END + + -- ------------------------------------------ + -- GET PATH ID LIST TO DELETE + -- ------------------------------------------ + DECLARE @RootPathIdList TABLE (ROOT_PATH_ID INT); + INSERT INTO @RootPathIdList (ROOT_PATH_ID) SELECT REG_PATH_ID FROM REG_PATH WHERE REG_PATH_VALUE = '/_system/governance/repository/identity/provider/saml'; + + DECLARE @PathIdList TABLE (REG_PATH_ID INT); + INSERT INTO @PathIdList (REG_PATH_ID) SELECT REG_PATH_ID FROM REG_PATH WHERE REG_PATH_PARENT_ID IN (SELECT ROOT_PATH_ID FROM @RootPathIdList); + + -- ------------------------------------------ + -- BACKUP DATA + -- ------------------------------------------ + IF (@backupTables = 1) + BEGIN + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] TABLE BACKUP STARTED ... !' AS 'INFO LOG' + END + + IF (EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'BAK_REG_RESOURCE')) + BEGIN + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] DELETING OLD BACKUP...' AS 'INFO LOG' + END + DROP TABLE BAK_REG_RESOURCE; + DROP TABLE BAK_REG_CONTENT; + END + + -- BACKUP REG_RESOURCE TABLE + SELECT * INTO BAK_REG_RESOURCE FROM REG_RESOURCE WHERE REG_PATH_ID IN (SELECT REG_PATH_ID FROM @PathIdList); + + -- BACKUP REG_CONTENT TABLE + DECLARE @ContentIdList TABLE (REG_CONTENT_ID INT); + INSERT INTO @ContentIdList (REG_CONTENT_ID) SELECT DISTINCT REG_CONTENT_ID FROM BAK_REG_RESOURCE WHERE REG_CONTENT_ID IS NOT NULL; + SELECT * INTO BAK_REG_CONTENT FROM REG_CONTENT WHERE REG_CONTENT_ID IN (SELECT REG_CONTENT_ID FROM @ContentIdList); + END + + -- ------------------------------------------ + -- CLEANUP DATA + -- ------------------------------------------ + + BEGIN TRANSACTION + + WHILE (@chunkCount > 0) + BEGIN + -- CREATE CHUNK TABLE + DROP TABLE IF EXISTS REG_RESOURCE_CHUNK_TMP; + DROP TABLE IF EXISTS REG_CONTENT_CHUNK_TMP; + + CREATE TABLE REG_RESOURCE_CHUNK_TMP(REG_VERSION INT, REG_TENANT_ID INT, REG_CONTENT_ID INT); + CREATE TABLE REG_CONTENT_CHUNK_TMP(REG_CONTENT_ID INT); + + INSERT INTO REG_RESOURCE_CHUNK_TMP(REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID) SELECT TOP (@chunkSize) REG_VERSION, + REG_TENANT_ID, REG_CONTENT_ID FROM REG_RESOURCE WHERE REG_PATH_ID IN (SELECT REG_PATH_ID FROM @PathIdList); + SET @chunkCount = @@ROWCOUNT; + INSERT INTO REG_CONTENT_CHUNK_TMP(REG_CONTENT_ID) SELECT REG_CONTENT_ID FROM REG_RESOURCE_CHUNK_TMP + WHERE REG_CONTENT_ID IS NOT NULL; + + IF (@chunkCount = 0) + BEGIN + BREAK; + END + + CREATE INDEX REG_RESOURCE_CHUNK_TMP on REG_RESOURCE_CHUNK_TMP (REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID); + CREATE INDEX REG_CONTENT_CHUNK_TMP on REG_CONTENT_CHUNK_TMP (REG_CONTENT_ID); + + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] CREATED REG_RESOURCE_CHUNK_TMP...' AS 'INFO LOG' + END + + -- BATCH LOOP + SET @batchCount = 1 + WHILE (@batchCount > 0) + BEGIN + -- CREATE BATCH TABLE + DROP TABLE IF EXISTS REG_RESOURCE_BATCH_TMP; + DROP TABLE IF EXISTS REG_CONTENT_BATCH_TMP; + + CREATE TABLE REG_RESOURCE_BATCH_TMP(REG_VERSION INT, REG_TENANT_ID INT, REG_CONTENT_ID INT); + CREATE TABLE REG_CONTENT_BATCH_TMP(REG_CONTENT_ID INT); + + INSERT INTO REG_RESOURCE_BATCH_TMP(REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID) SELECT TOP (@batchSize) REG_VERSION, + REG_TENANT_ID, REG_CONTENT_ID FROM REG_RESOURCE_CHUNK_TMP; + SET @batchCount = @@ROWCOUNT; + INSERT INTO REG_CONTENT_BATCH_TMP(REG_CONTENT_ID) SELECT REG_CONTENT_ID FROM REG_RESOURCE_BATCH_TMP + WHERE REG_CONTENT_ID IS NOT NULL; + + IF (@batchCount = 0) + BEGIN + BREAK; + END + + CREATE INDEX REG_RESOURCE_BATCH_TMP on REG_RESOURCE_BATCH_TMP (REG_VERSION, REG_TENANT_ID); + CREATE INDEX REG_CONTENT_BATCH_TMP on REG_CONTENT_BATCH_TMP (REG_CONTENT_ID); + + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] CREATED REG_RESOURCE_BATCH_TMP...' AS 'INFO LOG' + END + + -- BATCH DELETION + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] BATCH DELETE STARTED ON REG_RESOURCE...' AS 'INFO LOG' + END + + DELETE r FROM REG_RESOURCE r INNER JOIN REG_RESOURCE_BATCH_TMP tmp ON r.REG_VERSION = tmp.REG_VERSION + AND r.REG_TENANT_ID = tmp.REG_TENANT_ID; + SET @rowCount = @@ROWCOUNT; + DELETE FROM REG_CONTENT WHERE REG_CONTENT_ID IN (SELECT REG_CONTENT_ID FROM REG_CONTENT_BATCH_TMP); + + IF (@enableLog = 1) + BEGIN + SELECT CONCAT('BATCH DELETE FINISHED ON REG_RESOURCE : ', @rowCount) AS 'INFO LOG' + END + + -- DELETE FROM CHUNK + DELETE r FROM REG_RESOURCE_CHUNK_TMP r INNER JOIN REG_RESOURCE_BATCH_TMP tmp ON + r.REG_VERSION = tmp.REG_VERSION AND r.REG_TENANT_ID = tmp.REG_TENANT_ID; + END + END + + -- DELETE TEMP TABLES + DROP TABLE IF EXISTS REG_RESOURCE_BATCH_TMP; + DROP TABLE IF EXISTS REG_CONTENT_BATCH_TMP; + DROP TABLE IF EXISTS REG_RESOURCE_CHUNK_TMP; + DROP TABLE IF EXISTS REG_CONTENT_CHUNK_TMP; + + COMMIT TRANSACTION + + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] CLEANUP COMPLETED...!' AS 'INFO_LOG' + END + +END TRY +BEGIN CATCH + + IF (@enableLog = 1) + BEGIN + SELECT '[' + convert(varchar, getdate(), 121) + '] ERROR OCCURRED: ' + ERROR_MESSAGE() AS 'ERROR LOG' + END + + ROLLBACK TRANSACTION +END CATCH diff --git a/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mysql/saml-idp-metadata-cleanup/mysql-saml-idp-metadata-cleanup-restore.sql b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mysql/saml-idp-metadata-cleanup/mysql-saml-idp-metadata-cleanup-restore.sql new file mode 100644 index 000000000000..bb44cd73c22b --- /dev/null +++ b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mysql/saml-idp-metadata-cleanup/mysql-saml-idp-metadata-cleanup-restore.sql @@ -0,0 +1,50 @@ +DROP PROCEDURE IF EXISTS `WSO2_SAML_IDP_METADATA_CLEANUP_RESTORE`; +DELIMITER $$ +CREATE PROCEDURE `WSO2_SAML_IDP_METADATA_CLEANUP_RESTORE`() +BEGIN + -- ------------------------------------------ + -- DECLARE VARIABLES + -- ------------------------------------------ + DECLARE rowCount INT; + DECLARE enableLog BOOLEAN; + + -- ------------------------------------------ + -- CONFIGURABLE ATTRIBUTES + -- ------------------------------------------ + SET enableLog = TRUE; -- ENABLE LOGGING [DEFAULT : TRUE] + + IF (enableLog) + THEN + SELECT 'SAML IDP METADATA CLEANUP DATA RESTORATION STARTED...!' AS 'INFO LOG'; + END IF; + + SELECT COUNT(1) INTO rowCount FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA IN (SELECT DATABASE()) AND TABLE_NAME IN ('REG_RESOURCE'); + IF (rowCount = 1) + THEN + SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; + START TRANSACTION; + + INSERT INTO REG_RESOURCE (REG_PATH_ID, REG_NAME, REG_VERSION, REG_MEDIA_TYPE, REG_CREATOR, REG_CREATED_TIME, + REG_LAST_UPDATOR, REG_LAST_UPDATED_TIME, REG_DESCRIPTION, REG_CONTENT_ID, REG_TENANT_ID, REG_UUID) SELECT + A.REG_PATH_ID, A.REG_NAME, A.REG_VERSION, A.REG_MEDIA_TYPE, A.REG_CREATOR, A.REG_CREATED_TIME, A.REG_LAST_UPDATOR, + A.REG_LAST_UPDATED_TIME, A.REG_DESCRIPTION, A.REG_CONTENT_ID, A.REG_TENANT_ID, A.REG_UUID FROM BAK_REG_RESOURCE A LEFT + JOIN REG_RESOURCE B ON A.REG_VERSION = B.REG_VERSION AND A.REG_TENANT_ID = B.REG_TENANT_ID WHERE B.REG_VERSION IS NULL; + SELECT row_count() INTO rowCount; + + INSERT INTO REG_CONTENT (REG_CONTENT_ID, REG_CONTENT_DATA, REG_TENANT_ID) SELECT A.REG_CONTENT_ID, + A.REG_CONTENT_DATA, A.REG_TENANT_ID FROM BAK_REG_CONTENT A LEFT JOIN REG_CONTENT B ON + A.REG_CONTENT_ID = B.REG_CONTENT_ID WHERE B.REG_CONTENT_ID IS NULL; + IF (enableLog) + THEN + SELECT 'CLEANUP DATA RESTORATION COMPLETED ON REG_RESOURCE WITH %' + rowCount AS 'INFO LOG'; + END IF; + COMMIT; + END IF; + + IF (enableLog) + THEN + SELECT 'CLEANUP DATA RESTORATION COMPLETED...!'; + END IF; +END$$ + +DELIMITER ; diff --git a/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mysql/saml-idp-metadata-cleanup/mysql-saml-idp-metadata-cleanup.sql b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mysql/saml-idp-metadata-cleanup/mysql-saml-idp-metadata-cleanup.sql new file mode 100644 index 000000000000..f4c948a6a21a --- /dev/null +++ b/features/identity-core/org.wso2.carbon.identity.core.server.feature/resources/dbscripts/stored-procedures/mysql/saml-idp-metadata-cleanup/mysql-saml-idp-metadata-cleanup.sql @@ -0,0 +1,175 @@ +-- NOTE: This procedure assumes that the SAML IDP metadata is stored under the path +-- '/_system/governance/repository/identity/provider/saml' in the registry and only two corresponding RESOURCE entries +-- (one for the collection and one for the resource object) and one CONTENT entry exist for each SAML IDP metadata file. + +DROP PROCEDURE IF EXISTS `WSO2_SAML_IDP_METADATA_CLEANUP`; +DELIMITER $$ +CREATE PROCEDURE `WSO2_SAML_IDP_METADATA_CLEANUP`() +BEGIN + + -- ------------------------------------------ + -- DECLARE VARIABLES + -- ------------------------------------------ + DECLARE batchSize INT; + DECLARE chunkSize INT; + DECLARE batchCount INT; + DECLARE chunkCount INT; + DECLARE rowCount INT; + DECLARE enableLog BOOLEAN; + DECLARE backupTables BOOLEAN; + + -- ------------------------------------------ + -- CONFIGURABLE VARIABLES + -- ------------------------------------------ + SET batchSize = 10000; -- SET BATCH SIZE TO AVOID TABLE LOCKS [DEFAULT : 10000] + SET chunkSize = 500000; -- CHUNK WISE DELETE FOR LARGE TABLES [DEFAULT : 500000] + SET enableLog = TRUE; -- ENABLE LOGGING [DEFAULT : TRUE] + SET backupTables = TRUE; -- SET IF REGISTRY TABLES NEEDS TO BE BACKED-UP BEFORE DELETE [DEFAULT : TRUE]. + + SET rowCount = 0; + SET batchCount = 1; + SET chunkCount = 1; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + SELECT 'ERROR OCCURRED' AS error_message; + ROLLBACK; + END; + + IF (enableLog) + THEN + SELECT 'WSO2_SAML_IDP_METADATA_CLEANUP() STARTED...!' AS 'INFO LOG'; + END IF; + + -- ------------------------------------------ + -- GET PATH ID LIST TO DELETE + -- ------------------------------------------ + CREATE TEMPORARY TABLE RootPathIdList (ROOT_PATH_ID INT); + INSERT INTO RootPathIdList (ROOT_PATH_ID) SELECT REG_PATH_ID FROM REG_PATH WHERE REG_PATH_VALUE = '/_system/governance/repository/identity/provider/saml'; + + CREATE TEMPORARY TABLE PathIdList (REG_PATH_ID INT); + INSERT INTO PathIdList (REG_PATH_ID) SELECT REG_PATH_ID FROM REG_PATH WHERE REG_PATH_PARENT_ID IN (SELECT ROOT_PATH_ID FROM RootPathIdList); + + -- ------------------------------------------ + -- BACKUP DATA + -- ------------------------------------------ + IF (backupTables) + THEN + IF (enableLog) + BEGIN + SELECT 'TABLE BACKUP STARTED ... !' AS 'INFO LOG'; + END IF; + + IF (EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'BAK_REG_RESOURCE' and TABLE_SCHEMA in (SELECT DATABASE()))) + THEN + IF (enableLog) + THEN + SELECT 'DELETING OLD BACKUP...' AS 'INFO LOG'; + END IF; + DROP TABLE BAK_REG_RESOURCE; + DROP TABLE BAK_REG_CONTENT; + END IF; + + -- BACKUP REG_RESOURCE TABLE + CREATE TABLE BAK_REG_RESOURCE AS SELECT * FROM REG_RESOURCE WHERE REG_PATH_ID IN (SELECT REG_PATH_ID FROM PathIdList); + + -- BACKUP REG_CONTENT TABLE + CREATE TEMPORARY TABLE ContentIdList (REG_CONTENT_ID INT); + INSERT INTO ContentIdList (REG_CONTENT_ID) SELECT DISTINCT REG_CONTENT_ID FROM BAK_REG_RESOURCE WHERE + REG_CONTENT_ID IS NOT NULL; + CREATE TABLE BAK_REG_CONTENT AS SELECT * FROM REG_CONTENT WHERE REG_CONTENT_ID IN (SELECT REG_CONTENT_ID FROM ContentIdList); + END IF; + + -- ------------------------------------------ + -- CLEANUP DATA + -- ------------------------------------------ + + START TRANSACTION + + CONFIRMATION_CHUNK_LOOP : WHILE (chunkCount > 0) + DO + -- CREATE CHUNK TABLE + DROP TABLE IF EXISTS REG_RESOURCE_CHUNK_TMP; + DROP TABLE IF EXISTS REG_CONTENT_CHUNK_TMP; + + CREATE TABLE REG_RESOURCE_CHUNK_TMP AS SELECT REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID FROM REG_RESOURCE WHERE + REG_PATH_ID IN (SELECT REG_PATH_ID FROM PathIdList) LIMIT chunkSize; + SELECT row_count() INTO chunkCount; + CREATE TABLE REG_CONTENT_CHUNK_TMP AS SELECT REG_CONTENT_ID FROM REG_RESOURCE_CHUNK_TMP WHERE REG_CONTENT_ID IS NOT NULL; + + IF (chunkCount = 0) + THEN + LEAVE CONFIRMATION_CHUNK_LOOP; + END IF; + + CREATE INDEX REG_RESOURCE_CHUNK_TMP on REG_RESOURCE_CHUNK_TMP (REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID); + CREATE INDEX REG_CONTENT_CHUNK_TMP on REG_CONTENT_CHUNK_TMP (REG_CONTENT_ID); + + IF (enableLog) + THEN + SELECT 'CREATED REG_RESOURCE_CHUNK_TMP...' AS 'INFO LOG'; + END IF; + + -- BATCH LOOP + SET batchCount = 1; + CONFIRMATION_BATCH_LOOP : WHILE (batchCount > 0) + DO + -- CREATE BATCH TABLE + DROP TABLE IF EXISTS REG_RESOURCE_BATCH_TMP; + DROP TABLE IF EXISTS REG_CONTENT_BATCH_TMP; + + CREATE TABLE REG_RESOURCE_BATCH_TMP AS SELECT REG_VERSION, REG_TENANT_ID, REG_CONTENT_ID FROM REG_RESOURCE_CHUNK_TMP LIMIT batchSize; + SELECT row_count() INTO batchCount; + CREATE TABLE REG_CONTENT_BATCH_TMP AS SELECT REG_CONTENT_ID FROM REG_RESOURCE_BATCH_TMP WHERE REG_CONTENT_ID IS NOT NULL; + + IF (batchCount = 0) + THEN + LEAVE CONFIRMATION_BATCH_LOOP; + END IF; + + CREATE INDEX REG_RESOURCE_BATCH_TMP on REG_RESOURCE_BATCH_TMP (REG_VERSION, REG_TENANT_ID); + CREATE INDEX REG_CONTENT_BATCH_TMP on REG_CONTENT_BATCH_TMP (REG_CONTENT_ID); + + IF (enableLog) + THEN + SELECT 'CREATED REG_RESOURCE_BATCH_TMP...' AS 'INFO LOG'; + END IF; + + -- BATCH DELETION + IF (enableLog) + THEN + SELECT 'BATCH DELETE STARTED ON REG_RESOURCE...' AS 'INFO LOG'; + END IF; + + DELETE r FROM REG_RESOURCE r INNER JOIN REG_RESOURCE_BATCH_TMP tmp ON r.REG_VERSION = tmp.REG_VERSION + AND r.REG_TENANT_ID = tmp.REG_TENANT_ID; + SELECT row_count() INTO rowCount; + DELETE FROM REG_CONTENT WHERE REG_CONTENT_ID IN (SELECT REG_CONTENT_ID FROM REG_CONTENT_BATCH_TMP); + + IF (enableLog) + THEN + SELECT 'BATCH DELETE FINISHED ON REG_RESOURCE : ' AS 'INFO LOG', rowCount; + END IF; + + -- DELETE FROM CHUNK + DELETE r FROM REG_RESOURCE_CHUNK_TMP r INNER JOIN REG_RESOURCE_BATCH_TMP tmp ON + r.REG_VERSION = tmp.REG_VERSION AND r.REG_TENANT_ID = tmp.REG_TENANT_ID; + END WHILE; + END WHILE; + + -- DELETE TEMP TABLES + DROP TABLE IF EXISTS REG_RESOURCE_BATCH_TMP; + DROP TABLE IF EXISTS REG_CONTENT_BATCH_TMP; + DROP TABLE IF EXISTS REG_RESOURCE_CHUNK_TMP; + DROP TABLE IF EXISTS REG_CONTENT_CHUNK_TMP; + + COMMIT; + + IF (enableLog) + THEN + SELECT 'CLEANUP COMPLETED...!' AS 'INFO LOG'; + END IF; + +END$$ + +DELIMITER ;