From f93fed1c01bd89ee5845efe12802e827104ba1a7 Mon Sep 17 00:00:00 2001 From: Rahul Chauhan Date: Wed, 19 Jun 2024 15:40:52 +0200 Subject: [PATCH 1/2] Fix probes --- common/check_deletable_replicas | 31 ++++++++++++++++--------------- common/check_obsolete_replicas | 24 +++++++++++++++--------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/common/check_deletable_replicas b/common/check_deletable_replicas index 30dd1931..15a4fae2 100755 --- a/common/check_deletable_replicas +++ b/common/check_deletable_replicas @@ -10,17 +10,22 @@ # - Cedric Serfon, , 2018 ''' -Probe to check the queues of messages to submit by Hermes to the broker +Probe to update the RSE usage for expired replicas ''' from __future__ import print_function import sys -from rucio.db.sqla.session import get_session +from rucio.db.sqla.session import BASE, get_session +from sqlalchemy.sql import text # Exit statuses OK, WARNING, CRITICAL, UNKNOWN = 0, 1, 2, 3 +if BASE.metadata.schema: + schema = BASE.metadata.schema + '.' +else: + schema = '' if __name__ == "__main__": try: @@ -36,35 +41,31 @@ if __name__ == "__main__": SELECT id AS rse_id FROM - atlas_rucio.rses + {schema}rses WHERE deleted=0) a LEFT OUTER JOIN ( - SELECT /*+ INDEX_FFS(replicas REPLICAS_TOMBSTONE_IDX) */ + SELECT rse_id, COUNT(1) AS files, SUM(bytes) AS bytes FROM - ATLAS_RUCIO.REPLICAS + {schema}REPLICAS WHERE - ( - CASE - WHEN tombstone IS NOT NULL - THEN rse_id - END) IS NOT NULL + tombstone IS NOT NULL AND tombstone < sys_extract_utc(localtimestamp) GROUP BY rse_id) b ON a.rse_id=b.rse_id) LOOP - MERGE INTO atlas_rucio.RSE_USAGE + MERGE INTO {schema}RSE_USAGE USING DUAL - ON (atlas_rucio.RSE_USAGE.rse_id = u.rse_id and source = 'expired') + ON ({schema}RSE_USAGE.rse_id = u.rse_id and source = 'expired') WHEN NOT MATCHED THEN INSERT(rse_id, source, used, files, updated_at, created_at) VALUES (u.rse_id, 'expired', u.bytes, u.files, u.updated_at, u.updated_at) WHEN MATCHED THEN UPDATE SET used=u.bytes, files=u.files, updated_at=u.updated_at; - MERGE INTO ATLAS_RUCIO.RSE_USAGE_HISTORY H + MERGE INTO {schema}RSE_USAGE_HISTORY H USING DUAL ON (h.rse_id = u.rse_id and h.source = 'expired' and h.updated_at = u.updated_at) WHEN NOT MATCHED THEN INSERT(rse_id, source, used, files, updated_at, created_at) @@ -73,8 +74,8 @@ if __name__ == "__main__": COMMIT; END LOOP; END; -''' - SESSION.execute(QUERY) +'''.format(schema=schema) + SESSION.execute(text(QUERY)) except Exception as error: print(error) sys.exit(UNKNOWN) diff --git a/common/check_obsolete_replicas b/common/check_obsolete_replicas index efd83943..82187df8 100755 --- a/common/check_obsolete_replicas +++ b/common/check_obsolete_replicas @@ -12,14 +12,20 @@ ''' Probe to check the backlog of obsolete replicas. ''' +from __future__ import print_function import sys -from rucio.db.sqla.session import get_session +from rucio.db.sqla.session import BASE, get_session +from sqlalchemy.sql import text # Exit statuses OK, WARNING, CRITICAL, UNKNOWN = 0, 1, 2, 3 +if BASE.metadata.schema: + schema = BASE.metadata.schema + '.' +else: + schema = '' if __name__ == "__main__": try: @@ -35,7 +41,7 @@ if __name__ == "__main__": SELECT id AS rse_id FROM - atlas_rucio.rses + {schema}rses WHERE deleted=0) a LEFT OUTER JOIN @@ -46,7 +52,7 @@ if __name__ == "__main__": COUNT(1) AS files, SUM(bytes) AS bytes FROM - atlas_rucio.replicas + {schema}replicas WHERE ( CASE @@ -60,14 +66,14 @@ if __name__ == "__main__": a.rse_id=b.rse_id) LOOP - MERGE INTO atlas_rucio.RSE_USAGE + MERGE INTO {schema}RSE_USAGE USING DUAL - ON (atlas_rucio.RSE_USAGE.rse_id = u.rse_id and source = 'obsolete') + ON ({schema}RSE_USAGE.rse_id = u.rse_id and source = 'obsolete') WHEN NOT MATCHED THEN INSERT(rse_id, source, used, files, updated_at, created_at) VALUES (u.rse_id, 'obsolete', u.bytes, u.files, u.updated_at, u.updated_at) WHEN MATCHED THEN UPDATE SET used=u.bytes, files=u.files, updated_at=u.updated_at; - MERGE INTO ATLAS_RUCIO.RSE_USAGE_HISTORY H + MERGE INTO {schema}RSE_USAGE_HISTORY H USING DUAL ON (h.rse_id = u.rse_id and h.source = 'obsolete' and h.updated_at = u.updated_at) WHEN NOT MATCHED THEN INSERT(rse_id, source, used, files, updated_at, created_at) @@ -76,9 +82,9 @@ if __name__ == "__main__": COMMIT; END LOOP; END; -''' - SESSION.execute(QUERY) +'''.format(schema=schema) + SESSION.execute(text(QUERY)) except Exception as error: - print error + print(error) sys.exit(UNKNOWN) sys.exit(OK) From 86987f8d7c05090eb0820ee417f64d4cab2d05c3 Mon Sep 17 00:00:00 2001 From: Eric Vaandering Date: Mon, 24 Jun 2024 15:33:57 -0500 Subject: [PATCH 2/2] Move to python3 explicitly, minor code updates --- {common => cms}/check_deletable_replicas | 7 +++---- {common => cms}/check_obsolete_replicas | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) rename {common => cms}/check_deletable_replicas (97%) rename {common => cms}/check_obsolete_replicas (97%) diff --git a/common/check_deletable_replicas b/cms/check_deletable_replicas similarity index 97% rename from common/check_deletable_replicas rename to cms/check_deletable_replicas index 15a4fae2..2e5254a8 100755 --- a/common/check_deletable_replicas +++ b/cms/check_deletable_replicas @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright European Organization for Nuclear Research (CERN) 2013 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -9,11 +9,10 @@ # - Mario Lassnig, , 2013-2014 # - Cedric Serfon, , 2018 -''' +""" Probe to update the RSE usage for expired replicas -''' +""" -from __future__ import print_function import sys from rucio.db.sqla.session import BASE, get_session diff --git a/common/check_obsolete_replicas b/cms/check_obsolete_replicas similarity index 97% rename from common/check_obsolete_replicas rename to cms/check_obsolete_replicas index 82187df8..6c599ed7 100755 --- a/common/check_obsolete_replicas +++ b/cms/check_obsolete_replicas @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright European Organization for Nuclear Research (CERN) 2013 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -9,10 +9,9 @@ # - Vincent Garonne, , 2015 # - Cedric Serfon, , 2018 -''' +""" Probe to check the backlog of obsolete replicas. -''' -from __future__ import print_function +""" import sys