From 935e957ae2546f23bf4e44dd39ba40c47fcc1429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=98JoinTyang=E2=80=99?= Date: Fri, 10 Jan 2025 17:31:27 +0800 Subject: [PATCH] test dm --- content_scanner/models.py | 4 ++-- db.py | 20 +++++++++++++++++++- events/models.py | 14 +++++++------- main.py | 2 +- repo_data/db.py | 18 ++++++++++++++++-- statistics/models.py | 4 ++-- utils/ccnet_db.py | 8 ++++---- utils/seafile_db.py | 19 +++++++++++-------- virus_scanner/models.py | 4 ++-- 9 files changed, 64 insertions(+), 29 deletions(-) diff --git a/content_scanner/models.py b/content_scanner/models.py index e7065077..d0a36745 100644 --- a/content_scanner/models.py +++ b/content_scanner/models.py @@ -6,7 +6,7 @@ class ContentScanRecord(Base): - __tablename__ = 'ContentScanRecord' + __tablename__ = 'CONTENTSCANRECORD' id = mapped_column(Integer, primary_key=True, autoincrement=True) repo_id = mapped_column(String(length=36), nullable=False, index=True) @@ -21,7 +21,7 @@ def __init__(self, repo_id, commit_id, timestamp): class ContentScanResult(Base): - __tablename__ = 'ContentScanResult' + __tablename__ = 'CONTENTSCANRESULT' id = mapped_column(Integer, primary_key=True, autoincrement=True) repo_id = mapped_column(String(length=36), nullable=False, index=True) diff --git a/db.py b/db.py index 2e42ae52..b9e135a5 100644 --- a/db.py +++ b/db.py @@ -81,6 +81,23 @@ def create_engine_from_conf(config, db='seafevent'): db_url = "oracle://%s:%s@%s:%s/%s" % (username, quote_plus(passwd), host, port, service_name) + elif backend == 'dm': + if config.has_option(db_sec, 'host'): + host = config.get(db_sec, 'host').lower() + else: + host = 'localhost' + + if config.has_option(db_sec, 'port'): + port = config.getint(db_sec, 'port') + else: + port = 5236 + username = config.get(db_sec, user) + passwd = config.get(db_sec, 'password') + service_name = config.get(db_sec, db_name) + if db == 'seafile': + service_name = config.get(db_sec, user) + + db_url = "dm+dmPython://%s:%s@%s:%s/?schema=%s" % (username, quote_plus(passwd), host, port, service_name) else: logger.error("Unknown database backend: %s" % backend) raise RuntimeError("Unknown database backend: %s" % backend) @@ -104,7 +121,7 @@ def init_db_session_class(config, db='seafevent'): try: engine = create_engine_from_conf(config, db) except (configparser.NoOptionError, configparser.NoSectionError) as e: - logger.error(e) + logger.exception(e) raise RuntimeError("create db engine error: %s" % e) Session = sessionmaker(bind=engine) @@ -134,6 +151,7 @@ def prepare_db_tables(seafile_config): logger.error(e) raise RuntimeError("create db engine error: %s" % e) + # SeafBase.prepare(autoload_with=engine, schema='SYSDBA') SeafBase.prepare(autoload_with=engine) diff --git a/events/models.py b/events/models.py index 35815192..3d096ea2 100644 --- a/events/models.py +++ b/events/models.py @@ -12,7 +12,7 @@ class Activity(Base): """ """ - __tablename__ = 'Activity' + __tablename__ = 'ACTIVITY' id = mapped_column(BigInteger, primary_key=True, autoincrement=True) op_type = mapped_column(String(length=128), nullable=False) @@ -51,7 +51,7 @@ def __str__(self): class UserActivity(Base): """ """ - __tablename__ = 'UserActivity' + __tablename__ = 'USERACTIVITY' id = mapped_column(BigInteger, primary_key=True, autoincrement=True) username = mapped_column(String(length=255), nullable=False) @@ -73,7 +73,7 @@ def __str__(self): class FileHistory(Base): - __tablename__ = 'FileHistory' + __tablename__ = 'FILEHISTORY' id = mapped_column(BigInteger, primary_key=True, autoincrement=True) op_type = mapped_column(String(length=128), nullable=False) @@ -105,7 +105,7 @@ def __init__(self, record): class FileAudit(Base): - __tablename__ = 'FileAudit' + __tablename__ = 'FILEAUDIT' eid = mapped_column(BigInteger, primary_key=True, autoincrement=True) timestamp = mapped_column(DateTime, nullable=False, index=True) @@ -149,7 +149,7 @@ def __str__(self): class FileUpdate(Base): - __tablename__ = 'FileUpdate' + __tablename__ = 'FILEUPDATE' eid = mapped_column(BigInteger, primary_key=True, autoincrement=True) timestamp = mapped_column(DateTime, nullable=False, index=True) @@ -186,7 +186,7 @@ def __str__(self): class PermAudit(Base): - __tablename__ = 'PermAudit' + __tablename__ = 'PERMAUDIT' eid = mapped_column(BigInteger, primary_key=True, autoincrement=True) timestamp = mapped_column(DateTime, nullable=False) @@ -246,7 +246,7 @@ def __init__(self, login_date, username, login_ip, login_success): self.login_success = login_success class FileTrash(Base): - __tablename__ = 'FileTrash' + __tablename__ = 'FILETRASH' id = mapped_column(Integer, primary_key=True, autoincrement=True) user = mapped_column(String(length=255), nullable=False) diff --git a/main.py b/main.py index 40ecd073..17c40aad 100644 --- a/main.py +++ b/main.py @@ -37,7 +37,7 @@ def main(background_tasks_only=False): seafile_config = get_config(seafile_conf_path) config = get_config(args.config_file) try: - create_db_tables(config) + # create_db_tables(config) prepare_db_tables(seafile_config) except Exception as e: logging.error('Failed create tables, error: %s' % e) diff --git a/repo_data/db.py b/repo_data/db.py index 2c766c4d..1ddec672 100644 --- a/repo_data/db.py +++ b/repo_data/db.py @@ -36,6 +36,20 @@ def create_engine_from_conf(config_file): db_url = "mysql+pymysql://%s:%s@%s:%s/%s?charset=utf8" % \ (db_username, quote_plus(db_passwd), db_server, db_port, db_name) + + elif backend == 'dm': + db_server = 'localhost' + db_port = 5236 + + if seaf_conf.has_option('database', 'host'): + db_server = seaf_conf.get('database', 'host') + if seaf_conf.has_option('database', 'port'): + db_port = seaf_conf.getint('database', 'port') + db_username = seaf_conf.get('database', 'user') + db_passwd = seaf_conf.get('database', 'password') + db_name = seaf_conf.get('database', 'user') + db_url = "dm+dmPython://%s:%s@%s:%s?schema=%s" % (db_username, quote_plus(db_passwd), + db_server, db_port, db_name) else: logger.critical("Unknown Database backend: %s" % backend) raise RuntimeError("Unknown Database backend: %s" % backend) @@ -57,7 +71,7 @@ def init_db_session_class(config_file): except (configparser.NoOptionError, configparser.NoSectionError) as e: logger.error(e) raise RuntimeError("invalid config file %s", config_file) - + Session = sessionmaker(bind=engine) return Session @@ -74,6 +88,6 @@ def ping_connection(dbapi_connection, connection_record, connection_proxy): # py except: logger.info('fail to ping database server, disposing all cached connections') connection_proxy._pool.dispose() # pylint: disable=protected-access - + # Raise DisconnectionError so the pool would create a new connection raise DisconnectionError() diff --git a/statistics/models.py b/statistics/models.py index bf056066..ab063590 100644 --- a/statistics/models.py +++ b/statistics/models.py @@ -6,7 +6,7 @@ class TotalStorageStat(Base): - __tablename__ = 'TotalStorageStat' + __tablename__ = 'TOTALSTORAGESTAT' id = mapped_column(Integer, primary_key=True, autoincrement=True) timestamp = mapped_column(DateTime, nullable=False) @@ -30,7 +30,7 @@ class FileOpsStat(Base): op_type = mapped_column(String(length=16), nullable=False) number = mapped_column(Integer, nullable=False) org_id = mapped_column(Integer, nullable=False) - + __table_args__ = (Index('idx_file_ops_time_org', 'timestamp', 'org_id'), ) def __init__(self, org_id, timestamp, op_type, number): diff --git a/utils/ccnet_db.py b/utils/ccnet_db.py index c7fdcc42..763ce964 100644 --- a/utils/ccnet_db.py +++ b/utils/ccnet_db.py @@ -8,7 +8,7 @@ def get_ccnet_db_name(): - return os.environ.get('SEAFILE_MYSQL_DB_CCNET_DB_NAME', '') or 'ccnet_db' + return os.environ.get('SEAFILE_MYSQL_DB_CCNET_DB_NAME', '') or 'SYSDBA' class CcnetDB(object): @@ -88,7 +88,7 @@ def get_groups_by_ids(self, group_ids): sql = f""" SELECT * FROM - `{self.db_name}`.`Group` + {self.db_name}.Group WHERE group_id IN ({group_ids_str}) """ @@ -105,7 +105,7 @@ def get_groups_by_ids(self, group_ids): def get_org_user_count(self, org_id): sql = f""" - SELECT COUNT(1) FROM `{self.db_name}`.`OrgUser` WHERE org_id={org_id} + SELECT COUNT(1) FROM {self.db_name}.OrgUser WHERE org_id={org_id} """ with self.ccnet_db_cursor as cursor: cursor.execute(sql) @@ -114,7 +114,7 @@ def get_org_user_count(self, org_id): def get_user_role(self, email): sql = f""" - SELECT role FROM `{self.db_name}`.`UserRole` + SELECT role FROM {self.db_name}.UserRole WHERE email="{email}" """ with self.ccnet_db_cursor as cursor: diff --git a/utils/seafile_db.py b/utils/seafile_db.py index b1f61e73..d6792704 100644 --- a/utils/seafile_db.py +++ b/utils/seafile_db.py @@ -18,9 +18,11 @@ def get_seafile_db_name(): config.read(seafile_conf_path) if config.has_section('database'): - db_name = config.get('database', 'db_name', fallback='seafile') + # 暂时的调整， + # db_name = config.get('database', 'db_name', fallback='seafile') + db_name = config.get('database', 'user', fallback='seafile') else: - db_name = 'seafile' + db_name = 'sysdba' if config.get('database', 'type') != 'mysql': error_msg = 'Failed to init seafile db, only mysql db supported.' @@ -72,6 +74,7 @@ def init_seafile_db(self): db_port = seafile_config.getint('database', 'port', fallback=3306) db_user = seafile_config.get('database', 'user') db_passwd = seafile_config.get('database', 'password') + # 也需要改掉，不能用pymysql，改成能适配达梦数据库的形式 try: self.seafile_db_conn = pymysql.connect(host=db_host, port=db_port, user=db_user, @@ -100,15 +103,15 @@ def get_repo_info_by_ids(self, repo_ids): repo_ids_str = ','.join(["'%s'" % str(repo_id) for repo_id in repo_ids]) sql1 = f""" SELECT r.repo_id, name, owner_id - FROM `{self.db_name}`.`RepoInfo` r - LEFT JOIN `{self.db_name}`.`RepoOwner` o + FROM {self.db_name}.RepoInfo r + LEFT JOIN {self.db_name}.RepoOwner o ON o.repo_id = r.repo_id WHERE r.repo_id IN ({repo_ids_str}) """ sql2 = f""" SELECT r.repo_id, name, user - FROM `{self.db_name}`.`RepoInfo` r - LEFT JOIN `{self.db_name}`.`OrgRepo` o + FROM {self.db_name}.RepoInfo r + LEFT JOIN {self.db_name}.OrgRepo o ON o.repo_id = r.repo_id WHERE r.repo_id IN ({repo_ids_str}) """ @@ -129,10 +132,10 @@ def get_repo_info_by_ids(self, repo_ids): def reset_download_rate_limit(self): sql1 = f""" - TRUNCATE TABLE `{self.db_name}`.`UserDownloadRateLimit`; + TRUNCATE TABLE {self.db_name}.UserDownloadRateLimit; """ sql2 = f""" - TRUNCATE TABLE `{self.db_name}`.`OrgDownloadRateLimit` + TRUNCATE TABLE {self.db_name}.OrgDownloadRateLimit """ with self.seafile_db_cursor as cursor: cursor.execute(sql1) diff --git a/virus_scanner/models.py b/virus_scanner/models.py index e2dfae40..22a032be 100644 --- a/virus_scanner/models.py +++ b/virus_scanner/models.py @@ -5,7 +5,7 @@ class VirusScanRecord(Base): - __tablename__ = 'VirusScanRecord' + __tablename__ = 'VIRUSSCANRECORD' repo_id = mapped_column(String(length=36), nullable=False, primary_key=True) scan_commit_id = mapped_column(String(length=40), nullable=False) @@ -18,7 +18,7 @@ def __init__(self, repo_id, scan_commit_id): class VirusFile(Base): - __tablename__ = 'VirusFile' + __tablename__ = 'VIRUSFILE' vid = mapped_column(Integer, primary_key=True, autoincrement=True) repo_id = mapped_column(String(length=36), nullable=False, index=True)