Skip to content

Commit

Permalink
修复了一些已知问题
Browse files Browse the repository at this point in the history
  • Loading branch information
@mikigo
mikigo committed Nov 12, 2024
1 parent 9b5d6df commit c8f8b9b
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 140 deletions.
22 changes: 0 additions & 22 deletions src/rtk/local_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,6 @@


class LocalRunner:
"""
本地执行器
"""

__author__ = "Mikigo <[email protected]>"

Expand Down Expand Up @@ -85,16 +82,6 @@ def __init__(
**kwargs,
):
logger("INFO")
# try:
# github_tags = RequestX().open_url(
# f"https://api.github.com/repos/linuxdeepin/youqu/tags", timeout=1
# )
# latest_tag = json.loads(github_tags)[0].get("name")
# if GlobalConfig.current_tag != latest_tag:
# print(f"YouQu最新版本为: {latest_tag}，当前使用版本为: {GlobalConfig.current_tag}")
# print(f"建议使用：sudo pip3 install youqu=={latest_tag} 升级版本")
# except Exception:
# pass
self.default = {
Args.app_name.value: transform_app_name(
app_name if app_name or case_file else GlobalConfig.APP_NAME
Expand Down Expand Up @@ -282,15 +269,6 @@ def create_pytest_cmd(self, app_dir, default=None, proj_path=None):
GlobalConfig.ReportFormat.JSON not in report_formats
):
self.make_allure_report(cmd, GlobalConfig.ReportFormat.ALLURE, proj_path)
# xml
# if GlobalConfig.ReportFormat.XML in report_formats:
# self.make_xml_report(
# app_dir,
# default.get(Args.case_file.value),
# cmd,
# GlobalConfig.ReportFormat.XML,
# proj_path,
# )
# json
if (GlobalConfig.ReportFormat.ALLURE not in report_formats) and (
GlobalConfig.ReportFormat.JSON in report_formats
Expand Down
122 changes: 4 additions & 118 deletions src/rtk/remote_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,6 @@


class RemoteRunner:
"""
远程执行器：控制多台测试机远程执行用例。
在 setting/remote.ini 里面配置要执行的测试机信息。
"""

__author__ = "Mikigo <[email protected]>"

Expand Down Expand Up @@ -102,7 +98,6 @@ def __init__(
Args.json_backfill_password.value: remote_kwargs.get("json_backfill_password"),
Args.json_backfill_custom_api.value: remote_kwargs.get("json_backfill_custom_api") or "api/youqu/yqresult/"
}
# 客户端地址
if "/home/" not in GlobalConfig.ROOT_DIR:
raise EnvironmentError
self.server_project_path = "/".join(GlobalConfig.ROOT_DIR.split("/")[3:])
Expand All @@ -121,12 +116,6 @@ def __init__(
)
self.strf_time = strftime("%m%d%p%I%M%S")
self.server_detail_json_path = f"{GlobalConfig.REPORT_PATH}/json/{self.strf_time}_remote"
# self.client_xml_report_path = (
# lambda
# x: f"/home/{x}/{self.server_project_path}/{GlobalConfig.report_cfg.get('XML_REPORT_PATH', default='report')}/xml".replace(
# "//", "/"
# )
# )
self.client_list = list(self.default.get(Args.clients.value).keys())
_pty = "t"
if len(self.client_list) >= 2:
Expand All @@ -142,13 +131,6 @@ def __init__(
self.pms_password = None

def send_code_to_client(self, user, _ip, password):
"""
发送代码到测试机
:param user: 用户名
:param _ip: 测试机IP
:param password: 测试机密码
:return:
"""
logger.info(f"发送代码到测试机 - < {user}@{_ip} >")
system(
f"{self.ssh % password} {user}@{_ip} "
Expand All @@ -161,7 +143,6 @@ def send_code_to_client(self, user, _ip, password):
system(
f'{self.ssh % password} {user}@{_ip} "mkdir -p ~/{self.server_project_path}" {self.empty}'
)
# 过滤目录
app_name: str = self.default.get(Args.app_name.value)
exclude = ""
for i in [
Expand Down Expand Up @@ -195,29 +176,18 @@ def send_code_to_client(self, user, _ip, password):
continue
if app_name.replace("-", "_") != i:
exclude += f"--exclude='{i}' "
status = system(
system(
f"{self.rsync % (password,)} {exclude} {GlobalConfig.ROOT_DIR}/* "
f"{user}@{_ip}:~/{self.server_project_path}/ {self.empty}"
)
self.set_youqu_run_exitcode(status)
status = system(
system(
f"{self.rsync % (password,)} {exclude} {GlobalConfig.ROOT_DIR}/.env "
f"{user}@{_ip}:~/{self.server_project_path}/ {self.empty}"
)
self.set_youqu_run_exitcode(status)
if status != 0:
logger.error(f"代码发送失败 - < {user}@{_ip} >")
else:
logger.info(f"代码发送成功 - < {user}@{_ip} >")

logger.info(f"代码发送完成 - < {user}@{_ip} >")

def build_client_env(self, user, _ip, password):
"""
测试机环境安装
:param user: 用户名
:param _ip: 测试机IP
:param password: 测试机密码
:return:
"""
logger.info(f"安装环境 - < {user}@{_ip} >")
system(
f"{self.ssh % password} {user}@{_ip} "
Expand All @@ -230,24 +200,10 @@ def build_client_env(self, user, _ip, password):
logger.info(f"环境安装完成 - < {user}@{_ip} >")

def send_code_and_env(self, user, _ip, password):
"""
发送代码到测试机并且安装环境
:param user: 用户名
:param _ip: 测试机IP
:param password: 测试机密码
:return:
"""
self.send_code_to_client(user, _ip, password)
self.build_client_env(user, _ip, password)

def install_deb(self, user, _ip, password):
"""
安装 deb 包
:param user:
:param _ip:
:param password:
:return:
"""
logger.info(f"安装deb包 - < {user}@{_ip} >")
system(
f"{self.scp % password} {self.default.get(Args.deb_path.value)}/*.deb {user}@{_ip}:{self.default.get(Args.deb_path.value)}/"
Expand All @@ -258,12 +214,6 @@ def install_deb(self, user, _ip, password):
logger.info(f"deb包安装完成 - < {user}@{_ip} >")

def mul_do(self, func_obj, client_list):
"""
异步发送代码
:param func_obj: 函数对象
:param client_list: 测试机列表
:return:
"""
if len(client_list) >= 2:
executor = ThreadPoolExecutor()
_ps = []
Expand All @@ -279,33 +229,17 @@ def mul_do(self, func_obj, client_list):
func_obj(user, _ip, password)

def get_client_test_status(self, user, _ip, password):
"""
获取测试机是否有用例执行
:param user: 用户名
:param _ip: 测试机IP
:param password: 测试机密码
:return:
"""
status_test = popen(
f'{self.ssh % password} {user}@{_ip} "ps -aux | grep pytest | grep -v grep"'
).read()
return bool(status_test)

@staticmethod
def make_dir(dirs):
"""make_dir"""
if not exists(dirs):
makedirs(dirs)

def run_pytest_cmd(self, user, _ip, password):
"""
创建 Pytest 命令行参数
:param user: 用户名
:param _ip: 测试机IP
:param password: 测试机密码
:return:
"""
# pylint: disable=too-many-branches
cmd = [
self.ssh % password,
f"{user}@{_ip}",
Expand Down Expand Up @@ -372,10 +306,6 @@ def run_pytest_cmd(self, user, _ip, password):
system(cmd_str)

def pytest_co_cmd(self):
"""
创建收集用例的命令行参数
:return:
"""
app_dir = (
self.default.get(Args.app_name.value) if self.default.get(Args.app_name.value) else ""
)
Expand Down Expand Up @@ -409,41 +339,25 @@ def pytest_co_cmd(self):
return cases

def pre_env(self):
"""
前置环境处理
:return:
"""
# rm hosts
system(f"rm -rf ~/.ssh/known_hosts {self.empty}")
# rm server report
if self.clean_server_report_dir:
system(f"rm -rf {GlobalConfig.REPORT_PATH}/* {self.empty}")
# rm client report
if not self.default.get(Args.send_code.value) and self.clean_client_report_dir:
for client in self.default.get(Args.clients.value):
user, _ip, password = self.default.get(Args.clients.value).get(client)
system(
f"""{self.ssh % password} {user}@{_ip} "rm -rf {self.client_report_path(user)}/*" {self.empty}"""
)
# delete ssh ask
sudo = f"echo '{GlobalConfig.PASSWORD}' | sudo -S"
if "StrictHostKeyChecking no" not in popen("cat /etc/ssh/ssh_config").read():
system(
f"""{sudo} sed -i "s/# StrictHostKeyChecking ask/ StrictHostKeyChecking no/g" /etc/ssh/ssh_config {self.empty}"""
)
# install sshpass
if "(C)" not in popen("sshpass -V").read():
system(f"{sudo} apt update {self.empty}")
system(f"{sudo} apt install sshpass {self.empty}")

def scp_report(self, user, _ip, password):
"""
远程复制报告
:param user:
:param _ip:
:param password:
:return:
"""
html_dir_endswith = f"_{self.default.get(Args.app_name.value)}" if self.default.get(Args.app_name.value) else ""
if not self.default.get(Args.parallel.value):
self.nginx_server_allure_path = f"{GlobalConfig.REPORT_PATH}/allure/{self.strf_time}{html_dir_endswith}"
Expand Down Expand Up @@ -514,12 +428,6 @@ def remote_finish_send_to_pms(self):
).remote_finish_push(res)

def get_report(self, client_list):
"""
回传测试报告
:param client_list: 客户端列表
:return:
"""
# mul get report
if len(self.default.get(Args.clients.value)) >= 2:
_ps = []
executor = ThreadPoolExecutor()
Expand Down Expand Up @@ -555,7 +463,6 @@ def get_report(self, client_list):
except Exception as e:
logger.error(e)
sys.exit(1)
# 分布式执行的情况下需要汇总结果
if not self.default.get(Args.parallel.value):
summarize = {
"total": 0,
Expand All @@ -576,11 +483,6 @@ def get_report(self, client_list):
AllureCustom.gen(self.nginx_server_allure_path, generate_allure_html)

def parallel_run(self, client_list):
"""
并行跑
:param client_list:
:return:
"""
_ps = []
executor = ThreadPoolExecutor()
for client in client_list[:-1]:
Expand All @@ -598,22 +500,14 @@ def parallel_run(self, client_list):
sleep(5)

def nginx_run(self, client_list):
"""
分布式执行
:param client_list: 客户端列表
:return:
"""
# pylint: disable=too-many-nested-blocks
case_files = self.pytest_co_cmd()
logger.info(f"Collected {len(case_files)} case.")
# sort case
case_files.sort(key=lambda x: int(re.findall(r"(\d+)", x)[0]))
_ps = []
executor = ThreadPoolExecutor()
for case in case_files:
counter = {}
try:
# pylint: disable=unsubscriptable-object
for client in cycle(client_list)[::-1]:
user, _ip, password = self.default.get(Args.clients.value).get(client)
if not self.get_client_test_status(user, _ip, password):
Expand All @@ -631,26 +525,19 @@ def nginx_run(self, client_list):
if self.get_client_test_status(user, _ip, password):
counter[client] = 0
break
# else:
sleep(1)
else:
client_list.remove(client)
else:
# relax
counter[client] = counter.get(client, 0) + 1
if counter.get(client) >= self.scan:
client_list.remove(client)
sleep(1)
except ValueError:
break
# Wait for all child processes to end
wait(_ps, return_when=ALL_COMPLETED)

def remote_run(self):
"""
远程执行主函数
:return:
"""
client_list = list(self.default.get(Args.clients.value).keys())
self.pre_env()
logger.info(
Expand All @@ -668,6 +555,5 @@ def remote_run(self):
self.parallel_run(client_list)
else:
self.nginx_run(client_list)
# collect and integrate result data after all tests.
self.get_report(client_list)
self.exit_with_youqu_run_exitcode()

0 comments on commit c8f8b9b

Please sign in to comment.