Merge pull request linuxkerneltravel#760 from ziyangfu/develop

MagicEyes：provide magic_eyes_cli tool, a unified entry to all of backend tools

MagicEyes/CMakeLists.txt

@@ -137,4 +137,5 @@ if (NOT BpfObject_FOUND)
 endif ()
 
 add_subdirectory(src/backend)
-add_subdirectory(src/bridge)
+add_subdirectory(src/bridge)
+add_subdirectory(src/magic_eyes_cli)

MagicEyes/src/magic_eyes_cli/CMakeLists.txt

@@ -0,0 +1,17 @@
+set(MAGIC_EYES_CLI_INSTALL_DIR magic_eyes_cli)
+
+# 安装 MagicEyesCli 到 <intsall_dir>/sbin/
+
+# 安装magic_eyes_cli文件夹到 <install_dir>/sbin/
+
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
+        DESTINATION ${MAGIC_EYES_CLI_INSTALL_DIR}
+)
+
+install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/magic_eyes_cli
+        PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ WORLD_EXECUTE
+        DESTINATION ${MAGIC_EYES_CLI_INSTALL_DIR})
+
+install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/before_running.sh
+PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ WORLD_EXECUTE
+DESTINATION ${MAGIC_EYES_CLI_INSTALL_DIR})

MagicEyes/src/magic_eyes_cli/README.md

@@ -0,0 +1,72 @@
+## magic_eyes_cli 命令行前端
+
+### 1. 简述
+
+将所有的后端工具统一到一个命令行前端，并且具备自动补全功能。
+
+Tips：**记得Tab**
+
+### 2. 使用之前
+
+```bash
+mkdir build && cd build
+cmake .. && make && make install
+cd ./install/magic_eyes_cli
+# 运行前置条件脚本
+source ./before_running.sh
+```
+
+### 3. 使用
+
+```bash
+(venv) $ ./magic_eyes_cli -h
+/home/fzy/Downloads/04_bcc_ebpf/MagicEyes
+usage: magic_eyes_cli [-h] [-l | -c] {net,memory,system_diagnosis,process} ...
+
+magic_eyes_cli: command tools for Linux kernel diagnosis and optimization
+
+positional arguments:
+  {net,memory,system_diagnosis,process}
+    net                 tool for Linux net subsystem
+    memory              tool for Linux memory subsystem
+    system_diagnosis    tool for Linux system_diagnosis subsystem
+    process             tool for Linux process subsystem
+
+optional arguments:
+  -h, --help            show this help message and exit
+
+all of common options:
+  -l                    list all avaliable tools
+  -c                    check all tools dependency, and whether it can be run in current platform
+
+eg: magic_eyes_cli -l
+```
+
+**固定命令**
+
+magic_eyes_cli具有2个固定命令， 即
+
+```bash
+-l : 即list， 列出所有可用的后端命令
+-c : 即check， 检查所有运行依赖项（暂未实现）
+```
+
+**动态命令**
+
+{net,memory,system_diagnosis,process}为动态命令，会根据backend文件夹下的情况动态调整。
+
+### 4. 例程
+
+```bash
+magic_eyes_cli process cpu_watcher -h
+# <------------------ 自动补全 | 非自动补全
+```
+
+### 5.其他
+
+```bash
+# 生成requirements.txt
+pip3 freeze > requirements.txt
+#  安装
+pip3 install -r requiredments.txt
+```

MagicEyes/src/magic_eyes_cli/backend_tool_modules/README.md

@@ -0,0 +1,7 @@
+## 后端工具描述脚本存放文件夹
+
+1. 每个后端工具均有一个后端工具描述脚本
+   1. 描述后端工具名称
+   2. 描述后端工具依赖项，如内核配置选项，某些需要下载的依赖等
+   3. 原始文件请存放在各自工具文件夹的scripts中，在此文件夹创建软链接
+

MagicEyes/src/magic_eyes_cli/backend_tool_modules/cpu_watcher.py

@@ -0,0 +1,19 @@
+"""
+方案2: 采用后端工具描述文件，后续读取该文件，尤其是其中的依赖项，
+        并使用 magic_eyes_cli check 进行运行环境检查
+描述:
+        1. 工具名，所属子系统
+        2. 工具的简要描述
+        3. 是否具有运行依赖项，依赖项是什么？
+        
+"""
+
+class CpuWatcher():
+    def __init__(self):
+        self.tool_name = "cpu_watcher"
+        self.belong_to_subsystem = "process"
+        self.description = "A tool for analyzing CPU running status"
+        self.dependencies = {
+            """ 工具运行的依赖项 """
+        }
+

MagicEyes/src/magic_eyes_cli/before_running.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+MAGIC_EYES_CLI_INSTALL_DIR=$(dirname $(realpath $0))
+
+check_conditions() {
+	# 判断python3是否存在
+	if /usr/bin/env python3 -V > /dev/null; then
+		echo "python 已安装"
+	else
+		echo "python 未安装"
+		return 1
+	fi
+	# 进入python venv环境
+	if . ./venv/bin/activate; then
+		echo "成功进入venv环境"
+	else
+		echo "进入venv环境失败"
+		return 1
+	fi
+	# 使用pip列出所有已安装的包，并搜索argcomplete，判断是否argcomplete是否存在
+	if pip list --verbose | grep -q "argcomplete"; then
+		echo "argcomplete 已经安装."
+	else
+		echo "argcomplete 未安装."
+		echo "尝试安装argcomplete"
+		pip3 install -r ./requirements.txt
+		if pip list --verbose | grep -q "argcomplete"; then
+			echo "argcomplete安装完成"
+		else
+			return 1
+		fi
+	fi
+	# 注册 magic_eyes_cli
+	if eval "$(register-python-argcomplete ./magic_eyes_cli)"; then
+		echo "magic_eyes_cli注册成功"
+	else
+		echo "magic_eyes_cli注册失败"
+		return 1
+	fi
+	return 0
+}
+
+# 调用函数并获取返回值
+check_conditions
+exit_status=$?
+# 根据返回值输出最终结果
+if [ $exit_status -eq 0 ]; then
+    echo "OK"
+else
+    echo "条件不满足"
+fi
+
+
+
+
+

MagicEyes/src/magic_eyes_cli/core/cli_arg_parser.py

@@ -0,0 +1,103 @@
+"""
+    for argc parse
+"""
+import argparse
+import argcomplete
+import os
+import sys
+import subprocess
+import multiprocessing
+from core.filesystem import get_all_tools
+from core.filesystem import print_all_tools
+
+class CliArgParser:
+    """
+    class of commandline argc praser
+    """
+    def __init__(self) -> None:
+        """Initialize the parser """
+        self._arg_parser = argparse.ArgumentParser(
+            description=''' magic_eyes_cli: command tools for Linux kernel diagnosis and optimization ''',
+            add_help=True,
+            epilog='''eg: magic_eyes_cli -l''')
+        self._parsed_args = None
+        self._setup_args()
+        argcomplete.autocomplete(self._arg_parser)
+
+    def parse_args(self, args):
+        """ Parse the given arguments and return them """
+        self._parsed_args = self._arg_parser.parse_args(args[:2])  # 只解析到 net net_watcher
+        if self._parsed_args.list:
+            print_all_tools()
+        elif self._parsed_args.check:
+            print("will to do in future")
+        elif hasattr(self._parsed_args, 'func'):
+            self._parsed_args.func(args)
+        else:
+            self._arg_parser.print_help()
+
+    def handle_args(self, args):
+        backend_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 
+                                    '..', '..', 'backend')
+        tool_path = os.path.join(backend_path, args[0], args[1], 'bin', args[1])
+        if not os.path.exists(tool_path):
+            print(f"Error: Tool {args[1]} not found in {args[0]}")
+        # [tool] 后面的参数
+        tool_args = args[2:]
+        if (len(tool_args) == 1) and (tool_args[0] == '-h' or tool_args[0] == '--help'):
+            cmd = [tool_path] + tool_args               # -h 不需要超级权限
+        else:
+            cmd = ['sudo'] + [tool_path] + tool_args
+        try:
+            subprocess.run(cmd, check=True)
+        except KeyboardInterrupt:
+            print("Operation was cancelled by the user.")
+        except subprocess.CalledProcessError as e:
+            print(f"Error: {e}")
+            sys.exit(1)
+
+    def _setup_args(self):
+        # 通用参数部分，组内选项是互斥的  
+        common_opts_group = self._arg_parser.add_argument_group("all of common options")
+        comm_opts = common_opts_group.add_mutually_exclusive_group()
+        comm_opts.add_argument(
+            "-l", action='store_true', dest='list',
+            help=" list all avaliable tools ")
+        comm_opts.add_argument(
+            "-c", action='store_true', dest='check',
+            help="check all tools dependency, and whether it can be run in current platform"
+        )
+        subparser = self._arg_parser.add_subparsers(dest='command')
+        # 获取subsystem以及下属的工具清单
+        tools_lists = get_all_tools()
+        for subsystem, tools in tools_lists:
+            subsystem_parser = "subsystem_" + str({subsystem})
+            subsystem_parser = subparser.add_parser(
+                f'{subsystem}',
+                help=f"tool for Linux {subsystem} subsystem"
+            )
+            subtool_parser = subsystem_parser.add_subparsers(dest='tools')
+            for tool in tools:
+                tool_parser = "tool_" + str({tool})
+                tool_parser = subtool_parser.add_parser(
+                    f'{tool}',
+                    add_help=True,
+                    help=f"tool within {subsystem}"
+                )
+                #tool_parser.add_argument(
+                #    'tool_args', 
+                #    nargs='*',
+                #    help="tool all args"
+                #)
+                tool_parser.set_defaults(func=self.handle_args)
+
+    def exit(self, status=os.EX_OK, message=None):
+        self._arg_parser.exit(status=status, message=message)
+
+
+def cpu_num():
+    try:
+        num = multiprocessing.cpu_count()
+    except BaseException:
+        num = 1
+    return int(num)

MagicEyes/src/magic_eyes_cli/core/filesystem.py

@@ -0,0 +1,99 @@
+"""part of file system handle like files, directories and paths """
+
+import logging
+import os
+import shutil
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def delete_file_or_directory(path, ignore_errors=False, log_function=LOGGER.debug):
+    """
+    Used as a universal delete function. Also deletes non-empty directories recursively.
+
+    Args:
+        path (str): Path to the file or directory to be deleted
+
+    Keyword Args:
+        ignore_errors (bool): Catches and ignores the exceptions if True
+        log_function (Callable): Log messages will be passed to this callable
+
+    Returns:
+        bool: True if file or directory is removed
+              False if an exception occurred during deletion or the file/directory does not exist
+    """
+    logger = log_function or (lambda *x: None)
+    result = False
+
+    try:
+        if os.path.isfile(path):
+            logger("Deleting file %s", path)
+            os.remove(path)
+            result = True
+        elif os.path.isdir(path):
+            logger("Deleting directory %s", path)
+            shutil.rmtree(path)
+            result = True
+        else:
+            logger("The path %s does not exist", path)
+    except BaseException:
+        if not ignore_errors:
+            logger("Error during deletion of %s", path)
+            raise
+
+    return result
+
+
+def write_to_file(filepath, file_content):
+    """
+    Writes file_content to the file in filepath. Creates directories for filepath if they do not already exist.
+    Overwrites the file in filepath if it already exists.
+
+    Args:
+        filepath (str): Path to the file
+        file_content (str | list): If it is of str type, it is directly written to the file,
+            if it is of list type, each element is converted to str and written to the file separeted by a newline
+    """
+    file_directory = os.path.dirname(filepath)
+    if file_directory and not os.path.isdir(file_directory):
+        os.makedirs(file_directory)
+
+    with open(filepath, 'w') as file_:
+        if isinstance(file_content, list):
+            file_.write("\n".join(file_content))
+        else:
+            file_.write(str(file_content))
+
+
+# 列出所有子系统以及子系统下属的所有工具
+def get_all_tools():
+    backend_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 
+                                '..', '..' ,'backend')
+    if not os.path.isdir(backend_path):
+        print("invalid path")
+        return []
+    tools_lists = []
+    # 遍历backend目录下的所有文件和文件夹
+    for item in os.listdir(backend_path):
+        item_path = os.path.join(backend_path, item)
+        if os.path.isdir(item_path):
+            # 添加子系统
+            tools_lists.append([item, []])
+            # 添加子系统下的工具
+            for sub_item in os.listdir(item_path):
+                sub_item_path = os.path.join(item_path, sub_item)
+                if os.path.isdir(sub_item_path):
+                    tools_lists[-1][1].append(sub_item)
+    return tools_lists
+
+
+def print_all_tools():
+    print("list all avaliable tools:")
+    tools_lists = get_all_tools()
+    for subsystem, tools in tools_lists:
+        print(f"{' '.ljust(2)}[{subsystem}]")
+        for tool in tools:
+            print(f"{' '.ljust(8)}{tool}")
+        print()
+

MagicEyes/src/magic_eyes_cli/core/tool_module_index.py

@@ -0,0 +1,4 @@
+"""
+方案2: 采用后端工具描述文件，后续读取该文件，尤其是其中的依赖项，
+        并使用 magic_eyes_cli check 进行运行环境检查
+"""