diff --git a/hapray-gui/core/file_utils.py b/hapray-gui/core/file_utils.py index 60b68296..93d7285c 100644 --- a/hapray-gui/core/file_utils.py +++ b/hapray-gui/core/file_utils.py @@ -4,6 +4,7 @@ import logging import subprocess +import sys from pathlib import Path from typing import Optional @@ -72,11 +73,20 @@ def list_directories(directory: str) -> list[str]: def get_project_root() -> Path: """获取项目根目录""" # 从当前文件向上查找，直到找到包含特定标记文件的目录 - current = Path(__file__).resolve() + # 打包后默认路径：_internal/core + logger.info('当前文件路径：%s', __file__) + current = Path(__file__).resolve().parent + if hasattr(sys, 'frozen') and hasattr(sys, '_MEIPASS'): + return current.parent.parent + + # 向上遍历直到文件系统根目录（当到达根目录时，parent 等于 current） while current != current.parent: - if (current / 'hapray-gui').exists() or (current / 'perf_testing').exists(): - return current + if (current / 'tools').exists() or (current / 'perf_testing').exists(): + return current / 'dist' current = current.parent + + # 如果找不到，记录警告并返回当前工作目录 + logger.warning('未找到项目根目录，返回当前工作目录 ：%s', Path.cwd()) return Path.cwd() @staticmethod diff --git a/hapray-gui/core/plugin_loader.py b/hapray-gui/core/plugin_loader.py index 41393fb4..8dd4a8ae 100644 --- a/hapray-gui/core/plugin_loader.py +++ b/hapray-gui/core/plugin_loader.py @@ -46,7 +46,7 @@ def __init__(self, plugins_dir: Optional[Path] = None): plugins_dir = exe_dir.parent else: # 开发环境 - plugins_dir = Path(__file__).parent.parent.parent / 'tools' + plugins_dir = Path(__file__).parent.parent.parent / 'dist' / 'tools' self.plugins_dir = Path(plugins_dir).resolve() self.plugins: dict[str, BaseTool] = {} self.plugin_metadata: dict[str, dict[str, Any]] = {} diff --git a/hapray-gui/core/result_processor.py b/hapray-gui/core/result_processor.py index 38bd8804..9f3d65f9 100644 --- a/hapray-gui/core/result_processor.py +++ b/hapray-gui/core/result_processor.py @@ -23,10 +23,24 @@ def __init__(self, output_dir: Optional[str] = None): else: self.output_dir = Path(self.config.get_output_dir()) - def save_result(self, tool_name: str, result: ToolResult, params: dict[str, Any]) -> str: + def save_result( + self, + tool_name: str, + result: ToolResult, + params: dict[str, Any], + action_name: str = None, + menu_category: str = None, + ) -> str: """ 保存执行结果 + Args: + tool_name: 工具名称 + result: 执行结果 + params: 参数 + action_name: 动作名称（可选） + menu_category: 菜单分类（可选） + Returns: 保存的文件路径 """ @@ -38,6 +52,8 @@ def save_result(self, tool_name: str, result: ToolResult, params: dict[str, Any] result_file = result_dir / 'result.json' result_data = { 'tool_name': tool_name, + 'action_name': action_name, + 'menu_category': menu_category, 'timestamp': timestamp, 'success': result.success, 'message': result.message, diff --git a/hapray-gui/gui/main_window.py b/hapray-gui/gui/main_window.py index 9dcab09e..cda9fa5a 100644 --- a/hapray-gui/gui/main_window.py +++ b/hapray-gui/gui/main_window.py @@ -219,6 +219,7 @@ def build_function_tree(self): 'opt': '⚡', # SO编译优化 'static': '📱', # 应用技术栈分析 'symbol-recovery': '🔧', # 符号恢复 + 'ui-compare': '📌', # UI组件树对比 } # 定义菜单结构映射：plugin_id -> {action_key -> display_name} @@ -232,6 +233,7 @@ def build_function_tree(self): 'ui-tech-stack': '页面技术栈动态识别', 'update': '更新测试报告', 'compare': '对比报告', + 'ui-compare': 'UI组件树对比', } } }, @@ -280,7 +282,15 @@ def build_function_tree(self): icon = action_icons.get(action_key, '⚙️') action_item.setText(0, f'{icon} {display_name}') action_item.setData( - 0, Qt.UserRole, {'type': 'action', 'plugin_id': plugin_id, 'action': action_key} + 0, + Qt.UserRole, + { + 'type': 'action', + 'plugin_id': plugin_id, + 'action': action_key, + 'action_name': display_name, + 'menu_category': menu_name, + }, ) # 如果一级菜单下没有子项，隐藏该菜单 @@ -308,16 +318,18 @@ def on_function_selected(self, item, column): elif function_type in ['plugin', 'action']: plugin_id = data.get('plugin_id') action = data.get('action') - self.show_tool_config(plugin_id, action) + action_name = data.get('action_name') + menu_category = data.get('menu_category') + self.show_tool_config(plugin_id, action, action_name, menu_category) - def show_tool_config(self, plugin_id, action=None): + def show_tool_config(self, plugin_id, action=None, action_name=None, menu_category=None): """显示工具配置界面""" tool = self.plugin_loader.get_plugin(plugin_id) if not tool: return # 创建工具页面 - tool_page = ToolPage(tool) + tool_page = ToolPage(tool, action_name=action_name, menu_category=menu_category) if action and hasattr(tool_page, 'current_action') and hasattr(tool_page, 'rebuild_param_form'): tool_page.current_action = action tool_page.rebuild_param_form() diff --git a/hapray-gui/gui/multi_select_combobox.py b/hapray-gui/gui/multi_select_combobox.py index 3abf3e2f..6e74e741 100644 --- a/hapray-gui/gui/multi_select_combobox.py +++ b/hapray-gui/gui/multi_select_combobox.py @@ -3,7 +3,12 @@ """ from PySide6.QtCore import Qt, Signal -from PySide6.QtWidgets import QComboBox, QListView, QStyledItemDelegate +from PySide6.QtGui import QMouseEvent +from PySide6.QtWidgets import QComboBox, QListView, QSizePolicy, QStyledItemDelegate + +from core.logger import get_logger + +logger = get_logger(__name__) class CheckableComboBox(QComboBox): @@ -112,9 +117,19 @@ class MultiSelectComboBox(QComboBox): def __init__(self, parent=None): super().__init__(parent) + logger.debug('MultiSelectComboBox 初始化开始') + # 设置为可编辑，但不允许用户输入 self.setEditable(True) - self.lineEdit().setReadOnly(True) + line_edit = self.lineEdit() + line_edit.setReadOnly(True) + line_edit.setMinimumWidth(500) # 设置内部 lineEdit 的最小宽度 + # 安装事件过滤器到 lineEdit，以便处理点击事件 + line_edit.installEventFilter(self) + + # 设置控件本身的最小宽度和大小策略 + self.setMinimumWidth(500) + self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) # 创建列表视图 list_view = QListView() @@ -128,34 +143,52 @@ def __init__(self, parent=None): # 初始化 self.update_text() + logger.debug(f'MultiSelectComboBox 初始化完成，当前项目数量: {self.count()}') + def handle_item_pressed(self, index): """处理项目点击""" item = self.model().itemFromIndex(index) - if item.checkState() == Qt.Checked: - item.setCheckState(Qt.Unchecked) - else: - item.setCheckState(Qt.Checked) - return False # 阻止关闭下拉框 + if item: + if item.checkState() == Qt.Checked: + item.setCheckState(Qt.Unchecked) + else: + item.setCheckState(Qt.Checked) + # 对于多选下拉框，点击选项后不自动关闭，让用户可以继续选择 + # 用户点击外部区域时会自动关闭 def update_text(self): """更新显示文本""" + row_count = self.model().rowCount() + logger.debug(f'MultiSelectComboBox.update_text 被调用，模型行数: {row_count}，控件 count: {self.count()}') checked_items = self.get_checked_items() if not checked_items: - self.lineEdit().setText('请选择测试用例...') + display_text = '请选择测试用例...' + self.lineEdit().setText(display_text) + logger.debug(f'更新显示文本为: "{display_text}"') elif len(checked_items) == 1: - self.lineEdit().setText(checked_items[0]) + display_text = checked_items[0] + self.lineEdit().setText(display_text) + logger.debug(f'更新显示文本为: "{display_text}"') else: - self.lineEdit().setText(f'已选择 {len(checked_items)} 个测试用例') + display_text = f'已选择 {len(checked_items)} 个测试用例' + self.lineEdit().setText(display_text) + logger.debug(f'更新显示文本为: "{display_text}"') self.selection_changed.emit(checked_items) def get_checked_items(self) -> list[str]: """获取所有选中的项目""" + row_count = self.model().rowCount() + logger.debug(f'MultiSelectComboBox.get_checked_items 被调用，模型行数: {row_count}') checked_items = [] - for i in range(self.model().rowCount()): + for i in range(row_count): item = self.model().item(i) - if item and item.checkState() == Qt.Checked: - checked_items.append(item.text()) + if item: + if item.checkState() == Qt.Checked: + checked_items.append(item.text()) + else: + logger.warning(f'第 {i} 行的模型项为 None') + logger.debug(f'选中的项目: {checked_items}，共 {len(checked_items)} 个') return checked_items def set_checked_items(self, items: list[str]): @@ -171,17 +204,61 @@ def set_checked_items(self, items: list[str]): def addItem(self, text, userData=None): """添加项目""" + logger.debug(f'MultiSelectComboBox.addItem 被调用，添加项目: {text}') super().addItem(text, userData) item = self.model().item(self.count() - 1, 0) - item.setFlags(Qt.ItemIsUserCheckable | Qt.ItemIsEnabled) - item.setCheckState(Qt.Unchecked) + if item: + item.setFlags(Qt.ItemIsUserCheckable | Qt.ItemIsEnabled) + item.setCheckState(Qt.Unchecked) + logger.debug(f'项目 "{text}" 添加成功，当前总数: {self.count()}') + else: + logger.error(f'项目 "{text}" 添加失败，无法获取模型项') def addItems(self, texts): """添加多个项目""" + logger.info(f'MultiSelectComboBox.addItems 被调用，准备添加 {len(texts)} 个项目') + logger.debug(f'项目列表: {texts}') + if not texts: + logger.warning('MultiSelectComboBox.addItems 接收到空列表') for text in texts: self.addItem(text) + logger.info(f'MultiSelectComboBox.addItems 完成，当前项目总数: {self.count()}') def clear(self): """清空所有项目""" + old_count = self.count() + logger.info(f'MultiSelectComboBox.clear 被调用，清空前项目数: {old_count}') super().clear() + new_count = self.count() + logger.info(f'MultiSelectComboBox.clear 完成，清空后项目数: {new_count}') self.update_text() + + def showPopup(self): + """显示下拉框""" + logger.debug('MultiSelectComboBox.showPopup 被调用') + super().showPopup() + + def hidePopup(self): + """隐藏下拉框""" + logger.debug('MultiSelectComboBox.hidePopup 被调用') + # 允许正常关闭下拉框（点击外部或按下 ESC 时） + super().hidePopup() + + def eventFilter(self, obj, event): + """事件过滤器，处理 lineEdit 的点击事件""" + if ( + obj == self.lineEdit() + and isinstance(event, QMouseEvent) + and event.type() == QMouseEvent.Type.MouseButtonPress + and event.button() == Qt.LeftButton + ): + # 点击 lineEdit 时显示下拉框（因为 lineEdit 是只读的，无法输入） + try: + view = self.view() + if not view.isVisible(): + logger.debug('点击 lineEdit，显示下拉框') + self.showPopup() + return True # 阻止默认行为 + except Exception as e: + logger.error(f'处理 lineEdit 点击事件时出错: {e}') + return super().eventFilter(obj, event) diff --git a/hapray-gui/gui/result_viewer.py b/hapray-gui/gui/result_viewer.py index 38607716..9624a1bf 100644 --- a/hapray-gui/gui/result_viewer.py +++ b/hapray-gui/gui/result_viewer.py @@ -13,11 +13,12 @@ QApplication, QHBoxLayout, QLabel, - QListWidget, QMessageBox, QPushButton, QSplitter, QTextEdit, + QTreeWidget, + QTreeWidgetItem, QVBoxLayout, QWidget, ) @@ -72,31 +73,32 @@ def init_ui(self): list_label.setStyleSheet('font-size: 16px; font-weight: bold; color: #667eea; padding: 8px 0px;') left_layout.addWidget(list_label) - self.result_list = QListWidget() - self.result_list.itemSelectionChanged.connect(self.on_result_selected) - self.result_list.setStyleSheet(""" - QListWidget { + self.result_tree = QTreeWidget() + self.result_tree.setHeaderHidden(True) + self.result_tree.itemSelectionChanged.connect(self.on_result_selected) + self.result_tree.setStyleSheet(""" + QTreeWidget { border: 1px solid #e5e7eb; border-radius: 8px; background-color: #ffffff; alternate-background-color: rgba(102, 126, 234, 0.02); } - QListWidget::item { + QTreeWidget::item { padding: 8px 12px; border-radius: 4px; margin: 2px 4px; border-bottom: 1px solid rgba(0, 0, 0, 0.05); } - QListWidget::item:selected { + QTreeWidget::item:selected { background-color: rgba(102, 126, 234, 0.15); color: #667eea; font-weight: bold; } - QListWidget::item:hover { + QTreeWidget::item:hover { background-color: rgba(102, 126, 234, 0.08); } """) - left_layout.addWidget(self.result_list) + left_layout.addWidget(self.result_tree) splitter.addWidget(left_widget) @@ -136,38 +138,59 @@ def init_ui(self): def refresh_results(self): """刷新结果列表""" - self.result_list.clear() + self.result_tree.clear() # 获取所有工具的结果历史 history = self.processor.get_result_history() + # 按时间戳降序排序（最新的在前） + history.sort(key=lambda x: x.get('timestamp', ''), reverse=True) + + # 按菜单分类组织数据 + menu_groups = {} for result_data in history: - tool_name = result_data.get('tool_name', 'Unknown') - timestamp = result_data.get('timestamp', '') - success = result_data.get('success', False) + menu_category = result_data.get('menu_category', '其他') + if menu_category not in menu_groups: + menu_groups[menu_category] = [] + menu_groups[menu_category].append(result_data) + + # 构建树形结构 + for menu_category in sorted(menu_groups.keys()): + results = menu_groups[menu_category] + + # 创建一级菜单项 + menu_item = QTreeWidgetItem(self.result_tree) + menu_item.setText(0, f'{menu_category} ({len(results)})') + menu_item.setExpanded(True) + + # 添加二级结果项 + for result_data in results: + action_name = result_data.get('action_name', '') + timestamp = result_data.get('timestamp', '') + success = result_data.get('success', False) - status = '✓' if success else '✗' - item_text = f'[{status}] {tool_name} - {timestamp}' + status = '✓' if success else '✗' + item_text = f'[{status}] {action_name} - {timestamp}' if action_name else f'[{status}] {timestamp}' - self.result_list.addItem(item_text) - # 保存结果数据到item - item = self.result_list.item(self.result_list.count() - 1) - item.setData(Qt.UserRole, result_data) + result_item = QTreeWidgetItem(menu_item) + result_item.setText(0, item_text) + result_item.setData(0, Qt.UserRole, result_data) def on_result_selected(self): """结果项被选中""" - current_item = self.result_list.currentItem() + current_item = self.result_tree.currentItem() if not current_item: # 没有选中项时禁用按钮 self.open_dir_button.setEnabled(False) self.copy_path_button.setEnabled(False) return - result_data = current_item.data(Qt.UserRole) + result_data = current_item.data(0, Qt.UserRole) if not result_data: - # 没有数据时禁用按钮 + # 没有数据时禁用按钮（可能是一级菜单项） self.open_dir_button.setEnabled(False) self.copy_path_button.setEnabled(False) + self.result_detail.clear() return # 显示结果详情 @@ -185,6 +208,16 @@ def format_result_detail(self, result_data: dict) -> str: lines = [] lines.append('=' * 60) lines.append(f'工具名称: {result_data.get("tool_name", "Unknown")}') + + # 显示菜单分类和动作名称 + menu_category = result_data.get('menu_category') + if menu_category: + lines.append(f'菜单分类: {menu_category}') + + action_name = result_data.get('action_name') + if action_name: + lines.append(f'动作名称: {action_name}') + lines.append(f'执行时间: {result_data.get("timestamp", "Unknown")}') lines.append(f'执行状态: {"成功" if result_data.get("success") else "失败"}') lines.append(f'消息: {result_data.get("message", "")}') @@ -224,11 +257,11 @@ def format_result_detail(self, result_data: dict) -> str: def open_output_directory(self): """打开输出目录""" - current_item = self.result_list.currentItem() + current_item = self.result_tree.currentItem() if not current_item: return - result_data = current_item.data(Qt.UserRole) + result_data = current_item.data(0, Qt.UserRole) if not result_data: return @@ -259,11 +292,11 @@ def open_output_directory(self): def copy_output_path(self): """复制输出路径到剪贴板""" - current_item = self.result_list.currentItem() + current_item = self.result_tree.currentItem() if not current_item: return - result_data = current_item.data(Qt.UserRole) + result_data = current_item.data(0, Qt.UserRole) if not result_data: return diff --git a/hapray-gui/gui/tool_pages.py b/hapray-gui/gui/tool_pages.py index fef70ce6..74b76f86 100644 --- a/hapray-gui/gui/tool_pages.py +++ b/hapray-gui/gui/tool_pages.py @@ -20,6 +20,7 @@ QMessageBox, QProgressBar, QPushButton, + QSizePolicy, QSpinBox, QTabWidget, QTextEdit, @@ -77,10 +78,12 @@ class ExecutionThread(QThread): output_received = Signal(str) finished = Signal(str, str) # tool_name, result_path - def __init__(self, tool: BaseTool, params: dict[str, Any]): + def __init__(self, tool: BaseTool, params: dict[str, Any], action_name: str = None, menu_category: str = None): super().__init__() self.tool = tool self.params = params + self.action_name = action_name + self.menu_category = menu_category self.executor = ToolExecutor() self.processor = ResultProcessor() @@ -145,7 +148,9 @@ def output_callback(text: str): ) # 保存结果 - result_path = self.processor.save_result(self.tool.get_name(), result, self.params) + result_path = self.processor.save_result( + self.tool.get_name(), result, self.params, action_name=self.action_name, menu_category=self.menu_category + ) self.finished.emit(self.tool.get_name(), result_path) @@ -155,9 +160,11 @@ class ToolPage(QWidget): execution_finished = Signal(str, str) # tool_name, result_path - def __init__(self, tool: BaseTool): + def __init__(self, tool: BaseTool, action_name: str = None, menu_category: str = None): super().__init__() self.tool = tool + self.action_name = action_name + self.menu_category = menu_category self.param_widgets: dict[str, Any] = {} self.execution_thread: ExecutionThread = None self.current_action: str = None # 当前选中的action @@ -317,8 +324,13 @@ def create_param_widget(self, param_name: str, param_def: dict[str, Any]) -> QWi """创建参数控件""" param_type = param_def.get('type', 'str') default = param_def.get('default') + nargs = param_def.get('nargs') param_def.get('required', False) + # 处理需要多个值的参数 + if nargs: + return self._create_multi_value_widget(param_name, param_def, param_type, default, nargs) + if param_type == 'bool': widget = QCheckBox() widget.setChecked(default if default is not None else False) @@ -337,11 +349,13 @@ def create_param_widget(self, param_name: str, param_def: dict[str, Any]) -> QWi layout.setContentsMargins(0, 0, 0, 0) line_edit = QLineEdit() line_edit.setText(str(default) if default else '') + line_edit.setMinimumWidth(500) # 设置最小宽度 + line_edit.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) # 允许水平扩展 browse_button = QPushButton('浏览...') browse_button.clicked.connect( lambda: self.browse_file(line_edit, param_def.get('filter', 'All Files (*.*)')) ) - layout.addWidget(line_edit) + layout.addWidget(line_edit, 1) # 设置拉伸因子，让 line_edit 占据更多空间 layout.addWidget(browse_button) return widget @@ -351,9 +365,11 @@ def create_param_widget(self, param_name: str, param_def: dict[str, Any]) -> QWi layout.setContentsMargins(0, 0, 0, 0) line_edit = QLineEdit() line_edit.setText(str(default) if default else '') + line_edit.setMinimumWidth(500) # 设置最小宽度 + line_edit.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) # 允许水平扩展 browse_button = QPushButton('浏览...') browse_button.clicked.connect(lambda: self.browse_directory(line_edit)) - layout.addWidget(line_edit) + layout.addWidget(line_edit, 1) # 设置拉伸因子，让 line_edit 占据更多空间 layout.addWidget(browse_button) return widget @@ -364,6 +380,8 @@ def create_param_widget(self, param_name: str, param_def: dict[str, Any]) -> QWi if multi_select: # 多选下拉框 widget = MultiSelectComboBox() + widget.setMinimumWidth(500) # 设置最小宽度 + widget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) # 允许水平扩展 choices = param_def.get('choices', []) # 如果choices是函数名，则异步加载选项 @@ -383,6 +401,8 @@ def create_param_widget(self, param_name: str, param_def: dict[str, Any]) -> QWi return widget # 单选下拉框 widget = QComboBox() + widget.setMinimumWidth(500) # 设置最小宽度 + widget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) # 允许水平扩展 choices = param_def.get('choices', []) # 如果choices是函数名，则异步加载选项 @@ -401,8 +421,126 @@ def create_param_widget(self, param_name: str, param_def: dict[str, Any]) -> QWi # str widget = QLineEdit() widget.setText(str(default) if default else '') + widget.setMinimumWidth(500) # 设置最小宽度 + widget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) # 允许水平扩展 + return widget + + def _create_multi_value_widget( + self, param_name: str, param_def: dict[str, Any], param_type: str, default: Any, nargs: str + ) -> QWidget: + """创建支持多个值的参数控件""" + widget = QWidget() + layout = QVBoxLayout(widget) + layout.setContentsMargins(0, 0, 0, 0) + + # 创建输入区域 + input_widget = QWidget() + input_layout = QHBoxLayout(input_widget) + input_layout.setContentsMargins(0, 0, 0, 0) + + # 根据参数类型创建不同的输入控件 + if param_type == 'int': + value_input = QSpinBox() + value_input.setMinimum(-999999) + value_input.setMaximum(999999) + value_input.setValue(0) + value_input.setMinimumWidth(200) # 设置最小宽度 + value_input.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) # 允许水平扩展 + else: # str, file, dir等 + value_input = QLineEdit() + value_input.setMinimumWidth(500) # 设置最小宽度 + value_input.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) # 允许水平扩展 + + # 如果是文件或目录类型，添加浏览按钮 + if param_def.get('type') == 'file': + browse_button = QPushButton('浏览文件...') + browse_button.clicked.connect( + lambda: self.browse_file(value_input, param_def.get('filter', 'All Files (*.*)')) + ) + input_layout.addWidget(browse_button) + elif param_def.get('type') == 'dir': + browse_button = QPushButton('浏览目录...') + browse_button.clicked.connect(lambda: self.browse_directory(value_input)) + input_layout.addWidget(browse_button) + + input_layout.addWidget(value_input, 1) # 设置拉伸因子，让输入控件占据更多空间 + + # 添加按钮 + add_button = QPushButton('添加') + add_button.clicked.connect(lambda: self._add_multi_value_item(layout, param_name, param_def, value_input)) + input_layout.addWidget(add_button) + + layout.addWidget(input_widget) + + # 如果有默认值，预先添加 + if default: + if isinstance(default, list): + for value in default: + self._add_multi_value_item(layout, param_name, param_def, value_input, str(value)) + else: + self._add_multi_value_item(layout, param_name, param_def, value_input, str(default)) + return widget + def _add_multi_value_item( + self, + parent_layout: QVBoxLayout, + param_name: str, + param_def: dict[str, Any], + value_input: QWidget, + value: str = None, + ): + """添加多值项目""" + # 如果没有提供值，从输入控件获取 + if value is None: + if isinstance(value_input, QSpinBox): + value = str(value_input.value()) + elif isinstance(value_input, QLineEdit): + value = value_input.text().strip() + if not value: + return # 空值不添加 + value_input.clear() # 清空输入框 + + if not value: + return + + # 创建项目控件 + item_widget = QWidget() + item_layout = QHBoxLayout(item_widget) + item_layout.setContentsMargins(0, 0, 0, 0) + + # 值标签 + value_label = QLabel(value) + item_layout.addWidget(value_label) + + # 删除按钮 + remove_button = QPushButton('删除') + remove_button.clicked.connect(lambda: self._remove_multi_value_item(parent_layout, item_widget, param_name)) + item_layout.addWidget(remove_button) + + # 调整间距 + item_layout.addStretch() + + parent_layout.addWidget(item_widget) + + def _remove_multi_value_item(self, parent_layout: QVBoxLayout, item_widget: QWidget, param_name: str): + """删除多值项目""" + parent_layout.removeWidget(item_widget) + item_widget.deleteLater() + + def _is_widget_valid(self, widget: QWidget) -> bool: + """检查 widget 是否仍然有效（未被删除）""" + try: + # 尝试访问对象的一个属性来检查是否已被删除 + _ = widget.isVisible() + return True + except RuntimeError: + # RuntimeError: Internal C++ object (xxx) already deleted. + return False + except Exception: + # 其他异常也视为无效 + return False + def _setup_dynamic_choices_async( self, param_name: str, choices_func_name: str, widget: QWidget, multi_select: bool = False, default: Any = None ): @@ -422,8 +560,20 @@ def _on_choices_loaded(self, param_name: str, choices: list, widget: QWidget, mu if param_name in self.dynamic_loaders: del self.dynamic_loaders[param_name] + # 检查 widget 是否仍然有效 + if not self._is_widget_valid(widget): + logger.warning(f'动态选项加载完成，但 widget 已被删除: {param_name}') + return + # 更新控件选项 if multi_select and isinstance(widget, MultiSelectComboBox): + # 确保最小宽度和大小策略设置 + widget.setMinimumWidth(500) + widget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + line_edit = widget.lineEdit() + if line_edit and self._is_widget_valid(line_edit): + line_edit.setMinimumWidth(500) + # 清空现有选项 widget.clear() widget.addItems(choices) @@ -448,8 +598,11 @@ def _on_choices_loaded(self, param_name: str, choices: list, widget: QWidget, mu logger.debug(f'动态选项加载完成: {param_name}, {len(choices)} 个选项') + except RuntimeError as e: + # RuntimeError: Internal C++ object (xxx) already deleted. + logger.warning(f'更新动态选项时 widget 已被删除: {param_name}, {e}') except Exception as e: - logger.error(f'更新动态选项失败: {e}') + logger.error(f'更新动态选项失败: {param_name}, {e}') def browse_file(self, line_edit: QLineEdit, filter: str): """浏览文件""" @@ -466,28 +619,100 @@ def browse_directory(self, line_edit: QLineEdit): def get_params(self) -> dict[str, Any]: """获取参数值""" params = {} + logger.debug(f'开始收集参数，共有 {len(self.param_widgets)} 个控件') for param_name, widget in self.param_widgets.items(): + logger.debug(f'处理参数: {param_name}, 控件类型: {type(widget).__name__}') if isinstance(widget, QCheckBox): params[param_name] = widget.isChecked() + logger.debug(f' -> QCheckBox 值: {params[param_name]}') elif isinstance(widget, QSpinBox): params[param_name] = widget.value() + logger.debug(f' -> QSpinBox 值: {params[param_name]}') elif isinstance(widget, MultiSelectComboBox): # 多选下拉框，返回选中项列表 checked_items = widget.get_checked_items() if checked_items: params[param_name] = checked_items + logger.debug(f' -> MultiSelectComboBox 值: {params[param_name]}') + else: + logger.debug(' -> MultiSelectComboBox 无选中项，跳过') elif isinstance(widget, QComboBox): - params[param_name] = widget.currentText() - elif isinstance(widget, QWidget): # file or dir - line_edit = widget.findChild(QLineEdit) - if line_edit: - value = line_edit.text().strip() - if value: - params[param_name] = value - else: # QLineEdit + current_text = widget.currentText() + logger.debug(f' -> QComboBox 原始值: "{current_text}"') + # 过滤掉占位符文本和空值 + if current_text and current_text not in ['加载中...', '请选择...', '']: + params[param_name] = current_text + logger.debug(f' -> QComboBox 最终值: "{current_text}"') + else: + logger.debug(f' -> QComboBox 值被过滤: "{current_text}"') + elif isinstance(widget, QLineEdit): + # 直接处理 QLineEdit（必须在 QWidget 之前检查） value = widget.text().strip() + logger.debug(f' -> QLineEdit 原始值: "{value}"') if value: params[param_name] = value + logger.debug(f' -> QLineEdit 最终值: "{value}"') + else: + logger.debug(' -> QLineEdit 值为空，跳过') + elif isinstance(widget, QWidget): + # 检查是否是多值控件（有QVBoxLayout） + layout = widget.layout() + if isinstance(layout, QVBoxLayout): + # 多值控件，收集所有值标签的内容 + values = [] + for i in range(layout.count()): + item_widget = layout.itemAt(i).widget() + if item_widget and hasattr(item_widget, 'layout'): + item_layout = item_widget.layout() + if isinstance(item_layout, QHBoxLayout): + # 查找值标签（第一个QLabel） + for j in range(item_layout.count()): + child_widget = item_layout.itemAt(j).widget() + if ( + isinstance(child_widget, QLabel) + and child_widget != item_layout.itemAt(item_layout.count() - 2).widget() + ): # 不是删除按钮 + value_text = child_widget.text().strip() + if value_text: + values.append(value_text) + break + if values: + # 根据参数定义决定返回格式 + param_def = ( + self.tool.get_parameters(self.current_action).get(param_name, {}) + if self.current_action + else {} + ) + param_type = param_def.get('type', 'str') + if param_type == 'int': + # 转换为整数列表 + try: + params[param_name] = [int(v) for v in values] + except ValueError: + params[param_name] = values + else: + params[param_name] = values + logger.debug(f' -> 多值控件 值: {params[param_name]}') + else: + logger.debug(' -> 多值控件 无值，跳过') + else: + # file or dir 单值控件 + line_edit = widget.findChild(QLineEdit) + if line_edit: + value = line_edit.text().strip() + logger.debug(f' -> file/dir控件 原始值: "{value}"') + if value: + params[param_name] = value + logger.debug(f' -> file/dir控件 最终值: "{value}"') + else: + logger.debug(' -> file/dir控件 值为空，跳过') + else: + logger.debug(' -> QWidget 未找到 QLineEdit 子控件') + else: + # 未知控件类型 + logger.warning(f' -> 未知控件类型: {type(widget).__name__}') + + logger.info(f'收集到的参数: {params}') return params def execute_tool(self): @@ -522,7 +747,12 @@ def execute_tool(self): self.cancel_button.setEnabled(True) # 创建执行线程 - self.execution_thread = ExecutionThread(self.tool, params) + self.execution_thread = ExecutionThread( + self.tool, + params, + action_name=action_name if self.current_action else None, + menu_category=self.menu_category, + ) self.execution_thread.output_received.connect(self.on_output_received) self.execution_thread.finished.connect(self.on_execution_finished) self.execution_thread.start() diff --git a/hapray-gui/package.json b/hapray-gui/package.json index 7f6e93dc..f881a52c 100644 --- a/hapray-gui/package.json +++ b/hapray-gui/package.json @@ -12,7 +12,7 @@ "setup:unix": "python3 setup_env.py", "build": "npm run build:onedir && npm run build:post", "build:onedir": "./.venv/bin/pyinstaller -y --distpath ../dist --clean hapray.spec || .\\\\.venv\\\\Scripts\\\\pyinstaller.exe -y --distpath ../dist --clean hapray.spec", - "build:copy-script": "cp ../scripts/run_macos.sh ../dist/run_macos.sh 2>/dev/null && chmod +x ../dist/run_macos.sh 2>/dev/null || true", + "build:copy-script": "webpack --config webpack.config.js", "build:post": "npm run build:copy-script", "clean": "rm -rf build/ dist/ *.egg-info .venv", "install:dev": "npm run setup", diff --git a/hapray-gui/webpack.config.js b/hapray-gui/webpack.config.js new file mode 100644 index 00000000..d00514a0 --- /dev/null +++ b/hapray-gui/webpack.config.js @@ -0,0 +1,59 @@ +const path = require('path'); +const fs = require('fs'); +const CopyPlugin = require('copy-webpack-plugin'); + +// 构建拷贝模式数组 +const copyPatterns = []; + +// Mac 平台时拷贝 run_macos.sh 文件 +if (process.platform === 'darwin') { + copyPatterns.push({ + from: path.resolve(__dirname, '../scripts/run_macos.sh'), + to: path.resolve(__dirname, '../dist/run_macos.sh'), + noErrorOnMissing: true, + }); +} + +// 构建插件数组 +const plugins = []; + +// 只在有拷贝模式时才添加 CopyPlugin +if (copyPatterns.length > 0) { + plugins.push( + // 拷贝 run_macos.sh 到目标目录 + new CopyPlugin({ + patterns: copyPatterns, + }) + ); +} + +// 确保 macOS 下 run_macos.sh 拷贝后仍然是可执行文件 +plugins.push({ + apply: (compiler) => { + compiler.hooks.afterEmit.tap('MakeRunMacosExecutable', () => { + if (process.platform !== 'darwin') { + return; + } + const dest = path.resolve(__dirname, '../dist/run_macos.sh'); + try { + if (fs.existsSync(dest)) { + fs.chmodSync(dest, 0o755); + } + } catch (e) { + // 这里静默失败即可，不影响构建 + } + }); + }, +}); + +module.exports = { + target: 'node', + mode: 'production', + entry: './webpack-entry.js', // 虚拟入口文件 + output: { + filename: 'bundle.js', + path: path.resolve(__dirname, 'dist'), + }, + plugins: plugins, +}; + diff --git a/package.json b/package.json index 1f1b2c37..87c6e02f 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,7 @@ "release": "node scripts/merge_duplicates.js && node scripts/zip.js ArkAnalyzer-HapRay", "lint": "npm run lint --ws", "lint:fix": "npm run lint:fix --ws", - "download:test-products": "node scripts/download_test_products.js", - "test": "npm run download:test-products && node scripts/e2e_test.js" + "test": "node scripts/e2e_test.js ./dist" }, "repository": { "type": "git", diff --git a/perf_testing/hapray/actions/ui_compare_action.py b/perf_testing/hapray/actions/ui_compare_action.py new file mode 100644 index 00000000..bc4a4b7e --- /dev/null +++ b/perf_testing/hapray/actions/ui_compare_action.py @@ -0,0 +1,334 @@ +""" +Copyright (c) 2025 Huawei Device Co., Ltd. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +import base64 +import glob +import logging +import os + +from hapray import VERSION +from hapray.ui_detector.ui_tree_comparator import UITreeComparator + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + +class UICompareAction: + """UI组件树对比工具入口""" + + @staticmethod + def _find_ui_files(report_dir: str) -> dict: + """从报告目录自动查找UI文件 + + Args: + report_dir: 报告根目录，如 PerfLoad_meituan_0010 + + Returns: + 包含截图和组件树文件路径的字典，按step组织 + """ + ui_dir = os.path.join(report_dir, 'ui') + if not os.path.exists(ui_dir): + logging.error(f'UI目录不存在: {ui_dir}') + return {} + + result = {} + # 遍历所有step目录 + for step_dir in sorted(glob.glob(os.path.join(ui_dir, 'step*'))): + step_name = os.path.basename(step_dir) + + # 查找所有截图和组件树文件（按阶段分组） + result[step_name] = { + 'screenshots': { + 'start': sorted(glob.glob(os.path.join(step_dir, 'screenshot_start_*.png'))), + 'end': sorted(glob.glob(os.path.join(step_dir, 'screenshot_end_*.png'))), + }, + 'trees': { + 'start': sorted(glob.glob(os.path.join(step_dir, 'element_tree_start_*.txt'))), + 'end': sorted(glob.glob(os.path.join(step_dir, 'element_tree_end_*.txt'))), + }, + } + + return result + + @staticmethod + def _image_to_base64(image_path: str) -> str: + """将图片转换为base64编码""" + try: + with open(image_path, 'rb') as f: + return base64.b64encode(f.read()).decode('utf-8') + except Exception as e: + logging.error(f'Failed to encode image {image_path}: {e}') + return '' + + @staticmethod + def execute(args): + """执行UI对比""" + parser = argparse.ArgumentParser( + description='UI组件树对比工具 - 对比两个报告的UI组件树', + prog='ArkAnalyzer-HapRay ui-compare', + ) + + parser.add_argument( + '-v', + '--version', + action='version', + version=f'%(prog)s {VERSION}', + help="Show program's version number and exit", + ) + + parser.add_argument('--base_dir', type=str, required=True, help='基准报告根目录') + parser.add_argument('--compare_dir', type=str, required=True, help='对比报告根目录') + parser.add_argument('-o', '--output', type=str, default='./ui_compare_output', help='输出目录') + + parsed_args = parser.parse_args(args) + + # 验证目录 + if not os.path.exists(parsed_args.base_dir): + logging.error(f'基准目录不存在: {parsed_args.base_dir}') + return None + if not os.path.exists(parsed_args.compare_dir): + logging.error(f'对比目录不存在: {parsed_args.compare_dir}') + return None + + # 自动查找UI文件 + base_files = UICompareAction._find_ui_files(parsed_args.base_dir) + compare_files = UICompareAction._find_ui_files(parsed_args.compare_dir) + + if not base_files: + logging.error(f'基准目录未找到UI文件: {parsed_args.base_dir}') + return None + if not compare_files: + logging.error(f'对比目录未找到UI文件: {parsed_args.compare_dir}') + return None + + # 对比所有共同的step + all_results = {} + comparator = UITreeComparator() + + common_steps = set(base_files.keys()) & set(compare_files.keys()) + if not common_steps: + logging.error('两个报告没有共同的step') + return None + + for step in sorted(common_steps): + logging.info(f'正在对比 {step}...') + step_results = [] + + # 对比所有阶段（start和end） + for phase in ['start', 'end']: + base_screenshots = base_files[step]['screenshots'][phase] + base_trees = base_files[step]['trees'][phase] + compare_screenshots = compare_files[step]['screenshots'][phase] + compare_trees = compare_files[step]['trees'][phase] + + # 对比每一对截图和组件树 + for i in range(min(len(base_screenshots), len(compare_screenshots))): + if i >= len(base_trees) or i >= len(compare_trees): + continue + + logging.info(f' 对比 {phase}_{i + 1}...') + step_output = os.path.join(parsed_args.output, step, f'{phase}_{i + 1}') + + result = comparator.compare_ui_trees( + base_trees[i], + base_screenshots[i], + compare_trees[i], + compare_screenshots[i], + step_output, + filter_minor_changes=True, # 启用微小变化过滤 + ) + + # 将图片转换为base64用于HTML + result['marked_images_base64'] = [ + UICompareAction._image_to_base64(result['marked_images'][0]), + UICompareAction._image_to_base64(result['marked_images'][1]), + ] + result['phase'] = phase + result['index'] = i + 1 + + step_results.append(result) + logging.info(f' {phase}_{i + 1} 对比完成，发现 {result["diff_count"]} 处差异') + + all_results[step] = step_results + + # 生成HTML报告 + html_path = os.path.join(parsed_args.output, 'ui_compare_report.html') + UICompareAction._generate_html_report(all_results, html_path, parsed_args.base_dir, parsed_args.compare_dir) + + logging.info('所有对比完成') + logging.info(f'HTML报告: {html_path}') + + return all_results + + @staticmethod + def _generate_html_report(results: dict, output_path: str, base_dir: str, compare_dir: str): + """生成HTML对比报告""" + html = f""" + + + + + UI组件树对比报告 + + + +
+
+

🔍 UI组件树对比报告

+
+
基准版本: {os.path.abspath(base_dir)}
+
对比版本: {os.path.abspath(compare_dir)}
+
生成时间: {__import__('datetime').datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
+
+
+""" + + for step, step_results in sorted(results.items()): + total_diffs = sum(r['diff_count'] for r in step_results) + + # 生成选项列表 + options = [] + for idx, result in enumerate(step_results): + phase_label = '开始阶段' if result['phase'] == 'start' else '结束阶段' + options.append( + f'' + ) + + html += f''' +
+
📱 {step} - 共发现 {total_diffs} 处差异
+
+ + +
+''' + + for idx, result in enumerate(step_results): + phase_label = '开始阶段' if result['phase'] == 'start' else '结束阶段' + active_class = 'active' if idx == 0 else '' + + html += f''' +
+
{phase_label} #{result['index']}
+
+
🔍 发现差异: {result['diff_count']} 处 | + 总差异数: {result['total_differences']} | 已过滤: {result['filtered_count']}
+
+
+
+
📊 基准版本
+ Base +
+
+
📊 对比版本
+ Compare +
+
+''' + + if result['differences']: + html += """ +
+ + 📋 差异详情列表 + + + + + + + + + + + + +""" + for diff_idx, diff in enumerate(result['differences'], 1): + comp_type = diff.get('component', {}).get('type', '未知') + for attr_diff in diff.get('comparison_result', []): + val1 = str(attr_diff.get('value1', 'N/A')) + val2 = str(attr_diff.get('value2', 'N/A')) + html += f""" + + + + + + + +""" + html += """ + +
#组件类型属性名基准值对比值
{diff_idx}{comp_type}{attr_diff.get('attribute', 'N/A')}{val1}{val2}
+
+""" + else: + html += '
✅ 未发现差异
' + + html += """ +
+""" + + html += """ +
+""" + + html += """ +
+ + +""" + + with open(output_path, 'w', encoding='utf-8') as f: + f.write(html) diff --git a/perf_testing/hapray/analyze/__init__.py b/perf_testing/hapray/analyze/__init__.py index da007ac3..61f07feb 100644 --- a/perf_testing/hapray/analyze/__init__.py +++ b/perf_testing/hapray/analyze/__init__.py @@ -275,13 +275,14 @@ def _process_single_step(step_dir: str, scene_dir: str, analyzers: list[BaseAnal # Data conversion phase conversion_start_time = time.time() - logging.info('Converting perf to db for %s...', step_dir) - perf_convert_start = time.time() - if not ExeUtils.convert_data_to_db(perf_file, perf_db): - logging.error('Failed to convert perf to db for %s', step_dir) - return - perf_convert_time = time.time() - perf_convert_start - logging.info('Perf conversion for %s completed in %.2f seconds', step_dir, perf_convert_time) + if not os.path.exists(perf_db) and os.path.exists(perf_file): + logging.info('Converting perf to db for %s...', step_dir) + perf_convert_start = time.time() + if not ExeUtils.convert_data_to_db(perf_file, perf_db): + logging.error('Failed to convert perf to db for %s', step_dir) + return + perf_convert_time = time.time() - perf_convert_start + logging.info('Perf conversion for %s completed in %.2f seconds', step_dir, perf_convert_time) if not os.path.exists(trace_db) and os.path.exists(htrace_file): logging.info('Converting htrace to db for %s...', step_dir) diff --git a/perf_testing/hapray/analyze/memory_analyzer.py b/perf_testing/hapray/analyze/memory_analyzer.py index 37d89c9f..f0d2d95e 100644 --- a/perf_testing/hapray/analyze/memory_analyzer.py +++ b/perf_testing/hapray/analyze/memory_analyzer.py @@ -586,10 +586,12 @@ def _create_memory_data_dict_indexes(self): def _create_memory_meminfo_indexes(self): """Create indexes for memory_meminfo table to improve query performance""" + self.exec_sql('DROP INDEX IF EXISTS idx_memory_meminfo_step_id') + self.create_index('idx_memory_meminfo_step_id', 'memory_meminfo', 'step_id') self.exec_sql('DROP INDEX IF EXISTS idx_memory_meminfo_timestamp_epoch') - self.create_index('idx_memory_meminfo_timestamp_epoch', 'memory_meminfo', 'timestamp_epoch') + self.create_index('idx_memory_meminfo_timestamp_epoch', 'memory_meminfo', 'step_id, timestamp_epoch') self.exec_sql('DROP INDEX IF EXISTS idx_memory_meminfo_timestamp') - self.create_index('idx_memory_meminfo_timestamp', 'memory_meminfo', 'timestamp') + self.create_index('idx_memory_meminfo_timestamp', 'memory_meminfo', 'step_id, timestamp') def _save_step_data_to_db(self, step_id: int, step_data: dict): """Save step data to database diff --git a/perf_testing/hapray/analyze/ui_analyzer.py b/perf_testing/hapray/analyze/ui_analyzer.py index 752d51cd..c4a71e3c 100644 --- a/perf_testing/hapray/analyze/ui_analyzer.py +++ b/perf_testing/hapray/analyze/ui_analyzer.py @@ -14,6 +14,7 @@ """ import base64 +import glob import json import os import re @@ -751,5 +752,54 @@ def write_report(self, result: dict): except Exception as e: self.logger.exception('Failed to convert images to base64: %s', str(e)) - # 调用基类的 write_report 方法 + # 调用基类的 write_report 方法（保存ui_animate.json） super().write_report(result) + + # 额外保存UI原始数据（用于对比） + self._save_ui_raw_data() + + def _save_ui_raw_data(self): + """保存UI原始数据用于前端对比""" + ui_dir = os.path.join(self.scene_dir, 'ui') + if not os.path.exists(ui_dir): + return + + raw_data = {} + for step_dir in sorted(glob.glob(os.path.join(ui_dir, 'step*'))): + step_name = os.path.basename(step_dir) + + step_data = { + 'screenshots': {'start': [], 'end': []}, + 'trees': {'start': [], 'end': []}, + } + + # 保存所有截图 + for phase in ['start', 'end']: + screenshots = sorted(glob.glob(os.path.join(step_dir, f'screenshot_{phase}_*.png'))) + for screenshot in screenshots: + with open(screenshot, 'rb') as f: + img_data = f.read() + step_data['screenshots'][phase].append(base64.b64encode(img_data).decode('ascii')) + + # 保存所有组件树（只保存文本，前端不需要解析） + for phase in ['start', 'end']: + trees = sorted(glob.glob(os.path.join(step_dir, f'element_tree_{phase}_*.txt'))) + for tree in trees: + with open(tree, encoding='utf-8') as f: + step_data['trees'][phase].append(f.read()) + + if ( + step_data['screenshots']['start'] + or step_data['screenshots']['end'] + or step_data['trees']['start'] + or step_data['trees']['end'] + ): + raw_data[step_name] = step_data + + if raw_data: + report_dir = os.path.join(self.scene_dir, 'report') + os.makedirs(report_dir, exist_ok=True) + output_path = os.path.join(report_dir, 'ui_raw.json') + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(raw_data, f, ensure_ascii=False) + self.logger.info(f'UI原始数据已保存: {output_path}') diff --git a/perf_testing/hapray/analyze/unified_frame_analyzer.py b/perf_testing/hapray/analyze/unified_frame_analyzer.py index b029763c..0170f7a3 100644 --- a/perf_testing/hapray/analyze/unified_frame_analyzer.py +++ b/perf_testing/hapray/analyze/unified_frame_analyzer.py @@ -25,12 +25,18 @@ class UnifiedFrameAnalyzer(BaseAnalyzer): """统一帧分析器 - 整合帧负载、空帧、卡顿帧和VSync异常分析 - 合并了以下四个分析器的功能： + 合并了以下分析器的功能： 1. FrameLoadAnalyzer - 帧负载分析 - 2. EmptyFrameAnalyzer - 空帧分析 + 2. EmptyFrameAnalyzer - 空刷帧分析（包含正向检测 + RS Skip反向追溯） 3. FrameDropAnalyzer - 卡顿帧分析 4. VSyncAnomalyAnalyzer - VSync异常分析 + 注意：RSSkipFrameAnalyzer已合并到EmptyFrameAnalyzer中 + - EmptyFrameAnalyzer现在包含两种检测方法： + 1. 正向检测：flag=2帧 + 2. 反向追溯：RS skip → app frame + - 输出仍保留trace_rsSkip.json（向后兼容） + 主要职责： 1. 统一管理所有帧相关的分析 2. 使用FrameAnalyzerCore进行核心分析 @@ -55,12 +61,14 @@ def __init__(self, scene_dir: str, top_frames_count: int = 10): 'emptyFrame': 'trace/emptyFrame', 'frames': 'trace/frames', 'vsyncAnomaly': 'trace/vsyncAnomaly', + 'rsSkip': 'trace/rsSkip', } # 存储各分析器的结果 self.frame_loads_results = {} self.empty_frame_results = {} self.frame_drop_results = {} self.vsync_anomaly_results = {} + self.rs_skip_results = {} def _analyze_impl( self, step_dir: str, trace_db_path: str, perf_db_path: str, app_pids: list @@ -82,9 +90,9 @@ def _analyze_impl( logging.warning('Trace database not found: %s', trace_db_path) return None - if not os.path.exists(perf_db_path): - logging.warning('Perf database not found: %s', perf_db_path) - return None + # perf_db_path现在是可选的，如果不存在会从trace.db读取perf数据 + if perf_db_path and not os.path.exists(perf_db_path): + logging.info('Perf database not found: %s, will try to use perf data from trace.db', perf_db_path) try: # 新建 FrameAnalyzerCore，传入 trace_db_path, perf_db_path, app_pids, step_dir @@ -116,12 +124,20 @@ def _analyze_impl( if vsync_anomaly_result: self.vsync_anomaly_results[step_dir] = vsync_anomaly_result + # 5. RS Skip分析（已合并到EmptyFrameAnalyzer，此处调用兼容接口） + # 注意：EmptyFrameAnalyzer.analyze_empty_frames()已包含RS Skip检测 + # 此处调用analyze_rs_skip_frames()仅用于生成独立的rsSkip报告（向后兼容） + rs_skip_result = self._analyze_rs_skip_frames(core_analyzer, step_dir) + if rs_skip_result: + self.rs_skip_results[step_dir] = rs_skip_result + # 返回合并结果（用于主报告） return { 'frameLoads': frame_loads_result, 'emptyFrame': empty_frame_result, 'frames': frame_drop_result, 'vsyncAnomaly': vsync_anomaly_result, + 'rsSkip': rs_skip_result, } except Exception as e: @@ -159,6 +175,31 @@ def _analyze_frame_drops(self, core_analyzer: FrameAnalyzerCore, step_dir: str) logging.error('Frame drop analysis failed for step %s: %s', step_dir, str(e)) return None + def _analyze_rs_skip_frames(self, core_analyzer: FrameAnalyzerCore, step_dir: str) -> Optional[dict[str, Any]]: + """分析RS Skip帧（向后兼容接口） + + 注意：RSSkipFrameAnalyzer已合并到EmptyFrameAnalyzer中 + 此方法调用EmptyFrameAnalyzer，然后提取RS traced相关统计 + 用于生成独立的trace_rsSkip.json报告（向后兼容） + """ + try: + result = core_analyzer.analyze_rs_skip_frames() + if result and result.get('summary'): + summary = result['summary'] + logging.info( + 'RS Skip analysis for %s: skip_frames=%d, traced=%d, accuracy=%.1f%%, cpu_waste=%d', + step_dir, + summary.get('total_skip_frames', 0), + summary.get('traced_success_count', 0), + summary.get('trace_accuracy', 0.0), + summary.get('total_wasted_cpu', 0), + ) + return result + + except Exception as e: + logging.error('RS Skip frame analysis failed for step %s: %s', step_dir, str(e)) + return None + def write_report(self, result: dict): """写入报告 - 支持多个输出路径 @@ -171,9 +212,8 @@ def write_report(self, result: dict): if self.frame_loads_results: self._write_single_report(self.frame_loads_results, self.report_paths['frameLoads']) - # 写入空帧分析结果 - if self.empty_frame_results: - self._write_single_report(self.empty_frame_results, self.report_paths['emptyFrame']) + # 写入空帧分析结果（即使为空也写入，以更新已存在的文件） + self._write_single_report(self.empty_frame_results, self.report_paths['emptyFrame']) # 写入卡顿帧分析结果 if self.frame_drop_results: @@ -183,6 +223,10 @@ def write_report(self, result: dict): if self.vsync_anomaly_results: self._write_single_report(self.vsync_anomaly_results, self.report_paths['vsyncAnomaly']) + # 写入RS Skip分析结果 + if self.rs_skip_results: + self._write_single_report(self.rs_skip_results, self.report_paths['rsSkip']) + # 同时更新主结果字典 if self.frame_loads_results: self._update_result_dict(result, self.frame_loads_results, self.report_paths['frameLoads']) @@ -192,16 +236,23 @@ def write_report(self, result: dict): self._update_result_dict(result, self.frame_drop_results, self.report_paths['frames']) if self.vsync_anomaly_results: self._update_result_dict(result, self.vsync_anomaly_results, self.report_paths['vsyncAnomaly']) + if self.rs_skip_results: + self._update_result_dict(result, self.rs_skip_results, self.report_paths['rsSkip']) def _write_single_report(self, results: dict, report_path: str): """写入单个报告文件""" if not results: self.logger.warning('No results to write. Skipping report generation for %s', report_path) - return + # return try: file_path = os.path.join(self.scene_dir, 'report', report_path.replace('/', '_') + '.json') os.makedirs(os.path.dirname(file_path), exist_ok=True) + + # 强制删除旧文件，确保重新写入（避免缓存问题） + if os.path.exists(file_path): + os.remove(file_path) + with open(file_path, 'w', encoding='utf-8') as f: json.dump(results, f, ensure_ascii=False, sort_keys=True) except Exception as e: diff --git a/perf_testing/hapray/core/common/__init__.py b/perf_testing/hapray/core/common/__init__.py index d66f88e4..bf30c2e9 100644 --- a/perf_testing/hapray/core/common/__init__.py +++ b/perf_testing/hapray/core/common/__init__.py @@ -1,22 +1,13 @@ # Common 模块导出 -# Frame Analyzer 模块化组件（从frame子模块导入） # 工具模块 from .common_utils import CommonUtils from .coordinate_adapter import CoordinateAdapter from .excel_utils import ExcelReportSaver from .exe_utils import ExeUtils from .folder_utils import delete_folder, read_json_arrays_from_dir, scan_folders -from .frame import ( - # 核心组件 - FrameAnalyzerCore, - FrameCacheManager, -) __all__ = [ - # Frame Analyzer 组件 - 'FrameAnalyzerCore', - 'FrameCacheManager', # 工具模块 'CommonUtils', 'CoordinateAdapter', diff --git a/perf_testing/hapray/core/common/frame/__init__.py b/perf_testing/hapray/core/common/frame/__init__.py index db10aec8..5f6807af 100644 --- a/perf_testing/hapray/core/common/frame/__init__.py +++ b/perf_testing/hapray/core/common/frame/__init__.py @@ -1,13 +1,10 @@ # Frame Analyzer 模块化组件导出 -# 兼容性包装器（保持向后兼容） from .frame_core_analyzer import FrameAnalyzerCore -from .frame_core_cache_manager import FrameCacheManager # pylint: disable=duplicate-code __all__ = [ # 核心组件 'FrameAnalyzerCore', - 'FrameCacheManager', ] # pylint: enable=duplicate-code diff --git a/perf_testing/hapray/core/common/frame/frame_analyzer_empty.py b/perf_testing/hapray/core/common/frame/frame_analyzer_empty.py index 5d4b5b76..762b6430 100644 --- a/perf_testing/hapray/core/common/frame/frame_analyzer_empty.py +++ b/perf_testing/hapray/core/common/frame/frame_analyzer_empty.py @@ -15,7 +15,8 @@ import logging import time -import traceback +from bisect import bisect_left +from collections import defaultdict from typing import Optional import pandas as pd @@ -23,16 +24,42 @@ from .frame_constants import TOP_FRAMES_FOR_CALLCHAIN from .frame_core_cache_manager import FrameCacheManager from .frame_core_load_calculator import FrameLoadCalculator +from .frame_empty_common import ( + EmptyFrameCallchainAnalyzer, + EmptyFrameCPUCalculator, + EmptyFrameResultBuilder, + calculate_process_instructions, +) +from .frame_empty_framework_specific import detect_framework_specific_empty_frames +from .frame_rs_skip_backtrack_api import ( + preload_caches as preload_rs_api_caches, +) +from .frame_rs_skip_backtrack_api import ( + trace_rs_skip_to_app_frame as trace_rs_api, +) +from .frame_rs_skip_backtrack_nw import ( + preload_caches as preload_nw_caches, +) +from .frame_rs_skip_backtrack_nw import ( + trace_rs_skip_to_app_frame as trace_nw_api, +) from .frame_time_utils import FrameTimeUtils +from .frame_utils import is_system_thread class EmptyFrameAnalyzer: - """空帧分析器 + """统一空刷帧分析器（EmptyFrame + RSSkip合并） - 专门用于分析空帧（flag=2, type=0）的负载情况，包括： - 1. 空帧负载计算 - 2. 主线程vs后台线程分析 - 3. 空帧调用链分析 + 业务目标：检测所有空刷帧，使用两种检测方法： + 1. 正向检测：分析应用进程的flag=2空刷帧 + 2. 反向追溯：从RS skip事件追溯到应用帧（补充漏检） + + 核心功能： + 1. 空帧检测（正向 + 反向） + 2. 帧合并去重（避免重复计数） + 3. CPU浪费计算（应用进程 + RS进程） + 4. 调用链分析 + 5. 重要性标记（traced_count） 帧标志 (flag) 定义： - flag = 0: 实际渲染帧不卡帧（正常帧） @@ -43,7 +70,7 @@ class EmptyFrameAnalyzer: def __init__(self, debug_vsync_enabled: bool = False, cache_manager: FrameCacheManager = None): """ - 初始化空帧分析器 + 初始化统一空刷帧分析器 Args: debug_vsync_enabled: VSync调试开关 @@ -52,13 +79,50 @@ def __init__(self, debug_vsync_enabled: bool = False, cache_manager: FrameCacheM self.cache_manager = cache_manager self.load_calculator = FrameLoadCalculator(debug_vsync_enabled, cache_manager) + # 使用公共模块（模块化重构） + self.cpu_calculator = EmptyFrameCPUCalculator(cache_manager) + self.callchain_analyzer = EmptyFrameCallchainAnalyzer(cache_manager) + self.result_builder = EmptyFrameResultBuilder(cache_manager) + + # 检测方法开关 + self.direct_detection_enabled = True # 正向检测（flag=2） + self.rs_traced_detection_enabled = True # 反向追溯（RS skip） + self.framework_detection_enabled = True # 框架特定检测（Flutter/RN等） + + # 算法升级开关 + self.false_positive_filter_enabled = True # 默认启用假阳性过滤 + self.wakeup_chain_enabled = True # 默认启用唤醒链分析（只对Top N帧） + + # RS Skip追溯配置 + self.rs_api_enabled = True # 是否启用RS系统API追溯 + self.nw_api_enabled = True # 是否启用NativeWindow API追溯 + self.top_n = 10 # Top N帧数量 + + # 框架特定检测配置 + self.framework_types = ['flutter'] # 支持的框架类型 + + # 三个检测器的原始检测数量（在合并去重之前，不处理 overlap） + self.direct_detected_count = 0 # flag=2 检测器的原始检测数量 + self.rs_detected_count = 0 # RS skip 检测器的原始检测数量 + self.framework_detected_counts = {} # 框架特定检测器的原始检测数量 + def analyze_empty_frames(self) -> Optional[dict]: - """分析空帧（flag=2, type=0）的负载情况 + """分析空刷帧（统一方法：正向检测 + 反向追溯） + + 三个核心阶段： + 1. 找到空刷的帧：通过正向检测（flag=2）和反向追溯（RS skip → app frame） + 2. 算出帧的浪费的进程级别的cpu指令数： + - 2.1 计算每个空刷帧在[ts, ts+dur]时间范围内的CPU浪费（进程级别） + - 2.2 分析Top N帧的调用链，定位CPU浪费的具体代码路径 + 3. 统计整个trace浪费指令数占比：empty_frame_load / total_load * 100 - 空帧定义：flag=2 表示数据不需要绘制（没有frameNum信息） + 检测方法： + 1. 正向检测：flag=2 帧（direct detection） + 2. 反向追溯：RS skip → app frame（RS traced） + 3. 框架特定检测：Flutter/RN 等框架的空刷检测（framework specific） 返回: - - dict，包含分析结果 + - dict，包含统一的分析结果 """ # 从cache_manager获取数据库连接和参数 trace_conn = self.cache_manager.trace_conn @@ -66,29 +130,172 @@ def analyze_empty_frames(self) -> Optional[dict]: app_pids = self.cache_manager.app_pids if not trace_conn or not perf_conn: - logging.error('数据库连接未建立') + # logging.error('数据库连接未建立') return None total_start_time = time.time() timing_stats = {} try: - # 阶段1：加载数据 - trace_df, total_load, perf_df = self._load_empty_frame_data(app_pids, timing_stats) - if trace_df is None or trace_df.empty: - return None + # ========== 方法1：正向检测（Direct Detection）========== + direct_frames_df = pd.DataFrame() + direct_frames_count = 0 + + if self.direct_detection_enabled: + # 阶段1A：加载flag=2帧 + trace_df, total_load, perf_df, merged_time_ranges = self._load_empty_frame_data(app_pids, timing_stats) + + # 保存原始 total_load（用于日志对比） + timing_stats['original_total_load'] = total_load - # 阶段2：数据预处理 - self._prepare_frame_data(trace_df, timing_stats) + if not trace_df.empty: + # 假阳性过滤 + if self.false_positive_filter_enabled: + filter_start = time.time() + trace_df = self._filter_false_positives(trace_df, trace_conn) + timing_stats['false_positive_filter'] = time.time() - filter_start + # logging.info('假阳性过滤耗时: %.3f秒', timing_stats['false_positive_filter']) + + if not trace_df.empty: + direct_frames_df = trace_df + direct_frames_count = len(trace_df) + # 保存 flag=2 检测器的原始检测数量（第一次计算的位置） + self.direct_detected_count = direct_frames_count + # logging.info('正向检测到 %d 个空刷帧', direct_frames_count) + + # 方法2：反向追溯（RS Traced） + rs_traced_results = [] + rs_traced_count = 0 + rs_skip_cpu = 0 + + if self.rs_traced_detection_enabled: + # 阶段1B：检测RS skip事件并追溯到应用帧（反向追溯） + rs_skip_frames = self._detect_rs_skip_frames(trace_conn, timing_stats) + + if rs_skip_frames: + # logging.info('检测到 %d 个RS skip帧', len(rs_skip_frames)) + + # 计算RS进程CPU + if perf_conn: + rs_cpu_start = time.time() + rs_skip_cpu = self._calculate_rs_skip_cpu(rs_skip_frames) + timing_stats['rs_cpu_calculation'] = time.time() - rs_cpu_start + + # 预加载缓存并追溯到应用帧 + caches = self._preload_rs_caches(trace_conn, rs_skip_frames, timing_stats) + rs_traced_results = self._trace_rs_to_app_frames( + trace_conn, perf_conn, rs_skip_frames, caches, timing_stats + ) + + # 保存 RS skip 检测器的原始检测数量（第一次计算的位置） + self.rs_detected_count = len(rs_traced_results) if rs_traced_results else 0 + + # 统计追溯成功数 + rs_traced_count = sum( + 1 for r in rs_traced_results if r.get('trace_result') and r['trace_result'].get('app_frame') + ) + # logging.info('反向追溯成功 %d 个空刷帧', rs_traced_count) - # 阶段3：快速帧负载计算 - frame_loads = self._calculate_frame_loads(trace_df, perf_df, timing_stats) + # 方法3：框架特定检测（Flutter/RN等） + framework_frames_df = pd.DataFrame() + framework_frames_count = 0 - # 阶段4：Top帧调用链分析 - self._analyze_top_frames_callchains(frame_loads, trace_df, perf_df, perf_conn, timing_stats) + if self.framework_detection_enabled: + framework_start = time.time() + framework_frames_df = detect_framework_specific_empty_frames( + trace_conn=trace_conn, + app_pids=app_pids, + framework_types=self.framework_types, + timing_stats=timing_stats, + ) + framework_frames_count = len(framework_frames_df) + # 保存框架特定检测器的原始检测数量（第一次计算的位置） + self.framework_detected_counts = {} + for framework_type in self.framework_types: + detected_count = timing_stats.get(f'{framework_type}_detected_count', 0) + self.framework_detected_counts[framework_type] = detected_count + if detected_count > 0: + logging.info( + f'{framework_type} 检测器原始检测结果：{detected_count} 个空刷帧（在合并去重之前）' + ) + if framework_frames_count > 0: + # logging.info('框架特定检测到 %d 个空刷帧', framework_frames_count) + pass - # 阶段5：结果构建 - result = self._build_result(frame_loads, trace_df, total_load, timing_stats) + # ========== 合并检测结果 ========== + # 如果三种方法都没有检测到帧，返回空结果 + if direct_frames_count == 0 and rs_traced_count == 0 and framework_frames_count == 0: + # logging.info('未检测到空刷帧，返回空结果') + # 构建空的 detection_stats，包含三个检测器的原始检测结果 + empty_detection_stats = { + 'direct_detected_count': self.direct_detected_count, + 'rs_detected_count': self.rs_detected_count, + 'framework_detected_counts': self.framework_detected_counts.copy() + if self.framework_detected_counts + else {}, + } + return self._build_empty_result_unified(total_load, timing_stats, rs_skip_cpu, empty_detection_stats) + + # 阶段2：合并并去重 + merge_start = time.time() + all_frames_df, detection_stats = self._merge_and_deduplicate_frames( + direct_frames_df, rs_traced_results, framework_frames_df, timing_stats + ) + timing_stats['merge_deduplicate'] = time.time() - merge_start + + # 将三个检测器的原始检测结果记录到 detection_stats 中（在合并去重之前，不处理 overlap） + # 使用类属性中保存的值（在第一次计算的位置保存的） + detection_stats['direct_detected_count'] = self.direct_detected_count + detection_stats['rs_detected_count'] = self.rs_detected_count + detection_stats['framework_detected_counts'] = ( + self.framework_detected_counts.copy() if self.framework_detected_counts else {} + ) + + logging.info(f'flag=2 检测器原始检测结果：{self.direct_detected_count} 个空刷帧（在合并去重之前）') + logging.info(f'RS skip 检测器原始检测结果：{self.rs_detected_count} 个空刷帧（在合并去重之前）') + for framework_type, detected_count in self.framework_detected_counts.items(): + logging.info( + f'{framework_type} 检测器原始检测结果：{detected_count} 个空刷帧（在合并去重之前，可能与 flag=2 或 RS skip 重叠）' + ) + # logging.info('帧合并去重完成: 正向%d + 反向%d → 总计%d（去重后）', + # direct_frames_count, rs_traced_count, len(all_frames_df)) + + # 数据预处理 + self._prepare_frame_data(all_frames_df, timing_stats) + + # ========== 阶段2：算出帧的浪费的进程级别的CPU指令数 ========== + # 2.1 计算每个空刷帧的CPU浪费（进程级别） + # 确保perf_df已加载（如果为空或未定义，重新获取） + if 'perf_df' not in locals() or perf_df is None or perf_df.empty: + perf_load_start = time.time() + perf_df = self.cache_manager.get_perf_samples() + timing_stats['perf_reload'] = time.time() - perf_load_start + if timing_stats.get('perf_reload', 0) > 0.1: # 如果耗时超过0.1秒，记录日志 + # logging.info('重新加载perf数据耗时: %.3f秒', timing_stats['perf_reload']) + pass + frame_loads = self._calculate_frame_loads(all_frames_df, perf_df, timing_stats) + + # 2.2 分析Top N帧的调用链（核心功能，用于定位CPU浪费的具体代码路径） + self._analyze_top_frames_callchains(frame_loads, all_frames_df, perf_df, perf_conn, timing_stats) + + # 唤醒链分析（辅助分析，用于理解线程唤醒关系） + if self.wakeup_chain_enabled: + wakeup_start = time.time() + self._analyze_top_frames_wakeup_chain(frame_loads, all_frames_df, trace_conn) + timing_stats['wakeup_chain'] = time.time() - wakeup_start + + # ========== 阶段3：统计整个trace浪费指令数占比 ========== + # 在_build_result_unified中计算 empty_frame_percentage = empty_frame_load / total_load * 100 + result = self._build_result_unified( + frame_loads, + all_frames_df, + total_load, + timing_stats, + detection_stats, + rs_skip_cpu, + rs_traced_results, + merged_time_ranges, + ) # 总耗时统计 total_time = time.time() - total_start_time @@ -96,31 +303,31 @@ def analyze_empty_frames(self) -> Optional[dict]: return result - except Exception as e: - logging.error('分析空帧时发生异常: %s', str(e)) - logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + except Exception: + # logging.error('分析空帧时发生异常: %s', str(e)) + # logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) return None def _load_empty_frame_data(self, app_pids: list, timing_stats: dict) -> tuple: """加载空帧相关数据 Args: - trace_conn: trace数据库连接 - perf_conn: perf数据库连接 app_pids: 应用进程ID列表 timing_stats: 耗时统计字典，函数内部会设置相关时间值 Returns: - tuple: (trace_df, total_load, perf_df) + tuple: (trace_df, total_load, perf_df, merged_time_ranges) + - trace_df: 空帧数据 + - total_load: 总负载 + - perf_df: 性能样本 + - merged_time_ranges: 去重后的时间范围列表 """ - # 获取空帧数据 + # 获取空帧数据和去重后的时间范围 query_start = time.time() - trace_df = self.cache_manager.get_empty_frames_with_details(app_pids) + trace_df, merged_time_ranges = self.cache_manager.get_empty_frames_with_details(app_pids) timing_stats['query'] = time.time() - query_start - if trace_df.empty: - return None, None, None - + # 即使没有空刷帧，也获取总负载和perf数据，以便构建空结果 # 获取总负载数据 total_load_start = time.time() total_load = self.cache_manager.get_total_load_for_pids(app_pids) @@ -131,7 +338,7 @@ def _load_empty_frame_data(self, app_pids: list, timing_stats: dict) -> tuple: perf_df = self.cache_manager.get_perf_samples() if self.cache_manager else pd.DataFrame() timing_stats['perf'] = time.time() - perf_start - return trace_df, total_load, perf_df + return trace_df, total_load, perf_df, merged_time_ranges def _prepare_frame_data(self, trace_df: pd.DataFrame, timing_stats: dict) -> None: """数据预处理 @@ -146,28 +353,37 @@ def _prepare_frame_data(self, trace_df: pd.DataFrame, timing_stats: dict) -> Non timing_stats['data_prep'] = time.time() - data_prep_start def _calculate_frame_loads(self, trace_df: pd.DataFrame, perf_df: pd.DataFrame, timing_stats: dict) -> list: - """快速帧负载计算 + """阶段2.1：计算帧的浪费的进程级别的CPU指令数 + + 对于每个空刷帧，计算其在[ts, ts+dur]时间范围内（扩展±1ms）的进程级别CPU浪费。 + 进程级别意味着统计该进程所有线程的CPU指令数，不区分具体线程。 + + 注意：调用链分析（阶段2.2）会在此基础上对Top N帧进行详细分析。 Args: - trace_df: 帧数据DataFrame - perf_df: 性能数据DataFrame + trace_df: 帧数据DataFrame（阶段1找到的空刷帧） + perf_df: 性能数据DataFrame（perf_sample表数据） timing_stats: 耗时统计字典，函数内部会设置相关时间值 Returns: - list: 帧负载数据列表 + list: 帧负载数据列表，每个元素包含frame_load字段（CPU指令数） """ fast_calc_start = time.time() - frame_loads = self.load_calculator.calculate_all_frame_loads_fast(trace_df, perf_df) + # 使用公共模块EmptyFrameCPUCalculator + frame_loads = self.cpu_calculator.calculate_frame_loads(trace_df, perf_df) timing_stats['fast_calc'] = time.time() - fast_calc_start return frame_loads def _analyze_top_frames_callchains( self, frame_loads: list, trace_df: pd.DataFrame, perf_df: pd.DataFrame, perf_conn, timing_stats: dict ) -> None: - """识别Top帧并进行调用链分析 + """阶段2.2：分析Top N帧的调用链（核心功能） + + 对Top N（默认10个）CPU浪费最高的空刷帧进行调用链分析， + 定位CPU浪费的具体代码路径，帮助开发者找到优化点。 Args: - frame_loads: 帧负载数据列表 + frame_loads: 帧负载数据列表（已计算CPU浪费） trace_df: 帧数据DataFrame perf_df: 性能数据DataFrame perf_conn: perf数据库连接 @@ -175,41 +391,72 @@ def _analyze_top_frames_callchains( """ top_analysis_start = time.time() - # 按负载排序，获取前N帧进行详细调用链分析 - sorted_frames = sorted(frame_loads, key=lambda x: x['frame_load'], reverse=True) - top_10_frames = sorted_frames[:TOP_FRAMES_FOR_CALLCHAIN] + # 使用公共模块EmptyFrameCallchainAnalyzer + self.callchain_analyzer.analyze_callchains( + frame_loads=frame_loads, + trace_df=trace_df, + perf_df=perf_df, + perf_conn=perf_conn, + top_n=TOP_FRAMES_FOR_CALLCHAIN, + ) - # 只对Top N帧进行调用链分析 - for frame_data in top_10_frames: - # 找到对应的原始帧数据 - frame_mask = ( - (trace_df['ts'] == frame_data['ts']) - & (trace_df['dur'] == frame_data['dur']) - & (trace_df['tid'] == frame_data['thread_id']) - ) - original_frame = trace_df[frame_mask].iloc[0] if not trace_df[frame_mask].empty else None + timing_stats['top_analysis'] = time.time() - top_analysis_start - if original_frame is not None: - try: - _, sample_callchains = self.load_calculator.analyze_single_frame( - original_frame, perf_df, perf_conn, None - ) - frame_data['sample_callchains'] = sample_callchains - except Exception as e: - logging.warning('帧调用链分析失败: ts=%s, error=%s', frame_data['ts'], str(e)) - frame_data['sample_callchains'] = [] - else: - frame_data['sample_callchains'] = [] + def _build_empty_result(self, total_load: int, timing_stats: dict) -> dict: + """构建空结果（当没有空刷帧时） - # 对于非Top 10帧，设置空的调用链信息 - for frame_data in frame_loads: - if frame_data not in top_10_frames: - frame_data['sample_callchains'] = [] + Args: + total_load: 总负载 + timing_stats: 耗时统计字典 - timing_stats['top_analysis'] = time.time() - top_analysis_start + Returns: + dict: 空结果字典 + """ + result_build_start = time.time() + + result = { + 'status': 'success', + 'summary': { + 'total_load': int(total_load) if total_load is not None else 0, + 'empty_frame_load': 0, + 'empty_frame_percentage': 0.0, + 'background_thread_load': 0, + 'background_thread_percentage': 0.0, + 'total_empty_frames': 0, + 'empty_frames_with_load': 0, + }, + 'top_frames': [], # 统一列表，不再区分主线程和后台线程 + } + + timing_stats['result_build'] = time.time() - result_build_start + return result def _build_result(self, frame_loads: list, trace_df: pd.DataFrame, total_load: int, timing_stats: dict) -> dict: - """构建分析结果 + """构建分析结果（使用公共模块） + + Args: + frame_loads: 帧负载数据列表 + trace_df: 帧数据DataFrame + total_load: 总负载 + timing_stats: 耗时统计字典，函数内部会设置相关时间值 + + Returns: + dict: 分析结果 + """ + result_build_start = time.time() + + # 使用公共模块EmptyFrameResultBuilder + result = self.result_builder.build_result( + frame_loads=frame_loads, + total_load=total_load, + detection_stats=None, # EmptyFrameAnalyzer无追溯统计 + ) + + timing_stats['result_build'] = time.time() - result_build_start + return result + + def _build_result_old(self, frame_loads: list, trace_df: pd.DataFrame, total_load: int, timing_stats: dict) -> dict: + """构建分析结果（旧实现，保留作为参考） Args: frame_loads: 帧负载数据列表 @@ -224,8 +471,9 @@ def _build_result(self, frame_loads: list, trace_df: pd.DataFrame, total_load: i result_df = pd.DataFrame(frame_loads) if result_df.empty: + # 如果没有帧负载数据，返回空结果 timing_stats['result_build'] = time.time() - result_build_start - return None + return self._build_empty_result(total_load, timing_stats) # 获取第一帧时间戳用于相对时间计算 first_frame_time = self.cache_manager.get_first_frame_timestamp() if self.cache_manager else 0 @@ -245,7 +493,11 @@ def _build_result(self, frame_loads: list, trace_df: pd.DataFrame, total_load: i processed_bg_thread_frames = self._process_frame_timestamps(background_thread_frames, first_frame_time) # 计算统计信息 - empty_frame_load = int(sum(f['frame_load'] for f in frame_loads if f.get('is_main_thread') == 1)) + # 注意：根据新的五阶段算法，CPU计算是进程级的，所以empty_frame_load应该包括所有空刷帧的CPU浪费 + # 但是为了保持输出格式兼容性，我们仍然分别计算主线程和后台线程的负载 + # empty_frame_load：所有空刷帧的进程级CPU浪费（主线程+后台线程） + empty_frame_load = int(sum(f['frame_load'] for f in frame_loads)) + # background_thread_load：后台线程的空刷帧CPU浪费（用于兼容性，实际已包含在empty_frame_load中） background_thread_load = int(sum(f['frame_load'] for f in frame_loads if f.get('is_main_thread') != 1)) if total_load > 0: @@ -255,24 +507,73 @@ def _build_result(self, frame_loads: list, trace_df: pd.DataFrame, total_load: i empty_frame_percentage = 0.0 background_thread_percentage = 0.0 + # 处理占比超过100%的情况 + # 原因：时间窗口扩展（±1ms）导致多个帧的CPU计算存在重叠，这是正常的设计 + percentage_warning = None + display_percentage = empty_frame_percentage + severity_level = None + severity_description = None + + # 配置选项：是否限制显示占比为100%（当占比超过100%时） + # 设置为True时，显示占比将被限制为100%，但原始占比值仍保留在empty_frame_percentage中 + # 设置为False时，显示占比等于原始占比值 + LIMIT_DISPLAY_PERCENTAGE = True # 默认限制显示为100% + + # 根据占比判断严重程度 + if empty_frame_percentage < 3.0: + severity_level = 'normal' + severity_description = '正常：空刷帧CPU占比小于3%，属于正常范围。' + elif empty_frame_percentage < 10.0: + severity_level = 'moderate' + severity_description = '较为严重：空刷帧CPU占比在3%-10%之间，建议关注并优化。' + elif empty_frame_percentage <= 100.0: + severity_level = 'severe' + severity_description = '严重：空刷帧CPU占比超过10%，需要优先优化。' + else: # > 100% + severity_level = 'extreme' + severity_description = ( + f'极端异常：空刷帧CPU占比超过100% ({empty_frame_percentage:.2f}%)。' + f'这是因为时间窗口扩展（±1ms）导致多个帧的CPU计算存在重叠。' + f'这是正常的设计，不影响单个帧的CPU计算准确性。' + f'建议查看原始占比值（empty_frame_percentage）和警告信息（percentage_warning）了解详情。' + ) + percentage_warning = ( + f'注意：空刷帧占比超过100% ({empty_frame_percentage:.2f}%)，这是因为时间窗口扩展（±1ms）' + f'导致多个帧的CPU计算存在重叠。这是正常的设计，不影响单个帧的CPU计算准确性。' + ) + # logging.warning(percentage_warning) + + # 如果启用限制，将显示占比限制为100% + if LIMIT_DISPLAY_PERCENTAGE: + display_percentage = 100.0 + # logging.info(f"显示占比已限制为100%（原始占比：{empty_frame_percentage:.2f}%）") + # 构建结果字典 result = { 'status': 'success', 'summary': { 'total_load': int(total_load), 'empty_frame_load': int(empty_frame_load), - 'empty_frame_percentage': float(empty_frame_percentage), + 'empty_frame_percentage': float(empty_frame_percentage), # 原始占比值 + 'empty_frame_percentage_display': float(display_percentage), # 显示占比（如果超过100%可能被限制） 'background_thread_load': int(background_thread_load), 'background_thread_percentage': float(background_thread_percentage), - 'total_empty_frames': int(len(trace_df[trace_df['is_main_thread'] == 1])), - 'empty_frames_with_load': int(len([f for f in frame_loads if f.get('is_main_thread') == 1])), - }, - 'top_frames': { - 'main_thread_empty_frames': processed_main_thread_frames, - 'background_thread': processed_bg_thread_frames, + 'total_empty_frames': int(len(trace_df)), # 所有空刷帧数量（包括主线程和后台线程） + 'empty_frames_with_load': int( + len([f for f in frame_loads if f.get('frame_load', 0) > 0]) + ), # 有CPU负载的空刷帧数量 + # 严重程度评估 + 'severity_level': severity_level, # 严重程度级别：normal, moderate, severe, extreme + 'severity_description': severity_description, # 严重程度说明 }, + 'top_frames': processed_main_thread_frames + + processed_bg_thread_frames, # 统一列表，不再区分主线程和后台线程 } + # 如果占比超过100%，添加警告信息 + if percentage_warning: + result['summary']['percentage_warning'] = percentage_warning + timing_stats['result_build'] = time.time() - result_build_start return result @@ -304,23 +605,1144 @@ def _log_analysis_complete(self, total_time: float, timing_stats: dict) -> None: total_time: 总耗时 timing_stats: 各阶段耗时统计 """ - logging.info('空帧分析总耗时: %.3f秒', total_time) - logging.info( - '各阶段耗时占比: ' - '缓存检查%.1f%%, ' - '查询%.1f%%, ' - '总负载%.1f%%, ' - '性能样本%.1f%%, ' - '预处理%.1f%%, ' - '快速计算%.1f%%, ' - 'Top帧分析%.1f%%, ' - '结果构建%.1f%%', - timing_stats.get('cache_check', 0) / total_time * 100 if total_time > 0 else 0, - timing_stats.get('query', 0) / total_time * 100 if total_time > 0 else 0, - timing_stats.get('total_load', 0) / total_time * 100 if total_time > 0 else 0, - timing_stats.get('perf', 0) / total_time * 100 if total_time > 0 else 0, - timing_stats.get('data_prep', 0) / total_time * 100 if total_time > 0 else 0, - timing_stats.get('fast_calc', 0) / total_time * 100 if total_time > 0 else 0, - timing_stats.get('top_analysis', 0) / total_time * 100 if total_time > 0 else 0, - timing_stats.get('result_build', 0) / total_time * 100 if total_time > 0 else 0, + # logging.info('空帧分析总耗时: %.3f秒', total_time) + # logging.info( + # '各阶段耗时占比: ' + # '缓存检查%.1f%%, ' + # '查询%.1f%%, ' + # '总负载%.1f%%, ' + # '性能样本%.1f%%, ' + # '假阳性过滤%.1f%%, ' + # '预处理%.1f%%, ' + # '快速计算%.1f%%, ' + # 'Top帧分析%.1f%%, ' + # '唤醒链分析%.1f%%, ' + # '结果构建%.1f%%', + # timing_stats.get('cache_check', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('query', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('total_load', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('perf', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('false_positive_filter', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('data_prep', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('fast_calc', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('top_analysis', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('wakeup_chain', 0) / total_time * 100 if total_time > 0 else 0, + # timing_stats.get('result_build', 0) / total_time * 100 if total_time > 0 else 0, + # ) + + def _filter_false_positives(self, trace_df: pd.DataFrame, trace_conn) -> pd.DataFrame: + """过滤假阳性帧（对所有flag=2帧进行过滤） + + 假阳性定义：flag=2的帧，但通过NativeWindow API成功提交了内容 + + 检测方法： + 1. 预加载 NativeWindow API 事件（RequestBuffer, FlushBuffer等） + 2. 对每个 flag=2 帧，检查帧时间范围内是否有NativeWindow API提交事件 + 3. 如果有，标记为假阳性并从DataFrame中过滤 + + Args: + trace_df: 包含flag=2帧的DataFrame + trace_conn: trace数据库连接 + + Returns: + 过滤假阳性后的DataFrame + """ + if trace_df.empty: + return trace_df + + # 计算时间范围 + # 注意：处理dur可能为NaN的情况，并确保类型为int + min_ts = int(trace_df['ts'].min()) + # 过滤掉dur为NaN的帧，避免计算错误 + valid_dur_df = trace_df[trace_df['dur'].notna()] + if valid_dur_df.empty: + # logging.warning('所有帧的dur都为NaN，无法计算时间范围') + return trace_df + max_ts = int((valid_dur_df['ts'] + valid_dur_df['dur']).max()) + + # 预加载 NativeWindow API 事件 + # 注意：使用已验证的SQL结构（参考 wakeup_chain_analyzer.py:890） + nw_events_query = """ + SELECT + c.callid as itid, + c.ts + FROM callstack c + INNER JOIN thread t ON c.callid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE c.ts >= ? AND c.ts <= ? + AND p.name NOT IN ('render_service', 'rmrenderservice', 'ohos.sceneboard') + AND p.name NOT LIKE 'system_%' + AND p.name NOT LIKE 'com.ohos.%' + AND (c.name LIKE '%RequestBuffer%' + OR c.name LIKE '%FlushBuffer%' + OR c.name LIKE '%DoFlushBuffer%' + OR c.name LIKE '%NativeWindow%') + """ + cursor = trace_conn.cursor() + cursor.execute(nw_events_query, (min_ts, max_ts)) + nw_records = cursor.fetchall() + + # 构建事件缓存（按线程ID索引） + nw_events_cache = defaultdict(list) + for itid, ts in nw_records: + nw_events_cache[itid].append(ts) + + # 排序以便二分查找 + for itid in nw_events_cache: + nw_events_cache[itid].sort() + + # logging.info(f'预加载NativeWindow API事件: {len(nw_records)}条记录, {len(nw_events_cache)}个线程') + + # 过滤假阳性（使用DataFrame操作） + def is_false_positive(row): + try: + frame_itid = row['itid'] + frame_start = row['ts'] + frame_end = row['ts'] + row['dur'] + + if frame_itid in nw_events_cache: + event_timestamps = nw_events_cache[frame_itid] + idx = bisect_left(event_timestamps, frame_start) + if idx < len(event_timestamps) and event_timestamps[idx] <= frame_end: + return True + return False + except Exception: + # 如果检查过程中出现异常，记录日志但不标记为假阳性（保守策略） + # logging.warning(f'假阳性检查异常: {e}, row={row.to_dict() if hasattr(row, "to_dict") else row}') + return False + + # 标记假阳性 + false_positive_mask = trace_df.apply(is_false_positive, axis=1) + false_positive_mask.sum() + + # 过滤假阳性 + return trace_df[~false_positive_mask].copy() + + # logging.info( + # f'假阳性过滤: 过滤前={len(trace_df)}, 过滤后={len(filtered_df)}, ' + # f'假阳性={false_positive_count} ({false_positive_count/len(trace_df)*100:.1f}%)' + # ) + + def _analyze_top_frames_wakeup_chain(self, frame_loads: list, trace_df: pd.DataFrame, trace_conn) -> None: + """对Top N帧进行唤醒链分析，填充wakeup_threads字段 + + 注意： + - 只对Top N帧进行唤醒链分析（性能考虑） + - CPU计算是进程级的，不需要唤醒链分析 + - 唤醒链分析仅用于填充wakeup_threads字段和根因分析 + + Args: + frame_loads: 帧负载数据列表 + trace_df: 帧数据DataFrame + trace_conn: trace数据库连接 + """ + # 对所有空刷帧按frame_load排序，取Top N（不区分主线程和后台线程） + # 但排除系统线程（系统线程不计入占比，也不应该分析唤醒链） + non_system_frames = [ + f + for f in frame_loads + # 规则：由于frame_loads已经通过app_pids过滤，所有帧都属于应用进程 + # 因此，只排除进程名本身是系统进程的情况 + # 对于应用进程内的线程，即使名称像系统线程（如OS_VSyncThread），也不排除 + if not is_system_thread(f.get('process_name'), None) # 只检查进程名 + ] + sorted_frames = sorted(non_system_frames, key=lambda x: x['frame_load'], reverse=True) + top_n_frames = sorted_frames[:TOP_FRAMES_FOR_CALLCHAIN] + + # logging.info(f'开始对Top {len(top_n_frames)}帧进行唤醒链分析...') + + # 只对Top N帧进行唤醒链分析 + for frame_data in top_n_frames: + # 找到对应的原始帧数据 + # 优先使用vsync匹配（如果有vsync，一定能匹配到frame_slice中的帧） + vsync = frame_data.get('vsync') + matching_frames = pd.DataFrame() # 初始化为空DataFrame + + # 确保vsync不是'unknown'字符串，并且trace_df中有vsync列 + if vsync is not None and vsync != 'unknown' and 'vsync' in trace_df.columns: + try: + # 确保vsync是数值类型 + vsync_value = int(vsync) if not isinstance(vsync, (int, float)) else vsync + # 使用vsync匹配（最可靠的方式） + trace_vsync = pd.to_numeric(trace_df['vsync'], errors='coerce') + frame_mask = trace_vsync == vsync_value + matching_frames = trace_df[frame_mask] + if not matching_frames.empty: + # logging.debug(f'唤醒链分析：使用vsync匹配成功: vsync={vsync_value}, ts={frame_data["ts"]}, 找到{len(matching_frames)}个匹配帧') + pass + else: + # logging.warning(f'唤醒链分析：vsync匹配失败: vsync={vsync_value}, ts={frame_data["ts"]}') + pass + except (ValueError, TypeError): + # logging.warning(f'唤醒链分析：vsync类型转换失败: vsync={vsync}, error={e}, 使用备用匹配方式') + pass + matching_frames = pd.DataFrame() # 重置为空 + + # 如果vsync匹配失败，使用ts、dur、tid匹配 + if matching_frames.empty: + frame_mask = ( + (trace_df['ts'] == frame_data['ts']) + & (trace_df['dur'] == frame_data['dur']) + & (trace_df['tid'] == frame_data['thread_id']) + ) + matching_frames = trace_df[frame_mask] + + if not matching_frames.empty: + original_frame = matching_frames.iloc[0] + # 确保original_frame转换为字典格式，包含所有必要字段 + frame_dict = original_frame.to_dict() if isinstance(original_frame, pd.Series) else original_frame + + # 确保有itid字段（唤醒链分析需要） + # 检查itid是否存在且不是NaN（pandas的NaN值需要特殊处理） + frame_itid_value = frame_dict.get('itid') + # 检查itid是否为NaN或None + if 'itid' not in frame_dict or frame_itid_value is None or pd.isna(frame_itid_value): + # 如果itid是NaN或不存在，尝试从tid查询 + tid = frame_dict.get('tid') + if tid and pd.notna(tid) and trace_conn: + try: + cursor = trace_conn.cursor() + cursor.execute('SELECT id FROM thread WHERE tid = ? LIMIT 1', (int(tid),)) + result = cursor.fetchone() + if result: + frame_dict['itid'] = result[0] + # logging.debug(f'通过tid查询到itid: tid={tid}, itid={result[0]}') + except Exception: + # logging.warning(f'查询itid失败: tid={tid}, error={e}') + pass + else: + # 确保itid是整数类型（不是NaN） + frame_dict['itid'] = int(frame_itid_value) + + try: + # 调用简化的唤醒链分析（只获取线程列表，不计算CPU） + # logging.debug(f'开始唤醒链分析: ts={frame_data["ts"]}, tid={frame_data.get("thread_id")}, itid={frame_dict.get("itid")}, frame_dict keys={list(frame_dict.keys())[:10]}') + wakeup_threads = self._get_related_threads_simple(frame_dict, trace_conn) + frame_data['wakeup_threads'] = wakeup_threads if wakeup_threads else [] + if not wakeup_threads: + # logging.warning(f'唤醒链分析结果为空: ts={frame_data["ts"]}, tid={frame_data.get("thread_id")}, itid={frame_dict.get("itid")}, frame_dict={frame_dict}') + pass + else: + # logging.info(f'唤醒链分析成功: ts={frame_data["ts"]}, 找到 {len(related_threads)} 个相关线程') + pass + except Exception: + # logging.warning(f'唤醒链分析失败: ts={frame_data["ts"]}, error={e}, traceback={traceback.format_exc()}') + pass + frame_data['wakeup_threads'] = [] + else: + # logging.warning(f'未找到匹配的原始帧（唤醒链）: ts={frame_data["ts"]}, dur={frame_data["dur"]}, ' + # f'thread_id={frame_data.get("thread_id")}, trace_df中有{len(trace_df)}帧') + frame_data['wakeup_threads'] = [] + + # 对于非Top N帧，设置空的wakeup_threads + for frame_data in frame_loads: + if 'wakeup_threads' not in frame_data: + frame_data['wakeup_threads'] = [] + + # logging.info(f'完成Top {len(top_n_frames)}帧的唤醒链分析') + + def _get_related_threads_simple(self, frame, trace_conn) -> list: + """获取帧相关的线程列表（简化版唤醒链分析） + + Args: + frame: 帧数据（DataFrame行或字典） + trace_conn: trace数据库连接 + + Returns: + 线程列表 + """ + try: + from .frame_wakeup_chain import find_wakeup_chain # noqa: PLC0415 + except ImportError: + # 如果导入失败，返回空列表（唤醒链分析是可选的） + # logging.warning('无法导入wakeup_chain，跳过唤醒链分析') + return [] + + # 处理itid：如果不存在，尝试从tid查询 + # 确保frame是字典格式 + if isinstance(frame, pd.Series): + frame = frame.to_dict() + + frame_itid = frame.get('itid') + frame.get('ts') + frame.get('tid') + + # logging.debug(f'_get_related_threads_simple: 开始分析, ts={frame_ts}, tid={frame_tid}, itid={frame_itid}') + + # 检查itid是否为NaN或None（pandas的NaN值需要特殊处理） + if 'itid' not in frame or frame_itid is None or pd.isna(frame_itid): + # 如果itid是NaN或不存在，尝试从tid查询 + tid = frame.get('tid') + if tid and pd.notna(tid) and trace_conn: + try: + cursor = trace_conn.cursor() + cursor.execute('SELECT id FROM thread WHERE tid = ? LIMIT 1', (int(tid),)) + result = cursor.fetchone() + if result: + frame_itid = result[0] + # logging.debug(f'_get_related_threads_simple: 通过tid查询到itid, tid={tid}, itid={frame_itid}') + pass + except Exception: + # logging.warning(f'_get_related_threads_simple: 查询itid失败: tid={tid}, error={e}') + pass + else: + # 确保itid是整数类型（不是NaN） + frame_itid = int(frame_itid) + + if not frame_itid or pd.isna(frame_itid): + # 如果仍然没有itid，跳过唤醒链分析 + # logging.warning('帧缺少itid字段，跳过唤醒链分析: ts=%s, tid=%s', frame.get('ts'), frame.get('tid')) + return [] + + frame_start = frame.get('ts') or frame.get('start_time', 0) + frame_dur = frame.get('dur', 0) + frame_end = frame_start + frame_dur + app_pid = frame.get('pid') or frame.get('app_pid') + + # logging.debug(f'_get_related_threads_simple: 调用find_wakeup_chain, itid={frame_itid}, frame_start={frame_start}, frame_end={frame_end}, app_pid={app_pid}') + + # 调用唤醒链分析 + try: + related_itids_ordered = find_wakeup_chain( + trace_conn=trace_conn, + start_itid=frame_itid, + frame_start=frame_start, + frame_end=frame_end, + app_pid=app_pid, + ) + # related_itids_ordered 是 [(itid, depth), ...] 列表，按唤醒链顺序 + # logging.debug(f'_get_related_threads_simple: find_wakeup_chain返回 {len(related_itids_ordered)} 个线程') + except Exception: + # logging.warning(f'_get_related_threads_simple: find_wakeup_chain调用失败: error={e}') + related_itids_ordered = [] + + # 确保至少包含当前帧的线程（用户要求） + related_itids_set = {itid for itid, _ in related_itids_ordered} + if frame_itid and frame_itid not in related_itids_set: + related_itids_ordered.insert(0, (frame_itid, 0)) # 插入到最前面，深度为0 + related_itids_set.add(frame_itid) + # logging.debug(f'_get_related_threads_simple: 添加当前帧线程到related_itids, itid={frame_itid}') + + # 获取线程详细信息 + if not related_itids_ordered: + # 即使唤醒链为空，也要返回当前帧的线程 + # logging.debug(f'_get_related_threads_simple: related_itids_ordered为空，尝试返回当前帧线程, itid={frame_itid}') + if frame_itid: + try: + cursor = trace_conn.cursor() + cursor.execute( + """ + SELECT t.itid, t.tid, t.name, p.pid, p.name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE t.itid = ? + """, + (frame_itid,), + ) + result = cursor.fetchone() + if result: + itid, tid, thread_name, pid, process_name = result + # logging.debug(f'_get_related_threads_simple: 成功获取当前线程信息, itid={itid}, thread_name={thread_name}') + return [ + { + 'itid': itid, + 'tid': tid, + 'thread_name': thread_name, + 'pid': pid, + 'process_name': process_name, + 'is_system_thread': is_system_thread(process_name, thread_name), + } + ] + # logging.warning(f'_get_related_threads_simple: 查询当前线程信息无结果, itid={frame_itid}') + + pass + except Exception: + # logging.warning(f'_get_related_threads_simple: 获取当前线程信息失败: itid={frame_itid}, error={e}') + pass + # logging.warning(f'_get_related_threads_simple: 最终返回空列表, itid={frame_itid}') + return [] + + cursor = trace_conn.cursor() + # 按唤醒链顺序提取 itid 列表 + itids_list = [itid for itid, _ in related_itids_ordered] + placeholders = ','.join('?' * len(itids_list)) + + # logging.debug(f'_get_related_threads_simple: 查询 {len(itids_list)} 个线程的详细信息') + + cursor.execute( + f""" + SELECT t.itid, t.tid, t.name, p.pid, p.name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE t.itid IN ({placeholders}) + """, + itids_list, + ) + + thread_results = cursor.fetchall() + + # logging.debug(f'_get_related_threads_simple: 查询到 {len(thread_results)} 个线程结果') + + # 构建 itid 到线程信息的映射 + thread_info_map = {} + for itid, tid, thread_name, pid, process_name in thread_results: + thread_info_map[itid] = { + 'tid': tid, + 'thread_name': thread_name, + 'pid': pid, + 'process_name': process_name, + 'is_system_thread': is_system_thread(process_name, thread_name), + } + + # 按唤醒链顺序构建结果列表 + # 注意：使用 thread_id 而不是 tid，以保持与 sample_callchains 中字段命名的一致性 + # thread_id 和 tid 在语义上相同，都是线程号（来自 thread.tid 或 perf_sample.thread_id） + wakeup_threads = [] + for itid, depth in related_itids_ordered: + if itid in thread_info_map: + thread_info = thread_info_map[itid] + wakeup_threads.append( + { + 'itid': itid, + 'thread_id': thread_info['tid'], # 使用 thread_id 保持与 sample_callchains 一致 + 'thread_name': thread_info['thread_name'], + 'pid': thread_info['pid'], + 'process_name': thread_info['process_name'], + 'is_system_thread': thread_info['is_system_thread'], + 'wakeup_depth': depth, # 添加唤醒链深度信息 + } + ) + + # logging.debug(f'_get_related_threads_simple: 返回 {len(wakeup_threads)} 个唤醒链线程（按顺序）') + return wakeup_threads + + # ==================== RS Skip 检测方法（合并功能）==================== + + def _detect_rs_skip_frames(self, trace_conn, timing_stats: dict) -> list: + """检测RS skip事件并分组到帧 + + Args: + trace_conn: trace数据库连接 + timing_stats: 耗时统计字典 + + Returns: + list: RS skip帧列表（每个帧可能包含多个skip事件） + """ + detect_start = time.time() + + try: + cursor = trace_conn.cursor() + + # 步骤1: 查找所有DisplayNode skip事件 + skip_events_query = """ + SELECT + c.callid, + c.ts, + c.dur, + c.name + FROM callstack c + WHERE c.name LIKE '%DisplayNode skip%' + AND c.callid IN ( + SELECT t.id + FROM thread t + WHERE t.ipid IN ( + SELECT p.ipid + FROM process p + WHERE p.name = 'render_service' + ) + ) + ORDER BY c.ts + """ + cursor.execute(skip_events_query) + skip_events = cursor.fetchall() + + if not skip_events: + # logging.info('未找到DisplayNode skip事件') + timing_stats['detect_rs_skip'] = time.time() - detect_start + return [] + + # logging.info('找到 %d 个DisplayNode skip事件', len(skip_events)) + + # 步骤2: 获取RS进程的所有帧 + rs_frames_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.vsync, + fs.flag, + fs.type + FROM frame_slice fs + WHERE fs.ipid IN ( + SELECT p.ipid + FROM process p + WHERE p.name = 'render_service' + ) + AND fs.type = 0 + AND fs.flag IS NOT NULL + ORDER BY fs.ts + """ + cursor.execute(rs_frames_query) + rs_frames = cursor.fetchall() + + if not rs_frames: + # logging.warning('未找到RS进程的帧') + timing_stats['detect_rs_skip'] = time.time() - detect_start + return [] + + # logging.info('找到 %d 个RS进程帧', len(rs_frames)) + + # 步骤3: 将skip事件分配到对应的RS帧 + skip_frame_dict = {} + + for frame_data in rs_frames: + frame_id, frame_ts, frame_dur, frame_vsync, frame_flag, frame_type = frame_data + frame_dur = frame_dur if frame_dur else 0 + frame_end = frame_ts + frame_dur + + # 查找此帧时间窗口内的skip事件 + frame_skip_events = [] + for event in skip_events: + event_callid, event_ts, event_dur, event_name = event + if frame_ts <= event_ts < frame_end: + frame_skip_events.append( + { + 'callid': event_callid, + 'ts': event_ts, + 'dur': event_dur if event_dur else 0, + 'name': event_name, + } + ) + + # 如果此帧有skip事件，记录 + if frame_skip_events: + skip_frame_dict[frame_id] = { + 'frame_id': frame_id, + 'ts': frame_ts, + 'dur': frame_dur, + 'vsync': frame_vsync, + 'flag': frame_flag, + 'type': frame_type, + 'skip_events': frame_skip_events, + 'skip_event_count': len(frame_skip_events), + } + + result = list(skip_frame_dict.values()) + + # logging.info('检测完成: %d 个RS帧包含skip事件（共%d个skip事件）', + # len(result), len(skip_events)) + + timing_stats['detect_rs_skip'] = time.time() - detect_start + return result + + except Exception: + # logging.error('检测RS skip帧失败: %s', str(e)) + # logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + timing_stats['detect_rs_skip'] = time.time() - detect_start + return [] + + def _get_rs_process_pid(self) -> int: + """获取RS进程PID""" + try: + cursor = self.cache_manager.trace_conn.cursor() + cursor.execute(""" + SELECT pid FROM process + WHERE name LIKE '%render_service%' + LIMIT 1 + """) + result = cursor.fetchone() + return result[0] if result else 0 + except Exception: + # logging.error('获取RS进程PID失败: %s', e) + return 0 + + def _calculate_rs_skip_cpu(self, skip_frames: list) -> int: + """计算RS进程skip帧的CPU浪费""" + if not skip_frames: + return 0 + + rs_pid = self._get_rs_process_pid() + if not rs_pid: + # logging.warning('无法获取RS进程PID，跳过RS进程CPU计算') + return 0 + + total_rs_cpu = 0 + calculated_count = 0 + + for skip_frame in skip_frames: + frame_start = skip_frame['ts'] + frame_dur = skip_frame.get('dur', 0) or 16_666_666 # 默认16.67ms + frame_end = frame_start + frame_dur + + try: + app_instructions, system_instructions = calculate_process_instructions( + perf_conn=self.cache_manager.perf_conn, + trace_conn=self.cache_manager.trace_conn, + app_pid=rs_pid, + frame_start=frame_start, + frame_end=frame_end, + ) + if app_instructions > 0: + total_rs_cpu += app_instructions + calculated_count += 1 + except Exception: + # logging.warning('计算RS skip帧CPU失败: frame_id=%s, error=%s', + # skip_frame.get("frame_id"), e) + pass + + # logging.info('RS进程CPU统计: %d个skip帧, 成功计算%d个, 总CPU=%d 指令', + # len(skip_frames), calculated_count, total_rs_cpu) + return total_rs_cpu + + def _preload_rs_caches(self, trace_conn, skip_frames: list, timing_stats: dict) -> dict: + """预加载RS追溯需要的缓存""" + if not skip_frames: + return {} + + preload_start = time.time() + min_ts = min(f['ts'] for f in skip_frames) - 500_000_000 + max_ts = max(f['ts'] + f['dur'] for f in skip_frames) + 50_000_000 + + caches = {} + + if self.rs_api_enabled: + try: + caches['rs_api'] = preload_rs_api_caches(trace_conn, min_ts, max_ts) + except Exception: + # logging.error('加载RS API缓存失败: %s', e) + caches['rs_api'] = None + + if self.nw_api_enabled: + try: + caches['nw'] = preload_nw_caches(trace_conn, min_ts, max_ts) + except Exception: + # logging.error('加载NativeWindow API缓存失败: %s', e) + caches['nw'] = None + + timing_stats['preload_rs_caches'] = time.time() - preload_start + return caches + + def _trace_rs_to_app_frames( + self, trace_conn, perf_conn, skip_frames: list, caches: dict, timing_stats: dict + ) -> list: + """追溯RS skip事件到应用帧""" + trace_start = time.time() + traced_results = [] + rs_success = 0 + nw_success = 0 + failed = 0 + + for skip_frame in skip_frames: + rs_frame_id = skip_frame.get('frame_id') + trace_result = None + trace_method = None + + # 尝试RS API追溯 + if self.rs_api_enabled and caches.get('rs_api'): + try: + trace_result = trace_rs_api(trace_conn, rs_frame_id, caches=caches['rs_api'], perf_conn=perf_conn) + if trace_result and trace_result.get('app_frame'): + trace_method = 'rs_api' + rs_success += 1 + except Exception: + # logging.warning('RS API追溯失败: frame_id=%s, error=%s', rs_frame_id, e) + pass + + # 如果RS API失败，尝试NativeWindow API + if not trace_method and self.nw_api_enabled and caches.get('nw'): + try: + trace_result = trace_nw_api( + trace_conn, + rs_frame_id, + nativewindow_events_cache=caches['nw'].get('nativewindow_events'), + app_frames_cache=caches['nw'].get('app_frames'), + tid_to_info_cache=caches['nw'].get('tid_to_info'), + perf_conn=perf_conn, + ) + if trace_result and trace_result.get('app_frame'): + trace_method = 'nw_api' + nw_success += 1 + except Exception: + # logging.warning('NativeWindow API追溯失败: frame_id=%s, error=%s', rs_frame_id, e) + pass + + if not trace_method: + failed += 1 + + traced_results.append({'rs_frame': skip_frame, 'trace_result': trace_result, 'trace_method': trace_method}) + + timing_stats['trace_rs_to_app'] = time.time() - trace_start + # logging.info('RS追溯完成: RS API成功%d, NW API成功%d, 失败%d', + # rs_success, nw_success, failed) + + return traced_results + + def _merge_and_deduplicate_frames( + self, + direct_frames_df: pd.DataFrame, + rs_traced_results: list, + framework_frames_df: pd.DataFrame, + timing_stats: dict, + ) -> tuple: + """合并并去重空刷帧（核心方法） + + Args: + direct_frames_df: 正向检测的flag=2帧 + rs_traced_results: RS skip追溯结果 + framework_frames_df: 框架特定检测的帧（Flutter/RN等） + timing_stats: 耗时统计字典 + + Returns: + tuple: (合并后的DataFrame, 检测统计信息) + """ + trace_conn = self.cache_manager.trace_conn + frame_map = {} # key: (pid, ts, vsync) + detection_stats = { + 'direct_only': 0, + 'rs_traced_only': 0, + 'framework_specific_only': 0, + 'both': 0, + 'framework_and_direct': 0, + 'framework_and_rs': 0, + 'all_three': 0, + 'total_rs_skip_events': len(rs_traced_results), + 'total_framework_events': len(framework_frames_df) if not framework_frames_df.empty else 0, + } + + # 1. 处理正向检测的帧 + for _, row in direct_frames_df.iterrows(): + key = (row['pid'], row['ts'], row['vsync']) + # 使用"线程名=进程名"规则判断主线程（规则完全成立，无需查询is_main_thread字段） + thread_name = row.get('thread_name', '') + process_name = row.get('process_name', '') + is_main_thread = ( + 1 if thread_name == process_name else (row.get('is_main_thread', 0) if 'is_main_thread' in row else 0) + ) + + frame_map[key] = { + 'ts': row['ts'], + 'dur': row['dur'], + 'vsync': row['vsync'], + 'pid': row['pid'], + 'tid': row['tid'], + 'process_name': process_name, + 'thread_name': thread_name, + 'flag': row['flag'], + 'type': row.get('type', 0), + 'is_main_thread': is_main_thread, # 使用规则判断，如果规则不适用则使用原有值 + 'callstack_id': row.get('callstack_id'), + 'detection_method': 'direct', + 'traced_count': 0, + 'rs_skip_events': [], + 'trace_method': None, + } + + detection_stats['direct_only'] = len(frame_map) + + # 2. 处理RS traced帧 + for trace_result_wrapper in rs_traced_results: + trace_result = trace_result_wrapper.get('trace_result') + if not trace_result or not trace_result.get('app_frame'): + continue + + app_frame = trace_result['app_frame'] + rs_frame = trace_result_wrapper['rs_frame'] + trace_method = trace_result_wrapper.get('trace_method') + + # 构建key（注意：backtrack返回的字段名是frame_ts, frame_dur, frame_vsync） + key = ( + app_frame.get('app_pid') or app_frame.get('process_pid') or app_frame.get('pid'), + app_frame.get('frame_ts') or app_frame.get('ts'), + app_frame.get('frame_vsync') or app_frame.get('vsync'), + ) + + if key in frame_map: + # 已存在：更新为both + if frame_map[key]['detection_method'] == 'direct': + frame_map[key]['detection_method'] = 'both' + detection_stats['direct_only'] -= 1 + detection_stats['both'] += 1 + + frame_map[key]['traced_count'] += 1 + frame_map[key]['rs_skip_events'].append(rs_frame) + if not frame_map[key]['trace_method']: + frame_map[key]['trace_method'] = trace_method + else: + # 新帧：添加为rs_traced（注意字段名映射） + # 需要查询itid和tid（唤醒链分析需要） + # 注意：根据文档，RS追溯返回的thread_id是itid（TS内部线程ID，对应thread表的id字段） + # instant.wakeup_from字段明确说明是"唤醒当前线程的内部线程号（itid）" + # 而tid是线程号（thread表的tid字段），perf_sample.thread_id也是线程号 + itid = app_frame.get('thread_id') or app_frame.get('tid') + tid = None + + if itid and trace_conn: + try: + cursor = trace_conn.cursor() + # RS追溯返回的thread_id是itid（thread表的id字段），直接查询对应的tid + cursor.execute( + """ + SELECT id, tid FROM thread WHERE id = ? LIMIT 1 + """, + (itid,), + ) + result = cursor.fetchone() + if result: + itid = result[0] # thread.id（确认是itid） + tid = result[1] # thread.tid（线程号） + else: + # logging.warning('未找到itid=%s对应的线程信息', itid) + pass + except Exception: + # logging.warning('查询线程信息失败: itid=%s, error=%s', itid, e) + pass + + # 使用"线程名=进程名"规则判断主线程（规则完全成立，无需查询is_main_thread字段） + thread_name = app_frame.get('thread_name', '') + process_name = app_frame.get('process_name', '') + is_main_thread = 1 if thread_name == process_name else 0 + + frame_map[key] = { + 'ts': app_frame.get('frame_ts') or app_frame.get('ts'), + 'dur': app_frame.get('frame_dur') or app_frame.get('dur'), + 'vsync': app_frame.get('frame_vsync') or app_frame.get('vsync'), + 'pid': app_frame.get('app_pid') or app_frame.get('process_pid') or app_frame.get('pid'), + 'tid': tid, + 'itid': itid, # 添加itid字段（唤醒链分析需要） + 'process_name': app_frame.get('process_name', ''), + 'thread_name': app_frame.get('thread_name', ''), + 'flag': app_frame.get('frame_flag') or app_frame.get('flag', 2), # 空刷帧标记 + 'type': 0, + 'is_main_thread': is_main_thread, # 从thread表查询得到，或使用app_frame中的值 + 'callstack_id': app_frame.get('callstack_id'), + 'detection_method': 'rs_traced', + 'traced_count': 1, + 'rs_skip_events': [rs_frame], + 'trace_method': trace_method, + } + detection_stats['rs_traced_only'] += 1 + + # 3. 处理框架特定检测的帧（Flutter/RN等） + if framework_frames_df is not None and not framework_frames_df.empty: + for _, row in framework_frames_df.iterrows(): + # 使用 (pid, ts, vsync) 作为 key，如果没有 vsync 则使用 (pid, ts, None) + vsync = row.get('vsync') if pd.notna(row.get('vsync')) else None + key = (row['pid'], row['ts'], vsync) + + if key in frame_map: + # 已存在：更新检测方法 + existing_method = frame_map[key]['detection_method'] + if existing_method == 'direct': + frame_map[key]['detection_method'] = 'framework_and_direct' + detection_stats['direct_only'] -= 1 + detection_stats['framework_and_direct'] += 1 + elif existing_method == 'rs_traced': + frame_map[key]['detection_method'] = 'framework_and_rs' + detection_stats['rs_traced_only'] -= 1 + detection_stats['framework_and_rs'] += 1 + elif existing_method == 'both': + frame_map[key]['detection_method'] = 'all_three' + detection_stats['both'] -= 1 + detection_stats['all_three'] += 1 + # 如果是 framework_specific，则保持不变（理论上不会发生） + else: + # 新帧：添加为 framework_specific + frame_map[key] = { + 'ts': row['ts'], + 'dur': row['dur'], + 'vsync': vsync, + 'pid': row['pid'], + 'tid': row.get('tid'), + 'itid': row.get('itid'), + 'process_name': row.get('process_name', 'unknown'), + 'thread_name': row.get('thread_name', 'unknown'), + 'flag': row.get('flag', 2), + 'type': row.get('type', 0), + 'is_main_thread': row.get('is_main_thread', 0), + 'callstack_id': row.get('callstack_id'), + 'detection_method': 'framework_specific', + 'framework_type': row.get('framework_type', 'unknown'), + 'frame_damage': row.get('frame_damage'), # Flutter 特有 + 'beginframe_id': row.get('beginframe_id'), # Flutter 特有 + 'traced_count': 0, + 'rs_skip_events': [], + 'trace_method': None, + } + detection_stats['framework_specific_only'] += 1 + + # 4. 转换为DataFrame + merged_df = pd.DataFrame(list(frame_map.values())) if frame_map else pd.DataFrame() + + # 统计信息 + # logging.info('帧合并统计: 仅正向=%d, 仅反向=%d, 重叠=%d, 总计=%d', + # detection_stats['direct_only'], + # detection_stats['rs_traced_only'], + # detection_stats['both'], + # len(merged_df)) + + return merged_df, detection_stats + + def _merge_time_ranges(self, time_ranges: list[tuple[int, int]]) -> list[tuple[int, int]]: + """合并重叠的时间范围 + + Args: + time_ranges: 时间范围列表，格式为 [(start_ts, end_ts), ...] + + Returns: + 合并后的时间范围列表（无重叠） + """ + if not time_ranges: + return [] + + # 按开始时间排序 + sorted_ranges = sorted(time_ranges, key=lambda x: x[0]) + merged = [sorted_ranges[0]] + + for current_start, current_end in sorted_ranges[1:]: + last_start, last_end = merged[-1] + + # 如果当前范围与最后一个合并范围重叠 + if current_start <= last_end: + # 合并：扩展结束时间 + merged[-1] = (last_start, max(last_end, current_end)) + else: + # 无重叠，添加新范围 + merged.append((current_start, current_end)) + + return merged + + def _calculate_merged_time_ranges(self, frame_loads: list) -> list[tuple[int, int]]: + """计算所有帧的合并时间范围（使用原始时间戳，不含扩展） + + Args: + frame_loads: 帧负载数据列表 + + Returns: + 合并后的时间范围列表 + """ + if not frame_loads: + return [] + + # 收集所有帧的原始时间范围（使用 frame_slice 表中的 ts 和 dur，不含扩展） + time_ranges = [] + for frame in frame_loads: + ts = frame.get('ts', 0) + dur = frame.get('dur', 0) + + if ts > 0 and dur >= 0: + # 使用原始时间戳（frame_slice 表中的 ts 和 dur） + # 不扩展±1ms，避免重叠问题 + frame_start = ts + frame_end = ts + dur + time_ranges.append((frame_start, frame_end)) + + # 合并重叠的时间范围 + return self._merge_time_ranges(time_ranges) + + def _build_result_unified( + self, + frame_loads: list, + trace_df: pd.DataFrame, + total_load: int, + timing_stats: dict, + detection_stats: dict, + rs_skip_cpu: int, + rs_traced_results: list, + merged_time_ranges: list[tuple[int, int]], + ) -> dict: + """构建统一的分析结果（使用去重后的时间范围） + + Args: + frame_loads: 帧负载数据 + trace_df: 帧数据DataFrame + total_load: 总负载 + timing_stats: 耗时统计 + detection_stats: 检测方法统计 + rs_skip_cpu: RS进程CPU浪费 + rs_traced_results: RS追溯结果（用于统计） + merged_time_ranges: 去重后的时间范围列表（从 get_empty_frames_with_details 获取） + + Returns: + dict: 统一的分析结果 + """ + # === 使用去重后的时间范围计算所有汇总值（去除重叠区域） === + original_empty_frame_load = int(sum(f['frame_load'] for f in frame_loads)) if frame_loads else 0 + deduplicated_empty_frame_load = None + deduplicated_main_thread_load = None + deduplicated_background_thread_load = None + deduplicated_thread_loads = None # {thread_id: load, ...} + + if merged_time_ranges: + recalc_start = time.time() + + # 扩展合并后的时间范围（±1ms），与 frame_load 计算保持一致 + extended_merged_ranges = [] + for start_ts, end_ts in merged_time_ranges: + extended_start = start_ts - 1_000_000 + extended_end = end_ts + 1_000_000 + extended_merged_ranges.append((extended_start, extended_end)) + + # 计算去重后的 empty_frame_load + deduplicated_empty_frame_load = self.cache_manager.get_total_load_for_pids( + self.cache_manager.app_pids, time_ranges=extended_merged_ranges + ) + + # 计算去重后的主线程负载 + deduplicated_main_thread_load_dict = self.cache_manager.get_thread_loads_for_pids( + self.cache_manager.app_pids, time_ranges=extended_merged_ranges, filter_main_thread=True + ) + deduplicated_main_thread_load = sum(deduplicated_main_thread_load_dict.values()) + + # 计算去重后的后台线程负载 + deduplicated_background_thread_load_dict = self.cache_manager.get_thread_loads_for_pids( + self.cache_manager.app_pids, time_ranges=extended_merged_ranges, filter_main_thread=False + ) + deduplicated_background_thread_load = sum(deduplicated_background_thread_load_dict.values()) + + # 计算去重后的所有线程负载（用于 thread_statistics） + deduplicated_thread_loads = self.cache_manager.get_thread_loads_for_pids( + self.cache_manager.app_pids, time_ranges=extended_merged_ranges, filter_main_thread=None + ) + + timing_stats['recalc_deduplicated_loads'] = time.time() - recalc_start + + # 保存原始 empty_frame_load 到 timing_stats（用于日志对比） + timing_stats['original_empty_frame_load'] = original_empty_frame_load + + # 构建 tid_to_info 映射（用于 thread_statistics） + tid_to_info = {} + if self.cache_manager and self.cache_manager.trace_conn: + try: + trace_cursor = self.cache_manager.trace_conn.cursor() + app_pids = self.cache_manager.app_pids or [] + if app_pids: + placeholders = ','.join('?' * len(app_pids)) + trace_cursor.execute( + f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid IN ({placeholders}) + """, + app_pids, + ) + + for tid, thread_name, process_name in trace_cursor.fetchall(): + tid_to_info[tid] = { + 'thread_name': thread_name, + 'process_name': process_name, + 'is_main_thread': 1 if thread_name == process_name else 0, + } + except Exception: + pass + + # === 使用公共模块构建基础结果 === + base_result = self.result_builder.build_result( + frame_loads=frame_loads, + total_load=total_load, # total_load 保持不变（整个trace的CPU） + detection_stats=detection_stats, # 传递检测统计信息 + deduplicated_empty_frame_load=deduplicated_empty_frame_load, # 传递去重后的 empty_frame_load + deduplicated_main_thread_load=deduplicated_main_thread_load, # 传递去重后的主线程负载 + deduplicated_background_thread_load=deduplicated_background_thread_load, # 传递去重后的后台线程负载 + deduplicated_thread_loads=deduplicated_thread_loads, # 传递去重后的所有线程负载 + tid_to_info=tid_to_info, # 传递线程信息映射 ) + + # 注意：timing_stats 仅用于调试日志，不添加到最终结果中 + + # 增强summary：添加检测方法统计 + if base_result and 'summary' in base_result: + summary = base_result['summary'] + + # 添加检测方法分解 + summary['detection_breakdown'] = { + 'direct_only': detection_stats['direct_only'], + 'rs_traced_only': detection_stats['rs_traced_only'], + 'both': detection_stats['both'], + } + + # 添加RS追溯统计 + total_skip_events = detection_stats.get('total_rs_skip_events', 0) + total_skip_frames = len([r for r in rs_traced_results if r.get('rs_frame')]) + traced_success = sum( + 1 for r in rs_traced_results if r.get('trace_result') and r['trace_result'].get('app_frame') + ) + rs_api_success = sum(1 for r in rs_traced_results if r.get('trace_method') == 'rs_api') + nw_api_success = sum(1 for r in rs_traced_results if r.get('trace_method') == 'nw_api') + + summary['rs_trace_stats'] = { + 'total_skip_events': total_skip_events, + 'total_skip_frames': total_skip_frames, + 'traced_success_count': traced_success, + 'trace_accuracy': (traced_success / total_skip_frames * 100) if total_skip_frames > 0 else 0.0, + 'rs_api_success': rs_api_success, + 'nativewindow_success': nw_api_success, + 'failed': total_skip_frames - traced_success, + 'rs_skip_cpu': rs_skip_cpu, + } + + # 添加traced_count统计 + if frame_loads: + multi_traced_frames = [f for f in frame_loads if f.get('traced_count', 0) > 1] + max_traced = max((f.get('traced_count', 0) for f in frame_loads), default=0) + + summary['traced_count_stats'] = { + 'frames_traced_multiple_times': len(multi_traced_frames), + 'max_traced_count': max_traced, + } + + # 确保三个检测器的原始检测结果被保存 + summary['direct_detected_count'] = detection_stats.get('direct_detected_count', 0) if detection_stats else 0 + summary['rs_detected_count'] = detection_stats.get('rs_detected_count', 0) if detection_stats else 0 + framework_counts = detection_stats.get('framework_detected_counts', {}) if detection_stats else {} + summary['framework_detection_counts'] = framework_counts if framework_counts is not None else {} + + return base_result + + def _build_empty_result_unified( + self, total_load: int, timing_stats: dict, rs_skip_cpu: int, detection_stats: Optional[dict] = None + ) -> dict: + """构建空结果（统一格式）""" + result = { + 'status': 'success', + 'summary': { + 'total_empty_frames': 0, + 'empty_frame_load': 0, + 'empty_frame_percentage': 0.0, + 'total_load': total_load, + 'severity_level': 'normal', + 'severity_description': '正常：未检测到空刷帧', + 'detection_breakdown': {'direct_only': 0, 'rs_traced_only': 0, 'both': 0}, + 'rs_trace_stats': { + 'total_skip_events': 0, + 'total_skip_frames': 0, + 'traced_success_count': 0, + 'trace_accuracy': 0.0, + 'rs_api_success': 0, + 'nativewindow_success': 0, + 'failed': 0, + 'rs_skip_cpu': rs_skip_cpu, + }, + }, + 'top_frames': [], # 统一列表，不再区分主线程和后台线程 + # 注意：timing_stats 仅用于调试日志，不添加到最终结果中 + } + + # === 三个检测器的原始检测结果（在合并去重之前，不处理 overlap）=== + # 无论 detection_stats 是否存在，都设置这三个字段（确保总是存在） + if detection_stats: + direct_count = detection_stats.get('direct_detected_count', 0) + rs_count = detection_stats.get('rs_detected_count', 0) + framework_counts = detection_stats.get('framework_detected_counts') + framework_counts = framework_counts if framework_counts is not None else {} + + result['summary']['direct_detected_count'] = direct_count + result['summary']['rs_detected_count'] = rs_count + result['summary']['framework_detection_counts'] = framework_counts + else: + # 如果没有 detection_stats，设置默认值 + result['summary']['direct_detected_count'] = 0 + result['summary']['rs_detected_count'] = 0 + result['summary']['framework_detection_counts'] = {} + + return result diff --git a/perf_testing/hapray/core/common/frame/frame_analyzer_rs_skip.py b/perf_testing/hapray/core/common/frame/frame_analyzer_rs_skip.py new file mode 100644 index 00000000..f9e8792d --- /dev/null +++ b/perf_testing/hapray/core/common/frame/frame_analyzer_rs_skip.py @@ -0,0 +1,932 @@ +""" +Copyright (c) 2025 Huawei Device Co., Ltd. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import time +import traceback +from typing import Optional + +import pandas as pd + +from .frame_core_cache_manager import FrameCacheManager + +# 导入公共模块 +from .frame_empty_common import ( + EmptyFrameCallchainAnalyzer, + EmptyFrameCPUCalculator, + EmptyFrameResultBuilder, + calculate_process_instructions, +) + +# 导入RS skip追溯模块 +from .frame_rs_skip_backtrack_api import preload_caches as preload_rs_api_caches +from .frame_rs_skip_backtrack_api import trace_rs_skip_to_app_frame as trace_rs_api +from .frame_rs_skip_backtrack_nw import preload_caches as preload_nw_caches +from .frame_rs_skip_backtrack_nw import trace_rs_skip_to_app_frame as trace_nw_api + + +class RSSkipFrameAnalyzer: + """RS Skip帧分析器 + + 专门用于分析RS进程中的DisplayNode skip事件，包括： + 1. RS skip事件检测 + 2. 追溯到应用帧（RS API + NativeWindow API） + 3. CPU浪费统计 + 4. 追溯成功率分析 + + 与EmptyFrameAnalyzer的关系： + - EmptyFrameAnalyzer：分析应用进程的flag=2空刷帧 + - RSSkipFrameAnalyzer：分析RS进程的skip事件，追溯到应用帧 + - 两者互补，共同覆盖空刷帧检测 + + 完全遵循EmptyFrameAnalyzer的架构： + - 初始化参数相同：(debug_vsync_enabled, cache_manager) + - 主入口方法：analyze_rs_skip_frames() + - 分阶段处理：检测→预加载→追溯→计算→构建 + - 统一输出格式：{status, summary, top_frames} + - 错误处理一致：try-except返回None + """ + + def __init__(self, debug_vsync_enabled: bool = False, cache_manager: FrameCacheManager = None): + """初始化RS Skip分析器 + + 参数完全遵循EmptyFrameAnalyzer的设计 + + Args: + debug_vsync_enabled: VSync调试开关（保留以兼容接口，RS Skip分析不使用） + cache_manager: 缓存管理器实例 + """ + self.cache_manager = cache_manager + + # 使用公共模块（模块化重构） + self.cpu_calculator = EmptyFrameCPUCalculator(cache_manager) + self.callchain_analyzer = EmptyFrameCallchainAnalyzer(cache_manager) + self.result_builder = EmptyFrameResultBuilder(cache_manager) + + # 配置选项（参考EmptyFrameAnalyzer的设计） + self.rs_api_enabled = True # 是否启用RS系统API追溯 + self.nw_api_enabled = True # 是否启用NativeWindow API追溯 + self.max_frames = None # 最大处理帧数（None=全部） + self.top_n = 10 # Top N帧数量 + + def analyze_rs_skip_frames(self) -> Optional[dict]: + """分析RS Skip帧（主入口方法） + + 完全遵循EmptyFrameAnalyzer.analyze_empty_frames()的架构 + + Returns: + dict: 包含分析结果 + { + 'status': 'success', + 'summary': { + 'total_skip_frames': int, + 'traced_success_count': int, + 'trace_accuracy': float, + 'total_wasted_instructions': int, + 'severity_level': str, + 'severity_description': str, + }, + 'top_frames': { + 'top_skip_frames': list, + } + } + 或None（如果分析失败） + """ + # 从cache_manager获取数据库连接和参数（完全遵循EmptyFrameAnalyzer） + trace_conn = self.cache_manager.trace_conn if self.cache_manager else None + perf_conn = self.cache_manager.perf_conn if self.cache_manager else None + + if not trace_conn: + logging.error('trace数据库连接未建立') + return None + + total_start_time = time.time() + timing_stats = {} + + try: + # 阶段1：检测RS skip帧 + skip_frames = self._detect_skip_frames(trace_conn, timing_stats) + if not skip_frames: + logging.info('未检测到RS skip帧，返回空结果') + return self._build_empty_result(timing_stats) + + # 阶段2：计算RS进程的skip帧CPU浪费【新增】 + rs_skip_cpu = 0 + if perf_conn: + rs_cpu_start = time.time() + rs_skip_cpu = self._calculate_rs_skip_cpu(skip_frames) + timing_stats['rs_cpu_calculation'] = time.time() - rs_cpu_start + logging.info( + f'RS进程CPU计算完成: {rs_skip_cpu:,} 指令, 耗时: {timing_stats["rs_cpu_calculation"]:.3f}秒' + ) + + # 阶段3：预加载缓存 + caches = self._preload_caches(trace_conn, skip_frames, timing_stats) + + # 阶段4：追溯到应用帧 + traced_results = self._trace_to_app_frames(trace_conn, perf_conn, skip_frames, caches, timing_stats) + + # 阶段5：应用进程CPU计算（使用公共模块） + cpu_calc_start = time.time() + frame_loads_raw = self.cpu_calculator.extract_cpu_from_traced_results(traced_results) + + # 去重：同一个应用帧可能被多个RS skip事件追溯到 + frame_loads = self._deduplicate_app_frames(frame_loads_raw) + logging.info(f'应用进程CPU计算完成: {len(frame_loads_raw)}个帧（去重前）, {len(frame_loads)}个帧（去重后）') + + timing_stats['app_cpu_calculation'] = time.time() - cpu_calc_start + logging.info(f'应用进程CPU计算耗时: {timing_stats["app_cpu_calculation"]:.3f}秒') + + # 阶段6：调用链分析（使用公共模块）【新增】 + if perf_conn and frame_loads: + callchain_start = time.time() + self._analyze_callchains(frame_loads, traced_results, timing_stats) + timing_stats['callchain_analysis'] = time.time() - callchain_start + logging.info(f'调用链分析完成, 耗时: {timing_stats["callchain_analysis"]:.3f}秒') + + # 阶段7：结果构建（使用公共模块） + result = self._build_result_unified(skip_frames, traced_results, frame_loads, rs_skip_cpu, timing_stats) + + # 总耗时统计 + total_time = time.time() - total_start_time + self._log_analysis_complete(total_time, timing_stats) + + return result + + except Exception as e: + logging.error('分析RS Skip帧时发生异常: %s', str(e)) + logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return None + + # ==================== 私有方法：分阶段处理 ==================== + + def _get_rs_process_pid(self) -> int: + """获取RS进程PID + + Returns: + int: RS进程PID，如果未找到返回0 + """ + try: + cursor = self.cache_manager.trace_conn.cursor() + cursor.execute(""" + SELECT pid FROM process + WHERE name LIKE '%render_service%' + LIMIT 1 + """) + result = cursor.fetchone() + if result: + return result[0] + logging.warning('未找到render_service进程') + return 0 + except Exception as e: + logging.error(f'获取RS进程PID失败: {e}') + return 0 + + def _calculate_rs_skip_cpu(self, skip_frames: list) -> int: + """计算RS进程skip帧的CPU浪费【新增方法】 + + 注意：skip_frames是从callstack表检测的skip事件，不是frame_slice表的帧。 + 每个skip事件有ts和dur，我们计算这个时间窗口内RS进程的CPU。 + + Args: + skip_frames: RS skip帧列表（实际是skip事件列表） + + Returns: + int: RS进程CPU浪费（指令数） + """ + if not skip_frames: + return 0 + + # 获取RS进程PID + rs_pid = self._get_rs_process_pid() + if not rs_pid: + logging.warning('无法获取RS进程PID，跳过RS进程CPU计算') + return 0 + + total_rs_cpu = 0 + calculated_count = 0 + + for skip_frame in skip_frames: + frame_start = skip_frame['ts'] + frame_dur = skip_frame.get('dur', 0) + + # 如果dur=0或None，使用默认的16.67ms（一帧时间） + if not frame_dur or frame_dur <= 0: + frame_dur = 16_666_666 # 16.67ms + + frame_end = frame_start + frame_dur + + # 使用frame_empty_common的calculate_process_instructions计算RS进程CPU + try: + app_instructions, system_instructions = calculate_process_instructions( + perf_conn=self.cache_manager.perf_conn, + trace_conn=self.cache_manager.trace_conn, + app_pid=rs_pid, # RS进程PID + frame_start=frame_start, + frame_end=frame_end, + ) + if app_instructions > 0: + total_rs_cpu += app_instructions + calculated_count += 1 + except Exception as e: + logging.warning(f'计算RS skip帧CPU失败: frame_id={skip_frame.get("frame_id")}, error={e}') + continue + + logging.info( + f'RS进程CPU统计: {len(skip_frames)}个skip帧, 成功计算{calculated_count}个, 总CPU={total_rs_cpu:,} 指令' + ) + return total_rs_cpu + + def _deduplicate_app_frames(self, frame_loads: list) -> list: + """去重并统计追溯次数：同一个应用帧可能被多个RS skip事件追溯到 + + 同一个帧被多次追溯到说明它导致了多次RS skip，这是重要的严重性指标。 + 去重时保留CPU数据（只计算一次），但记录追溯次数。 + + Args: + frame_loads: 帧负载数据列表（可能包含重复） + + Returns: + list: 去重后的帧负载数据列表（包含traced_count字段） + """ + if not frame_loads: + return [] + + seen_frames = {} + trace_counts = {} + + for frame in frame_loads: + frame_id = frame.get('frame_id') + # 使用frame_id作为唯一标识，如果没有frame_id，使用(ts, dur, thread_id)作为唯一标识 + key = f'fid_{frame_id}' if frame_id else (frame.get('ts'), frame.get('dur'), frame.get('thread_id')) + + if key not in seen_frames: + seen_frames[key] = frame + trace_counts[key] = 1 + else: + # 重复帧，增加追溯次数 + trace_counts[key] += 1 + + # 添加traced_count字段 + unique_frames = [] + for key, frame in seen_frames.items(): + frame['traced_count'] = trace_counts[key] # 添加追溯次数 + unique_frames.append(frame) + + # 统计信息 + total_traces = len(frame_loads) + unique_count = len(unique_frames) + duplicate_count = total_traces - unique_count + + # 统计被多次追溯的帧 + multi_traced = [f for f in unique_frames if f['traced_count'] > 1] + max_traced = max((f['traced_count'] for f in unique_frames), default=1) + + logging.info( + f'去重统计: {total_traces}个帧 → {unique_count}个唯一帧 ' + f'(去除{duplicate_count}个重复，{len(multi_traced)}个帧被多次追溯，最多{max_traced}次)' + ) + + return unique_frames + + def _analyze_callchains(self, frame_loads: list, traced_results: list, timing_stats: dict) -> None: + """分析调用链（新增功能） + + Args: + frame_loads: 帧负载数据列表 + traced_results: 追溯结果列表 + timing_stats: 耗时统计字典 + """ + if not frame_loads: + return + + # 将追溯结果转换为DataFrame格式（用于调用链分析） + app_frames_data = [] + for result in traced_results: + if result.get('trace_result') and result['trace_result'].get('app_frame'): + app_frame = result['trace_result']['app_frame'] + app_frames_data.append( + { + 'ts': app_frame.get('frame_ts'), + 'dur': app_frame.get('frame_dur'), + 'tid': app_frame.get('thread_id'), + 'thread_id': app_frame.get('thread_id'), + } + ) + + if not app_frames_data: + logging.warning('没有成功追溯的应用帧，跳过调用链分析') + return + + trace_df = pd.DataFrame(app_frames_data) + perf_df = self.cache_manager.get_perf_samples() + perf_conn = self.cache_manager.perf_conn + + # 使用公共模块EmptyFrameCallchainAnalyzer + self.callchain_analyzer.analyze_callchains( + frame_loads=frame_loads, trace_df=trace_df, perf_df=perf_df, perf_conn=perf_conn, top_n=self.top_n + ) + + logging.info(f'调用链分析完成: Top {self.top_n}帧') + + def _detect_skip_frames(self, trace_conn, timing_stats: dict) -> list[dict]: + """阶段1：检测RS skip帧 + + 调用原有的detect_displaynode_skip逻辑（修改为接收连接对象） + + Args: + trace_conn: trace数据库连接 + timing_stats: 耗时统计字典 + + Returns: + list: skip帧列表 + """ + detect_start = time.time() + + try: + cursor = trace_conn.cursor() + + # 步骤1: 查找所有DisplayNode skip事件 + skip_events_query = """ + SELECT + c.callid, + c.ts, + c.dur, + c.name + FROM callstack c + WHERE c.name LIKE '%DisplayNode skip%' + AND c.callid IN ( + SELECT t.id + FROM thread t + WHERE t.ipid IN ( + SELECT p.ipid + FROM process p + WHERE p.name = 'render_service' + ) + ) + ORDER BY c.ts + """ + cursor.execute(skip_events_query) + skip_events = cursor.fetchall() + + if not skip_events: + logging.info('未找到DisplayNode skip事件') + timing_stats['detect_skip'] = time.time() - detect_start + return [] + + logging.info('找到 %d 个DisplayNode skip事件', len(skip_events)) + + # 步骤2: 获取RS进程的所有帧 + rs_frames_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.vsync, + fs.flag, + fs.type + FROM frame_slice fs + WHERE fs.ipid IN ( + SELECT p.ipid + FROM process p + WHERE p.name = 'render_service' + ) + AND fs.type = 0 + AND fs.flag IS NOT NULL + ORDER BY fs.ts + """ + cursor.execute(rs_frames_query) + rs_frames = cursor.fetchall() + + if not rs_frames: + logging.warning('未找到RS进程的帧') + timing_stats['detect_skip'] = time.time() - detect_start + return [] + + logging.info('找到 %d 个RS进程帧', len(rs_frames)) + + # 步骤3: 将skip事件匹配到对应的帧（使用SQL JOIN优化） + frame_skip_map_query = """ + SELECT + fs.rowid as frame_rowid, + c.callid, + c.ts as skip_ts, + COALESCE(c.dur, 0) as skip_dur, + c.name as skip_name + FROM callstack c + INNER JOIN thread t ON c.callid = t.id + INNER JOIN process p ON t.ipid = p.ipid + INNER JOIN frame_slice fs ON fs.ipid = p.ipid + WHERE c.name LIKE '%DisplayNode skip%' + AND p.name = 'render_service' + AND fs.type = 0 + AND fs.flag IS NOT NULL + AND fs.ts <= c.ts + AND (fs.ts + COALESCE(fs.dur, 0)) >= c.ts + ORDER BY fs.rowid, c.ts + """ + + cursor.execute(frame_skip_map_query) + matched_results = cursor.fetchall() + + # 构建frame_skip_map + frame_skip_map = {} + for row in matched_results: + frame_rowid = row[0] + if frame_rowid not in frame_skip_map: + frame_skip_map[frame_rowid] = [] + frame_skip_map[frame_rowid].append({'ts': row[2], 'dur': row[3], 'name': row[4], 'callid': row[1]}) + + # 步骤4: 构建结果 + frame_info_map = {frame[0]: frame for frame in rs_frames} + + result = [] + for frame_rowid, skip_events_list in frame_skip_map.items(): + frame_info = frame_info_map.get(frame_rowid) + if frame_info: + result.append( + { + 'frame_id': frame_rowid, + 'ts': frame_info[1], + 'dur': frame_info[2] if frame_info[2] is not None else 0, + 'vsync': frame_info[3], + 'flag': frame_info[4], + 'skip_count': len(skip_events_list), + 'skip_events': skip_events_list, + } + ) + + result.sort(key=lambda x: x['skip_count'], reverse=True) + + # 限制处理数量 + if self.max_frames and result: + original_count = len(result) + result = result[: self.max_frames] + logging.info('限制处理数量: %d -> %d', original_count, len(result)) + + timing_stats['detect_skip'] = time.time() - detect_start + logging.info('检测到%d个RS skip帧，耗时: %.3f秒', len(result), timing_stats['detect_skip']) + + return result + + except Exception as e: + logging.error('检测RS skip帧失败: %s', str(e)) + logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + timing_stats['detect_skip'] = time.time() - detect_start + return [] + + def _preload_caches(self, trace_conn, skip_frames: list, timing_stats: dict) -> dict: + """阶段2：预加载缓存 + + Args: + trace_conn: trace数据库连接 + skip_frames: skip帧列表 + timing_stats: 耗时统计字典 + + Returns: + dict: 缓存字典 + """ + preload_start = time.time() + + # 计算时间范围 + if not skip_frames: + timing_stats['preload_cache'] = 0 + return {} + + min_ts = min(f['ts'] for f in skip_frames) - 500_000_000 # 前500ms + max_ts = max(f['ts'] + f['dur'] for f in skip_frames) + 50_000_000 # 后50ms + + caches = {} + + # 预加载RS API缓存 + if self.rs_api_enabled: + rs_api_start = time.time() + try: + caches['rs_api'] = preload_rs_api_caches(trace_conn, min_ts, max_ts) + timing_stats['preload_rs_api'] = time.time() - rs_api_start + logging.info('预加载RS API缓存完成，耗时: %.3f秒', timing_stats['preload_rs_api']) + except Exception as e: + logging.error('加载RS API缓存失败: %s', str(e)) + caches['rs_api'] = None + + # 预加载NativeWindow API缓存 + if self.nw_api_enabled: + nw_start = time.time() + try: + caches['nw'] = preload_nw_caches(trace_conn, min_ts, max_ts) + timing_stats['preload_nw'] = time.time() - nw_start + logging.info('预加载NativeWindow API缓存完成，耗时: %.3f秒', timing_stats['preload_nw']) + except Exception as e: + logging.error('加载NativeWindow API缓存失败: %s', str(e)) + caches['nw'] = None + + timing_stats['preload_cache'] = time.time() - preload_start + + return caches + + def _trace_to_app_frames(self, trace_conn, perf_conn, skip_frames: list, caches: dict, timing_stats: dict) -> list: + """阶段3：追溯到应用帧 + + Args: + trace_conn: trace数据库连接 + perf_conn: perf数据库连接 + skip_frames: skip帧列表 + caches: 预加载的缓存 + timing_stats: 耗时统计字典 + + Returns: + list: 追溯结果列表 + """ + trace_start = time.time() + + # 导入追溯函数 + # 追溯模块已在文件开头导入（trace_rs_api, trace_nw_api） + + traced_results = [] + rs_success = 0 + nw_success = 0 + failed = 0 + + for skip_frame in skip_frames: + trace_result = None + trace_method = None + + # 提取rs_frame_id（函数期望的是整数ID，而不是整个skip_frame字典） + rs_frame_id = skip_frame.get('frame_id') + + # 尝试RS API追溯 + trace_result = None + if self.rs_api_enabled and caches.get('rs_api'): + try: + trace_result = trace_rs_api( + trace_conn, + rs_frame_id, # 传递整数ID，而不是整个字典 + caches=caches['rs_api'], + perf_conn=perf_conn, + ) + # 只有当trace_result存在且包含app_frame时，才算RS API成功 + # 这与原始算法保持一致（RS_skip_checker.py第429行） + if trace_result and trace_result.get('app_frame'): + trace_method = 'rs_api' + rs_success += 1 + # RS API成功，不需要尝试NativeWindow API + traced_results.append( + {'skip_frame': skip_frame, 'trace_result': trace_result, 'trace_method': trace_method} + ) + continue + except Exception as e: + logging.debug('RS API追溯失败: %s', str(e)) + trace_result = None + + # 如果RS API追溯失败（返回None或没有app_frame），尝试NativeWindow API追溯 + if (not trace_result or not trace_result.get('app_frame')) and self.nw_api_enabled and caches.get('nw'): + try: + # NativeWindow API的函数签名： + # trace_rs_skip_to_app_frame(trace_conn, rs_frame_id, + # nativewindow_events_cache, app_frames_cache, perf_conn) + nw_cache = caches['nw'] + trace_result = trace_nw_api( + trace_conn, + rs_frame_id, # 传递整数ID + nativewindow_events_cache=nw_cache.get('nativewindow_events_cache'), + app_frames_cache=nw_cache.get('app_frames_cache'), + perf_conn=perf_conn, + ) + # 只有当trace_result存在且包含app_frame时，才算NativeWindow API成功 + # 这与原始算法保持一致（RS_skip_checker.py第468行） + if trace_result and trace_result.get('app_frame'): + trace_method = 'nativewindow' + nw_success += 1 + # NativeWindow API成功 + traced_results.append( + {'skip_frame': skip_frame, 'trace_result': trace_result, 'trace_method': trace_method} + ) + continue + except Exception as e: + logging.debug('NativeWindow API追溯失败: %s', str(e)) + + # 两种方法都失败 + failed += 1 + traced_results.append( + { + 'skip_frame': skip_frame, + 'trace_result': trace_result, # 可能是None或没有app_frame的结果 + 'trace_method': trace_method, + } + ) + + timing_stats['trace_to_app'] = time.time() - trace_start + timing_stats['rs_api_success'] = rs_success + timing_stats['nw_api_success'] = nw_success + timing_stats['trace_failed'] = failed + + trace_accuracy = (rs_success + nw_success) / len(skip_frames) * 100 if skip_frames else 0.0 + logging.info('追溯完成，耗时: %.3f秒，成功率: %.1f%%', timing_stats['trace_to_app'], trace_accuracy) + + return traced_results + + def _calculate_cpu_waste( + self, trace_conn, perf_conn, traced_results: list, caches: dict, timing_stats: dict + ) -> None: + """阶段4：计算CPU浪费 + + 注意：backtrack模块已经在追溯时计算了CPU浪费，这里只需要统计 + + Args: + trace_conn: trace数据库连接 + perf_conn: perf数据库连接 + traced_results: 追溯结果列表 + caches: 预加载的缓存 + timing_stats: 耗时统计字典 + """ + cpu_start = time.time() + + # backtrack模块已经在追溯时计算了CPU浪费 + # 这里只需要统计有CPU数据的帧数 + frames_with_cpu = 0 + + for result in traced_results: + if result['trace_result'] and result['trace_result'].get('app_frame'): + app_frame = result['trace_result']['app_frame'] + cpu_waste = app_frame.get('cpu_waste', {}) + + # backtrack模块已经计算了CPU浪费，直接使用 + if cpu_waste and cpu_waste.get('has_perf_data'): + wasted = cpu_waste.get('wasted_instructions', 0) + if wasted > 0: + frames_with_cpu += 1 + + timing_stats['cpu_calculation'] = time.time() - cpu_start + timing_stats['frames_with_cpu'] = frames_with_cpu + + logging.info( + 'CPU浪费统计完成，耗时: %.3f秒，有CPU数据的帧: %d', timing_stats['cpu_calculation'], frames_with_cpu + ) + + def _build_result_unified( + self, skip_frames: list, traced_results: list, frame_loads: list, rs_skip_cpu: int, timing_stats: dict + ) -> dict: + """阶段7：结果构建（使用公共模块，统一输出格式） + + Args: + skip_frames: skip帧列表 + traced_results: 追溯结果列表 + frame_loads: 帧负载数据列表（已包含frame_load字段） + rs_skip_cpu: RS进程CPU浪费 + timing_stats: 耗时统计字典 + + Returns: + dict: 统一格式的分析结果 + """ + result_build_start = time.time() + + # 统计 + total_skip_frames = len(skip_frames) + total_skip_events = sum(f.get('skip_count', 0) for f in skip_frames) + + traced_success = sum(1 for r in traced_results if r['trace_result'] and r['trace_result'].get('app_frame')) + trace_accuracy = (traced_success / total_skip_frames * 100) if total_skip_frames > 0 else 0.0 + + rs_api_success = timing_stats.get('rs_api_success', 0) + nw_api_success = timing_stats.get('nw_api_success', 0) + failed = timing_stats.get('trace_failed', 0) + + # 应用进程CPU统计 + app_empty_cpu = sum(f['frame_load'] for f in frame_loads) if frame_loads else 0 + + # 总CPU浪费 + total_wasted_cpu = rs_skip_cpu + app_empty_cpu + + # 使用公共模块EmptyFrameResultBuilder构建结果 + result = self.result_builder.build_result( + frame_loads=frame_loads, + total_load=0, # RS无total_load + detection_stats={ + # 追溯统计 + 'total_skip_frames': total_skip_frames, + 'total_skip_events': total_skip_events, + 'trace_accuracy': trace_accuracy, + 'traced_success_count': traced_success, + 'rs_api_success': rs_api_success, + 'nativewindow_success': nw_api_success, + 'failed': failed, + # RS进程CPU统计【新增】 + 'rs_skip_cpu': rs_skip_cpu, + 'rs_skip_percentage': 0.0, # 可选，需要rs_total_cpu + # 应用进程CPU统计 + 'app_empty_cpu': app_empty_cpu, + # 总计 + 'total_wasted_cpu': total_wasted_cpu, + }, + ) + + timing_stats['result_build'] = time.time() - result_build_start + return result + + def _build_result(self, skip_frames: list, traced_results: list, timing_stats: dict) -> dict: + """阶段5：结果构建（旧实现，已弃用，保留作为参考） + + 注意：此方法已被_build_result_unified替代，使用公共模块EmptyFrameResultBuilder + 完全遵循EmptyFrameAnalyzer的输出格式 + + Args: + skip_frames: skip帧列表 + traced_results: 追溯结果列表 + timing_stats: 耗时统计字典 + + Returns: + dict: 分析结果（统一格式） + """ + result_build_start = time.time() + + # 统计 + total_skip_frames = len(skip_frames) + total_skip_events = sum(f.get('skip_count', 0) for f in skip_frames) + + traced_success = sum(1 for r in traced_results if r['trace_result'] and r['trace_result'].get('app_frame')) + trace_accuracy = (traced_success / total_skip_frames * 100) if total_skip_frames > 0 else 0.0 + + rs_api_success = timing_stats.get('rs_api_success', 0) + nw_api_success = timing_stats.get('nw_api_success', 0) + failed = timing_stats.get('trace_failed', 0) + + # CPU统计 + total_wasted = 0 + frames_with_cpu = 0 + for r in traced_results: + if r['trace_result'] and r['trace_result'].get('app_frame'): + app_frame = r['trace_result']['app_frame'] + cpu_waste = app_frame.get('cpu_waste', {}) + if cpu_waste.get('has_perf_data'): + total_wasted += cpu_waste.get('wasted_instructions', 0) + frames_with_cpu += 1 + + avg_wasted = (total_wasted / frames_with_cpu) if frames_with_cpu > 0 else 0.0 + + # 严重程度评估（参考EmptyFrameAnalyzer的方式） + severity_level, severity_description = self._assess_severity(total_skip_frames, trace_accuracy, traced_success) + + # 提取Top N帧 + top_skip_frames = self._extract_top_frames(traced_results, self.top_n) + + # 构建结果（完全遵循EmptyFrameAnalyzer的格式） + result = { + 'status': 'success', + 'summary': { + 'total_skip_frames': int(total_skip_frames), + 'total_skip_events': int(total_skip_events), + 'traced_success_count': int(traced_success), + 'trace_accuracy': float(trace_accuracy), + 'rs_api_success': int(rs_api_success), + 'nativewindow_success': int(nw_api_success), + 'failed': int(failed), + 'total_wasted_instructions': int(total_wasted), + 'avg_wasted_instructions': float(avg_wasted), + 'frames_with_cpu_data': int(frames_with_cpu), + # 严重程度评估（参考EmptyFrameAnalyzer） + 'severity_level': severity_level, + 'severity_description': severity_description, + }, + 'top_frames': { + 'top_skip_frames': top_skip_frames, # Top N skip帧详情 + }, + } + + # 添加警告（如果追溯成功率低） + if trace_accuracy < 50.0: + trace_warning = ( + f'警告：RS Skip帧追溯成功率较低({trace_accuracy:.1f}%)，可能导致CPU浪费统计不准确。建议检查数据质量。' + ) + result['summary']['trace_warning'] = trace_warning + logging.warning(trace_warning) + + timing_stats['result_build'] = time.time() - result_build_start + return result + + def _build_empty_result(self, timing_stats: dict) -> dict: + """构建空结果（完全遵循EmptyFrameAnalyzer的格式） + + Args: + timing_stats: 耗时统计字典 + + Returns: + dict: 空结果字典 + """ + result_build_start = time.time() + + result = { + 'status': 'success', + 'summary': { + 'total_skip_frames': 0, + 'total_skip_events': 0, + 'traced_success_count': 0, + 'trace_accuracy': 0.0, + 'rs_api_success': 0, + 'nativewindow_success': 0, + 'failed': 0, + 'total_wasted_instructions': 0, + 'avg_wasted_instructions': 0.0, + 'frames_with_cpu_data': 0, + 'severity_level': 'normal', + 'severity_description': '正常：未检测到RS Skip帧。', + }, + 'top_frames': { + 'top_skip_frames': [], + }, + } + + timing_stats['result_build'] = time.time() - result_build_start + return result + + def _assess_severity(self, total_skip: int, trace_accuracy: float, traced_success: int) -> tuple: + """评估严重程度（参考EmptyFrameAnalyzer的分级标准） + + 评估标准： + 1. 追溯成功率：< 50% = critical + 2. Skip帧数量：< 10 = normal, < 100 = moderate, >= 100 = severe + + Args: + total_skip: 总skip帧数 + trace_accuracy: 追溯成功率 + traced_success: 追溯成功数 + + Returns: + (severity_level, severity_description) + """ + # 追溯成功率评估 + if trace_accuracy < 50.0: + return ('critical', f'严重：RS Skip帧追溯成功率仅{trace_accuracy:.1f}%，数据质量差，无法准确评估CPU浪费。') + + # Skip帧数量评估 + if total_skip < 10: + return ('normal', f'正常：仅检测到{total_skip}个RS Skip帧，属于正常范围。追溯成功率{trace_accuracy:.1f}%。') + if total_skip < 100: + return ('moderate', f'较为严重：检测到{total_skip}个RS Skip帧，建议关注。追溯成功率{trace_accuracy:.1f}%。') + return ('severe', f'严重：检测到{total_skip}个RS Skip帧，需要优先优化。追溯成功率{trace_accuracy:.1f}%。') + + def _extract_top_frames(self, traced_results: list, top_n: int) -> list: + """提取Top N skip帧（按CPU浪费排序） + + Args: + traced_results: 追溯结果列表 + top_n: Top N数量 + + Returns: + list: Top N skip帧列表 + """ + # 提取有CPU数据的帧 + frames_with_cpu = [] + for result in traced_results: + if result['trace_result'] and result['trace_result'].get('app_frame'): + skip_frame = result['skip_frame'] + app_frame = result['trace_result']['app_frame'] + cpu_waste = app_frame.get('cpu_waste', {}) + + if cpu_waste.get('has_perf_data'): + frames_with_cpu.append( + { + 'rs_frame_id': skip_frame['frame_id'], + 'rs_frame_ts': skip_frame['ts'], + 'skip_count': skip_frame['skip_count'], + 'app_frame_id': app_frame.get('frame_id'), + 'app_frame_ts': app_frame.get('frame_ts'), + 'trace_method': result['trace_method'], + 'wasted_instructions': cpu_waste.get('wasted_instructions', 0), + 'thread_name': app_frame.get('thread_name', 'N/A'), + 'process_name': app_frame.get('process_name', 'N/A'), + 'pid': app_frame.get('pid'), + } + ) + + # 排序并返回Top N + frames_with_cpu.sort(key=lambda x: x['wasted_instructions'], reverse=True) + return frames_with_cpu[:top_n] + + def _log_analysis_complete(self, total_time: float, timing_stats: dict) -> None: + """记录分析完成日志（完全遵循EmptyFrameAnalyzer的风格） + + Args: + total_time: 总耗时 + timing_stats: 各阶段耗时统计 + """ + logging.info('RS Skip分析总耗时: %.3f秒', total_time) + logging.info( + '各阶段耗时占比: 检测%.1f%%, 预加载缓存%.1f%%, 追溯%.1f%%, CPU计算%.1f%%, 结果构建%.1f%%', + timing_stats.get('detect_skip', 0) / total_time * 100 if total_time > 0 else 0, + timing_stats.get('preload_cache', 0) / total_time * 100 if total_time > 0 else 0, + timing_stats.get('trace_to_app', 0) / total_time * 100 if total_time > 0 else 0, + timing_stats.get('cpu_calculation', 0) / total_time * 100 if total_time > 0 else 0, + timing_stats.get('result_build', 0) / total_time * 100 if total_time > 0 else 0, + ) diff --git a/perf_testing/hapray/core/common/frame/frame_core_analyzer.py b/perf_testing/hapray/core/common/frame/frame_core_analyzer.py index fc9a3161..3ca43d09 100644 --- a/perf_testing/hapray/core/common/frame/frame_core_analyzer.py +++ b/perf_testing/hapray/core/common/frame/frame_core_analyzer.py @@ -20,6 +20,7 @@ from .frame_analyzer_stuttered import StutteredFrameAnalyzer from .frame_analyzer_vsync import VSyncAnomalyAnalyzer +# RSSkipFrameAnalyzer已合并到EmptyFrameAnalyzer中 # 导入新的模块化组件 from .frame_constants import TOP_FRAMES_FOR_CALLCHAIN from .frame_core_cache_manager import FrameCacheManager @@ -76,6 +77,7 @@ def __init__( self.empty_frame_analyzer = EmptyFrameAnalyzer(debug_vsync_enabled, self.cache_manager) self.stuttered_frame_analyzer = StutteredFrameAnalyzer(debug_vsync_enabled, self.cache_manager) self.vsync_anomaly_analyzer = VSyncAnomalyAnalyzer(self.cache_manager) + # RSSkipFrameAnalyzer已合并到EmptyFrameAnalyzer中 def analyze_empty_frames(self) -> Optional[dict]: """分析空帧（flag=2, type=0）的负载情况 @@ -101,6 +103,47 @@ def analyze_vsync_anomalies(self) -> Optional[dict[str, Any]]: """ return self.vsync_anomaly_analyzer.analyze_vsync_anomalies(self.app_pids) + def analyze_rs_skip_frames(self) -> Optional[dict]: + """分析RS Skip帧（已合并到EmptyFrameAnalyzer） + + 为了向后兼容，保留此方法，但实际调用EmptyFrameAnalyzer + EmptyFrameAnalyzer现在包含了正向检测和反向追溯两种方法 + + Returns: + dict: 包含分析结果（从统一的空刷帧结果中提取RS traced部分） + """ + # 调用统一的空刷帧分析 + result = self.empty_frame_analyzer.analyze_empty_frames() + + # 提取RS traced相关的统计信息（用于单独展示） + if result and 'summary' in result: + rs_stats = result['summary'].get('rs_trace_stats', {}) + detection_breakdown = result['summary'].get('detection_breakdown', {}) + + # 构建兼容的RS Skip结果格式 + return { + 'status': result['status'], + 'summary': { + 'total_skip_events': rs_stats.get('total_skip_events', 0), + 'total_skip_frames': rs_stats.get('total_skip_frames', 0), + 'traced_success_count': rs_stats.get('traced_success_count', 0), + 'trace_accuracy': rs_stats.get('trace_accuracy', 0.0), + 'rs_api_success': rs_stats.get('rs_api_success', 0), + 'nativewindow_success': rs_stats.get('nativewindow_success', 0), + 'failed': rs_stats.get('failed', 0), + 'total_wasted_cpu': rs_stats.get('rs_skip_cpu', 0), + # 添加空刷帧相关统计 + 'total_empty_frames': detection_breakdown.get('rs_traced_only', 0) + + detection_breakdown.get('both', 0), + 'empty_frame_load': result['summary'].get('empty_frame_load', 0), + 'severity_level': result['summary'].get('severity_level', 'normal'), + 'severity_description': result['summary'].get('severity_description', ''), + }, + 'top_frames': result.get('top_frames', {}), + } + + return result + def analyze_frame_loads_fast(self, step_id: str = None) -> dict[str, Any]: """快速分析所有帧的负载值（不分析调用链） diff --git a/perf_testing/hapray/core/common/frame/frame_core_cache_manager.py b/perf_testing/hapray/core/common/frame/frame_core_cache_manager.py index dd1df05e..0f282917 100644 --- a/perf_testing/hapray/core/common/frame/frame_core_cache_manager.py +++ b/perf_testing/hapray/core/common/frame/frame_core_cache_manager.py @@ -17,6 +17,7 @@ import logging import os import sqlite3 +import traceback from typing import Any, Callable, Optional import pandas as pd @@ -105,6 +106,37 @@ class FrameCacheManager(FramePerfAccessor, FrameTraceAccessor): # pylint: disab 注意：本类继承自FrameTraceAccessor和FramePerfAccessor，可以直接使用它们的方法 """ + def _trace_db_has_perf_tables(self) -> bool: + """检查trace.db是否包含必要的perf数据表 + + Returns: + bool: 如果trace.db包含perf_sample, perf_callchain, perf_files表则返回True + """ + if not self.trace_conn: + return False + + required_tables = ['perf_sample', 'perf_callchain', 'perf_files'] + try: + cursor = self.trace_conn.cursor() + # 查询所有表名 + cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") + existing_tables = {row[0] for row in cursor.fetchall()} + + # 检查是否包含所有必需的perf表 + has_all_tables = all(table in existing_tables for table in required_tables) + + if has_all_tables: + # 进一步检查表是否有数据（至少perf_sample表不为空） + cursor.execute('SELECT COUNT(*) FROM perf_sample') + sample_count = cursor.fetchone()[0] + return sample_count > 0 + + return False + + except Exception as e: + logging.warning('检查trace.db中的perf表失败: %s', str(e)) + return False + def __init__(self, trace_db_path: str = None, perf_db_path: str = None, app_pids: list = None): """初始化FrameCacheManager @@ -134,10 +166,19 @@ def __init__(self, trace_db_path: str = None, perf_db_path: str = None, app_pids self.perf_conn = sqlite3.connect(perf_db_path) except Exception as e: logging.error('建立perf数据库连接失败: %s', str(e)) + elif self.trace_conn and self._trace_db_has_perf_tables(): + # 如果perf.db不存在但trace.db包含perf数据，则使用trace.db作为perf数据源 + logging.info('perf.db不存在，使用trace.db中的perf数据进行帧分析') + self.perf_conn = self.trace_conn + self.perf_db_path = trace_db_path # 更新perf_db_path指向trace.db FramePerfAccessor.__init__(self, self.perf_conn) FrameTraceAccessor.__init__(self, self.trace_conn) + # 【新增】自动查找并添加ArkWeb render进程 + if self.trace_conn and self.app_pids: + self._expand_arkweb_render_processes() + # ==================== 实例变量：缓存存储 ==================== self._callchain_cache = None self._files_cache = None @@ -257,6 +298,343 @@ def get_files_cache(self) -> pd.DataFrame: return pd.DataFrame() return FramePerfAccessor.get_files_cache(self) + @cached('_tid_to_info_cache', 'tid_to_info') + def get_tid_to_info(self) -> dict: + """获取tid到线程信息的映射（带缓存） + + 查询thread表获取所有thread_id和thread_name的对应关系，并缓存结果。 + 这个映射用于快速查找线程名称，避免重复查询数据库。 + 注意：查询所有线程信息，不仅限于应用进程，因为perf_sample可能包含系统线程。 + + Returns: + dict: {tid: {'thread_name': str, 'process_name': str}} 的字典 + """ + if not self.trace_conn: + logging.warning('trace_conn未建立，无法获取线程信息') + return {} + + tid_to_info = {} + try: + trace_cursor = self.trace_conn.cursor() + # 查询所有线程信息，不仅限于应用进程 + # 因为perf_sample可能包含系统线程或其他进程的线程 + trace_cursor.execute(""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + """) + + for tid, thread_name, process_name in trace_cursor.fetchall(): + tid_to_info[tid] = {'thread_name': thread_name, 'process_name': process_name} + except Exception as e: + logging.warning('查询线程信息失败: %s', str(e)) + + return tid_to_info + + def get_total_load_for_pids(self, app_pids: list[int], time_ranges: Optional[list[tuple[int, int]]] = None) -> int: + """获取指定进程的总负载（重写父类方法，使用trace_conn关联thread表） + + 修复：原方法错误地用进程ID直接匹配thread_id，现在通过trace数据库的thread表 + 找到属于指定进程的所有线程ID，然后统计这些线程的perf_sample总和。 + + Args: + app_pids: 应用进程ID列表 + time_ranges: 可选的时间范围列表，格式为 [(start_ts, end_ts), ...] + 如果为None，则统计整个trace的CPU + 如果提供，只统计这些时间范围内的CPU + + Returns: + int: 总负载值（只包含指定进程的线程的perf_sample） + """ + # 验证app_pids参数 + if not app_pids or not isinstance(app_pids, (list, tuple)) or len(app_pids) == 0: + logging.warning('app_pids参数无效，返回0') + return 0 + + # 过滤掉无效的PID值 + valid_pids = [pid for pid in app_pids if pd.notna(pid) and isinstance(pid, (int, float))] + if not valid_pids: + logging.warning('没有有效的PID值，返回0') + return 0 + + if not self.trace_conn or not self.perf_conn: + logging.warning('数据库连接未建立，返回0') + return 0 + + try: + # 步骤1：通过trace数据库的thread表，找到属于指定进程的所有线程ID（tid） + cursor = self.trace_conn.cursor() + placeholders = ','.join('?' * len(valid_pids)) + thread_query = f""" + SELECT DISTINCT t.tid + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid IN ({placeholders}) + """ + cursor.execute(thread_query, valid_pids) + thread_rows = cursor.fetchall() + thread_ids = [row[0] for row in thread_rows if row[0] is not None] + + if not thread_ids: + logging.warning('未找到进程 %s 的线程，返回0', valid_pids) + return 0 + + # 步骤2：统计这些线程的perf_sample总和 + perf_cursor = self.perf_conn.cursor() + thread_placeholders = ','.join('?' * len(thread_ids)) + + if time_ranges: + # 使用时间范围过滤 + # 如果时间范围太多，SQLite 的表达式树会太大（最大深度 1000） + # 解决方案：分批查询，每批最多 200 个时间范围 + MAX_RANGES_PER_BATCH = 200 + total_load = 0 + + if len(time_ranges) <= MAX_RANGES_PER_BATCH: + # 时间范围不多，直接查询 + time_conditions = [] + time_params = [] + + for start_ts, end_ts in time_ranges: + time_conditions.append('(timestamp_trace >= ? AND timestamp_trace <= ?)') + time_params.extend([start_ts, end_ts]) + + time_condition_str = ' OR '.join(time_conditions) + perf_query = f""" + SELECT SUM(event_count) as total_load + FROM perf_sample + WHERE thread_id IN ({thread_placeholders}) + AND ({time_condition_str}) + """ + perf_cursor.execute(perf_query, thread_ids + time_params) + result = perf_cursor.fetchone() + total_load = result[0] if result and result[0] else 0 + else: + # 时间范围太多，分批查询 + logging.info('时间范围数量 (%d) 超过限制 (%d)，分批查询', len(time_ranges), MAX_RANGES_PER_BATCH) + + for i in range(0, len(time_ranges), MAX_RANGES_PER_BATCH): + batch_ranges = time_ranges[i : i + MAX_RANGES_PER_BATCH] + time_conditions = [] + time_params = [] + + for start_ts, end_ts in batch_ranges: + time_conditions.append('(timestamp_trace >= ? AND timestamp_trace <= ?)') + time_params.extend([start_ts, end_ts]) + + time_condition_str = ' OR '.join(time_conditions) + perf_query = f""" + SELECT SUM(event_count) as total_load + FROM perf_sample + WHERE thread_id IN ({thread_placeholders}) + AND ({time_condition_str}) + """ + perf_cursor.execute(perf_query, thread_ids + time_params) + result = perf_cursor.fetchone() + batch_load = result[0] if result and result[0] else 0 + total_load += batch_load + + logging.info( + '分批查询完成: %d 批，总负载: %d', + (len(time_ranges) + MAX_RANGES_PER_BATCH - 1) // MAX_RANGES_PER_BATCH, + total_load, + ) + else: + # 原有逻辑：统计整个trace + perf_query = f""" + SELECT SUM(event_count) as total_load + FROM perf_sample + WHERE thread_id IN ({thread_placeholders}) + """ + perf_cursor.execute(perf_query, thread_ids) + result = perf_cursor.fetchone() + total_load = result[0] if result and result[0] else 0 + + if time_ranges: + total_time_ns = sum(end - start for start, end in time_ranges) + logging.info( + '获取总负载: %d (进程ID: %s, 线程数: %d, 时间范围: %d个区间, 总时长: %.2f秒)', + total_load, + valid_pids, + len(thread_ids), + len(time_ranges), + total_time_ns / 1_000_000_000, + ) + else: + logging.info('获取总负载: %d (进程ID: %s, 线程数: %d)', total_load, valid_pids, len(thread_ids)) + + return int(total_load) + except Exception as e: + logging.error('获取总负载失败: %s', str(e)) + logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return 0 + + def get_thread_loads_for_pids( + self, + app_pids: list[int], + time_ranges: Optional[list[tuple[int, int]]] = None, + filter_main_thread: Optional[bool] = None, + ) -> dict[int, int]: + """获取指定进程的按线程分组的负载（支持时间范围去重） + + Args: + app_pids: 应用进程ID列表 + time_ranges: 时间范围列表，格式为 [(start_ts, end_ts), ...] + 如果为None，则统计整个trace的CPU + 如果提供，只统计这些时间范围内的CPU（已去重） + filter_main_thread: 是否过滤主线程 + None: 不过滤，返回所有线程 + True: 只返回主线程（thread_name == process_name） + False: 只返回后台线程（thread_name != process_name） + + Returns: + dict[int, int]: {thread_id: total_load, ...} + """ + # 验证app_pids参数 + if not app_pids or not isinstance(app_pids, (list, tuple)) or len(app_pids) == 0: + logging.warning('app_pids参数无效，返回空字典') + return {} + + # 过滤掉无效的PID值 + valid_pids = [pid for pid in app_pids if pd.notna(pid) and isinstance(pid, (int, float))] + if not valid_pids: + logging.warning('没有有效的PID值，返回空字典') + return {} + + if not self.trace_conn or not self.perf_conn: + logging.warning('数据库连接未建立，返回空字典') + return {} + + try: + # 步骤1：通过trace数据库的thread表，找到属于指定进程的所有线程信息 + cursor = self.trace_conn.cursor() + placeholders = ','.join('?' * len(valid_pids)) + + # 根据filter_main_thread构建查询条件 + if filter_main_thread is True: + # 只要主线程 + thread_query = f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid IN ({placeholders}) + AND t.name = p.name + """ + elif filter_main_thread is False: + # 只要后台线程 + thread_query = f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid IN ({placeholders}) + AND t.name != p.name + """ + else: + # 不过滤，返回所有线程 + thread_query = f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid IN ({placeholders}) + """ + + cursor.execute(thread_query, valid_pids) + thread_rows = cursor.fetchall() + thread_ids = [row[0] for row in thread_rows if row[0] is not None] + + if not thread_ids: + logging.warning('未找到进程 %s 的线程，返回空字典', valid_pids) + return {} + + # 步骤2：统计这些线程的perf_sample总和（按线程分组） + perf_cursor = self.perf_conn.cursor() + thread_placeholders = ','.join('?' * len(thread_ids)) + + if time_ranges: + # 使用时间范围过滤 + MAX_RANGES_PER_BATCH = 200 + thread_loads = {} # {thread_id: total_load} + + if len(time_ranges) <= MAX_RANGES_PER_BATCH: + # 时间范围不多，直接查询 + time_conditions = [] + time_params = [] + + for start_ts, end_ts in time_ranges: + time_conditions.append('(timestamp_trace >= ? AND timestamp_trace <= ?)') + time_params.extend([start_ts, end_ts]) + + time_condition_str = ' OR '.join(time_conditions) + perf_query = f""" + SELECT thread_id, SUM(event_count) as total_load + FROM perf_sample + WHERE thread_id IN ({thread_placeholders}) + AND ({time_condition_str}) + GROUP BY thread_id + """ + perf_cursor.execute(perf_query, thread_ids + time_params) + results = perf_cursor.fetchall() + + for thread_id, total_load in results: + if thread_id is not None and total_load is not None: + thread_loads[thread_id] = int(total_load) + else: + # 时间范围太多，分批查询 + logging.info('时间范围数量 (%d) 超过限制 (%d)，分批查询', len(time_ranges), MAX_RANGES_PER_BATCH) + + for i in range(0, len(time_ranges), MAX_RANGES_PER_BATCH): + batch_ranges = time_ranges[i : i + MAX_RANGES_PER_BATCH] + time_conditions = [] + time_params = [] + + for start_ts, end_ts in batch_ranges: + time_conditions.append('(timestamp_trace >= ? AND timestamp_trace <= ?)') + time_params.extend([start_ts, end_ts]) + + time_condition_str = ' OR '.join(time_conditions) + perf_query = f""" + SELECT thread_id, SUM(event_count) as total_load + FROM perf_sample + WHERE thread_id IN ({thread_placeholders}) + AND ({time_condition_str}) + GROUP BY thread_id + """ + perf_cursor.execute(perf_query, thread_ids + time_params) + batch_results = perf_cursor.fetchall() + + for thread_id, batch_load in batch_results: + if thread_id is not None and batch_load is not None: + if thread_id not in thread_loads: + thread_loads[thread_id] = 0 + thread_loads[thread_id] += int(batch_load) + + logging.info( + '分批查询完成: %d 批，线程数: %d', + (len(time_ranges) + MAX_RANGES_PER_BATCH - 1) // MAX_RANGES_PER_BATCH, + len(thread_loads), + ) + else: + # 原有逻辑：统计整个trace + perf_query = f""" + SELECT thread_id, SUM(event_count) as total_load + FROM perf_sample + WHERE thread_id IN ({thread_placeholders}) + GROUP BY thread_id + """ + perf_cursor.execute(perf_query, thread_ids) + results = perf_cursor.fetchall() + thread_loads = {} + + for thread_id, total_load in results: + if thread_id is not None and total_load is not None: + thread_loads[thread_id] = int(total_load) + + return thread_loads + except Exception as e: + logging.error('获取线程负载失败: %s', str(e)) + logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return {} + @cached('_process_cache', 'process') def get_process_cache(self) -> pd.DataFrame: """获取进程缓存数据（带缓存） @@ -572,6 +950,132 @@ def _find_insert_position(self, frame_list: list, frame_load: int) -> int: # ==================== Private 方法：数据库检查 ==================== + def _expand_arkweb_render_processes(self) -> None: + """自动查找并添加ArkWeb render进程到app_pids列表 + + ArkWeb使用多进程架构：主进程和render进程 + - 主进程：如 com.jd.hm.mall + - render进程：如 .hm.mall:render 或 com.jd.hm.mall:render + + 此方法会自动识别ArkWeb应用的render进程并添加到app_pids中 + """ + if not self.trace_conn or not self.app_pids: + return + + try: + cursor = self.trace_conn.cursor() + original_pids = list(self.app_pids) + + # 查找所有以":render"结尾的进程（ArkWeb render进程） + cursor.execute(""" + SELECT DISTINCT p.pid, p.name + FROM process p + WHERE p.name LIKE '%:render' + AND p.name != 'render_service' + AND p.name != 'rmrenderservice' + """) + + all_render_processes = cursor.fetchall() + + if not all_render_processes: + return + + # 对每个主进程，尝试匹配对应的render进程 + for main_pid in original_pids: + # 查找主进程名 + cursor.execute('SELECT name FROM process WHERE pid = ? LIMIT 1', (main_pid,)) + result = cursor.fetchone() + if not result: + continue + + main_process_name = result[0] + + # 如果主进程名不包含"."，可能不是ArkWeb，跳过 + if '.' not in main_process_name: + continue + + # 提取主进程名的关键部分（去掉com.前缀） + main_key = main_process_name.split('.', 1)[-1] if '.' in main_process_name else main_process_name + + # 尝试匹配render进程 + for render_pid, render_process_name in all_render_processes: + # 检查render进程名是否包含主进程的关键部分 + if ( + main_key in render_process_name or main_process_name.split('.')[-1] in render_process_name + ) and render_pid not in self.app_pids: + self.app_pids.append(render_pid) + logging.info( + f'检测到ArkWeb render进程: {render_process_name} (PID={render_pid})，已添加到app_pids' + ) + + except Exception as e: + logging.warning(f'查找ArkWeb render进程失败: {e}') + + def _expand_arkweb_render_processes(self) -> None: + """自动查找并添加ArkWeb render进程到app_pids列表 + + ArkWeb使用多进程架构：主进程和render进程 + - 主进程：如 com.jd.hm.mall (PID=5489) + - render进程：如 .hm.mall:render (PID=35301) + + 此方法会自动识别ArkWeb应用的render进程并添加到app_pids中， + 确保CPU计算包含render进程的开销 + """ + if not self.trace_conn or not self.app_pids: + return + + try: + cursor = self.trace_conn.cursor() + original_pids = list(self.app_pids) + + # 查找所有以":render"结尾的进程（ArkWeb render进程） + cursor.execute(""" + SELECT DISTINCT p.pid, p.name + FROM process p + WHERE p.name LIKE '%:render' + AND p.name != 'render_service' + AND p.name != 'rmrenderservice' + """) + + all_render_processes = cursor.fetchall() + + if not all_render_processes: + return + + # 对每个主进程，尝试匹配对应的render进程 + for main_pid in original_pids: + # 查找主进程名 + cursor.execute('SELECT name FROM process WHERE pid = ? LIMIT 1', (main_pid,)) + result = cursor.fetchone() + if not result: + continue + + main_process_name = result[0] + + # 如果主进程名不包含"."，可能不是ArkWeb，跳过 + if '.' not in main_process_name: + continue + + # 提取主进程名的关键部分（去掉com.前缀） + main_key = main_process_name.split('.', 1)[-1] if '.' in main_process_name else main_process_name + + # 尝试匹配render进程 + # 可能的匹配模式： + # 1. com.jd.hm.mall -> com.jd.hm.mall:render + # 2. com.jd.hm.mall -> .hm.mall:render (去掉com前缀) + # 3. com.jd.hm.mall -> hm.mall:render + for render_pid, render_process_name in all_render_processes: + if ( + main_key in render_process_name or main_process_name.split('.')[-1] in render_process_name + ) and render_pid not in self.app_pids: + self.app_pids.append(render_pid) + logging.info( + f'检测到ArkWeb render进程: {render_process_name} (PID={render_pid})，已添加到app_pids' + ) + + except Exception as e: + logging.warning(f'查找ArkWeb render进程失败: {e}') + def _check_perf_db_size(self, perf_db_path: str) -> None: """检查性能数据库文件大小 diff --git a/perf_testing/hapray/core/common/frame/frame_core_load_calculator.py b/perf_testing/hapray/core/common/frame/frame_core_load_calculator.py index 60ddc41f..31a44039 100644 --- a/perf_testing/hapray/core/common/frame/frame_core_load_calculator.py +++ b/perf_testing/hapray/core/common/frame/frame_core_load_calculator.py @@ -14,9 +14,12 @@ limitations under the License. """ +import bisect import logging +import sqlite3 import time -from typing import Any +import traceback +from typing import TYPE_CHECKING, Any, Optional import pandas as pd @@ -27,7 +30,329 @@ VSYNC_SYMBOL_HANDLE, VSYNC_SYMBOL_ON_READABLE, ) -from .frame_core_cache_manager import FrameCacheManager +from .frame_utils import is_system_thread + +if TYPE_CHECKING: + from .frame_core_cache_manager import FrameCacheManager + + +# ============================================================================ +# CPU计算工具函数 +# ============================================================================ + + +def calculate_thread_instructions( + perf_conn: sqlite3.Connection, + trace_conn: sqlite3.Connection, + thread_ids: set[int], + frame_start: int, + frame_end: int, + tid_to_info_cache: Optional[dict] = None, + perf_sample_cache: Optional[dict] = None, + perf_timestamp_field: Optional[str] = None, + return_callchain_ids: bool = False, +) -> tuple[dict[int, int], dict[int, int]] | tuple[dict[int, int], dict[int, int], list[dict]]: + """计算线程在帧时间范围内的 CPU 指令数（区分应用线程和系统线程） + + 参考RN实现：只计算应用线程的指令数，排除系统线程 + 注意：扩展时间范围（前后各扩展1ms）以包含时间戳对齐问题导致的perf_sample + + Args: + perf_conn: perf 数据库连接 + trace_conn: trace 数据库连接（用于获取线程信息） + thread_ids: 线程 ID 集合（perf_thread 表的 thread_id，对应trace thread.tid） + frame_start: 帧开始时间 + frame_end: 帧结束时间 + tid_to_info_cache: tid到线程信息的缓存 + perf_sample_cache: perf_sample缓存 + perf_timestamp_field: perf_sample时间戳字段名 + + Returns: + (app_thread_instructions, system_thread_instructions) + - app_thread_instructions: 应用线程的指令数字典 {thread_id: instruction_count} + - system_thread_instructions: 系统线程的指令数字典 {thread_id: instruction_count} + """ + if not perf_conn or not trace_conn: + return {}, {} + + try: + perf_cursor = perf_conn.cursor() + trace_cursor = trace_conn.cursor() + + if not thread_ids: + return {}, {} + + # 性能优化：使用预加载的缓存，避免数据库查询 + extended_start = frame_start - 1_000_000 # 1ms before + extended_end = frame_end + 1_000_000 # 1ms after + + # 如果提供了缓存，从缓存中查询 + if perf_sample_cache is not None and perf_timestamp_field: + # 从缓存中查找和聚合（使用二分查找优化） + thread_instruction_map = {} + for thread_id in thread_ids: + if thread_id in perf_sample_cache: + # 在缓存中查找时间范围内的数据 + samples = perf_sample_cache[thread_id] + if not samples: + continue + + # 使用二分查找找到起始位置 + # samples是(timestamp, event_count)的列表，已按timestamp排序 + timestamps = [s[0] for s in samples] + start_idx = bisect.bisect_left(timestamps, extended_start) + end_idx = bisect.bisect_right(timestamps, extended_end) + + # 聚合范围内的event_count + total_instructions = sum(samples[i][1] for i in range(start_idx, end_idx)) + + if total_instructions > 0: + thread_instruction_map[thread_id] = total_instructions + + # 区分应用线程和系统线程 + app_thread_instructions = {} + system_thread_instructions = {} + + for thread_id, instruction_count in thread_instruction_map.items(): + thread_info = tid_to_info_cache.get(thread_id, {}) if tid_to_info_cache else {} + process_name = thread_info.get('process_name') + thread_name = thread_info.get('thread_name') + + if is_system_thread(process_name, thread_name): + system_thread_instructions[thread_id] = instruction_count + else: + app_thread_instructions[thread_id] = instruction_count + + return app_thread_instructions, system_thread_instructions + + # 向后兼容：如果没有缓存，查询数据库 + # 检查 perf_sample 表字段名（只检查一次，可以缓存） + perf_cursor.execute('PRAGMA table_info(perf_sample)') + columns = [row[1] for row in perf_cursor.fetchall()] + timestamp_field = ( + perf_timestamp_field + if perf_timestamp_field + else ('timestamp_trace' if 'timestamp_trace' in columns else 'timeStamp') + ) + + # 获取线程信息（进程名和线程名），用于判断是否为系统线程 + thread_ids_list = list(thread_ids) + if not thread_ids_list: + return {}, {} + + # 定义placeholders（在后续查询中使用） + placeholders = ','.join('?' * len(thread_ids_list)) + + # 性能优化：使用预加载的缓存，避免数据库查询 + thread_info_map = {} + if tid_to_info_cache: + # 从缓存中查找 + for tid in thread_ids_list: + if tid in tid_to_info_cache: + thread_info_map[tid] = tid_to_info_cache[tid] + else: + # 向后兼容：如果没有缓存，查询数据库 + trace_cursor.execute( + f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE t.tid IN ({placeholders}) + """, + thread_ids_list, + ) + + for tid, thread_name, process_name in trace_cursor.fetchall(): + thread_info_map[tid] = {'thread_name': thread_name, 'process_name': process_name} + + # 优化：使用索引提示，批量查询所有线程的指令数 + # 使用EXPLAIN QUERY PLAN优化查询性能 + if return_callchain_ids: + # 如果需要返回callchain_id，查询所有样本（不GROUP BY），在Python中处理 + instructions_query = f""" + SELECT + ps.thread_id, + ps.event_count, + ps.callchain_id + FROM perf_sample ps + WHERE ps.thread_id IN ({placeholders}) + AND ps.{timestamp_field} >= ? AND ps.{timestamp_field} <= ? + """ + else: + # 原有逻辑：只查询汇总结果 + instructions_query = f""" + SELECT + ps.thread_id, + SUM(ps.event_count) as total_instructions + FROM perf_sample ps + WHERE ps.thread_id IN ({placeholders}) + AND ps.{timestamp_field} >= ? AND ps.{timestamp_field} <= ? + GROUP BY ps.thread_id + """ + + params = thread_ids_list + [extended_start, extended_end] + perf_cursor.execute(instructions_query, params) + + results = perf_cursor.fetchall() + + app_thread_instructions = {} + system_thread_instructions = {} + sample_details = [] # 保存所有样本的详细信息：thread_id, event_count, callchain_id + + if return_callchain_ids: + # 一轮遍历：累加event_count，同时保存每个样本的详细信息 + for thread_id, event_count, callchain_id in results: + # 保存样本详细信息（只保存有callchain_id的样本，用于后续调用链分析） + if callchain_id is not None: + sample_details.append( + {'thread_id': thread_id, 'event_count': event_count, 'callchain_id': callchain_id} + ) + + # 累加event_count（区分应用线程和系统线程） + thread_info = thread_info_map.get(thread_id, {}) + process_name = thread_info.get('process_name') + thread_name = thread_info.get('thread_name') + + if is_system_thread(process_name, thread_name): + system_thread_instructions[thread_id] = system_thread_instructions.get(thread_id, 0) + event_count + else: + app_thread_instructions[thread_id] = app_thread_instructions.get(thread_id, 0) + event_count + + # 打印保存的样本详细信息（用于调试） + if sample_details: + # logging.info( + # f'[第一轮筛选] 保存样本详细信息: 总样本数={len(sample_details)}, ' + # f'涉及线程数={len(set(s["thread_id"] for s in sample_details))}, ' + # f'唯一callchain_id数={len(set(s["callchain_id"] for s in sample_details))}, ' + # f'时间范围=[{extended_start}, {extended_end}]' + # ) + # 打印前5个样本的详细信息 + for _i, _sample in enumerate(sample_details[:5], 1): + # logging.info( + # f'[第一轮筛选] 样本{i}: thread_id={sample["thread_id"]}, ' + # f'event_count={sample["event_count"]}, callchain_id={sample["callchain_id"]}' + # ) + pass + if len(sample_details) > 5: + # logging.info(f'[第一轮筛选] ... 还有 {len(sample_details) - 5} 个样本') + pass + else: + # 原有逻辑：直接使用汇总结果 + for thread_id, instruction_count in results: + thread_info = thread_info_map.get(thread_id, {}) + process_name = thread_info.get('process_name') + thread_name = thread_info.get('thread_name') + + if is_system_thread(process_name, thread_name): + system_thread_instructions[thread_id] = instruction_count + else: + app_thread_instructions[thread_id] = instruction_count + + if return_callchain_ids: + return app_thread_instructions, system_thread_instructions, sample_details + return app_thread_instructions, system_thread_instructions + + except Exception as e: + logging.error('计算线程指令数失败: %s', str(e)) + logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return {}, {} + + +def calculate_process_instructions( + perf_conn: sqlite3.Connection, + trace_conn: sqlite3.Connection, + app_pid: int, + frame_start: int, + frame_end: int, + perf_sample_cache: Optional[dict] = None, + perf_timestamp_field: Optional[str] = None, + tid_to_info_cache: Optional[dict] = None, + return_callchain_ids: bool = False, +) -> tuple[int, int] | tuple[int, int, list[dict]]: + """计算应用进程在帧时间范围内的所有CPU指令数（进程级统计） + + 这是推荐的CPU计算方式：直接统计应用进程所有线程的CPU指令数，简单、完整、性能好。 + 不需要通过唤醒链分析找到相关线程，直接统计进程级别的CPU。 + + Args: + perf_conn: perf数据库连接 + trace_conn: trace数据库连接（用于获取进程的线程列表） + app_pid: 应用进程ID（process.pid） + frame_start: 帧开始时间 + frame_end: 帧结束时间 + perf_sample_cache: perf_sample缓存（可选，用于性能优化） + perf_timestamp_field: perf_sample时间戳字段名（可选） + tid_to_info_cache: tid到线程信息的缓存（可选） + + Returns: + (app_instructions, system_instructions) + - app_instructions: 应用线程的总指令数 + - system_instructions: 系统线程的总指令数（通常为0，因为已过滤系统线程） + """ + if not perf_conn or not trace_conn: + return 0, 0 + + try: + # 扩展时间范围（前后各扩展1ms）以包含时间戳对齐问题导致的perf_sample + frame_start - 1_000_000 # 1ms before + frame_end + 1_000_000 # 1ms after + + # 步骤1: 查找应用进程的所有线程ID（tid） + trace_cursor = trace_conn.cursor() + trace_cursor.execute( + """ + SELECT DISTINCT t.tid + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid = ? + """, + (app_pid,), + ) + + thread_tids = [row[0] for row in trace_cursor.fetchall()] + + if not thread_tids: + logging.debug(f'进程 {app_pid} 没有找到任何线程') + return 0, 0 + + # 步骤2: 计算这些线程在帧时间范围内的CPU指令数 + # 注意：calculate_thread_instructions内部会使用扩展时间窗口（±1ms）， + # 所以这里直接传递原始的frame_start和frame_end即可 + thread_ids = set(thread_tids) + result = calculate_thread_instructions( + perf_conn=perf_conn, + trace_conn=trace_conn, + thread_ids=thread_ids, + frame_start=frame_start, + frame_end=frame_end, + tid_to_info_cache=tid_to_info_cache, + perf_sample_cache=perf_sample_cache, + perf_timestamp_field=perf_timestamp_field, + return_callchain_ids=return_callchain_ids, + ) + + if return_callchain_ids: + app_instructions_dict, system_instructions_dict, sample_details = result + else: + app_instructions_dict, system_instructions_dict = result + sample_details = [] + + # 步骤3: 汇总所有线程的指令数 + app_instructions = sum(app_instructions_dict.values()) + system_instructions = sum(system_instructions_dict.values()) + + logging.debug( + f'进程 {app_pid} CPU统计: 应用线程={app_instructions:,} 指令, 系统线程={system_instructions:,} 指令' + ) + + if return_callchain_ids: + return app_instructions, system_instructions, sample_details + return app_instructions, system_instructions + + except Exception as e: + logging.error(f'计算进程CPU指令数失败 (PID={app_pid}): {e}') + logging.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return 0, 0 class FrameLoadCalculator: @@ -40,7 +365,7 @@ class FrameLoadCalculator: 4. VSync过滤 """ - def __init__(self, debug_vsync_enabled: bool = False, cache_manager: FrameCacheManager = None): + def __init__(self, debug_vsync_enabled: bool = False, cache_manager: 'FrameCacheManager' = None): """ 初始化帧负载计算器 @@ -50,6 +375,8 @@ def __init__(self, debug_vsync_enabled: bool = False, cache_manager: FrameCacheM """ self.debug_vsync_enabled = debug_vsync_enabled self.cache_manager = cache_manager + # 新增：进程级CPU计算开关（默认启用） + self.use_process_level_cpu = True def analyze_perf_callchain( self, @@ -200,12 +527,15 @@ def analyze_single_frame(self, frame, perf_df, perf_conn, step_id): time_calc_time = time.time() - time_calc_start # 数据过滤 + # 注意：一帧可能包含多个线程的样本，每个样本都有自己的callchain_id和thread_id + # 因此只按时间范围过滤，不限制线程，确保能分析该帧时间范围内所有线程的样本 filter_start = time.time() - mask = ( - (perf_df['timestamp_trace'] >= frame_start_time_ts) - & (perf_df['timestamp_trace'] <= frame_end_time_ts) - & (perf_df['thread_id'] == frame['tid']) - ) + mask = (perf_df['timestamp_trace'] >= frame_start_time_ts) & (perf_df['timestamp_trace'] <= frame_end_time_ts) + # 如果提供了app_pid，可以进一步过滤应用进程的样本（可选优化） + if 'app_pid' in frame and frame['app_pid']: + # 这里可以添加进程过滤，但需要perf_df中有pid字段 + # 暂时只按时间范围过滤，因为perf_df中可能没有pid字段 + pass frame_samples = perf_df[mask] filter_time = time.time() - filter_start @@ -222,12 +552,15 @@ def analyze_single_frame(self, frame, perf_df, perf_conn, step_id): ) return frame_load, sample_callchains - # 样本分析 + # 样本分析：收集所有调用链 sample_analysis_start = time.time() callchain_analysis_total = 0 vsync_filter_total = 0 sample_count = 0 + # 存储每个sample的调用链信息 + sample_callchain_list = [] + for _, sample in frame_samples.iterrows(): sample_count += 1 if not pd.notna(sample['callchain_id']): @@ -266,41 +599,75 @@ def analyze_single_frame(self, frame, perf_df, perf_conn, step_id): if not is_vsync_chain: frame_load += sample['event_count'] - try: - sample_load_percentage = (sample['event_count'] / frame_load) * 100 - sample_callchains.append( - { - 'timestamp': int(sample['timestamp_trace']), - 'event_count': int(sample['event_count']), - 'load_percentage': float(sample_load_percentage), - 'callchain': callchain_info, - } - ) - except Exception as e: - logging.error( - '处理样本时出错: %s, sample: %s, frame_load: %s', str(e), sample.to_dict(), frame_load - ) - continue + + # 直接保存sample信息，每个深度的value就是sample的event_count + # 火焰图的宽度差异来自于前端合并不同调用栈时的累加 + sample_callchain_list.append( + { + 'timestamp': int(sample['timestamp_trace']), + 'event_count': int(sample['event_count']), + 'thread_id': int(sample['thread_id']), + 'callchain_info': callchain_info, + } + ) except Exception as e: logging.error('分析调用链时出错: %s', str(e)) continue + # 获取thread_id到thread_name的映射（从缓存中获取，避免重复查询） + tid_to_info = {} + if self.cache_manager: + tid_to_info = self.cache_manager.get_tid_to_info() + + # 为每个sample的调用链添加负载信息和thread_name + for sample_data in sample_callchain_list: + try: + sample_load_percentage = (sample_data['event_count'] / frame_load) * 100 if frame_load > 0 else 0 + + # 每个深度的value都是sample的event_count + # 火焰图会在前端合并时自动累加 + callchain_with_load = [] + for call in sample_data['callchain_info']: + call_with_load = call.copy() + call_with_load['value'] = sample_data['event_count'] + callchain_with_load.append(call_with_load) + + # 从缓存中获取thread_name + thread_id = sample_data['thread_id'] + thread_info = tid_to_info.get(thread_id, {}) + thread_name = thread_info.get('thread_name', 'unknown') + + sample_callchains.append( + { + 'timestamp': sample_data['timestamp'], + 'event_count': sample_data['event_count'], + 'load_percentage': float(sample_load_percentage), + 'thread_id': thread_id, + 'thread_name': thread_name, # 添加thread_name + 'callchain': callchain_with_load, + } + ) + except Exception as e: + logging.error('处理样本时出错: %s, sample: %s, frame_load: %s', str(e), sample_data, frame_load) + continue + sample_analysis_time = time.time() - sample_analysis_start # 保存帧负载数据到缓存 + # 注意：一帧是针对整个进程的，不是某个线程，因此不保存thread_id + # 每个sample_callchain都有自己的thread_id cache_save_start = time.time() frame_load_data = { 'ts': frame.get('ts', frame.get('start_time', 0)), 'dur': frame.get('dur', frame.get('end_time', 0) - frame.get('start_time', 0)), 'frame_load': frame_load, - 'thread_id': frame.get('tid'), 'thread_name': frame.get('thread_name', 'unknown'), 'process_name': frame.get('process_name', 'unknown'), 'type': frame.get('type', 0), 'vsync': frame.get('vsync', 'unknown'), 'flag': frame.get('flag'), - 'sample_callchains': sample_callchains, + 'sample_callchains': sample_callchains, # 每个callchain都有自己的thread_id } if self.cache_manager: self.cache_manager.add_frame_load(frame_load_data) @@ -336,15 +703,60 @@ def analyze_single_frame(self, frame, perf_df, perf_conn, step_id): return frame_load, sample_callchains - def calculate_frame_load_simple(self, frame: dict[str, Any], perf_df: pd.DataFrame) -> int: - """简单计算帧负载（不包含调用链分析） + def calculate_frame_load_multi_process( + self, frame: dict[str, Any], trace_conn, perf_conn, app_pids: list[int] + ) -> int: + """使用进程级统计计算帧负载，支持多进程（如ArkWeb） + + Args: + frame: 帧数据字典，必须包含 'ts', 'dur' 字段 + trace_conn: trace数据库连接 + perf_conn: perf数据库连接 + app_pids: 应用进程ID列表（支持多个进程，如ArkWeb的主进程+render进程） + + Returns: + int: 帧负载（多进程级统计，包含系统线程） + """ + frame_start = frame['ts'] + frame_end = frame['ts'] + frame['dur'] + + total_cpu = 0 + + # 对每个进程分别计算CPU，然后汇总 + for pid in app_pids: + # 使用进程级统计（包含系统线程，如OS_VSyncThread） + app_instructions, sys_instructions = calculate_process_instructions( + perf_conn=perf_conn, trace_conn=trace_conn, app_pid=pid, frame_start=frame_start, frame_end=frame_end + ) + + # 包含系统线程CPU（OS_VSyncThread等系统线程也在应用进程中运行） + total_cpu += app_instructions + sys_instructions + + return int(total_cpu) + + def calculate_frame_load_process_level(self, frame: dict[str, Any], trace_conn, perf_conn, app_pid: int) -> int: + """使用进程级统计计算帧负载（单进程版本，兼容旧代码） + + Args: + frame: 帧数据字典，必须包含 'ts', 'dur' 字段 + trace_conn: trace数据库连接 + perf_conn: perf数据库连接 + app_pid: 应用进程ID + + Returns: + int: 帧负载（进程级统计，包含系统线程） + """ + return self.calculate_frame_load_multi_process(frame, trace_conn, perf_conn, [app_pid]) + + def _calculate_single_thread_load(self, frame: dict[str, Any], perf_df: pd.DataFrame) -> int: + """单线程CPU计算（旧方法，用于回退） Args: frame: 帧数据字典，必须包含 'ts', 'dur', 'tid' 字段 perf_df: perf样本DataFrame Returns: - int: 帧负载 + int: 帧负载（单线程） """ frame_start_time = frame['ts'] frame_end_time = frame['ts'] + frame['dur'] @@ -362,9 +774,40 @@ def calculate_frame_load_simple(self, frame: dict[str, Any], perf_df: pd.DataFra return int(frame_samples['event_count'].sum()) + def calculate_frame_load_simple(self, frame: dict[str, Any], perf_df: pd.DataFrame) -> int: + """简单计算帧负载（支持进程级统计） + + Args: + frame: 帧数据字典，必须包含 'ts', 'dur', 'tid' 字段 + perf_df: perf样本DataFrame + + Returns: + int: 帧负载 + """ + # 如果启用了进程级CPU计算，且有cache_manager，则使用进程级统计 + if self.use_process_level_cpu and self.cache_manager: + app_pid = frame.get('app_pid') or frame.get('pid') + # 如果frame中没有app_pid，尝试从cache_manager获取 + if not app_pid and self.cache_manager.app_pids: + app_pid = self.cache_manager.app_pids[0] + + if app_pid and self.cache_manager.trace_conn and self.cache_manager.perf_conn: + try: + return self.calculate_frame_load_process_level( + frame, self.cache_manager.trace_conn, self.cache_manager.perf_conn, app_pid + ) + except Exception as e: + logging.warning(f'进程级CPU计算失败，回退到单线程计算: {e}') + # 继续执行单线程计算 + + # 向后兼容：使用原有的单线程计算方式 + return self._calculate_single_thread_load(frame, perf_df) + def calculate_all_frame_loads_fast(self, frames: pd.DataFrame, perf_df: pd.DataFrame) -> list[dict[str, Any]]: """快速计算所有帧的负载值，不分析调用链 + 注意：如果启用了进程级CPU计算，将使用数据库查询而不是perf_df + Args: frames: 帧数据DataFrame perf_df: perf样本DataFrame @@ -374,10 +817,37 @@ def calculate_all_frame_loads_fast(self, frames: pd.DataFrame, perf_df: pd.DataF """ frame_loads = [] + # 检查是否使用进程级CPU计算 + use_process_level = ( + self.use_process_level_cpu + and self.cache_manager + and self.cache_manager.trace_conn + and self.cache_manager.perf_conn + and self.cache_manager.app_pids + ) + + if use_process_level: + app_pids = self.cache_manager.app_pids + logging.info(f'使用进程级CPU计算（PIDs={app_pids}），共{len(frames)}帧') + else: + logging.info(f'使用单线程CPU计算，共{len(frames)}帧') + # 批量计算开始 for i, (_, frame) in enumerate(frames.iterrows()): - # 使用现有的简单方法，只计算负载值 - frame_load = self.calculate_frame_load_simple(frame, perf_df) + # 根据配置选择计算方法 + if use_process_level: + try: + # 计算所有app_pids的CPU总和（支持ArkWeb的多进程架构） + frame_load = self.calculate_frame_load_multi_process( + frame, self.cache_manager.trace_conn, self.cache_manager.perf_conn, app_pids + ) + except Exception as e: + if i < 5: # 只记录前5个错误，避免日志过多 + logging.warning(f'帧{i}进程级CPU计算失败，回退到单线程: {e}') + frame_load = self._calculate_single_thread_load(frame, perf_df) + else: + # 使用单线程计算（快速但不准确） + frame_load = self._calculate_single_thread_load(frame, perf_df) # 检查并记录NaN值 nan_fields = [] @@ -404,12 +874,14 @@ def calculate_all_frame_loads_fast(self, frames: pd.DataFrame, perf_df: pd.DataF ) # 确保时间戳字段正确，处理NaN值 + # 注意：一帧是针对整个进程的，不是某个线程，因此不保存thread_id + # 每个sample_callchain都有自己的thread_id frame_loads.append( { 'ts': int(frame['ts']) if pd.notna(frame['ts']) else 0, # 确保时间戳是整数 'dur': int(frame['dur']) if pd.notna(frame['dur']) else 0, # 确保持续时间是整数 'frame_load': frame_load, - 'thread_id': int(frame['tid']) if pd.notna(frame['tid']) else 0, # 确保线程ID是整数 + 'thread_id': int(frame['tid']) if pd.notna(frame['tid']) else 0, # 添加thread_id字段，用于帧匹配 'thread_name': frame.get('thread_name', 'unknown'), 'process_name': frame.get('process_name', 'unknown'), 'type': int(frame.get('type', 0)) if pd.notna(frame.get('type')) else 0, # 确保类型是整数 diff --git a/perf_testing/hapray/core/common/frame/frame_empty_common.py b/perf_testing/hapray/core/common/frame/frame_empty_common.py new file mode 100644 index 00000000..09608fa8 --- /dev/null +++ b/perf_testing/hapray/core/common/frame/frame_empty_common.py @@ -0,0 +1,1035 @@ +""" +Copyright (c) 2025 Huawei Device Co., Ltd. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import sqlite3 +import traceback +from typing import Optional + +import pandas as pd + +from .frame_constants import TOP_FRAMES_FOR_CALLCHAIN +from .frame_core_load_calculator import ( + FrameLoadCalculator, + calculate_process_instructions, + calculate_thread_instructions, +) +from .frame_time_utils import FrameTimeUtils +from .frame_utils import clean_frame_data, is_system_thread + +logger = logging.getLogger(__name__) + +"""空刷帧负载分析公共模块 + +包含： +1. CPU计算工具函数（线程级、进程级、应用帧级） +2. 空刷帧分析公共模块类（CPU计算、调用链分析、结果构建） + +用于EmptyFrameAnalyzer和RSSkipFrameAnalyzer复用。 +""" + +logger = logging.getLogger(__name__) + + +# ============================================================================ +# CPU计算工具函数 +# ============================================================================ + + +def calculate_app_frame_cpu_waste( + trace_conn: sqlite3.Connection, + perf_conn: Optional[sqlite3.Connection], + app_frame: dict, + perf_sample_cache: Optional[dict] = None, + perf_timestamp_field: Optional[str] = None, + tid_to_info_cache: Optional[dict] = None, + use_process_level: bool = True, +) -> dict: + """计算应用帧浪费的CPU指令数 + + Args: + trace_conn: trace数据库连接 + perf_conn: perf数据库连接（可为None） + app_frame: 应用帧信息字典，包含frame_ts, frame_dur, thread_id, app_pid等 + perf_sample_cache: perf_sample缓存 + perf_timestamp_field: perf_sample时间戳字段名 + tid_to_info_cache: tid到线程信息的缓存（tid可以直接用于perf_sample.thread_id） + use_process_level: 是否使用进程级统计（推荐，默认True） + - True: 直接统计应用进程所有线程的CPU指令数（简单、完整、性能好） + - False: 只计算帧对应线程的CPU指令数（向后兼容） + + Returns: + 包含CPU指令数信息的字典 + """ + if not perf_conn: + return {'wasted_instructions': 0, 'system_instructions': 0, 'has_perf_data': False} + + try: + frame_start = app_frame.get('frame_ts', 0) + frame_dur = app_frame.get('frame_dur', 0) + frame_end = frame_start + frame_dur + + # 推荐方式：进程级统计（直接统计应用进程所有线程的CPU） + if use_process_level: + app_pid = app_frame.get('app_pid') + if app_pid: + app_instructions, system_instructions = calculate_process_instructions( + perf_conn=perf_conn, + trace_conn=trace_conn, + app_pid=app_pid, + frame_start=frame_start, + frame_end=frame_end, + perf_sample_cache=perf_sample_cache, + perf_timestamp_field=perf_timestamp_field, + tid_to_info_cache=tid_to_info_cache, + ) + return { + 'wasted_instructions': app_instructions, + 'system_instructions': system_instructions, + 'has_perf_data': True, + } + logger.warning('use_process_level=True 但 app_frame 中没有 app_pid，回退到线程级统计') + + # 向后兼容：线程级统计（只计算帧对应线程的CPU） + itid = app_frame.get('thread_id') + if not itid: + return {'wasted_instructions': 0, 'system_instructions': 0, 'has_perf_data': False} + + # 从itid获取tid + tid = None + if tid_to_info_cache: + # 从缓存中查找：tid_to_info_cache的键是tid，值包含itid + # 需要反向查找：从itid找到tid + for tid_key, info in tid_to_info_cache.items(): + if info.get('itid') == itid: + tid = tid_key + break + else: + # 查询数据库：itid就是thread.id + cursor = trace_conn.cursor() + cursor.execute('SELECT tid FROM thread WHERE id = ?', (itid,)) + result = cursor.fetchone() + if result: + tid = result[0] + + if not tid: + logger.warning(f'无法从itid {itid} 获取tid') + return {'wasted_instructions': 0, 'system_instructions': 0, 'has_perf_data': False} + + # 计算线程在帧时间范围内的CPU指令数 + # tid可以直接用于perf_sample.thread_id + thread_ids = {tid} + app_instructions, system_instructions = calculate_thread_instructions( + perf_conn=perf_conn, + trace_conn=trace_conn, + thread_ids=thread_ids, + frame_start=frame_start, + frame_end=frame_end, + tid_to_info_cache=tid_to_info_cache, + perf_sample_cache=perf_sample_cache, + perf_timestamp_field=perf_timestamp_field, + ) + + wasted_instructions = app_instructions.get(tid, 0) + sys_instructions = system_instructions.get(tid, 0) + + return { + 'wasted_instructions': wasted_instructions, + 'system_instructions': sys_instructions, + 'has_perf_data': True, + } + except Exception as e: + logger.warning(f'计算CPU指令数失败: {e}') + return {'wasted_instructions': 0, 'system_instructions': 0, 'has_perf_data': False, 'error': str(e)} + + +# ============================================================================ +# 空刷帧分析公共模块类 +# ============================================================================ + + +class EmptyFrameCPUCalculator: + """空刷帧CPU计算模块 + + 统一处理EmptyFrameAnalyzer和RSSkipFrameAnalyzer的应用进程CPU计算。 + 追溯成功后，两个analyzer的app_frames格式完全一致，可以使用同一个CPU计算模块。 + """ + + def __init__(self, cache_manager): + """初始化CPU计算器 + + Args: + cache_manager: FrameCacheManager实例 + """ + self.cache_manager = cache_manager + + def calculate_frame_loads( + self, frames: pd.DataFrame | list[dict], perf_df: Optional[pd.DataFrame] = None + ) -> list[dict]: + """计算帧负载（统一接口） + + 这是统一的CPU计算接口，可以被EmptyFrameAnalyzer和RSSkipFrameAnalyzer复用。 + + Args: + frames: 帧数据（DataFrame或字典列表） + perf_df: perf样本DataFrame（可选，如果为None则从cache_manager获取） + + Returns: + list[dict]: 帧负载数据列表，每个元素包含frame_load字段 + """ + # 转换为DataFrame（如果需要） + if isinstance(frames, list): + if not frames: + return [] + frames_df = pd.DataFrame(frames) + else: + frames_df = frames + + if frames_df.empty: + return [] + + # 获取perf数据 + if perf_df is None: + perf_df = self.cache_manager.get_perf_samples() if self.cache_manager else pd.DataFrame() + + # 使用FrameLoadCalculator统一计算 + load_calculator = FrameLoadCalculator(cache_manager=self.cache_manager) + return load_calculator.calculate_all_frame_loads_fast(frames_df, perf_df) + + def extract_cpu_from_traced_results(self, traced_results: list[dict]) -> list[dict]: + """从追溯结果中提取CPU数据（RSSkipFrameAnalyzer专用） + + RSSkipFrameAnalyzer的追溯结果中，CPU数据已经由backtrack模块计算并嵌入到app_frame中。 + 这个方法提取这些数据并转换为统一的frame_load格式。 + + Args: + traced_results: 追溯结果列表，格式为： + [ + { + 'skip_frame': {...}, + 'trace_result': { + 'app_frame': { + 'cpu_waste': { + 'wasted_instructions': int, + 'system_instructions': int, + 'has_perf_data': bool + }, + ...其他帧信息 + } + } + }, + ... + ] + + Returns: + list[dict]: 统一的帧负载数据列表，每个元素包含frame_load字段 + """ + frame_loads = [] + + for result in traced_results: + if not result.get('trace_result') or not result['trace_result'].get('app_frame'): + continue + + app_frame = result['trace_result']['app_frame'] + cpu_waste = app_frame.get('cpu_waste', {}) + + if not cpu_waste.get('has_perf_data'): + continue + + # 转换格式：cpu_waste → frame_load + frame_data = { + 'frame_id': app_frame.get('frame_id'), + 'ts': app_frame.get('frame_ts'), + 'dur': app_frame.get('frame_dur'), + 'thread_id': app_frame.get('thread_id'), + 'thread_name': app_frame.get('thread_name', 'N/A'), + 'is_main_thread': app_frame.get('is_main_thread', 0), + 'pid': app_frame.get('pid'), + 'process_name': app_frame.get('process_name', 'N/A'), + 'frame_load': cpu_waste.get('wasted_instructions', 0), # 统一为frame_load + 'system_instructions': cpu_waste.get('system_instructions', 0), + } + + frame_loads.append(frame_data) + + return frame_loads + + +class EmptyFrameCallchainAnalyzer: + """空刷帧调用链分析模块 + + 统一处理EmptyFrameAnalyzer和RSSkipFrameAnalyzer的调用链分析。 + 追溯成功后，两个analyzer的app_frames格式完全一致，可以使用同一个调用链分析模块。 + """ + + def __init__(self, cache_manager): + """初始化调用链分析器 + + Args: + cache_manager: FrameCacheManager实例 + """ + self.cache_manager = cache_manager + self.load_calculator = FrameLoadCalculator(cache_manager=cache_manager) + + def analyze_callchains( + self, frame_loads: list[dict], trace_df: pd.DataFrame, perf_df: pd.DataFrame, perf_conn, top_n: int = 10 + ) -> None: + """分析调用链（统一接口） + + 只对Top N帧进行调用链分析（性能考虑）。 + 直接修改frame_loads，添加sample_callchains字段。 + + Args: + frame_loads: 帧负载数据列表（会被修改，添加sample_callchains字段） + trace_df: 原始帧数据DataFrame（用于匹配） + perf_df: perf样本DataFrame + perf_conn: perf数据库连接 + top_n: Top N帧数量（默认10） + + Returns: + None（直接修改frame_loads） + """ + if not frame_loads or trace_df.empty: + logger.warning('analyze_callchains: frame_loads为空或trace_df为空') + return + + if perf_df is None or perf_df.empty: + logger.warning('analyze_callchains: perf_df为空，尝试从cache_manager重新加载') + # 尝试从cache_manager重新加载perf_df + if self.cache_manager: + perf_df = self.cache_manager.get_perf_samples() + if perf_df is None or perf_df.empty: + logger.warning('analyze_callchains: 从cache_manager加载的perf_df仍为空，无法进行调用链分析') + # 即使perf_df为空，也要设置空的sample_callchains + for frame_data in frame_loads: + if 'sample_callchains' not in frame_data: + frame_data['sample_callchains'] = [] + return + else: + logger.warning('analyze_callchains: cache_manager为空，无法重新加载perf_df') + for frame_data in frame_loads: + if 'sample_callchains' not in frame_data: + frame_data['sample_callchains'] = [] + return + + # 对所有空刷帧按frame_load排序，取Top N（不区分主线程和后台线程） + # 但排除系统线程（系统线程不计入占比，也不应该分析callchain） + # 规则：由于frame_loads已经通过app_pids过滤，所有帧都属于应用进程 + # 因此，只排除进程名本身是系统进程的情况 + # 对于应用进程内的线程，即使名称像系统线程（如OS_VSyncThread），也不排除 + non_system_frames = [ + f + for f in frame_loads + if not is_system_thread(f.get('process_name'), None) # 只检查进程名 + ] + sorted_frames = sorted(non_system_frames, key=lambda x: x.get('frame_load', 0), reverse=True) + top_n_frames = sorted_frames[: min(top_n, TOP_FRAMES_FOR_CALLCHAIN)] + + # logger.info(f'analyze_callchains: 开始分析Top {len(top_n_frames)}帧的调用链, 总帧数={len(frame_loads)}') + # logger.info(f'analyze_callchains: Top N帧详情: {[(f.get("ts"), f.get("thread_id"), f.get("is_main_thread"), f.get("frame_load", 0), f.get("vsync")) for f in top_n_frames[:10]]}') + for _i, frame_data in enumerate(top_n_frames): + # logger.info(f'analyze_callchains: 处理第{i+1}帧: ts={frame_data.get("ts")}, tid={frame_data.get("thread_id")}, ' + # f'is_main_thread={frame_data.get("is_main_thread")}, frame_load={frame_data.get("frame_load", 0)}, vsync={frame_data.get("vsync")}') + # 找到对应的原始帧数据 + # 优先使用vsync匹配（如果有vsync，一定能匹配到frame_slice中的帧） + vsync = frame_data.get('vsync') + matching_frames = pd.DataFrame() # 初始化为空DataFrame + + # 确保vsync不是'unknown'字符串，并且trace_df中有vsync列 + if vsync is not None and vsync != 'unknown' and 'vsync' in trace_df.columns: + try: + # 确保vsync是数值类型（可能是int或str） + vsync_value = int(vsync) if not isinstance(vsync, (int, float)) else vsync + # 使用vsync匹配（最可靠的方式） + # 只匹配type=0的真实帧，不要期望帧 + # 确保trace_df中的vsync也是数值类型 + trace_vsync = pd.to_numeric(trace_df['vsync'], errors='coerce') + # 只匹配type=0的真实帧 + type_mask = ( + (trace_df['type'] == 0) + if 'type' in trace_df.columns + else pd.Series([True] * len(trace_df), index=trace_df.index) + ) + frame_mask = (trace_vsync == vsync_value) & type_mask + matching_frames = trace_df[frame_mask] + if not matching_frames.empty: + # logger.info(f'使用vsync匹配成功: vsync={vsync_value}, ts={frame_data["ts"]}, 找到{len(matching_frames)}个匹配帧（type=0）') + pass + else: + logger.warning( + f'vsync匹配失败: vsync={vsync_value}, ts={frame_data["ts"]}, trace_df中vsync范围={sorted(trace_vsync.dropna().unique().astype(int))[:10] if not trace_vsync.empty else []}, type=0的帧数={len(trace_df[type_mask]) if "type" in trace_df.columns else len(trace_df)}' + ) + except (ValueError, TypeError) as e: + logger.warning(f'vsync类型转换失败: vsync={vsync}, error={e}, 使用备用匹配方式') + matching_frames = pd.DataFrame() # 重置为空 + + # 如果vsync匹配失败，使用ts、dur、tid匹配 + if matching_frames.empty: + # 如果没有vsync或vsync匹配失败，使用ts、dur、tid匹配 + frame_mask = ( + (trace_df['ts'] == frame_data['ts']) + & (trace_df['dur'] == frame_data['dur']) + & (trace_df['tid'] == frame_data['thread_id']) + ) + matching_frames = trace_df[frame_mask] + + # 如果精确匹配失败，尝试只匹配ts和tid（dur可能有微小差异） + if matching_frames.empty: + logger.debug( + f'精确匹配失败，尝试宽松匹配: ts={frame_data["ts"]}, dur={frame_data["dur"]}, tid={frame_data.get("thread_id")}' + ) + loose_mask = (trace_df['ts'] == frame_data['ts']) & (trace_df['tid'] == frame_data['thread_id']) + matching_frames = trace_df[loose_mask] + if not matching_frames.empty: + logger.info(f'宽松匹配成功: ts={frame_data["ts"]}, 找到{len(matching_frames)}个匹配帧') + + if not matching_frames.empty: + original_frame = matching_frames.iloc[0] + # 确保original_frame转换为字典格式，包含所有必要字段 + if isinstance(original_frame, pd.Series): + frame_dict = original_frame.to_dict() + else: + frame_dict = dict(original_frame) if not isinstance(original_frame, dict) else original_frame + + # 确保有所有必要字段（analyze_single_frame需要） + if 'start_time' not in frame_dict: + frame_dict['start_time'] = frame_dict.get('ts', 0) + if 'end_time' not in frame_dict: + frame_dict['end_time'] = frame_dict.get('ts', 0) + frame_dict.get('dur', 0) + # 确保有tid字段（analyze_single_frame需要用于过滤perf样本） + if 'tid' not in frame_dict: + frame_dict['tid'] = frame_data.get('thread_id') + # 确保有其他必要字段（进程级查询需要pid） + if 'pid' not in frame_dict: + frame_dict['pid'] = frame_data.get('pid') or frame_data.get('process_id') + if 'app_pid' not in frame_dict and 'pid' in frame_dict: + frame_dict['app_pid'] = frame_dict['pid'] + if 'thread_name' not in frame_dict: + frame_dict['thread_name'] = frame_data.get('thread_name', 'unknown') + if 'process_name' not in frame_dict: + frame_dict['process_name'] = frame_data.get('process_name', 'unknown') + # 确保vsync字段被传递（frame_slice表必须包含vsync，用于日志和调试） + if 'vsync' not in frame_dict or frame_dict.get('vsync') is None: + frame_dict['vsync'] = ( + frame_data.get('vsync') or original_frame.get('vsync') + if hasattr(original_frame, 'get') + else None + ) + + # 关键：将frame_data中的_sample_details传递到frame_dict中，以便analyze_single_frame使用 + if '_sample_details' in frame_data: + frame_dict['_sample_details'] = frame_data['_sample_details'] + # logger.info(f'传递_sample_details到frame_dict: vsync={frame_dict.get("vsync")}, 样本数={len(frame_data["_sample_details"])}') + + try: + # 确保perf_df不为空（调用链分析需要） + if (perf_df is None or perf_df.empty) and self.cache_manager: + perf_df = self.cache_manager.get_perf_samples() + logger.info( + f'重新加载perf_df用于调用链分析: size={len(perf_df) if perf_df is not None and not perf_df.empty else 0}' + ) + + if perf_df is None or perf_df.empty: + logger.warning(f'perf_df仍为空，无法进行调用链分析: ts={frame_data["ts"]}') + frame_data['sample_callchains'] = [] + else: + _, sample_callchains = self.load_calculator.analyze_single_frame( + frame_dict, perf_df, perf_conn, None + ) + frame_data['sample_callchains'] = sample_callchains if sample_callchains else [] + if sample_callchains: + # 记录成功获取调用链 + # logger.info(f'帧调用链分析成功: ts={frame_data["ts"]}, tid={frame_dict.get("tid")}, ' + # f'调用链数={len(sample_callchains)}, frame_load={frame_data.get("frame_load", 0)}') + pass + else: + # 使用info级别，确保能看到日志 + logger.info( + f'帧调用链为空: ts={frame_data["ts"]}, tid={frame_dict.get("tid")}, ' + f'frame_start={frame_dict.get("start_time")}, frame_end={frame_dict.get("end_time")}, ' + f'perf_df_size={len(perf_df)}, perf_df_columns={list(perf_df.columns) if not perf_df.empty else []}, ' + f'frame_load={frame_data.get("frame_load", 0)}' + ) + except Exception as e: + logger.warning( + f'帧调用链分析失败: ts={frame_data["ts"]}, error={e}, frame_dict_keys={list(frame_dict.keys())}' + ) + logger.debug(f'异常堆栈: {traceback.format_exc()}') + frame_data['sample_callchains'] = [] + else: + logger.warning( + f'未找到匹配的原始帧: ts={frame_data["ts"]}, dur={frame_data["dur"]}, ' + f'thread_id={frame_data.get("thread_id")}, is_main_thread={frame_data.get("is_main_thread")}, ' + f'frame_load={frame_data.get("frame_load", 0)}, ' + f'trace_df中有{len(trace_df)}帧, trace_df的tid范围={sorted(trace_df["tid"].unique())[:10] if not trace_df.empty and "tid" in trace_df.columns else []}, ' + f'trace_df中该ts的帧数={len(trace_df[trace_df["ts"] == frame_data["ts"]]) if not trace_df.empty and "ts" in trace_df.columns else 0}' + ) + frame_data['sample_callchains'] = [] + + # 对于非Top N帧，设置空的调用链信息 + for frame_data in frame_loads: + if frame_data not in top_n_frames and 'sample_callchains' not in frame_data: + frame_data['sample_callchains'] = [] + + +class EmptyFrameResultBuilder: + """空刷帧结果构建模块（统一输出格式） + + 统一构建EmptyFrameAnalyzer和RSSkipFrameAnalyzer的分析结果。 + 支持RS特有的追溯统计和双重CPU统计。 + """ + + def __init__(self, cache_manager): + """初始化结果构建器 + + Args: + cache_manager: FrameCacheManager实例 + """ + self.cache_manager = cache_manager + + def build_result( + self, + frame_loads: list[dict], + total_load: int = 0, + detection_stats: Optional[dict] = None, + deduplicated_empty_frame_load: Optional[int] = None, + deduplicated_main_thread_load: Optional[int] = None, + deduplicated_background_thread_load: Optional[int] = None, + deduplicated_thread_loads: Optional[dict[int, int]] = None, + tid_to_info: Optional[dict] = None, + ) -> dict: + """构建统一的分析结果 + + Args: + frame_loads: 帧负载数据列表（格式完全一致，无论来自哪个analyzer） + total_load: 总负载（用于计算占比） + detection_stats: 检测统计信息（可选） + - EmptyFrameAnalyzer：可以为空 + - RSSkipFrameAnalyzer：包含追溯统计和RS进程CPU统计 + + Returns: + dict: 统一格式的分析结果 + """ + if not frame_loads: + return self._build_empty_result(detection_stats) + + # 获取第一帧时间戳 + first_frame_time = self.cache_manager.get_first_frame_timestamp() if self.cache_manager else 0 + + # === 通用处理：统一显示全局Top 10帧（与sample_callchains分析保持一致）=== + result_df = pd.DataFrame(frame_loads) + + # 排除系统线程，按frame_load排序，取全局Top 10（不区分主线程和后台线程） + # 规则：由于frame_loads已经通过app_pids过滤，所有帧都属于应用进程 + # 因此，只排除进程名本身是系统进程的情况 + # 对于应用进程内的线程，即使名称像系统线程（如OS_VSyncThread），也不排除 + non_system_frames = result_df[ + ~result_df.apply( + lambda row: is_system_thread( + row.get('process_name'), + None, # 只检查进程名，不检查线程名 + ), + axis=1, + ) + ] + + # 全局Top 10帧（与sample_callchains分析的帧保持一致） + # 不再区分主线程和后台线程，统一为一个列表 + top_frames_global = non_system_frames.sort_values('frame_load', ascending=False).head(TOP_FRAMES_FOR_CALLCHAIN) + + # 处理时间戳（统一处理，不区分主线程和后台线程） + processed_all = self._process_frame_timestamps(top_frames_global, first_frame_time) + + # === 通用统计 === + # 如果提供了去重后的 empty_frame_load，使用它（去除重叠区域） + # 否则使用累加的方式（包含重叠区域） + if deduplicated_empty_frame_load is not None: + empty_frame_load = int(deduplicated_empty_frame_load) + else: + empty_frame_load = int(sum(f['frame_load'] for f in frame_loads)) + + # 计算主线程负载：使用去重后的值（如果提供） + if deduplicated_main_thread_load is not None: + main_thread_load = int(deduplicated_main_thread_load) + else: + # 回退到未去重的方式（从_sample_details累加） + main_thread_load = 0 + + # 构建tid到线程信息的映射（用于判断线程是否为主线程） + if tid_to_info is None: + tid_to_info = {} + if self.cache_manager and self.cache_manager.trace_conn: + try: + trace_cursor = self.cache_manager.trace_conn.cursor() + app_pids = self.cache_manager.app_pids or [] + if app_pids: + placeholders = ','.join('?' * len(app_pids)) + trace_cursor.execute( + f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid IN ({placeholders}) + """, + app_pids, + ) + + for tid, thread_name, process_name in trace_cursor.fetchall(): + tid_to_info[tid] = { + 'thread_name': thread_name, + 'process_name': process_name, + 'is_main_thread': 1 if thread_name == process_name else 0, + } + except Exception: + # logging.warning(f'查询线程信息失败: {e}') + pass + + # 从所有空刷帧的_sample_details中提取主线程的CPU + for f in frame_loads: + # 排除系统进程 + if is_system_thread(f.get('process_name'), None): + continue + + # 从_sample_details中提取该帧的主线程CPU + sample_details = f.get('_sample_details', []) + if sample_details: + # 累加所有主线程的event_count（不区分帧是主线程还是后台线程的帧） + for sample in sample_details: + tid = sample.get('thread_id') + if tid is None: + continue + + # 判断该线程是否为主线程 + thread_info = tid_to_info.get(tid, {}) + thread_is_main = thread_info.get('is_main_thread', 0) + + # 只累加主线程的CPU（无论这个帧是主线程还是后台线程的帧） + if thread_is_main == 1: + event_count = sample.get('event_count', 0) + main_thread_load += event_count + + # 计算后台线程负载：使用去重后的值（如果提供） + if deduplicated_background_thread_load is not None: + background_thread_load = int(deduplicated_background_thread_load) + else: + # 回退到未去重的方式（从_sample_details累加） + background_thread_load = 0 + + # 构建tid到线程信息的映射（用于判断线程是否为主线程） + if tid_to_info is None: + tid_to_info = {} + if self.cache_manager and self.cache_manager.trace_conn: + try: + trace_cursor = self.cache_manager.trace_conn.cursor() + app_pids = self.cache_manager.app_pids or [] + if app_pids: + placeholders = ','.join('?' * len(app_pids)) + trace_cursor.execute( + f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid IN ({placeholders}) + """, + app_pids, + ) + + for tid, thread_name, process_name in trace_cursor.fetchall(): + tid_to_info[tid] = { + 'thread_name': thread_name, + 'process_name': process_name, + 'is_main_thread': 1 if thread_name == process_name else 0, + } + except Exception: + # logging.warning(f'查询线程信息失败: {e}') + pass + + # 从所有空刷帧的_sample_details中提取后台线程的CPU + for f in frame_loads: + # 排除系统进程 + if is_system_thread(f.get('process_name'), None): + continue + + # 从_sample_details中提取该帧的后台线程CPU + sample_details = f.get('_sample_details', []) + if sample_details: + # 累加所有后台线程的event_count（不区分帧是主线程还是后台线程的帧） + for sample in sample_details: + tid = sample.get('thread_id') + if tid is None: + continue + + # 判断该线程是否为主线程 + thread_info = tid_to_info.get(tid, {}) + thread_is_main = thread_info.get('is_main_thread', 0) + + # 只累加后台线程的CPU（无论这个帧是主线程还是后台线程的帧） + if thread_is_main == 0: + event_count = sample.get('event_count', 0) + background_thread_load += event_count + + # === 阶段3：统计整个trace浪费指令数占比 === + # empty_frame_percentage = empty_frame_load / total_load * 100 + # 表示空刷帧的CPU浪费占整个trace总CPU的百分比 + if total_load > 0: + empty_frame_percentage = (empty_frame_load / total_load) * 100 + main_thread_percentage = (main_thread_load / total_load) * 100 + background_thread_percentage = (background_thread_load / total_load) * 100 + else: + empty_frame_percentage = 0.0 + main_thread_percentage = 0.0 + background_thread_percentage = 0.0 + + # === 线程级别统计 === + # 使用去重后的线程负载（如果提供），否则回退到未去重的方式 + if deduplicated_thread_loads is not None and tid_to_info is not None: + # 从去重后的线程负载构建 thread_statistics + thread_loads = {} + for tid, load in deduplicated_thread_loads.items(): + thread_info = tid_to_info.get(tid, {}) + thread_name = thread_info.get('thread_name', f'thread_{tid}') + process_name = thread_info.get('process_name', 'N/A') + + # 排除系统进程 + if is_system_thread(process_name, None): + continue + + thread_key = (tid, thread_name, process_name) + thread_loads[thread_key] = { + 'thread_id': tid, + 'thread_name': thread_name, + 'process_name': process_name, + 'total_load': load, + 'frame_count': 0, # 去重后无法直接统计帧数，需要从frame_loads统计 + } + + # 统计涉及该线程的帧数（从frame_loads中统计） + for f in frame_loads: + sample_details = f.get('_sample_details', []) + if sample_details: + frame_tids = set(sample.get('thread_id') for sample in sample_details if sample.get('thread_id')) + for tid in frame_tids: + if tid in deduplicated_thread_loads: + thread_info = tid_to_info.get(tid, {}) + thread_name = thread_info.get('thread_name', f'thread_{tid}') + process_name = thread_info.get('process_name', 'N/A') + + if is_system_thread(process_name, None): + continue + + thread_key = (tid, thread_name, process_name) + if thread_key in thread_loads: + thread_loads[thread_key]['frame_count'] += 1 + else: + # 回退到未去重的方式（从_sample_details累加） + thread_loads = {} + + # 构建tid到线程信息的映射（用于从_sample_details中获取线程信息） + if tid_to_info is None: + tid_to_info = {} + if self.cache_manager and self.cache_manager.trace_conn: + try: + trace_cursor = self.cache_manager.trace_conn.cursor() + # 查询所有应用进程的线程信息 + app_pids = self.cache_manager.app_pids or [] + if app_pids: + placeholders = ','.join('?' * len(app_pids)) + trace_cursor.execute( + f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid IN ({placeholders}) + """, + app_pids, + ) + + for tid, thread_name, process_name in trace_cursor.fetchall(): + tid_to_info[tid] = {'thread_name': thread_name, 'process_name': process_name} + except Exception: + # logging.warning(f'查询线程信息失败: {e}') + pass + + # 从_sample_details中统计所有线程的负载 + # 遍历所有空刷帧，从每个帧的_sample_details中提取线程负载 + for f in frame_loads: + sample_details = f.get('_sample_details', []) + if not sample_details: + continue + + # 按thread_id分组，累加event_count + thread_event_counts = {} + for sample in sample_details: + tid = sample.get('thread_id') + if tid is None: + continue + + event_count = sample.get('event_count', 0) + if tid not in thread_event_counts: + thread_event_counts[tid] = 0 + thread_event_counts[tid] += event_count + + # 将每个线程的负载添加到thread_loads中 + for tid, event_count in thread_event_counts.items(): + # 从tid_to_info获取线程信息 + thread_info = tid_to_info.get(tid, {}) + thread_name = thread_info.get('thread_name', f'thread_{tid}') + process_name = thread_info.get('process_name', 'N/A') + + # 排除系统进程 + if is_system_thread(process_name, None): + continue + + # 使用(thread_id, thread_name, process_name)作为唯一标识 + thread_key = (tid, thread_name, process_name) + if thread_key not in thread_loads: + thread_loads[thread_key] = { + 'thread_id': tid, + 'thread_name': thread_name, + 'process_name': process_name, + 'total_load': 0, + 'frame_count': 0, + } + # 累加该线程在这个帧中的负载 + thread_loads[thread_key]['total_load'] += event_count + # 统计涉及该线程的帧数（每个帧只统计一次） + thread_loads[thread_key]['frame_count'] += 1 + + # 找出负载最多的线程 + top_threads = sorted(thread_loads.values(), key=lambda x: x['total_load'], reverse=True)[:10] # Top 10线程 + + # 计算每个线程的占比 + for thread_info in top_threads: + if total_load > 0: + thread_info['percentage'] = (thread_info['total_load'] / total_load) * 100 + else: + thread_info['percentage'] = 0.0 + + # 严重程度评估 + severity_level, severity_description = self._assess_severity(empty_frame_percentage, detection_stats) + + # === 构建结果（统一格式）=== + result = { + 'status': 'success', + 'summary': { + # 通用字段 + 'total_load': int(total_load), + 'empty_frame_load': int(empty_frame_load), + 'empty_frame_percentage': float(empty_frame_percentage), + 'total_empty_frames': int(len(frame_loads)), + 'empty_frames_with_load': int(len([f for f in frame_loads if f.get('frame_load', 0) > 0])), + 'main_thread_load': int(main_thread_load), + 'main_thread_percentage': float(main_thread_percentage), + 'background_thread_load': int(background_thread_load), + 'background_thread_percentage': float(background_thread_percentage), + 'severity_level': severity_level, + 'severity_description': severity_description, + }, + 'top_frames': processed_all, # 统一列表，不再区分主线程和后台线程 + 'thread_statistics': { + 'top_threads': [ + { + 'thread_id': t.get('thread_id'), + 'tid': t.get('thread_id'), # tid与thread_id相同（frame_loads中的thread_id就是tid） + 'thread_name': t.get('thread_name'), + 'process_name': t.get('process_name'), + 'total_load': int(t.get('total_load', 0)), + 'percentage': float(t.get('percentage', 0.0)), + 'frame_count': int(t.get('frame_count', 0)), + } + for t in top_threads + ] + }, + } + + # === 三个检测器的原始检测结果（在合并去重之前，不处理 overlap）=== + # 无论 detection_stats 是否存在，都设置这三个字段（确保总是存在） + if detection_stats: + # 1. flag=2 检测器的原始检测数量 + direct_count = detection_stats.get('direct_detected_count', 0) + result['summary']['direct_detected_count'] = direct_count + + # 2. RS skip 检测器的原始检测数量 + rs_count = detection_stats.get('rs_detected_count', 0) + result['summary']['rs_detected_count'] = rs_count + + # 3. 框架特定检测器的原始检测数量 + framework_counts = detection_stats.get('framework_detected_counts') + framework_counts = framework_counts if framework_counts is not None else {} + result['summary']['framework_detection_counts'] = framework_counts + else: + # 如果没有 detection_stats，设置默认值 + result['summary']['direct_detected_count'] = 0 + result['summary']['rs_detected_count'] = 0 + result['summary']['framework_detection_counts'] = {} + + # === RS特有字段（如果有）=== + if detection_stats: + # 追溯统计 + if 'total_skip_frames' in detection_stats: + result['summary'].update( + { + 'total_skip_frames': detection_stats.get('total_skip_frames', 0), + 'total_skip_events': detection_stats.get('total_skip_events', 0), + 'trace_accuracy': detection_stats.get('trace_accuracy', 0.0), + 'traced_success_count': detection_stats.get('traced_success_count', 0), + 'rs_api_success': detection_stats.get('rs_api_success', 0), + 'nativewindow_success': detection_stats.get('nativewindow_success', 0), + 'failed': detection_stats.get('failed', 0), + } + ) + + # 追溯成功率低的警告 + if detection_stats.get('trace_accuracy', 100.0) < 50.0: + trace_warning = ( + f'警告：RS Skip帧追溯成功率较低({detection_stats["trace_accuracy"]:.1f}%)，' + f'可能导致CPU浪费统计不准确。' + ) + result['summary']['trace_warning'] = trace_warning + + # RS进程CPU统计（新增） + if 'rs_skip_cpu' in detection_stats: + result['summary'].update( + { + 'rs_skip_cpu': detection_stats.get('rs_skip_cpu', 0), + 'rs_skip_percentage': detection_stats.get('rs_skip_percentage', 0.0), + 'app_empty_cpu': empty_frame_load, # 应用进程CPU = empty_frame_load + 'app_empty_percentage': empty_frame_percentage, + 'total_wasted_cpu': detection_stats.get('total_wasted_cpu', 0), + } + ) + + # 占比超过100%的警告 + if empty_frame_percentage > 100.0: + percentage_warning = ( + f'注意：空刷帧占比超过100% ({empty_frame_percentage:.2f}%)，' + f'这是因为时间窗口扩展（±1ms）导致多个帧的CPU计算存在重叠。' + ) + result['summary']['percentage_warning'] = percentage_warning + + return result + + def _build_empty_result(self, detection_stats: Optional[dict] = None) -> dict: + """构建空结果（当没有空刷帧时） + + Args: + detection_stats: 检测统计信息（可选） + + Returns: + dict: 空结果字典 + """ + result = { + 'status': 'success', + 'summary': { + 'total_load': 0, + 'empty_frame_load': 0, + 'empty_frame_percentage': 0.0, + 'total_empty_frames': 0, + 'empty_frames_with_load': 0, + 'background_thread_load': 0, + 'background_thread_percentage': 0.0, + 'severity_level': 'normal', + 'severity_description': '正常：未检测到空刷帧。', + }, + 'top_frames': [], # 统一列表，不再区分主线程和后台线程 + 'thread_statistics': {'top_threads': []}, + } + + # === 三个检测器的原始检测结果（在合并去重之前，不处理 overlap）=== + # 同时添加到 summary 和最外层（双重保障） + if detection_stats: + direct_count = detection_stats.get('direct_detected_count', 0) + rs_count = detection_stats.get('rs_detected_count', 0) + framework_counts = detection_stats.get('framework_detected_counts', {}) + framework_counts = framework_counts if framework_counts is not None else {} + + result['summary']['direct_detected_count'] = direct_count + result['summary']['rs_detected_count'] = rs_count + result['summary']['framework_detection_counts'] = framework_counts + else: + # 如果没有 detection_stats，设置默认值 + result['summary']['direct_detected_count'] = 0 + result['summary']['rs_detected_count'] = 0 + result['summary']['framework_detection_counts'] = {} + + # RS特有字段 + if detection_stats and 'total_skip_frames' in detection_stats: + result['summary'].update( + { + 'total_skip_frames': 0, + 'total_skip_events': 0, + 'traced_success_count': 0, + 'trace_accuracy': 0.0, + 'rs_api_success': 0, + 'nativewindow_success': 0, + 'failed': 0, + } + ) + + return result + + def _assess_severity( + self, empty_frame_percentage: float, detection_stats: Optional[dict] = None + ) -> tuple[str, str]: + """评估严重程度 + + Args: + empty_frame_percentage: 空刷帧CPU占比 + detection_stats: 检测统计信息（可选） + + Returns: + (severity_level, severity_description) + """ + # 如果是RS checker且追溯成功率低，优先标记为critical + if detection_stats and 'trace_accuracy' in detection_stats: + trace_accuracy = detection_stats.get('trace_accuracy', 100.0) + if trace_accuracy < 50.0: + return ( + 'critical', + f'严重：RS Skip帧追溯成功率仅{trace_accuracy:.1f}%，数据质量差，无法准确评估CPU浪费。', + ) + + # 根据占比判断严重程度 + if empty_frame_percentage < 3.0: + return ('normal', '正常：空刷帧CPU占比小于3%，属于正常范围。') + if empty_frame_percentage < 10.0: + return ('moderate', '较为严重：空刷帧CPU占比在3%-10%之间，建议关注并优化。') + if empty_frame_percentage <= 100.0: + return ('severe', '严重：空刷帧CPU占比超过10%，需要优先优化。') + # > 100% + return ( + 'extreme', + f'极端异常：空刷帧CPU占比超过100% ({empty_frame_percentage:.2f}%)。' + f'这是因为时间窗口扩展（±1ms）导致多个帧的CPU计算存在重叠。', + ) + + def _process_frame_timestamps(self, frames_df: pd.DataFrame, first_frame_time: int) -> list: + """处理帧时间戳，转换为相对时间 + + Args: + frames_df: 帧数据DataFrame + first_frame_time: 第一帧时间戳 + + Returns: + list: 处理后的帧数据列表 + """ + processed_frames = [] + for _, frame in frames_df.iterrows(): + frame_dict = clean_frame_data(frame.to_dict()) + # 转换时间戳为相对时间 + frame_dict['ts'] = FrameTimeUtils.convert_to_relative_nanoseconds(frame.get('ts', 0), first_frame_time) + processed_frames.append(frame_dict) + + return processed_frames diff --git a/perf_testing/hapray/core/common/frame/frame_empty_framework_specific.py b/perf_testing/hapray/core/common/frame/frame_empty_framework_specific.py new file mode 100644 index 00000000..d1cea649 --- /dev/null +++ b/perf_testing/hapray/core/common/frame/frame_empty_framework_specific.py @@ -0,0 +1,602 @@ +""" +Copyright (c) 2025 Huawei Device Co., Ltd. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import re +import sqlite3 +import time +from typing import Optional + +import pandas as pd + +"""框架特定空刷检测模块 + +本模块包含所有框架特定的空刷检测逻辑，包括： +1. Flutter 空刷检测（基于 SetPresentInfo frame_damage） +2. RN 空刷检测（待实现） + +设计原则： +- Self-contained：所有框架特定的检测逻辑都在本文件中 +- 统一接口：所有检测器返回统一格式的 DataFrame +- 易于扩展：新增框架只需添加新的检测器类 +- 独立维护：不影响其他模块的代码 +""" + +logger = logging.getLogger(__name__) + + +class FrameworkSpecificDetector: + """框架特定空刷检测器基类 + + 所有框架特定的检测器都应该继承此类，实现统一的接口。 + """ + + def __init__(self, trace_conn: sqlite3.Connection, app_pids: list[int]): + """ + 初始化框架检测器 + + Args: + trace_conn: trace 数据库连接 + app_pids: 应用进程 ID 列表 + """ + self.trace_conn = trace_conn + self.app_pids = app_pids + + def detect_empty_frames(self) -> pd.DataFrame: + """ + 检测空刷帧（抽象方法，子类必须实现） + + Returns: + DataFrame: 统一格式的空刷帧数据，包含以下字段： + - ts: 时间戳 + - dur: 持续时间 + - tid: 线程 ID + - itid: 内部线程 ID + - pid: 进程 ID + - ipid: 内部进程 ID + - vsync: VSync 标识（如果匹配到 frame_slice） + - flag: 帧标志（默认 2，表示空刷） + - type: 帧类型（0=实际帧） + - thread_name: 线程名 + - process_name: 进程名 + - is_main_thread: 是否主线程（0/1） + - detection_method: 检测方法（'framework_specific'） + - framework_type: 框架类型（'flutter'/'rn'） + - frame_damage: 原始 frame_damage 信息（Flutter 特有） + - callstack_id: callstack 事件 ID + - beginframe_id: BeginFrame 事件 ID（Flutter 特有） + """ + raise NotImplementedError('子类必须实现 detect_empty_frames 方法') + + def _match_frame_slice( + self, + itid: int, + ts: int, + time_tolerance_ns: int = 5000000, # 默认 ±5ms 容差 + ) -> Optional[dict]: + """ + 通过 callstack 事件匹配 frame_slice 表中的帧，获取 vsync 等信息 + + Args: + itid: 内部线程 ID（对应 callstack.callid） + ts: 时间戳（对应 callstack.ts） + time_tolerance_ns: 时间容差（纳秒），默认 ±5ms + + Returns: + dict: 匹配到的 frame_slice 信息，包含 vsync, flag, type, ts, dur 等 + 如果未匹配到，返回 None + """ + try: + cursor = self.trace_conn.cursor() + cursor.execute( + """ + SELECT fs.ts, fs.dur, fs.vsync, fs.flag, fs.type, fs.itid, fs.ipid, + t.tid, t.name as thread_name, p.pid, p.name as process_name + FROM frame_slice fs + INNER JOIN thread t ON fs.itid = t.id + INNER JOIN process p ON fs.ipid = p.ipid + WHERE fs.itid = ? + AND fs.ts BETWEEN ? - ? AND ? + ? + AND fs.type = 0 + ORDER BY ABS(fs.ts - ?) + LIMIT 1 + """, + (itid, ts, time_tolerance_ns, ts, time_tolerance_ns, ts), + ) + + row = cursor.fetchone() + if row: + return { + 'ts': row[0], + 'dur': row[1], + 'vsync': row[2], + 'flag': row[3], + 'type': row[4], + 'itid': row[5], + 'ipid': row[6], + 'tid': row[7], + 'thread_name': row[8], + 'pid': row[9], + 'process_name': row[10], + } + except Exception as e: + logger.warning(f'匹配 frame_slice 失败: itid={itid}, ts={ts}, error={e}') + + return None + + +class FlutterEmptyFrameDetector(FrameworkSpecificDetector): + """Flutter 空刷检测器 + + 基于 SetPresentInfo 事件的 frame_damage 检测空刷。 + 核心逻辑： + 1. 查找 SetPresentInfo 事件（在 1.raster 线程） + 2. 提取 frame_damage，判断是否为空刷 + 3. 匹配 frame_slice 获取 vsync 等信息 + 4. 追溯到最近的 BeginFrame（在 1.ui 线程） + 5. 转换为统一格式 + """ + + def __init__(self, trace_conn: sqlite3.Connection, app_pids: list[int]): + super().__init__(trace_conn, app_pids) + self.framework_type = 'flutter' + self.total_detected_count = 0 # 记录检测到的空刷帧数量（frame_damage为空） + + def detect_empty_frames(self) -> pd.DataFrame: + """检测 Flutter 空刷帧""" + try: + # 1. 查找 Flutter 应用进程和线程 + flutter_info = self._find_flutter_process_and_threads() + if not flutter_info: + logger.info('未找到 Flutter 应用进程或线程') + return pd.DataFrame() + + app_ipid, app_pid, app_name, raster_itid, raster_tid, ui_itid, ui_tid = flutter_info + + # 2. 查找 SetPresentInfo 事件 + setpresent_events = self._get_setpresent_events(raster_itid) + total_setpresent_count = len(setpresent_events) # 所有SetPresentInfo事件数量 + logger.info(f'找到 {total_setpresent_count} 个 SetPresentInfo 事件') + if not setpresent_events: + logger.info('未找到 SetPresentInfo 事件') + # 即使没有事件，也记录检测器运行了（检测到0个） + self.total_detected_count = 0 + return pd.DataFrame() + + # 3. 识别空刷并构建帧数据 + empty_frames = [] + for event_item in setpresent_events: + # 兼容不同的返回格式 + if len(event_item) == 5: + event_id, ts, dur, name, frame_damage = event_item + elif len(event_item) == 6: + event_id, ts, dur, name, frame_damage, is_empty = event_item + else: + logger.warning(f'意外的 SetPresentInfo 事件格式: {event_item}') + continue + # 提取 frame_damage 并判断是否为空刷 + # 如果 frame_damage 不为空（有脏区），跳过，不算作检测到一帧 + if not self._is_empty_frame_damage(frame_damage): + continue + + # 匹配 frame_slice 获取 vsync 等信息 + matched_frame = self._match_frame_slice(raster_itid, ts) + + # 追溯到最近的 BeginFrame + beginframe = self._find_nearest_beginframe(ui_itid, ts) + + # 构建统一格式的帧数据 + frame_data = self._build_frame_data( + event_id, + ts, + dur, + name, + frame_damage, + raster_itid, + raster_tid, + ui_itid, + ui_tid, + app_ipid, + app_pid, + app_name, + matched_frame, + beginframe, + ) + + if frame_data: + empty_frames.append(frame_data) + + # 统计检测到的空刷帧数量（frame_damage为空的事件） + self.total_detected_count = len(empty_frames) + if total_setpresent_count > 0: + logger.info( + f'其中 {self.total_detected_count} 个是空刷帧（frame_damage为空），{total_setpresent_count - self.total_detected_count} 个有脏区（非空刷，已跳过）' + ) + + if not empty_frames: + logger.info('Flutter 检测器：未检测到空刷帧（在合并去重之前）') + return pd.DataFrame() + + return pd.DataFrame(empty_frames) + # 注意：这里不输出日志，因为 detect_framework_specific_empty_frames 中已经会输出 + + except Exception as e: + logger.error(f'Flutter 空刷检测失败: {e}', exc_info=True) + return pd.DataFrame() + + def _find_flutter_process_and_threads(self) -> Optional[tuple]: + """ + 查找 Flutter 应用进程和关键线程（1.raster 和 1.ui） + + Returns: + tuple: (app_ipid, app_pid, app_name, raster_itid, raster_tid, ui_itid, ui_tid) + 如果未找到，返回 None + """ + cursor = self.trace_conn.cursor() + + # 查找 Flutter 应用进程（匹配 %_with_animation% 或 app_pids） + if self.app_pids: + cursor.execute( + """ + SELECT p.ipid, p.pid, p.name + FROM process p + WHERE p.pid IN ({}) + AND (p.name LIKE '%_with_animation%' OR p.name LIKE '%.flutter%') + LIMIT 1 + """.format(','.join('?' * len(self.app_pids))), + self.app_pids, + ) + else: + cursor.execute(""" + SELECT p.ipid, p.pid, p.name + FROM process p + WHERE p.name LIKE '%_with_animation%' + LIMIT 1 + """) + + process_row = cursor.fetchone() + if not process_row: + return None + + app_ipid, app_pid, app_name = process_row + + # 查找 1.raster 线程 + cursor.execute( + """ + SELECT t.id, t.tid + FROM thread t + WHERE t.name LIKE '%1.raster%' + AND t.ipid = ? + LIMIT 1 + """, + (app_ipid,), + ) + + raster_row = cursor.fetchone() + if not raster_row: + return None + + raster_itid, raster_tid = raster_row + + # 查找 1.ui 线程 + cursor.execute( + """ + SELECT t.id, t.tid + FROM thread t + WHERE t.name LIKE '%1.ui%' + AND t.ipid = ? + LIMIT 1 + """, + (app_ipid,), + ) + + ui_row = cursor.fetchone() + if not ui_row: + return None + + ui_itid, ui_tid = ui_row + + return (app_ipid, app_pid, app_name, raster_itid, raster_tid, ui_itid, ui_tid) + + def _get_setpresent_events(self, raster_itid: int) -> list[tuple]: + """ + 查找所有 SetPresentInfo 事件（包含 frame_damage） + + Args: + raster_itid: 1.raster 线程的内部 ID + + Returns: + List[Tuple]: [(event_id, ts, dur, name, frame_damage), ...] + """ + cursor = self.trace_conn.cursor() + cursor.execute( + """ + SELECT id, ts, dur, name + FROM callstack + WHERE callid = ? + AND name LIKE '%SetPresentInfo%' + AND name LIKE '%frame_damage%' + ORDER BY ts + """, + (raster_itid,), + ) + + results = [] + for event_id, ts, dur, name in cursor.fetchall(): + frame_damage = self._extract_frame_damage(name) + if frame_damage is not None: + results.append((event_id, ts, dur, name, frame_damage)) + + return results + + def _extract_frame_damage(self, name: str) -> Optional[str]: + """ + 从事件名称中提取 frame_damage 信息 + + Args: + name: 事件名称，如 "SetPresentInfo frame_damage:<0,0,0,0>" + + Returns: + str: frame_damage 字符串，如 "<0,0,0,0>"，如果未找到返回 None + """ + match = re.search(r'frame_damage:<([^>]+)>', name) + if match: + return f'<{match.group(1)}>' + return None + + def _is_empty_frame_damage(self, frame_damage: str) -> bool: + """ + 判断 frame_damage 是否表示空刷 + + Args: + frame_damage: frame_damage 字符串，如 "<0,0,0,0>" 或 "" + + Returns: + bool: True 表示空刷，False 表示非空刷 + """ + if not frame_damage: + return False + + # 提取 x, y, w, h + match = re.search(r'<(\d+),(\d+),(\d+),(\d+)>', frame_damage) + if not match: + return False + + x, y, w, h = map(int, match.groups()) + + # 空刷判断：w == 0 或 h == 0 或 (x == 0 and y == 0 and w == 0 and h == 0) + return w == 0 or h == 0 or (x == 0 and y == 0 and w == 0 and h == 0) + + def _find_nearest_beginframe(self, ui_itid: int, setpresent_ts: int) -> Optional[dict]: + """ + 查找最近的 BeginFrame 事件（在 1.ui 线程） + + Args: + ui_itid: 1.ui 线程的内部 ID + setpresent_ts: SetPresentInfo 的时间戳 + + Returns: + dict: BeginFrame 信息，包含 id, ts, dur, name + 如果未找到，返回 None + """ + try: + cursor = self.trace_conn.cursor() + cursor.execute( + """ + SELECT id, ts, dur, name + FROM callstack + WHERE callid = ? + AND name LIKE '%BeginFrame%' + AND ts <= ? + ORDER BY ts DESC + LIMIT 1 + """, + (ui_itid, setpresent_ts), + ) + + row = cursor.fetchone() + if row: + return {'id': row[0], 'ts': row[1], 'dur': row[2], 'name': row[3]} + except Exception as e: + logger.warning(f'查找 BeginFrame 失败: ui_itid={ui_itid}, ts={setpresent_ts}, error={e}') + + return None + + def _build_frame_data( + self, + setpresent_id: int, + setpresent_ts: int, + setpresent_dur: int, + setpresent_name: str, + frame_damage: str, + raster_itid: int, + raster_tid: int, + ui_itid: int, + ui_tid: int, + app_ipid: int, + app_pid: int, + app_name: str, + matched_frame: Optional[dict], + beginframe: Optional[dict], + ) -> Optional[dict]: + """ + 构建统一格式的帧数据 + + Args: + setpresent_id: SetPresentInfo 事件 ID + setpresent_ts: SetPresentInfo 时间戳 + setpresent_dur: SetPresentInfo 持续时间 + setpresent_name: SetPresentInfo 事件名称 + frame_damage: frame_damage 字符串 + raster_itid: 1.raster 线程的内部 ID + raster_tid: 1.raster 线程的系统 ID + ui_itid: 1.ui 线程的内部 ID + ui_tid: 1.ui 线程的系统 ID + app_ipid: 应用进程的内部 ID + app_pid: 应用进程的系统 ID + app_name: 应用进程名称 + matched_frame: 匹配到的 frame_slice 信息（可选） + beginframe: BeginFrame 信息（可选） + + Returns: + dict: 统一格式的帧数据 + """ + # 确定时间戳和持续时间 + if matched_frame: + # 优先使用匹配到的 frame_slice 的时间 + ts = matched_frame['ts'] + dur = matched_frame['dur'] + vsync = matched_frame['vsync'] + flag = matched_frame.get('flag', 2) + elif beginframe: + # 使用 BeginFrame 到 SetPresentInfo 的时间区间 + ts = beginframe['ts'] + dur = setpresent_ts + setpresent_dur - beginframe['ts'] + vsync = None + flag = 2 + else: + # 如果没有 BeginFrame，使用 SetPresentInfo 的时间 + ts = setpresent_ts + dur = setpresent_dur + vsync = None + flag = 2 + + # 判断是否主线程（线程名 == 进程名） + is_main_thread = 1 if app_name in {'1.ui', '1.raster'} else 0 + + return { + 'ts': ts, + 'dur': dur, + 'tid': raster_tid, # 使用 1.raster 线程的 tid + 'itid': raster_itid, # 使用 1.raster 线程的 itid + 'pid': app_pid, + 'ipid': app_ipid, + 'vsync': vsync, + 'flag': flag, + 'type': 0, # 实际帧 + 'thread_name': '1.raster', + 'process_name': app_name, + 'is_main_thread': is_main_thread, + 'callstack_id': setpresent_id, + 'detection_method': 'framework_specific', + 'framework_type': self.framework_type, + 'frame_damage': frame_damage, + 'beginframe_id': beginframe['id'] if beginframe else None, + } + + +class RNEmptyFrameDetector(FrameworkSpecificDetector): + """RN 空刷检测器（待实现） + + RN 的空刷检测逻辑需要根据实际数据验证后实现。 + 可能的检测方法： + 1. 检测 RNOH_JS 线程的特定事件 + 2. 检测渲染相关的空刷标志 + 3. 其他框架特定的检测方法 + """ + + def __init__(self, trace_conn: sqlite3.Connection, app_pids: list[int]): + super().__init__(trace_conn, app_pids) + self.framework_type = 'rn' + + def detect_empty_frames(self) -> pd.DataFrame: + """检测 RN 空刷帧（待实现）""" + # TODO: 实现 RN 空刷检测逻辑 + logger.warning('RN 空刷检测尚未实现') + return pd.DataFrame() + + +# ==================== 统一入口函数 ==================== + + +def detect_framework_specific_empty_frames( + trace_conn: sqlite3.Connection, app_pids: list[int], framework_types: list[str] = None, timing_stats: dict = None +) -> pd.DataFrame: + """ + 检测框架特定的空刷帧（统一入口函数） + + 本函数是 EmptyFrameAnalyzer 调用框架检测的唯一入口。 + 所有框架特定的检测逻辑都封装在本模块中，便于维护和扩展。 + + Args: + trace_conn: trace 数据库连接 + app_pids: 应用进程 ID 列表 + framework_types: 要检测的框架类型列表，如 ['flutter'] + 如果为 None，则自动检测所有支持的框架 + timing_stats: 耗时统计字典（可选，用于记录检测耗时） + + Returns: + DataFrame: 统一格式的空刷帧数据，包含所有检测到的框架特定空刷帧 + """ + if framework_types is None: + # 默认检测所有支持的框架 + framework_types = ['flutter'] + + all_frames = [] + + for framework_type in framework_types: + try: + if timing_stats: + start_time = time.time() + + detector = None + if framework_type == 'flutter': + detector = FlutterEmptyFrameDetector(trace_conn, app_pids) + elif framework_type == 'rn': + detector = RNEmptyFrameDetector(trace_conn, app_pids) + else: + logger.warning(f'不支持的框架类型: {framework_type}') + continue + + if detector: + frames_df = detector.detect_empty_frames() + empty_frame_count = len(frames_df) # 空刷帧数量（frame_damage为空） + + # 获取检测到的空刷帧数量（只统计frame_damage为空的事件） + # 有脏区的事件不算作检测到一帧 + if hasattr(detector, 'total_detected_count'): + detected_count = detector.total_detected_count + else: + # 兼容旧版本：如果没有total_detected_count，使用空刷帧数量 + detected_count = empty_frame_count + + # 输出检测结果 + if detected_count > 0: + logger.info( + f'{framework_type} 框架检测器：检测到 {detected_count} 个空刷帧（frame_damage为空，在合并去重之前，可能与 flag=2 或 RS skip 重叠）' + ) + else: + logger.info(f'{framework_type} 框架检测器：检测到 0 个空刷帧') + + if not frames_df.empty: + all_frames.append(frames_df) + + # 记录到 timing_stats 中，方便后续统计 + # detected_count 只记录空刷帧数量（frame_damage为空的事件） + # 有脏区的事件不算作检测到一帧 + if timing_stats: + timing_stats[f'{framework_type}_detection'] = time.time() - start_time + timing_stats[f'{framework_type}_detected_count'] = detected_count + + except Exception as e: + logger.error(f'{framework_type} 框架检测失败: {e}', exc_info=True) + if timing_stats: + timing_stats[f'{framework_type}_detection'] = -1 # 标记为失败 + + if not all_frames: + return pd.DataFrame() + + # 合并所有框架的检测结果 + return pd.concat(all_frames, ignore_index=True) diff --git a/perf_testing/hapray/core/common/frame/frame_perf_accessor.py b/perf_testing/hapray/core/common/frame/frame_perf_accessor.py index 43315067..71aa1964 100644 --- a/perf_testing/hapray/core/common/frame/frame_perf_accessor.py +++ b/perf_testing/hapray/core/common/frame/frame_perf_accessor.py @@ -162,18 +162,35 @@ def get_total_load_for_pids(self, app_pids: list[int]) -> int: logging.warning('没有有效的PID值，返回0') return 0 - query = f""" - SELECT SUM(event_count) as total_load - FROM perf_sample - WHERE thread_id IN ({','.join('?' * len(valid_pids))}) - """ - + # 修复：需要通过进程ID找到所有属于这些进程的线程ID + # perf_sample表只有thread_id，需要通过perf_thread表关联进程ID + # 先检查perf_thread表是否存在process_id字段 try: - result = self.perf_conn.execute(query, valid_pids).fetchone() - total_load = result[0] if result and result[0] else 0 - return int(total_load) + cursor = self.perf_conn.cursor() + cursor.execute('PRAGMA table_info(perf_thread)') + columns = [row[1] for row in cursor.fetchall()] + + if 'process_id' in columns: + # 使用perf_thread表的process_id字段 + query = f""" + SELECT SUM(ps.event_count) as total_load + FROM perf_sample ps + INNER JOIN perf_thread pt ON ps.thread_id = pt.thread_id + WHERE pt.process_id IN ({','.join('?' * len(valid_pids))}) + """ + result = cursor.execute(query, valid_pids).fetchone() + total_load = result[0] if result and result[0] else 0 + logging.info('通过perf_thread.process_id获取总负载: %d (进程ID: %s)', total_load, valid_pids) + return int(total_load) + # 如果没有process_id字段，需要通过trace数据库的thread表来关联 + # 但FramePerfAccessor没有trace_conn，所以需要从外部传入 + # 这里先记录警告，返回0，让调用者知道需要修复 + logging.warning('perf_thread表没有process_id字段，无法按进程ID过滤总负载，返回0') + logging.warning('建议：需要在FrameCacheManager中实现get_total_load_for_pids，使用trace_conn关联thread表') + return 0 except Exception as e: logging.error('获取总负载失败: %s', str(e)) + # 如果出错，返回0而不是所有数据，避免占比计算错误 return 0 # ==================== 数据标准化方法 ==================== diff --git a/perf_testing/hapray/core/common/frame/frame_rs_skip_backtrack_api.py b/perf_testing/hapray/core/common/frame/frame_rs_skip_backtrack_api.py new file mode 100644 index 00000000..41938289 --- /dev/null +++ b/perf_testing/hapray/core/common/frame/frame_rs_skip_backtrack_api.py @@ -0,0 +1,938 @@ +#!/usr/bin/env python3 +""" +RS系统API流程：从RS skip帧追溯到应用进程提交的帧 + +支持框架：ArkUI, RN, KMP + +追溯方法： +1. 优先：通过IPC线程的runnable状态，wakeup_from tid直接找到应用线程的帧 +2. 备选：通过唤醒链追溯（RS进程IPC线程 → 应用进程IPC线程 → UI线程） +""" + +import argparse +import logging +import re +import sqlite3 +import sys +import time +from typing import Optional + +# 注意：移除模块级别的stdout/stderr重定向和日志配置 +# 这些操作应该由主脚本统一管理，避免导入时的冲突 +# 导入CPU指令数计算函数（使用相对导入） +from .frame_empty_common import calculate_app_frame_cpu_waste + +logger = logging.getLogger(__name__) + + +def parse_unmarsh_pid(event_name: str) -> Optional[int]: + """从UnMarsh事件名称中提取pid""" + pattern = r'recv data from (\d+)' + match = re.search(pattern, event_name) + if match: + return int(match.group(1)) + return None + + +def find_unmarsh_events_in_rs_frame( + trace_conn: sqlite3.Connection, + rs_frame_ts: int, + rs_frame_dur: int, + time_window_before: int = 500_000_000, # 扩大到500ms，保证找到最近的buffer + unmarsh_events_cache: Optional[list[tuple]] = None, +) -> list[tuple]: + """ + 在RS帧时间窗口内查找UnMarsh事件（支持缓存） + + Args: + trace_conn: trace数据库连接 + rs_frame_ts: RS帧开始时间 + rs_frame_dur: RS帧持续时间 + time_window_before: RS帧开始前的时间窗口（纳秒） + unmarsh_events_cache: UnMarsh事件缓存列表，每个元素为 (name, ts, thread_id, thread_name, process_name, process_pid) + + Returns: + 事件列表 + """ + # 如果提供了缓存，从缓存中过滤，并选择最近的事件 + if unmarsh_events_cache is not None: + result = [] + frame_start = rs_frame_ts - time_window_before + frame_end = rs_frame_ts + rs_frame_dur + + for event in unmarsh_events_cache: + event_ts = event[1] # ts是第二个元素 + if frame_start <= event_ts <= frame_end: + result.append(event) + + # 按距离RS帧的时间排序，选择最近的5个 + if result: + result.sort(key=lambda x: abs(x[1] - rs_frame_ts)) + return result[:5] + return result + + # 没有缓存，查询数据库 + cursor = trace_conn.cursor() + + query = """ + SELECT + c.name, + c.ts, + c.callid as thread_id, + t.name as thread_name, + p.name as process_name, + p.pid as process_pid + FROM callstack c + INNER JOIN thread t ON c.callid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.name = 'render_service' + AND c.name LIKE '%UnMarsh RSTransactionData%' + AND c.ts >= ? - ? + AND c.ts <= ? + ? + ORDER BY ABS(c.ts - ?) + LIMIT 5 + """ + + cursor.execute( + query, + ( + rs_frame_ts, + time_window_before, + rs_frame_ts, + rs_frame_dur, + rs_frame_ts, # 用于ORDER BY ABS计算 + ), + ) + + return cursor.fetchall() + + +def trace_by_runnable_wakeup_from( + trace_conn: sqlite3.Connection, + rs_ipc_thread_id: int, + event_ts: int, + app_pid: int, + time_window: int = 10_000_000, + instant_cache: Optional[dict] = None, + thread_info_cache: Optional[dict] = None, + app_frames_cache: Optional[dict] = None, +) -> Optional[dict]: + """ + 通过IPC线程的runnable状态，wakeup_from tid追溯应用帧 + + Args: + trace_conn: trace数据库连接 + rs_ipc_thread_id: RS进程IPC线程ID + event_ts: UnMarsh事件时间戳 + app_pid: 应用进程PID + time_window: 时间窗口（纳秒） + + Returns: + 应用帧信息，如果找到；否则None + """ + # 步骤1: 查找IPC线程的runnable状态下的wakeup_from tid + # 在instant表中查找sched_wakeup事件，ref是IPC线程，wakeup_from是应用线程 + app_thread_id = None + wakeup_ts = None + + if instant_cache is not None: + # 从缓存中查找 + if rs_ipc_thread_id in instant_cache: + wakeup_events = instant_cache[rs_ipc_thread_id] + # 在时间窗口内查找最接近的wakeup事件 + best_wakeup = None + best_diff = float('inf') + for wf, ts in wakeup_events: + if event_ts - time_window <= ts <= event_ts + time_window: + diff = abs(ts - event_ts) + if diff < best_diff: + best_diff = diff + best_wakeup = (wf, ts) + if best_wakeup: + app_thread_id, wakeup_ts = best_wakeup + else: + # 查询数据库 + cursor = trace_conn.cursor() + wakeup_query = """ + SELECT + i.wakeup_from as app_thread_id, + i.ts as wakeup_ts + FROM instant i + WHERE i.name = 'sched_wakeup' + AND i.ref = ? + AND i.ts >= ? - ? + AND i.ts <= ? + ? + ORDER BY ABS(i.ts - ?) + LIMIT 1 + """ + + cursor.execute(wakeup_query, (rs_ipc_thread_id, event_ts, time_window, event_ts, time_window, event_ts)) + + wakeup_result = cursor.fetchone() + if wakeup_result: + app_thread_id = wakeup_result[0] + wakeup_ts = wakeup_result[1] + + if not app_thread_id or not wakeup_ts: + logger.debug(f'未找到IPC线程 {rs_ipc_thread_id} 的wakeup_from tid') + return None + + # 步骤2: 验证应用线程是否属于指定的应用进程 + thread_id = None + thread_name = None + process_name = None + process_pid = None + + if thread_info_cache is not None: + # 从缓存中查找 + if app_thread_id in thread_info_cache: + info = thread_info_cache[app_thread_id] + if info.get('pid') == app_pid: + thread_id = app_thread_id + thread_name = info.get('thread_name') + process_name = info.get('process_name') + process_pid = info.get('pid') + else: + # 查询数据库 + cursor = trace_conn.cursor() + thread_info_query = """ + SELECT t.id, t.name, p.name as process_name, p.pid + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE t.id = ? AND p.pid = ? + """ + + cursor.execute(thread_info_query, (app_thread_id, app_pid)) + thread_info = cursor.fetchone() + + if thread_info: + thread_id, thread_name, process_name, process_pid = thread_info + + if not thread_id: + logger.debug(f'线程 {app_thread_id} 不属于进程 {app_pid}') + return None + + # 步骤3: 在应用线程中查找对应的帧 + # 时间窗口：UnMarsh事件前500ms到后10ms + time_start = event_ts - 500_000_000 # 扩大到500ms + time_end = event_ts + 10_000_000 + + frame_id = None + frame_ts = None + frame_dur = None + frame_flag = None + frame_vsync = None + + if app_frames_cache is not None: + # 从缓存中查找 + if app_thread_id in app_frames_cache: + frames = app_frames_cache[app_thread_id] + best_frame = None + best_diff = float('inf') + for f in frames: + f_ts = f[1] # ts是第二个元素 + if time_start <= f_ts <= time_end: + diff = abs(f_ts - event_ts) + if diff < best_diff: + best_diff = diff + best_frame = f + if best_frame: + frame_id, frame_ts, frame_dur, frame_flag, frame_vsync = best_frame[:5] + else: + # 查询数据库 + cursor = trace_conn.cursor() + frame_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.flag, + fs.vsync + FROM frame_slice fs + WHERE fs.itid = ? + AND fs.ts >= ? + AND fs.ts <= ? + AND fs.type = 0 + ORDER BY ABS(fs.ts - ?) + LIMIT 1 + """ + + cursor.execute(frame_query, (app_thread_id, time_start, time_end, event_ts)) + frame_result = cursor.fetchone() + + if frame_result: + frame_id, frame_ts, frame_dur, frame_flag, frame_vsync = frame_result + + if not frame_id: + logger.debug(f'未找到应用线程 {thread_name} 的帧') + return None + + return { + 'frame_id': frame_id, + 'frame_ts': frame_ts, + 'frame_dur': frame_dur if frame_dur else 0, + 'frame_flag': frame_flag, + 'frame_vsync': frame_vsync, + 'thread_id': thread_id, + 'thread_name': thread_name, + 'process_name': process_name, + 'process_pid': process_pid, + 'app_pid': process_pid, # 添加app_pid字段，与process_pid相同 + 'pid': process_pid, # 添加pid字段（兼容） + 'wakeup_ts': wakeup_ts, + 'trace_method': 'runnable_wakeup_from', + } + + +def trace_by_wakeup_chain( + trace_conn: sqlite3.Connection, + rs_ipc_thread_id: int, + event_ts: int, + app_pid: int, + max_depth: int = 5, + time_window: int = 500_000_000, # 扩大到500ms + instant_cache: Optional[dict] = None, + thread_info_cache: Optional[dict] = None, + app_frames_cache: Optional[dict] = None, +) -> Optional[dict]: + """ + 通过唤醒链追溯应用帧（备选方法） + + 追溯路径：RS进程IPC线程 → 应用进程IPC线程 → UI线程 + """ + cursor = trace_conn.cursor() + + # 步骤1: 从RS进程IPC线程开始，向前追溯唤醒链 + # 找到应用进程的IPC线程 + current_thread_id = rs_ipc_thread_id + current_ts = event_ts + depth = 0 + app_ipc_thread_id = None + + while depth < max_depth and current_thread_id: + # 查找唤醒当前线程的事件 + waker_thread_id = None + wakeup_ts = None + + if instant_cache is not None: + # 从缓存中查找 + if current_thread_id in instant_cache: + wakeup_events = instant_cache[current_thread_id] + # 在时间窗口内查找最接近的wakeup事件 + best_wakeup = None + best_diff = float('inf') + for wf, ts in wakeup_events: + if current_ts - time_window <= ts <= current_ts: + diff = abs(ts - current_ts) + if diff < best_diff: + best_diff = diff + best_wakeup = (wf, ts) + if best_wakeup: + waker_thread_id, wakeup_ts = best_wakeup + else: + # 查询数据库 + cursor = trace_conn.cursor() + wakeup_query = """ + SELECT + i.wakeup_from as waker_thread_id, + i.ts as wakeup_ts + FROM instant i + WHERE i.name = 'sched_wakeup' + AND i.ref = ? + AND i.ts >= ? - ? + AND i.ts <= ? + ORDER BY i.ts DESC + LIMIT 1 + """ + + cursor.execute(wakeup_query, (current_thread_id, current_ts, time_window, current_ts)) + wakeup_result = cursor.fetchone() + + if wakeup_result: + waker_thread_id = wakeup_result[0] + wakeup_ts = wakeup_result[1] + + if not waker_thread_id or not wakeup_ts: + break + + if not waker_thread_id: + break + + # 检查唤醒者线程是否属于应用进程 + if thread_info_cache is not None: + # 从缓存中查找 + if waker_thread_id in thread_info_cache: + info = thread_info_cache[waker_thread_id] + if info.get('pid') == app_pid: + # 找到应用进程的线程 + app_ipc_thread_id = waker_thread_id + break + else: + # 查询数据库 + cursor = trace_conn.cursor() + thread_info_query = """ + SELECT t.id, t.name, p.name as process_name, p.pid + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE t.id = ? AND p.pid = ? + """ + + cursor.execute(thread_info_query, (waker_thread_id, app_pid)) + thread_info = cursor.fetchone() + + if thread_info: + # 找到应用进程的线程 + app_ipc_thread_id = waker_thread_id + break + + current_thread_id = waker_thread_id + current_ts = wakeup_ts + depth += 1 + + if not app_ipc_thread_id: + logger.debug(f'未找到应用进程 {app_pid} 的IPC线程') + return None + + # 步骤2: 从应用进程IPC线程开始，向前追溯找到UI线程 + # 查找应用进程的UI线程（名称与进程名相同，或包含"UI"） + ui_thread_id = None + ui_thread_name = None + process_name = None + + if thread_info_cache is not None: + # 从缓存中查找应用进程的UI线程 + for itid, info in thread_info_cache.items(): + if info.get('pid') == app_pid: + thread_name = info.get('thread_name', '') + proc_name = info.get('process_name', '') + if thread_name == proc_name or 'UI' in thread_name or 'ui' in thread_name: + ui_thread_id = itid + ui_thread_name = thread_name + process_name = proc_name + break + else: + # 查询数据库 + cursor = trace_conn.cursor() + ui_thread_query = """ + SELECT t.id, t.name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid = ? + AND (t.name = p.name OR t.name LIKE '%UI%' OR t.name LIKE '%ui%') + ORDER BY + CASE + WHEN t.name = p.name THEN 0 + WHEN t.name LIKE '%UI%' OR t.name LIKE '%ui%' THEN 1 + ELSE 2 + END + LIMIT 1 + """ + + cursor.execute(ui_thread_query, (app_pid,)) + ui_thread = cursor.fetchone() + + if ui_thread: + ui_thread_id, ui_thread_name, process_name = ui_thread + + if not ui_thread_id: + logger.debug(f'未找到应用进程 {app_pid} 的UI线程') + return None + + # 步骤3: 在UI线程中查找对应的帧 + time_start = event_ts - 500_000_000 # 扩大到500ms + time_end = event_ts + 10_000_000 + + frame_id = None + frame_ts = None + frame_dur = None + frame_flag = None + frame_vsync = None + + if app_frames_cache is not None: + # 从缓存中查找 + if ui_thread_id in app_frames_cache: + frames = app_frames_cache[ui_thread_id] + best_frame = None + best_diff = float('inf') + for f in frames: + f_ts = f[1] # ts是第二个元素 + if time_start <= f_ts <= time_end: + diff = abs(f_ts - event_ts) + if diff < best_diff: + best_diff = diff + best_frame = f + if best_frame: + frame_id, frame_ts, frame_dur, frame_flag, frame_vsync = best_frame[:5] + else: + # 查询数据库 + cursor = trace_conn.cursor() + frame_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.flag, + fs.vsync + FROM frame_slice fs + WHERE fs.itid = ? + AND fs.ts >= ? + AND fs.ts <= ? + AND fs.type = 0 + ORDER BY ABS(fs.ts - ?) + LIMIT 1 + """ + + cursor.execute(frame_query, (ui_thread_id, time_start, time_end, event_ts)) + frame_result = cursor.fetchone() + + if frame_result: + frame_id, frame_ts, frame_dur, frame_flag, frame_vsync = frame_result + + if not frame_id: + logger.debug(f'未找到UI线程 {ui_thread_name} 的帧') + return None + + return { + 'frame_id': frame_id, + 'frame_ts': frame_ts, + 'frame_dur': frame_dur if frame_dur else 0, + 'frame_flag': frame_flag, + 'frame_vsync': frame_vsync, + 'thread_id': ui_thread_id, + 'thread_name': ui_thread_name, + 'process_name': process_name, + 'process_pid': app_pid, + 'app_pid': app_pid, # 添加app_pid字段 + 'pid': app_pid, # 添加pid字段（兼容） + 'trace_method': 'wakeup_chain', + } + + +def preload_caches(trace_conn: sqlite3.Connection, min_ts: int, max_ts: int) -> dict: + """ + 预加载数据库表到内存（性能优化） + + Args: + trace_conn: trace数据库连接 + min_ts: 最小时间戳 + max_ts: 最大时间戳 + + Returns: + 包含所有缓存的字典 + """ + cursor = trace_conn.cursor() + caches = {} + + print(f'[性能优化] 开始预加载数据库表到内存 (时间范围: {min_ts} - {max_ts})...') + preload_start = time.time() + + # 1. 预加载UnMarsh事件 + print(' [预加载] 加载UnMarsh事件...') + unmarsh_query = """ + SELECT + c.name, + c.ts, + c.callid as thread_id, + t.name as thread_name, + p.name as process_name, + p.pid as process_pid + FROM callstack c + INNER JOIN thread t ON c.callid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.name = 'render_service' + AND c.name LIKE '%UnMarsh RSTransactionData%' + AND c.ts >= ? + AND c.ts <= ? + ORDER BY c.ts + """ + cursor.execute(unmarsh_query, (min_ts, max_ts)) + caches['unmarsh_events'] = cursor.fetchall() + print(f' [预加载] UnMarsh事件: {len(caches["unmarsh_events"])} 条记录') + + # 2. 预加载instant表（用于runnable和唤醒链） + print(' [预加载] 加载instant表...') + instant_query = """ + SELECT i.ref, i.wakeup_from, i.ts + FROM instant i + WHERE i.name = 'sched_wakeup' + AND i.ts >= ? + AND i.ts <= ? + AND i.ref_type = 'itid' + AND i.wakeup_from IS NOT NULL + AND i.ref IS NOT NULL + """ + cursor.execute(instant_query, (min_ts, max_ts)) + instant_data = cursor.fetchall() + + # 构建instant缓存：按ref分组，存储(wakeup_from, ts)列表 + instant_cache = {} + for ref, wakeup_from, ts in instant_data: + if ref not in instant_cache: + instant_cache[ref] = [] + instant_cache[ref].append((wakeup_from, ts)) + caches['instant'] = instant_cache + print(f' [预加载] instant表: {len(instant_data)} 条记录，{len(instant_cache)} 个线程') + + # 3. 预加载thread和process表 + print(' [预加载] 加载thread和process表...') + thread_process_query = """ + SELECT t.id, t.tid, t.name as thread_name, p.name as process_name, p.pid + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + """ + cursor.execute(thread_process_query) + thread_process_data = cursor.fetchall() + + # 构建thread_info缓存：itid -> {tid, thread_name, process_name, pid} + thread_info_cache = {} + # 同时构建tid_to_info缓存：tid -> {itid, thread_name, process_name, pid} + tid_to_info_cache = {} + for itid, tid, thread_name, process_name, pid in thread_process_data: + thread_info_cache[itid] = {'tid': tid, 'thread_name': thread_name, 'process_name': process_name, 'pid': pid} + tid_to_info_cache[tid] = {'itid': itid, 'thread_name': thread_name, 'process_name': process_name, 'pid': pid} + caches['thread_info'] = thread_info_cache + caches['tid_to_info'] = tid_to_info_cache + print(f' [预加载] thread/process表: {len(thread_process_data)} 个线程') + + # 4. 预加载应用帧（非RS进程的帧） + print(' [预加载] 加载应用帧...') + app_frames_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.flag, + fs.vsync, + fs.itid + FROM frame_slice fs + INNER JOIN thread t ON fs.itid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.name != 'render_service' + AND fs.type = 0 + AND fs.ts >= ? + AND fs.ts <= ? + """ + cursor.execute(app_frames_query, (min_ts, max_ts)) + app_frames_data = cursor.fetchall() + + # 构建app_frames缓存：按itid分组 + app_frames_cache = {} + for row in app_frames_data: + itid = row[5] # itid是最后一个元素 + if itid not in app_frames_cache: + app_frames_cache[itid] = [] + app_frames_cache[itid].append(row[:5]) # (rowid, ts, dur, flag, vsync) + caches['app_frames'] = app_frames_cache + print(f' [预加载] 应用帧: {len(app_frames_data)} 条记录，{len(app_frames_cache)} 个线程') + + preload_elapsed = time.time() - preload_start + print(f'[性能优化] 预加载完成，耗时: {preload_elapsed:.3f}秒\n') + + return caches + + +def trace_rs_skip_to_app_frame( + trace_conn: sqlite3.Connection, + rs_frame_id: int, + caches: Optional[dict] = None, + perf_conn: Optional[sqlite3.Connection] = None, + perf_sample_cache: Optional[dict] = None, + perf_timestamp_field: Optional[str] = None, +) -> Optional[dict]: + """ + 从RS skip帧追溯到应用进程提交的帧 + + Args: + trace_conn: trace数据库连接 + rs_frame_id: RS帧ID（frame_slice.rowid） + + Returns: + 追溯结果，包含RS帧和应用帧信息 + """ + # ========== 性能计时开始 ========== + perf_timings = {} + perf_start = time.time() + # ========== 性能计时开始 ========== + + cursor = trace_conn.cursor() + + # 步骤1: 获取RS帧信息 + perf_t1 = time.time() + perf_timings['get_rs_frame'] = perf_t1 - perf_start + rs_frame_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.flag, + fs.vsync, + fs.itid + FROM frame_slice fs + WHERE fs.rowid = ? + AND fs.ipid IN ( + SELECT p.ipid + FROM process p + WHERE p.name = 'render_service' + ) + """ + + cursor.execute(rs_frame_query, (rs_frame_id,)) + rs_frame = cursor.fetchone() + + if not rs_frame: + logger.error(f'未找到RS帧 {rs_frame_id}') + return None + + frame_id, frame_ts, frame_dur, frame_flag, frame_vsync, frame_itid = rs_frame + frame_dur = frame_dur if frame_dur else 0 + + # logger.info(f'RS帧 {frame_id}: 时间={frame_ts}, dur={frame_dur}, flag={frame_flag}, vsync={frame_vsync}') + + # 步骤2: 在RS帧时间窗口内查找UnMarsh事件 + perf_t2 = time.time() + perf_timings['find_unmarsh'] = perf_t2 - perf_t1 + unmarsh_events_cache = caches.get('unmarsh_events') if caches else None + unmarsh_events = find_unmarsh_events_in_rs_frame( + trace_conn, + frame_ts, + frame_dur, + time_window_before=500_000_000, # 扩大到500ms，保证找到最近的buffer + unmarsh_events_cache=unmarsh_events_cache, + ) + perf_t3 = time.time() + perf_timings['find_unmarsh_done'] = perf_t3 - perf_t2 + + if not unmarsh_events: + logger.warning(f'RS帧 {frame_id} 未找到UnMarsh事件') + return { + 'rs_frame': { + 'frame_id': frame_id, + 'ts': frame_ts, + 'dur': frame_dur, + 'flag': frame_flag, + 'vsync': frame_vsync, + }, + 'app_frame': None, + 'trace_method': None, + 'error': '未找到UnMarsh事件', + } + + # logger.info(f'找到 {len(unmarsh_events)} 个UnMarsh事件') + + # 步骤3: 对每个UnMarsh事件尝试追溯 + for unmarsh_event in unmarsh_events: + event_name = unmarsh_event[0] + event_ts = unmarsh_event[1] + rs_ipc_thread_id = unmarsh_event[2] + unmarsh_event[3] + + # 提取应用进程PID + app_pid = parse_unmarsh_pid(event_name) + if not app_pid: + continue + + # logger.info(f'UnMarsh事件: 时间={event_ts}, RS IPC线程={rs_ipc_thread_name}, 应用PID={app_pid}') + + # 方法1: 优先使用runnable方法 + perf_t4 = time.time() + perf_timings['runnable_start'] = perf_t4 - perf_t3 + instant_cache = caches.get('instant') if caches else None + thread_info_cache = caches.get('thread_info') if caches else None + app_frames_cache = caches.get('app_frames') if caches else None + app_frame = trace_by_runnable_wakeup_from( + trace_conn, + rs_ipc_thread_id, + event_ts, + app_pid, + instant_cache=instant_cache, + thread_info_cache=thread_info_cache, + app_frames_cache=app_frames_cache, + ) + perf_t5 = time.time() + perf_timings['runnable_done'] = perf_t5 - perf_t4 + + if app_frame: + # logger.info(f'通过runnable方法找到应用帧: {app_frame["frame_id"]}') + + # 计算CPU浪费 + tid_to_info_cache = caches.get('tid_to_info') if caches else None + cpu_waste = calculate_app_frame_cpu_waste( + trace_conn=trace_conn, + perf_conn=perf_conn, + app_frame=app_frame, + perf_sample_cache=perf_sample_cache, + perf_timestamp_field=perf_timestamp_field, + tid_to_info_cache=tid_to_info_cache, + ) + app_frame['cpu_waste'] = cpu_waste + + perf_end = time.time() + perf_timings['total'] = perf_end - perf_start + # print(f'[性能] RS帧{frame_id} 追溯耗时: {perf_timings["total"]*1000:.2f}ms | ' + # f'获取RS帧: {perf_timings["get_rs_frame"]*1000:.2f}ms | ' + # f'查找UnMarsh: {perf_timings["find_unmarsh"]*1000:.2f}ms | ' + # f'UnMarsh查询: {perf_timings["find_unmarsh_done"]*1000:.2f}ms | ' + # f'Runnable追溯: {perf_timings["runnable_done"]*1000:.2f}ms') + return { + 'rs_frame': { + 'frame_id': frame_id, + 'ts': frame_ts, + 'dur': frame_dur, + 'flag': frame_flag, + 'vsync': frame_vsync, + }, + 'app_frame': app_frame, + 'trace_method': 'runnable_wakeup_from', + } + + # 方法2: 备选使用唤醒链方法 + perf_t6 = time.time() + perf_timings['wakeup_chain_start'] = perf_t6 - perf_t5 + app_frame = trace_by_wakeup_chain( + trace_conn, + rs_ipc_thread_id, + event_ts, + app_pid, + instant_cache=instant_cache, + thread_info_cache=thread_info_cache, + app_frames_cache=app_frames_cache, + ) + perf_t7 = time.time() + perf_timings['wakeup_chain_done'] = perf_t7 - perf_t6 + + if app_frame: + # logger.info(f'通过唤醒链方法找到应用帧: {app_frame["frame_id"]}') + + # 计算CPU浪费 + tid_to_info_cache = caches.get('tid_to_info') if caches else None + cpu_waste = calculate_app_frame_cpu_waste( + trace_conn=trace_conn, + perf_conn=perf_conn, + app_frame=app_frame, + perf_sample_cache=perf_sample_cache, + perf_timestamp_field=perf_timestamp_field, + tid_to_info_cache=tid_to_info_cache, + ) + app_frame['cpu_waste'] = cpu_waste + + perf_end = time.time() + perf_timings['total'] = perf_end - perf_start + # print(f'[性能] RS帧{frame_id} 追溯耗时: {perf_timings["total"]*1000:.2f}ms | ' + # f'获取RS帧: {perf_timings["get_rs_frame"]*1000:.2f}ms | ' + # f'查找UnMarsh: {perf_timings["find_unmarsh"]*1000:.2f}ms | ' + # f'UnMarsh查询: {perf_timings["find_unmarsh_done"]*1000:.2f}ms | ' + # f'Runnable追溯: {perf_timings.get("runnable_done", 0)*1000:.2f}ms | ' + # f'唤醒链追溯: {perf_timings["wakeup_chain_done"]*1000:.2f}ms') + return { + 'rs_frame': { + 'frame_id': frame_id, + 'ts': frame_ts, + 'dur': frame_dur, + 'flag': frame_flag, + 'vsync': frame_vsync, + }, + 'app_frame': app_frame, + 'trace_method': 'wakeup_chain', + } + + # 所有方法都失败 + perf_end = time.time() + perf_timings['total'] = perf_end - perf_start + # print(f'[性能] RS帧{frame_id} 追溯失败，耗时: {perf_timings["total"]*1000:.2f}ms | ' + # f'获取RS帧: {perf_timings["get_rs_frame"]*1000:.2f}ms | ' + # f'查找UnMarsh: {perf_timings["find_unmarsh"]*1000:.2f}ms | ' + # f'UnMarsh查询: {perf_timings["find_unmarsh_done"]*1000:.2f}ms') + return { + 'rs_frame': {'frame_id': frame_id, 'ts': frame_ts, 'dur': frame_dur, 'flag': frame_flag, 'vsync': frame_vsync}, + 'app_frame': None, + 'trace_method': None, + 'error': '所有追溯方法都失败', + } + + +def main(): + parser = argparse.ArgumentParser(description='RS系统API流程：从RS skip帧追溯到应用进程提交的帧') + parser.add_argument('trace_db', help='trace数据库路径') + parser.add_argument('--rs-frame-id', type=int, help='RS帧ID（frame_slice.rowid）') + parser.add_argument('--verbose', '-v', action='store_true', help='详细输出') + + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # 连接数据库 + try: + trace_conn = sqlite3.connect(args.trace_db) + except Exception as e: + logger.error(f'无法连接数据库: {e}') + return 1 + + try: + if args.rs_frame_id: + # 获取RS帧信息以确定时间范围 + cursor = trace_conn.cursor() + rs_frame_query = """ + SELECT fs.ts, fs.dur + FROM frame_slice fs + WHERE fs.rowid = ? + AND fs.ipid IN ( + SELECT p.ipid FROM process p WHERE p.name = 'render_service' + ) + """ + cursor.execute(rs_frame_query, (args.rs_frame_id,)) + rs_frame_info = cursor.fetchone() + + # 预加载缓存（如果RS帧存在） + caches = None + if rs_frame_info: + frame_ts, frame_dur = rs_frame_info + frame_dur = frame_dur if frame_dur else 0 + min_ts = frame_ts - 200_000_000 # 前200ms + max_ts = frame_ts + frame_dur + 50_000_000 # 后50ms + caches = preload_caches(trace_conn, min_ts, max_ts) + + # 追溯指定的RS帧 + result = trace_rs_skip_to_app_frame(trace_conn, args.rs_frame_id, caches=caches) + + if result: + print(f'\n{"=" * 100}') + print('追溯结果') + print(f'{"=" * 100}\n') + + print('RS帧:') + print(f' 帧ID: {result["rs_frame"]["frame_id"]}') + print(f' 时间: {result["rs_frame"]["ts"]}') + print(f' 持续时间: {result["rs_frame"]["dur"] / 1_000_000:.2f}ms') + print(f' Flag: {result["rs_frame"]["flag"]}') + print(f' VSync: {result["rs_frame"]["vsync"]}') + + if result['app_frame']: + print('\n应用帧:') + print(f' 帧ID: {result["app_frame"]["frame_id"]}') + print(f' 时间: {result["app_frame"]["frame_ts"]}') + print(f' 持续时间: {result["app_frame"]["frame_dur"] / 1_000_000:.2f}ms') + print(f' Flag: {result["app_frame"]["frame_flag"]}') + print(f' VSync: {result["app_frame"]["frame_vsync"]}') + print(f' 线程: {result["app_frame"]["thread_name"]} (ID: {result["app_frame"]["thread_id"]})') + print(f' 进程: {result["app_frame"]["process_name"]} (PID: {result["app_frame"]["process_pid"]})') + print(f' 追溯方法: {result["trace_method"]}') + else: + print('\n应用帧: 未找到') + if 'error' in result: + print(f' 错误: {result["error"]}') + else: + # 查找所有包含skip的RS帧并追溯 + # logger.info('查找所有包含skip的RS帧...') + # 这里可以调用rs_skip_analyzer.py的逻辑来找到skip帧 + print('请使用 --rs-frame-id 指定要追溯的RS帧ID') + print('或者先运行 rs_skip_analyzer.py 找到skip帧，然后使用 --rs-frame-id 追溯') + + finally: + trace_conn.close() + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/perf_testing/hapray/core/common/frame/frame_rs_skip_backtrack_nw.py b/perf_testing/hapray/core/common/frame/frame_rs_skip_backtrack_nw.py new file mode 100644 index 00000000..4b2dd3fd --- /dev/null +++ b/perf_testing/hapray/core/common/frame/frame_rs_skip_backtrack_nw.py @@ -0,0 +1,586 @@ +#!/usr/bin/env python3 +""" +NativeWindow API流程：从RS skip帧追溯到应用进程提交的帧 + +支持框架：Flutter, ArkWeb + +追溯方法： +- 基于时间戳匹配（时间窗口匹配） +- 在RS帧时间窗口内查找AcquireBuffer/DoFlushBuffer事件 +- 在RS事件时间窗口内查找应用帧（前100ms到后10ms） +""" + +import argparse +import logging +import sqlite3 +import sys +import time +from typing import Optional + +# 移除模块级别的stdout/stderr重定向和日志配置 +# 这些操作应该由主脚本统一管理，避免导入时的冲突 +# 导入CPU指令数计算函数（使用相对导入） +from .frame_empty_common import calculate_app_frame_cpu_waste + +logger = logging.getLogger(__name__) + + +def find_rs_nativewindow_events_in_frame( + trace_conn: sqlite3.Connection, + rs_frame_ts: int, + rs_frame_dur: int, + time_window_before: int = 500_000_000, # 扩大到500ms，保证找到最近的buffer + nativewindow_events_cache: Optional[list[tuple]] = None, +) -> list[tuple]: + """ + 在RS帧时间窗口内查找NativeWindow API相关事件（支持缓存） + + Args: + trace_conn: trace数据库连接 + rs_frame_ts: RS帧开始时间 + rs_frame_dur: RS帧持续时间 + time_window_before: RS帧开始前的时间窗口（纳秒） + nativewindow_events_cache: NativeWindow事件缓存列表，每个元素为 (name, ts, dur, thread_id, thread_name, process_name) + + Returns: + 事件列表，每个元素为 (event_name, event_ts, event_dur, thread_id, thread_name, process_name) + """ + # 如果提供了缓存，从缓存中过滤，并选择最近的事件 + if nativewindow_events_cache is not None: + result = [] + frame_start = rs_frame_ts - time_window_before + frame_end = rs_frame_ts + rs_frame_dur + + for event in nativewindow_events_cache: + event_ts = event[1] # ts是第二个元素 + if frame_start <= event_ts <= frame_end: + result.append(event) + + # 按距离RS帧的时间排序，选择最近的5个 + if result: + result.sort(key=lambda x: abs(x[1] - rs_frame_ts)) + return result[:5] + return result + + # 没有缓存，查询数据库 + cursor = trace_conn.cursor() + + query = """ + SELECT + c.name, + c.ts, + c.dur, + c.callid as thread_id, + t.name as thread_name, + p.name as process_name + FROM callstack c + INNER JOIN thread t ON c.callid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.name = 'render_service' + AND ( + c.name LIKE '%AcquireBuffer%' + OR c.name LIKE '%DoFlushBuffer%' + OR c.name LIKE '%ConsumeAndUpdateAllNodes%' + ) + AND c.ts >= ? - ? + AND c.ts <= ? + ? + ORDER BY ABS(c.ts - ?) + LIMIT 5 + """ + + cursor.execute( + query, + ( + rs_frame_ts, + time_window_before, + rs_frame_ts, + rs_frame_dur, + rs_frame_ts, # 用于ORDER BY ABS计算 + ), + ) + + return cursor.fetchall() + + +def find_app_frames_near_rs_event( + trace_conn: sqlite3.Connection, + rs_event_ts: int, + time_window_before: int = 500_000_000, # 扩大到500ms + time_window_after: int = 10_000_000, + app_frames_cache: Optional[list[tuple]] = None, +) -> list[dict]: + """ + 在RS事件时间窗口内查找应用帧（支持缓存） + + Args: + trace_conn: trace数据库连接 + rs_event_ts: RS事件时间戳 + time_window_before: 事件前的时间窗口（纳秒，默认100ms） + time_window_after: 事件后的时间窗口（纳秒，默认10ms） + app_frames_cache: 应用帧缓存列表，每个元素为 (rowid, ts, dur, flag, vsync, thread_name, process_name, process_pid) + + Returns: + 应用帧列表，按时间差排序（最近的在前） + """ + # 如果提供了缓存，从缓存中过滤 + if app_frames_cache is not None: + result = [] + window_start = rs_event_ts - time_window_before + window_end = rs_event_ts + time_window_after + + for frame in app_frames_cache: + frame_ts = frame[1] # ts是第二个元素 + if window_start <= frame_ts <= window_end: + # frame格式: (rowid, ts, dur, flag, vsync, itid, thread_name, process_name, process_pid) + frame_id, frame_ts, frame_dur, frame_flag, frame_vsync, itid, thread_name, process_name, process_pid = ( + frame + ) + frame_dur = frame_dur if frame_dur else 0 + time_diff_ns = abs(frame_ts - rs_event_ts) + + result.append( + { + 'frame_id': frame_id, + 'frame_ts': frame_ts, + 'frame_dur': frame_dur, + 'frame_flag': frame_flag, + 'frame_vsync': frame_vsync, + 'thread_id': itid, # itid用于计算CPU指令数 + 'thread_name': thread_name, + 'process_name': process_name, + 'process_pid': process_pid, + 'time_diff_ns': time_diff_ns, + 'time_diff_ms': time_diff_ns / 1_000_000, + } + ) + + # 按时间差排序 + result.sort(key=lambda x: x['time_diff_ns']) + return result[:10] # 只返回前10个 + + # 没有缓存，查询数据库 + cursor = trace_conn.cursor() + + query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.flag, + fs.vsync, + fs.itid, + t.name as thread_name, + p.name as process_name, + p.pid as process_pid + FROM frame_slice fs + INNER JOIN thread t ON fs.itid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.name != 'render_service' + AND fs.type = 0 + AND fs.ts >= ? - ? + AND fs.ts <= ? + ? + ORDER BY ABS(fs.ts - ?) + LIMIT 10 + """ + + cursor.execute(query, (rs_event_ts, time_window_before, rs_event_ts, time_window_after, rs_event_ts)) + + frames = cursor.fetchall() + + result = [] + for frame in frames: + frame_id, frame_ts, frame_dur, frame_flag, frame_vsync, itid, thread_name, process_name, process_pid = frame + frame_dur = frame_dur if frame_dur else 0 + time_diff_ns = abs(frame_ts - rs_event_ts) + + result.append( + { + 'frame_id': frame_id, + 'frame_ts': frame_ts, + 'frame_dur': frame_dur, + 'frame_flag': frame_flag, + 'frame_vsync': frame_vsync, + 'thread_id': itid, # itid用于计算CPU指令数 + 'thread_name': thread_name, + 'process_name': process_name, + 'process_pid': process_pid, + 'app_pid': process_pid, # 添加app_pid字段，与process_pid相同 + 'pid': process_pid, # 添加pid字段（兼容） + 'time_diff_ns': time_diff_ns, + 'time_diff_ms': time_diff_ns / 1_000_000, + } + ) + + return result + + +def preload_caches(trace_conn: sqlite3.Connection, min_ts: int, max_ts: int) -> dict: + """ + 预加载数据库表到内存（性能优化） + + Args: + trace_conn: trace数据库连接 + min_ts: 最小时间戳 + max_ts: 最大时间戳 + + Returns: + 包含所有缓存的字典 + """ + cursor = trace_conn.cursor() + caches = {} + + print(f'[性能优化] 开始预加载数据库表到内存 (时间范围: {min_ts} - {max_ts})...') + preload_start = time.time() + + # 1. 预加载NativeWindow API事件 + print(' [预加载] 加载NativeWindow API事件...') + nativewindow_query = """ + SELECT + c.name, + c.ts, + c.dur, + c.callid as thread_id, + t.name as thread_name, + p.name as process_name + FROM callstack c + INNER JOIN thread t ON c.callid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.name = 'render_service' + AND ( + c.name LIKE '%AcquireBuffer%' + OR c.name LIKE '%DoFlushBuffer%' + OR c.name LIKE '%ConsumeAndUpdateAllNodes%' + ) + AND c.ts >= ? + AND c.ts <= ? + ORDER BY c.ts + """ + cursor.execute(nativewindow_query, (min_ts, max_ts)) + caches['nativewindow_events'] = cursor.fetchall() + print(f' [预加载] NativeWindow API事件: {len(caches["nativewindow_events"])} 条记录') + + # 2. 预加载应用帧（非RS进程的帧） + print(' [预加载] 加载应用帧...') + app_frames_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.flag, + fs.vsync, + fs.itid, + t.name as thread_name, + p.name as process_name, + p.pid as process_pid + FROM frame_slice fs + INNER JOIN thread t ON fs.itid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.name != 'render_service' + AND fs.type = 0 + AND fs.ts >= ? + AND fs.ts <= ? + """ + cursor.execute(app_frames_query, (min_ts, max_ts)) + caches['app_frames'] = cursor.fetchall() + print(f' [预加载] 应用帧: {len(caches["app_frames"])} 条记录') + + # 3. 预加载thread和process表（用于tid到itid的映射） + print(' [预加载] 加载thread和process表...') + thread_process_query = """ + SELECT t.id, t.tid, t.name as thread_name, p.name as process_name, p.pid + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + """ + cursor.execute(thread_process_query) + thread_process_data = cursor.fetchall() + + # 构建tid_to_info缓存：tid -> {itid, thread_name, process_name, pid} + tid_to_info_cache = {} + for itid, tid, thread_name, process_name, pid in thread_process_data: + tid_to_info_cache[tid] = {'itid': itid, 'thread_name': thread_name, 'process_name': process_name, 'pid': pid} + caches['tid_to_info'] = tid_to_info_cache + print(f' [预加载] thread/process表: {len(thread_process_data)} 个线程') + + preload_elapsed = time.time() - preload_start + print(f'[性能优化] 预加载完成，耗时: {preload_elapsed:.3f}秒\n') + + return caches + + +def trace_rs_skip_to_app_frame( + trace_conn: sqlite3.Connection, + rs_frame_id: int, + nativewindow_events_cache: Optional[list[tuple]] = None, + app_frames_cache: Optional[list[tuple]] = None, + perf_conn: Optional[sqlite3.Connection] = None, + perf_sample_cache: Optional[dict] = None, + perf_timestamp_field: Optional[str] = None, + tid_to_info_cache: Optional[dict] = None, +) -> Optional[dict]: + """ + 从RS skip帧追溯到应用进程提交的帧（NativeWindow API流程） + + Args: + trace_conn: trace数据库连接 + rs_frame_id: RS帧ID（frame_slice.rowid） + + Returns: + 追溯结果，包含RS帧和应用帧信息 + """ + # ========== 性能计时开始 ========== + perf_timings = {} + perf_start = time.time() + # ========== 性能计时开始 ========== + + cursor = trace_conn.cursor() + + # 步骤1: 获取RS帧信息 + perf_t1 = time.time() + perf_timings['get_rs_frame'] = perf_t1 - perf_start + rs_frame_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.flag, + fs.vsync, + fs.itid + FROM frame_slice fs + WHERE fs.rowid = ? + AND fs.ipid IN ( + SELECT p.ipid + FROM process p + WHERE p.name = 'render_service' + ) + """ + + cursor.execute(rs_frame_query, (rs_frame_id,)) + rs_frame = cursor.fetchone() + + if not rs_frame: + logger.error(f'未找到RS帧 {rs_frame_id}') + return None + + frame_id, frame_ts, frame_dur, frame_flag, frame_vsync, frame_itid = rs_frame + frame_dur = frame_dur if frame_dur else 0 + + # logger.info(f'RS帧 {frame_id}: 时间={frame_ts}, dur={frame_dur}, flag={frame_flag}, vsync={frame_vsync}') + + # 步骤2: 在RS帧时间窗口内查找NativeWindow API事件 + perf_t2 = time.time() + perf_timings['find_nativewindow_events'] = perf_t2 - perf_t1 + rs_events = find_rs_nativewindow_events_in_frame( + trace_conn, + frame_ts, + frame_dur, + time_window_before=500_000_000, # 扩大到500ms，保证找到最近的buffer + nativewindow_events_cache=nativewindow_events_cache, + ) + perf_t3 = time.time() + perf_timings['find_nativewindow_events_done'] = perf_t3 - perf_t2 + + if not rs_events: + logger.warning(f'RS帧 {frame_id} 未找到NativeWindow API事件') + return { + 'rs_frame': { + 'frame_id': frame_id, + 'ts': frame_ts, + 'dur': frame_dur, + 'flag': frame_flag, + 'vsync': frame_vsync, + }, + 'app_frame': None, + 'trace_method': None, + 'error': '未找到NativeWindow API事件', + } + + logger.info(f'找到 {len(rs_events)} 个NativeWindow API事件') + + # 步骤3: 对每个RS事件，在时间窗口内查找应用帧 + perf_t4 = time.time() + perf_timings['match_frames_start'] = perf_t4 - perf_t3 + best_match = None + best_time_diff = float('inf') + + for rs_event in rs_events: + event_name = rs_event[0] + event_ts = rs_event[1] + event_dur = rs_event[2] + rs_event[3] + thread_name = rs_event[4] + rs_event[5] + + # logger.info(f'RS事件: {event_name[:60]}... | 时间={event_ts}, 线程={thread_name}') + + # 在RS事件时间窗口内查找应用帧 + perf_t5 = time.time() + app_frames = find_app_frames_near_rs_event( + trace_conn, + event_ts, + time_window_before=500_000_000, # 扩大到500ms + time_window_after=10_000_000, # 10ms + app_frames_cache=app_frames_cache, + ) + perf_t6 = time.time() + perf_timings['find_app_frames'] = perf_timings.get('find_app_frames', 0) + (perf_t6 - perf_t5) + + if app_frames: + # 选择时间差最小的应用帧 + closest_frame = app_frames[0] # 已经按时间差排序 + + if closest_frame['time_diff_ms'] < best_time_diff: + best_match = { + 'rs_event': {'name': event_name, 'ts': event_ts, 'dur': event_dur, 'thread_name': thread_name}, + 'app_frame': closest_frame, + } + best_time_diff = closest_frame['time_diff_ms'] + + logger.info( + f'找到应用帧: 帧ID={closest_frame["frame_id"]}, ' + f'进程={closest_frame["process_name"]}, ' + f'时间差={closest_frame["time_diff_ms"]:.2f}ms' + ) + + perf_t7 = time.time() + perf_timings['match_frames_done'] = perf_t7 - perf_t4 + perf_end = time.time() + perf_timings['total'] = perf_end - perf_start + # print(f'[性能] RS帧{frame_id} 追溯耗时: {perf_timings["total"]*1000:.2f}ms | ' + # f'获取RS帧: {perf_timings["get_rs_frame"]*1000:.2f}ms | ' + # f'查找NativeWindow事件: {perf_timings["find_nativewindow_events"]*1000:.2f}ms | ' + # f'NativeWindow事件查询: {perf_timings["find_nativewindow_events_done"]*1000:.2f}ms | ' + # f'匹配应用帧: {perf_timings["match_frames_done"]*1000:.2f}ms | ' + # f'查找应用帧(累计): {perf_timings.get("find_app_frames", 0)*1000:.2f}ms') + + if best_match: + app_frame = best_match['app_frame'] + + # 计算CPU浪费 + cpu_waste = calculate_app_frame_cpu_waste( + trace_conn=trace_conn, + perf_conn=perf_conn, + app_frame=app_frame, + perf_sample_cache=perf_sample_cache, + perf_timestamp_field=perf_timestamp_field, + tid_to_info_cache=tid_to_info_cache, + ) + app_frame['cpu_waste'] = cpu_waste + + return { + 'rs_frame': { + 'frame_id': frame_id, + 'ts': frame_ts, + 'dur': frame_dur, + 'flag': frame_flag, + 'vsync': frame_vsync, + }, + 'app_frame': app_frame, + 'trace_method': 'time_window_matching', + 'rs_event': best_match['rs_event'], + 'time_diff_ms': best_time_diff, + } + return { + 'rs_frame': {'frame_id': frame_id, 'ts': frame_ts, 'dur': frame_dur, 'flag': frame_flag, 'vsync': frame_vsync}, + 'app_frame': None, + 'trace_method': None, + 'error': '所有时间窗口匹配都失败', + } + + +def main(): + parser = argparse.ArgumentParser(description='从RS skip帧追溯到应用进程提交的帧（NativeWindow API流程）') + parser.add_argument('trace_db', help='trace数据库路径') + parser.add_argument('--rs-frame-id', type=int, help='RS帧ID（frame_slice.rowid）') + parser.add_argument('--verbose', '-v', action='store_true', help='详细输出') + + args = parser.parse_args() + + if args.verbose: + logger.setLevel(logging.DEBUG) + + # 连接数据库 + try: + conn = sqlite3.connect(args.trace_db) + except Exception as e: + logger.error(f'无法连接数据库: {e}') + sys.exit(1) + + if args.rs_frame_id: + # 获取RS帧信息以确定时间范围 + cursor = conn.cursor() + rs_frame_query = """ + SELECT fs.ts, fs.dur + FROM frame_slice fs + WHERE fs.rowid = ? + AND fs.ipid IN ( + SELECT p.ipid FROM process p WHERE p.name = 'render_service' + ) + """ + cursor.execute(rs_frame_query, (args.rs_frame_id,)) + rs_frame_info = cursor.fetchone() + + # 预加载缓存（如果RS帧存在） + caches = None + if rs_frame_info: + frame_ts, frame_dur = rs_frame_info + frame_dur = frame_dur if frame_dur else 0 + min_ts = frame_ts - 200_000_000 # 前200ms + max_ts = frame_ts + frame_dur + 50_000_000 # 后50ms + caches = preload_caches(conn, min_ts, max_ts) + + # 追溯单个RS帧 + nativewindow_events_cache = caches.get('nativewindow_events') if caches else None + app_frames_cache = caches.get('app_frames') if caches else None + result = trace_rs_skip_to_app_frame( + conn, + args.rs_frame_id, + nativewindow_events_cache=nativewindow_events_cache, + app_frames_cache=app_frames_cache, + ) + + if result: + print(f'\n{"=" * 100}') + print('追溯结果') + print(f'{"=" * 100}\n') + + rs_frame = result['rs_frame'] + print('RS帧:') + print(f' 帧ID: {rs_frame["frame_id"]}') + print(f' 时间: {rs_frame["ts"]} ns ({rs_frame["ts"] / 1_000_000:.2f} ms)') + print(f' 持续时间: {rs_frame["dur"]} ns ({rs_frame["dur"] / 1_000_000:.2f} ms)') + print(f' Flag: {rs_frame["flag"]}') + print(f' VSync: {rs_frame["vsync"]}') + + if result.get('rs_event'): + rs_event = result['rs_event'] + print('\nRS事件:') + print(f' 名称: {rs_event["name"]}') + print(f' 时间: {rs_event["ts"]} ns ({rs_event["ts"] / 1_000_000:.2f} ms)') + print(f' 线程: {rs_event["thread_name"]}') + + app_frame = result.get('app_frame') + if app_frame: + print('\n应用帧:') + print(f' 帧ID: {app_frame["frame_id"]}') + print(f' 时间: {app_frame["ts"]} ns ({app_frame["ts"] / 1_000_000:.2f} ms)') + print(f' 持续时间: {app_frame["dur"]} ns ({app_frame["dur"] / 1_000_000:.2f} ms)') + print(f' Flag: {app_frame["flag"]}') + print(f' VSync: {app_frame["vsync"]}') + print(f' 进程: {app_frame["process_name"]} (PID: {app_frame["process_pid"]})') + print(f' 线程: {app_frame["thread_name"]}') + print(f' 时间差: {app_frame["time_diff_ms"]:.2f} ms') + print(f'\n追溯方法: {result["trace_method"]}') + else: + print('\n未找到应用帧') + if result.get('error'): + print(f'错误: {result["error"]}') + else: + print('追溯失败') + else: + print('请指定 --rs-frame-id 参数') + + conn.close() + + +if __name__ == '__main__': + main() diff --git a/perf_testing/hapray/core/common/frame/frame_trace_accessor.py b/perf_testing/hapray/core/common/frame/frame_trace_accessor.py index 5dbae320..46294366 100644 --- a/perf_testing/hapray/core/common/frame/frame_trace_accessor.py +++ b/perf_testing/hapray/core/common/frame/frame_trace_accessor.py @@ -229,21 +229,26 @@ def get_frame_statistics(self) -> dict: # ==================== 高级查询方法 ==================== - def get_empty_frames_with_details(self, app_pids: list[int]) -> pd.DataFrame: - """获取空帧详细信息（包含进程、线程、调用栈信息） + def get_empty_frames_with_details(self, app_pids: list[int]) -> tuple[pd.DataFrame, list[tuple[int, int]]]: + """获取空帧详细信息（包含进程、线程、调用栈信息）及其去重后的时间范围 Args: app_pids: 应用进程ID列表 Returns: - pd.DataFrame: 包含详细信息的空帧数据 + tuple: (frames_df, merged_time_ranges) + - frames_df: 包含详细信息的空帧数据（保留所有原始信息） + - merged_time_ranges: 去重后的时间范围列表 [(start_ts, end_ts), ...] """ + # 验证app_pids参数 valid_pids = validate_app_pids(app_pids) if not valid_pids: logging.warning('没有有效的PID值，返回空DataFrame') - return pd.DataFrame() + return pd.DataFrame(), [] + # 学习Checker的逻辑：排除系统进程，查找应用进程的空刷帧 + # 系统进程包括：render_service, rmrenderservice, ohos.sceneboard, system_*, com.ohos.* query = f""" WITH filtered_frames AS ( -- 首先获取符合条件的帧 @@ -253,12 +258,15 @@ def get_empty_frames_with_details(self, app_pids: list[int]) -> pd.DataFrame: AND fs.type = {FRAME_TYPE_ACTUAL} ), process_filtered AS ( - -- 通过process表过滤出app_pids中的帧 + -- 通过process表过滤出app_pids中的帧，并排除系统进程 SELECT ff.*, p.pid, p.name as process_name, t.tid, t.name as thread_name, t.is_main_thread FROM filtered_frames ff JOIN process p ON ff.ipid = p.ipid JOIN thread t ON ff.itid = t.itid WHERE p.pid IN ({','.join('?' * len(valid_pids))}) + AND p.name NOT IN ('render_service', 'rmrenderservice', 'ohos.sceneboard') + AND p.name NOT LIKE 'system_%' + AND p.name NOT LIKE 'com.ohos.%' ) -- 最后获取调用栈信息 SELECT pf.*, cs.name as callstack_name @@ -268,10 +276,55 @@ def get_empty_frames_with_details(self, app_pids: list[int]) -> pd.DataFrame: """ try: - return pd.read_sql_query(query, self.trace_conn, params=valid_pids) + frames_df = pd.read_sql_query(query, self.trace_conn, params=valid_pids) + + # 计算合并后的时间范围（去重） + merged_time_ranges = [] + if not frames_df.empty: + time_ranges = [] + for _, row in frames_df.iterrows(): + ts = row.get('ts', 0) + dur = row.get('dur', 0) + if ts > 0 and dur >= 0: + time_ranges.append((ts, ts + dur)) + + # 合并重叠的时间范围 + if time_ranges: + merged_time_ranges = self._merge_time_ranges(time_ranges) + + return frames_df, merged_time_ranges except Exception as e: logging.error('获取空帧详细信息失败: %s', str(e)) - return pd.DataFrame() + return pd.DataFrame(), [] + + def _merge_time_ranges(self, time_ranges: list[tuple[int, int]]) -> list[tuple[int, int]]: + """合并重叠的时间范围 + + Args: + time_ranges: 时间范围列表，格式为 [(start_ts, end_ts), ...] + + Returns: + 合并后的时间范围列表（无重叠） + """ + if not time_ranges: + return [] + + # 按开始时间排序 + sorted_ranges = sorted(time_ranges, key=lambda x: x[0]) + merged = [sorted_ranges[0]] + + for current_start, current_end in sorted_ranges[1:]: + last_start, last_end = merged[-1] + + # 如果当前范围与最后一个合并范围重叠 + if current_start <= last_end: + # 合并：扩展结束时间 + merged[-1] = (last_start, max(last_end, current_end)) + else: + # 无重叠，添加新范围 + merged.append((current_start, current_end)) + + return merged # ==================== 工具方法 ==================== diff --git a/perf_testing/hapray/core/common/frame/frame_utils.py b/perf_testing/hapray/core/common/frame/frame_utils.py index 9c3d87d3..8322e65e 100644 --- a/perf_testing/hapray/core/common/frame/frame_utils.py +++ b/perf_testing/hapray/core/common/frame/frame_utils.py @@ -13,15 +13,22 @@ limitations under the License. """ +import logging from typing import Any, Optional import pandas as pd -"""帧分析工具函数模块 +"""帧分析基础工具函数模块 -提供帧分析中常用的工具函数，避免代码重复。 +提供帧分析中常用的基础工具函数，避免代码重复。 +包括：数据清理、验证、系统线程判断等。 + +注意：CPU计算相关函数和空刷帧分析公共模块类已移至frame_empty_common.py +为了向后兼容，这里在文件末尾重新导出这些函数和类。 """ +logger = logging.getLogger(__name__) + def clean_frame_data(frame_data: dict[str, Any]) -> dict[str, Any]: """清理帧数据中的NaN值，确保JSON序列化安全 @@ -36,6 +43,9 @@ def clean_frame_data(frame_data: dict[str, Any]) -> dict[str, Any]: skip_fields = {'frame_samples', 'index'} for key, value in frame_data.items(): + # 跳过内部字段（以下划线开头的字段，如 _sample_details, _callchain_ids） + if key.startswith('_'): + continue if key in skip_fields: continue cleaned_data[key] = clean_single_value(value) @@ -111,3 +121,146 @@ def validate_app_pids(app_pids: Optional[list]) -> list[int]: return [] return [int(pid) for pid in app_pids if pd.notna(pid) and isinstance(pid, (int, float))] + + +def is_system_thread(process_name: Optional[str], thread_name: Optional[str]) -> bool: + """判断线程是否为系统线程（参考RN实现） + + 系统线程包括： + - hiperf（性能采集工具） + - render_service（渲染服务） + - sysmgr-main（系统管理） + - OS_开头的线程 + - ohos.开头的线程 + - pp.开头的线程 + - 其他系统服务 + + Args: + process_name: 进程名 + thread_name: 线程名 + + Returns: + True 如果是系统线程，False 如果是应用线程 + """ + if not process_name: + process_name = '' + if not thread_name: + thread_name = '' + + # 系统进程名模式 + system_process_patterns = [ + 'hiperf', + 'render_service', + 'sysmgr-main', + 'OS_%', + 'ohos.%', + 'pp.%', + 'hilogd', + 'hiprofiler', + 'hiprofiler_cmd', + 'hiprofiler_plug', + 'hiprofilerd', + 'kworker', + 'hdcd', + 'hiview', + 'foundation', + 'resource_schedu', + 'netmanager', + 'wifi_manager', + 'telephony', + 'sensors', + 'multimodalinput', + 'accountmgr', + 'accesstoken_ser', + 'samgr', + 'memmgrservice', + 'distributeddata', + 'privacy_service', + 'security_guard', + 'time_service', + 'bluetooth_servi', + 'media_service', + 'audio_server', + 'rmrenderservice', + 'ohos.sceneboard', + ] + + # 系统线程名模式 + system_thread_patterns = [ + 'OS_%', + 'ohos.%', + 'pp.%', + 'hiperf', + 'hiprofiler', + 'hiprofiler_cmd', + 'hiprofiler_plug', + 'hiprofilerd', + 'HmTraceReader', + 'kworker', + 'hilogd', + 'render_service', + 'VSyncGenerator', + 'RSUniRenderThre', + 'RSHardwareThrea', + 'RSBackgroundThr', + 'Present Fence', + 'Acquire Fence', + 'Release Fence', + 'gpu-work-server', + 'tppmgr-sched-in', + 'tppmgr-misc', + 'fs-kmsgfd', + 'dh-irq-bind', + 'dpu_gfx_primary', + 'display_engine_', + 'gpu-pm-release', + 'hisi_frw', + 'rcu_sched', + 'effect thread', + 'gpu-wq-id', + 'gpu-token-id', + 'irq/', + 'ksoftirqd', + 'netlink_handle', + 'hisi_tx_sch', + 'hisi_hcc', + 'hisi_rxdata', + 'spi', + 'gpufreq', + 'npu_excp', + 'dra_thread', + 'agent_vltmm', + 'tuid', + 'hw_kstate', + 'pci_ete_rx0', + 'wlan_bus_rx', + 'bbox_main', + 'kthread-joind', + 'dmabuf-deferred', + 'chr_web_thread', + 'ldk-kallocd', + ] + + # 检查进程名 + for pattern in system_process_patterns: + if pattern.endswith('%'): + if process_name.startswith(pattern[:-1]): + return True + elif pattern.endswith('.'): + if process_name.startswith(pattern): + return True + elif pattern in process_name or process_name == pattern: + return True + + # 检查线程名 + for pattern in system_thread_patterns: + if pattern.endswith('%'): + if thread_name.startswith(pattern[:-1]): + return True + elif pattern.endswith('_'): + if thread_name.startswith(pattern): + return True + elif pattern in thread_name or thread_name == pattern: + return True + + return False diff --git a/perf_testing/hapray/core/common/frame/frame_wakeup_chain.py b/perf_testing/hapray/core/common/frame/frame_wakeup_chain.py new file mode 100644 index 00000000..612ad0ee --- /dev/null +++ b/perf_testing/hapray/core/common/frame/frame_wakeup_chain.py @@ -0,0 +1,1806 @@ +""" +应用层空刷帧唤醒链分析：通过线程唤醒关系找到空刷帧相关的所有线程 + +分析逻辑： +1. 找到 flag=2 的空刷帧（没有送到 RS 线程） +2. 找到这个帧最后事件绑定的线程 +3. 通过线程唤醒关系（sched_wakeup事件，使用instant.wakeup_from字段）找到所有相关的线程 +4. 统计这些线程在帧时间范围内的 CPU 指令数（浪费的 CPU） +""" + +import json +import logging +import os +import random +import sqlite3 +import sys +import time +import traceback +from bisect import bisect_left +from collections import defaultdict, deque +from typing import Any, Optional + +# 注意：移除模块级别的日志配置，避免导入时的冲突 +# 从frame_empty_common和frame_utils导入函数（使用相对导入） +from .frame_core_load_calculator import calculate_process_instructions, calculate_thread_instructions +from .frame_utils import is_system_thread + +logger = logging.getLogger(__name__) + + +def find_frame_last_event( + trace_conn, + frame: dict[str, Any], + callstack_id_cache: Optional[dict] = None, + callstack_events_cache: Optional[dict] = None, +) -> Optional[dict[str, Any]]: + """找到帧的最后事件（性能优化：支持缓存） + + 策略： + 1. 优先使用 callstack_id 对应的事件（这是帧记录时绑定的调用栈） + 2. 如果没有 callstack_id，找该线程在帧时间范围内的最后一个事件 + + Args: + trace_conn: trace 数据库连接 + frame: 帧信息字典，包含 ts, dur, itid, callstack_id + callstack_id_cache: callstack_id -> callstack记录的缓存 + callstack_events_cache: (itid, min_ts, max_ts) -> callstack记录列表的缓存 + + Returns: + 最后事件的信息，包含 ts, name, callid, itid + """ + try: + frame_start = frame['ts'] + frame_end = frame['ts'] + frame.get('dur', 0) + frame_itid = frame['itid'] + + # 策略1: 优先使用 callstack_id 对应的事件（使用缓存） + if frame.get('callstack_id'): + if callstack_id_cache and frame['callstack_id'] in callstack_id_cache: + result = callstack_id_cache[frame['callstack_id']] + return { + 'id': result[0], + 'ts': result[1], + 'dur': result[2] if result[2] else 0, + 'name': result[3], + 'callid': result[4], + 'itid': result[4], + } + # 向后兼容：如果没有缓存，查询数据库 + cursor = trace_conn.cursor() + callstack_query = """ + SELECT + c.id, + c.ts, + c.dur, + c.name, + c.callid + FROM callstack c + WHERE c.id = ? + """ + cursor.execute(callstack_query, (frame['callstack_id'],)) + result = cursor.fetchone() + if result: + return { + 'id': result[0], + 'ts': result[1], + 'dur': result[2] if result[2] else 0, + 'name': result[3], + 'callid': result[4], + 'itid': result[4], + } + + # 策略2: 找该线程在帧时间范围内的最后一个事件（使用缓存） + if callstack_events_cache: + # 从缓存中查找：查找该线程在时间范围内的最后一个事件 + cache_key = (frame_itid, frame_start, frame_end) + if cache_key in callstack_events_cache: + events = callstack_events_cache[cache_key] + if events: + # 取最后一个（按ts排序） + result = max(events, key=lambda x: x[1]) # x[1]是ts + return { + 'id': result[0], + 'ts': result[1], + 'dur': result[2] if result[2] else 0, + 'name': result[3], + 'callid': result[4], + 'itid': result[4], + } + else: + # 向后兼容：如果没有缓存，查询数据库 + cursor = trace_conn.cursor() + last_event_query = """ + SELECT + c.id, + c.ts, + c.dur, + c.name, + c.callid + FROM callstack c + WHERE c.callid = ? + AND c.ts >= ? AND c.ts <= ? + ORDER BY c.ts DESC + LIMIT 1 + """ + cursor.execute(last_event_query, (frame_itid, frame_start, frame_end)) + result = cursor.fetchone() + if result: + return { + 'id': result[0], + 'ts': result[1], + 'dur': result[2] if result[2] else 0, + 'name': result[3], + 'callid': result[4], + 'itid': result[4], + } + + # 如果都找不到，返回帧的线程信息作为最后事件 + logger.warning('帧 %d 未找到最后事件，使用线程信息', frame.get('frame_id', 'unknown')) + return { + 'id': None, + 'ts': frame_start, + 'dur': frame.get('dur', 0), + 'name': 'frame_end', + 'callid': frame_itid, + 'itid': frame_itid, + } + + except Exception as e: + logger.error('查找帧最后事件失败: %s', str(e)) + return None + + +def filter_threads_executed_in_frame( + trace_conn, + itids: set[int], + frame_start: int, + frame_end: int, + callstack_cache: Optional[set[int]] = None, + thread_state_cache: Optional[set[int]] = None, +) -> set[int]: + """过滤线程，保留在帧时间范围内实际在CPU上执行过的线程，以及唤醒链中的关键线程 + + 策略： + 1. 保留在帧时间范围内有 Running/Runnable 状态的线程 + 2. 保留在帧时间范围内有 callstack 活动的线程 + 3. 对于唤醒链中的线程，即使帧内未执行，如果： + - 在帧时间前后被唤醒（扩展时间范围） + - 或者在帧时间范围内有任何状态记录（包括sleep状态） + 也应该保留，因为它们是唤醒链的一部分 + + 根据文档： + - thread_state 表：记录线程状态，state='R' (Runnable) 或 'Runing' (Running) 表示在CPU上执行 + - callstack 表：记录调用堆栈，callid 对应线程的 itid，有活动说明线程执行过 + + Args: + trace_conn: trace 数据库连接 + itids: 线程 itid 集合（来自唤醒链） + frame_start: 帧开始时间 + frame_end: 帧结束时间 + + Returns: + 与帧相关的线程 itid 集合（包括执行过的和唤醒链中的关键线程） + """ + if not itids: + return set() + + try: + cursor = trace_conn.cursor() + executed_threads = set() + + # 性能优化：减少时间范围扩展，只扩展10ms + extended_start = frame_start - 10_000_000 # 10ms + extended_end = frame_end + 5_000_000 # 5ms + + # 性能优化：优先使用缓存，如果没有缓存才查询数据库 + if callstack_cache is not None and thread_state_cache is not None: + # 使用缓存：直接检查itids是否在缓存中 + for itid in itids: + if itid in callstack_cache or itid in thread_state_cache: + executed_threads.add(itid) + # 向后兼容：如果没有缓存，查询数据库 + elif itids: + itids_list = list(itids) + placeholders = ','.join('?' * len(itids_list)) + + # 合并查询：同时检查帧内和扩展时间范围 + callstack_query = f""" + SELECT DISTINCT callid + FROM callstack + WHERE callid IN ({placeholders}) + AND ts >= ? + AND ts <= ? + """ + + params = itids_list + [extended_start, extended_end] + cursor.execute(callstack_query, params) + callstack_results = cursor.fetchall() + for (callid,) in callstack_results: + executed_threads.add(callid) + + # 方法2: 检查 thread_state 表（只查Running/Runnable状态，减少查询） + thread_state_query = f""" + SELECT DISTINCT itid + FROM thread_state + WHERE itid IN ({placeholders}) + AND state IN ('R', 'Runing') + AND ts <= ? + AND ts + dur >= ? + """ + + params = itids_list + [extended_end, extended_start] + cursor.execute(thread_state_query, params) + state_results = cursor.fetchall() + for (itid,) in state_results: + executed_threads.add(itid) + + logger.debug(f'帧时间范围内相关的线程: {len(executed_threads)}/{len(itids)}') + return executed_threads + + except Exception as e: + logger.warning('过滤执行线程失败，返回所有线程: %s', str(e)) + return itids # 如果出错，返回所有线程 + + +def find_wakeup_chain( + trace_conn, + start_itid: int, + frame_start: int, + frame_end: int, + max_depth: int = 20, + instant_cache: Optional[dict] = None, + app_pid: Optional[int] = None, +) -> list[tuple[int, int]]: + """通过线程唤醒关系找到所有相关的线程（唤醒链） + + 策略：从最后的线程开始，反向追溯唤醒链，直到找到系统VSync线程或达到最大深度 + 时间范围：扩展到帧开始前20ms（约1-2个VSync周期），以确保能找到完整的唤醒链 + + Args: + trace_conn: trace 数据库连接 + start_itid: 起始线程的 itid（通常是帧最后事件绑定的线程） + frame_start: 帧开始时间 + frame_end: 帧结束时间 + max_depth: 最大搜索深度（默认10） + instant_cache: instant表数据缓存 {(ref, ts_range): [(wakeup_from, ts), ...]} + + Returns: + 所有相关线程的 itid 列表，按唤醒链顺序（从起始线程到最远的唤醒线程） + 每个元素为 (itid, depth)，depth 表示在唤醒链中的深度（0为起始线程） + """ + try: + # 扩展时间范围：往前追溯50ms，确保能找到完整的唤醒链（参考RN实现） + search_start = frame_start - 50_000_000 # 50ms in nanoseconds + search_end = frame_end + + # 如果提供了app_pid，预先查询应用进程的所有线程itid（用于优先查找） + app_thread_itids = set() + if app_pid: + cursor = trace_conn.cursor() + cursor.execute( + """ + SELECT t.itid + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid = ? + """, + (app_pid,), + ) + app_thread_itids = {row[0] for row in cursor.fetchall()} + logger.debug(f'应用进程 {app_pid} 包含 {len(app_thread_itids)} 个线程') + + # 使用 BFS 反向搜索唤醒链（优先反向，找到谁唤醒了当前线程） + visited = set() + queue = deque([(start_itid, 0, frame_start)]) # (itid, depth, last_seen_ts) + related_threads_ordered = [(start_itid, 0)] # 按唤醒链顺序存储 (itid, depth) + related_threads_set = {start_itid} # 用于快速查找是否已包含 + + logger.debug(f'开始追溯唤醒链，起始线程 itid={start_itid}，搜索时间范围: {search_start} - {search_end}') + + # 如果没有提供缓存，使用数据库查询（向后兼容） + use_cache = instant_cache is not None + + while queue: + current_itid, depth, current_ts = queue.popleft() + + if depth >= max_depth: + logger.debug(f'达到最大深度 {max_depth}，停止搜索') + continue + + if current_itid in visited: + continue + + visited.add(current_itid) + + # 性能优化：从内存缓存中查找，而不是查询数据库 + # 优先查找应用进程的线程（除了ArkWeb，其他框架都在应用进程内开新线程） + woken_by_results = [] + if use_cache: + # 从缓存中查找：instant_cache[ref] = [(wakeup_from, ts), ...] + if current_itid in instant_cache: + # 过滤出在时间范围内的所有事件 + filtered = [(wf, ts) for wf, ts in instant_cache[current_itid] if search_start <= ts <= current_ts] + if filtered: + # 如果有app_pid，优先选择应用进程的线程 + if app_pid and app_thread_itids: + app_thread_wakers = [(wf, ts) for wf, ts in filtered if wf in app_thread_itids] + if app_thread_wakers: + # 优先使用应用进程的线程，按时间倒序取最近的 + app_thread_wakers.sort(key=lambda x: x[1], reverse=True) + woken_by_results = app_thread_wakers[:1] + else: + # 如果没有应用进程的线程，使用系统线程 + filtered.sort(key=lambda x: x[1], reverse=True) + woken_by_results = filtered[:1] + else: + # 没有app_pid，只取最近的1个 + filtered.sort(key=lambda x: x[1], reverse=True) + woken_by_results = filtered[:1] + else: + # 向后兼容：如果没有缓存，查询数据库 + cursor = trace_conn.cursor() + if app_pid and app_thread_itids: + # 优先查找应用进程的线程 + app_thread_ids_list = list(app_thread_itids) + placeholders = ','.join('?' * len(app_thread_ids_list)) + woken_by_query = f""" + SELECT i.wakeup_from, i.ts + FROM instant i + WHERE i.name IN ('sched_wakeup', 'sched_waking') + AND i.ref = ? + AND i.ts >= ? + AND i.ts <= ? + AND i.ref_type = 'itid' + AND i.wakeup_from IS NOT NULL + AND i.wakeup_from IN ({placeholders}) + ORDER BY i.ts DESC + LIMIT 1 + """ + cursor.execute( + woken_by_query, (current_itid, search_start, current_ts) + tuple(app_thread_ids_list) + ) + woken_by_results = cursor.fetchall() + + # 如果没找到应用进程的线程，查找所有线程 + if not woken_by_results: + woken_by_query = """ + SELECT i.wakeup_from, i.ts + FROM instant i + WHERE i.name IN ('sched_wakeup', 'sched_waking') + AND i.ref = ? + AND i.ts >= ? + AND i.ts <= ? + AND i.ref_type = 'itid' + AND i.wakeup_from IS NOT NULL + ORDER BY i.ts DESC + LIMIT 1 + """ + cursor.execute(woken_by_query, (current_itid, search_start, current_ts)) + woken_by_results = cursor.fetchall() + else: + woken_by_query = """ + SELECT i.wakeup_from, i.ts + FROM instant i + WHERE i.name IN ('sched_wakeup', 'sched_waking') + AND i.ref = ? + AND i.ts >= ? + AND i.ts <= ? + AND i.ref_type = 'itid' + AND i.wakeup_from IS NOT NULL + ORDER BY i.ts DESC + LIMIT 1 + """ + cursor.execute(woken_by_query, (current_itid, search_start, current_ts)) + woken_by_results = cursor.fetchall() + + for waker_itid_raw, wakeup_ts in woken_by_results: + waker_itid = int(waker_itid_raw) + if waker_itid and waker_itid not in visited: + # 按唤醒链顺序添加到列表（深度递增） + related_threads_ordered.append((waker_itid, depth + 1)) + related_threads_set.add(waker_itid) + # 继续从这个唤醒时间点往前追溯 + queue.append((waker_itid, depth + 1, wakeup_ts)) + + logger.debug(f'唤醒链追溯完成，共找到 {len(related_threads_ordered)} 个相关线程') + return related_threads_ordered + + except Exception as e: + logger.error('查找唤醒链失败: %s', str(e)) + logger.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return {start_itid} # 至少返回起始线程 + + +# is_system_thread 和 calculate_thread_instructions 已移动到 frame_utils，直接使用导入的版本 + + +def calculate_thread_instructions_batch( + perf_conn, trace_conn, frame_data_list: list[dict] +) -> dict[int, tuple[dict[int, int], dict[int, int]]]: + """批量计算多个帧的线程指令数（性能优化版本） + + 收集所有帧的线程ID和时间范围，批量查询perf_sample，在内存中按帧分组计算指令数。 + 这样可以大大减少数据库查询次数。 + + Args: + perf_conn: perf 数据库连接 + trace_conn: trace 数据库连接 + frame_data_list: 帧数据列表，每个元素包含： + - frame_id: 帧ID + - thread_ids: 线程ID集合 + - frame_start: 帧开始时间 + - frame_end: 帧结束时间 + + Returns: + {frame_id: (app_thread_instructions, system_thread_instructions)} + """ + if not perf_conn or not trace_conn or not frame_data_list: + return {} + + try: + perf_cursor = perf_conn.cursor() + trace_cursor = trace_conn.cursor() + + # 检查 perf_sample 表字段名（使用缓存，避免重复查询） + # 如果外部传入了字段名，直接使用；否则查询一次 + if hasattr(calculate_thread_instructions, '_timestamp_field_cache'): + timestamp_field = calculate_thread_instructions._timestamp_field_cache + else: + perf_cursor.execute('PRAGMA table_info(perf_sample)') + columns = [row[1] for row in perf_cursor.fetchall()] + timestamp_field = 'timestamp_trace' if 'timestamp_trace' in columns else 'timeStamp' + calculate_thread_instructions._timestamp_field_cache = timestamp_field + + # 收集所有唯一的线程ID和时间范围 + all_thread_ids = set() + min_time = float('inf') + max_time = float('-inf') + + for frame_data in frame_data_list: + all_thread_ids.update(frame_data['thread_ids']) + extended_start = frame_data['frame_start'] - 1_000_000 + extended_end = frame_data['frame_end'] + 1_000_000 + min_time = min(min_time, extended_start) + max_time = max(max_time, extended_end) + + if not all_thread_ids: + return {} + + # 批量获取所有线程信息（一次性查询） + thread_ids_list = list(all_thread_ids) + placeholders = ','.join('?' * len(thread_ids_list)) + + trace_cursor.execute( + f""" + SELECT DISTINCT t.tid, t.name as thread_name, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE t.tid IN ({placeholders}) + """, + thread_ids_list, + ) + + thread_info_map = {} + for tid, thread_name, process_name in trace_cursor.fetchall(): + thread_info_map[tid] = {'thread_name': thread_name, 'process_name': process_name} + + # 批量查询所有帧时间范围内的perf_sample数据 + # 使用更大的时间范围，然后在内存中按帧分组 + batch_query = f""" + SELECT + ps.thread_id, + ps.{timestamp_field} as ts, + ps.event_count + FROM perf_sample ps + WHERE ps.thread_id IN ({placeholders}) + AND ps.{timestamp_field} >= ? AND ps.{timestamp_field} <= ? + """ + + params = thread_ids_list + [min_time, max_time] + perf_cursor.execute(batch_query, params) + + # 在内存中按帧分组计算指令数 + frame_results = {} + for frame_data in frame_data_list: + frame_id = frame_data['frame_id'] + frame_thread_ids = frame_data['thread_ids'] + extended_start = frame_data['frame_start'] - 1_000_000 + extended_end = frame_data['frame_end'] + 1_000_000 + + # 为每个帧初始化结果字典 + app_thread_instructions = {} + system_thread_instructions = {} + + # 重置游标，重新执行查询（或者使用已加载的数据） + # 为了性能，我们重新查询该帧的时间范围（但线程ID已经批量查询了） + frame_placeholders = ','.join('?' * len(frame_thread_ids)) + frame_thread_ids_list = list(frame_thread_ids) + frame_query = f""" + SELECT + ps.thread_id, + SUM(ps.event_count) as total_instructions + FROM perf_sample ps + WHERE ps.thread_id IN ({frame_placeholders}) + AND ps.{timestamp_field} >= ? AND ps.{timestamp_field} <= ? + GROUP BY ps.thread_id + """ + + frame_params = frame_thread_ids_list + [extended_start, extended_end] + perf_cursor.execute(frame_query, frame_params) + frame_results_db = perf_cursor.fetchall() + + for thread_id, instruction_count in frame_results_db: + thread_info = thread_info_map.get(thread_id, {}) + process_name = thread_info.get('process_name') + thread_name = thread_info.get('thread_name') + + if is_system_thread(process_name, thread_name): + system_thread_instructions[thread_id] = instruction_count + else: + app_thread_instructions[thread_id] = instruction_count + + frame_results[frame_id] = (app_thread_instructions, system_thread_instructions) + + return frame_results + + except Exception as e: + logger.error('批量计算线程指令数失败: %s', str(e)) + logger.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return {} + + +def _check_perf_sample_has_data(conn) -> bool: + """检查 perf_sample 表是否存在且有数据（参考 detect_rn_empty_render.py） + + Args: + conn: 数据库连接 + + Returns: + 如果表存在且有数据返回 True，否则返回 False + """ + try: + cursor = conn.cursor() + # 检查表是否存在 + cursor.execute(""" + SELECT name FROM sqlite_master + WHERE type='table' AND name='perf_sample' + """) + if not cursor.fetchone(): + return False + + # 检查表中是否有数据 + cursor.execute(""" + SELECT COUNT(*) FROM perf_sample + """) + count = cursor.fetchone()[0] + return count > 0 + except Exception: + return False + + +def map_itid_to_perf_thread_id( + trace_conn, perf_conn, itids: set[int], itid_to_tid_cache: dict = None +) -> dict[int, int]: + """将 trace 数据库的 itid 映射到 perf 数据库的 thread_id（参考RN实现） + + 映射逻辑（参考RN实现）： + 1. 从 trace 数据库的 thread 表获取 tid（线程号） + 2. 直接使用 tid 作为 perf_sample.thread_id（不需要通过 perf_thread 表） + 3. perf_sample.thread_id 直接对应 trace thread.tid + + Args: + trace_conn: trace 数据库连接 + perf_conn: perf 数据库连接 + itids: trace 数据库的 itid 集合 + + Returns: + 映射字典 {itid: tid}，其中 tid 可以直接用于查询 perf_sample.thread_id + """ + if not itids or not perf_conn: + return {} + + try: + # 性能优化：使用预加载的缓存，避免数据库查询 + itid_to_perf_thread = {} + if itid_to_tid_cache: + # 从缓存中查找 + for itid in itids: + if itid in itid_to_tid_cache: + itid_to_perf_thread[itid] = itid_to_tid_cache[itid] + else: + # 向后兼容：如果没有缓存，查询数据库 + trace_cursor = trace_conn.cursor() + itids_list = list(itids) + placeholders = ','.join('?' * len(itids_list)) + tid_query = f""" + SELECT t.itid, t.tid + FROM thread t + WHERE t.itid IN ({placeholders}) + """ + trace_cursor.execute(tid_query, itids_list) + thread_info = trace_cursor.fetchall() + + # 直接使用 tid 作为 perf_sample.thread_id（参考RN实现） + for itid, tid in thread_info: + itid_to_perf_thread[itid] = tid + + return itid_to_perf_thread + + except Exception as e: + logger.error('映射 itid 到 perf_thread_id 失败: %s', str(e)) + logger.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return {} + + +def analyze_empty_frame_wakeup_chain( + trace_db_path: str, perf_db_path: str, app_pids: list[int] = None, sample_size: int = None +) -> Optional[list[dict[str, Any]]]: + """分析空刷帧的唤醒链和 CPU 浪费 + + Args: + trace_db_path: trace 数据库路径 + perf_db_path: perf 数据库路径 + app_pids: 应用进程 ID 列表（可选） + sample_size: 随机采样的帧数（可选），如果为 None 则分析所有帧 + + Returns: + 分析结果列表，每个元素包含： + - frame_id: 帧 ID + - frame_info: 帧基本信息 + - last_event: 最后事件信息 + - wakeup_threads: 唤醒链线程列表（通过唤醒链分析找到的线程） + - total_wasted_instructions: 浪费的总指令数 + - thread_instructions: 每个线程的指令数 + + 注意：会过滤假阳性帧（flag=2但实际通过NativeWindow API成功提交了帧） + """ + try: + # 检查输入参数 + if not trace_db_path or not os.path.exists(trace_db_path): + logger.error('Trace数据库文件不存在: %s', trace_db_path) + return None + + trace_conn = sqlite3.connect(trace_db_path) + perf_conn = None + + # 处理 perf 数据库（参考 detect_rn_empty_render.py 的实现） + if perf_db_path and os.path.exists(perf_db_path): + # 独立的 perf.db 文件 + perf_conn = sqlite3.connect(perf_db_path) + # 检查 perf.db 中是否有 perf_sample 表且有数据 + if not _check_perf_sample_has_data(perf_conn): + perf_conn.close() + perf_conn = None + logger.warning('perf.db 文件存在但 perf_sample 表为空，将无法计算CPU指令数和占比') + else: + # 检查 trace.db 中是否有 perf_sample 表且有数据 + cursor = trace_conn.cursor() + cursor.execute(""" + SELECT name FROM sqlite_master + WHERE type='table' AND name='perf_sample' + """) + if cursor.fetchone(): + # perf_sample 表存在，检查是否有数据 + if _check_perf_sample_has_data(trace_conn): + # perf_sample 在 trace.db 中且有数据，复用连接 + perf_conn = trace_conn + logger.info('perf_sample 表在 trace.db 中，且包含数据') + else: + # 表存在但没有数据 + perf_conn = None + logger.warning('trace.db 中的 perf_sample 表为空，将无法计算CPU指令数和占比') + + if not perf_conn: + logger.warning('未找到可用的 perf_sample 数据，将无法计算CPU指令数和占比') + logger.warning('提示: 需要 perf+trace 数据才能计算CPU指令数和占比') + + cursor = trace_conn.cursor() + + # 步骤1: 找到所有 flag=2 的空刷帧 + empty_frames_query = """ + SELECT + fs.rowid, + fs.ts, + fs.dur, + fs.itid, + fs.callstack_id, + fs.vsync, + fs.flag, + t.tid, + t.name as thread_name, + p.pid, + p.name as process_name + FROM frame_slice fs + INNER JOIN thread t ON fs.itid = t.itid + INNER JOIN process p ON fs.ipid = p.ipid + WHERE fs.flag = 2 + AND fs.type = 0 + """ + + if app_pids: + empty_frames_query += f' AND p.pid IN ({",".join("?" * len(app_pids))})' + + empty_frames_query += ' ORDER BY fs.ts' + + params = list(app_pids) if app_pids else [] + cursor.execute(empty_frames_query, params) + empty_frames = cursor.fetchall() + + if not empty_frames: + print('未找到 flag=2 的空刷帧') + trace_conn.close() + if perf_conn: + perf_conn.close() + return [] + + print(f'找到 {len(empty_frames)} 个 flag=2 的空刷帧') + + # 重置假阳性计数器 + if hasattr(analyze_empty_frame_wakeup_chain, '_false_positive_count'): + analyze_empty_frame_wakeup_chain._false_positive_count = 0 + + # 统一过滤假阳性（在分析开始前，与update保持一致） + # 计算所有帧的时间范围 + if empty_frames: + min_ts = min(ts for _, ts, _, _, _, _, _, _, _, _, _ in empty_frames) - 10_000_000 + max_ts = max(ts + (dur if dur else 0) for _, ts, dur, _, _, _, _, _, _, _, _ in empty_frames) + 5_000_000 + + # 预加载 NativeWindow API 事件 + cursor = trace_conn.cursor() + nw_query = """ + SELECT + c.callid as itid, + c.ts + FROM callstack c + INNER JOIN thread t ON c.callid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE c.ts >= ? AND c.ts <= ? + AND p.name NOT IN ('render_service', 'rmrenderservice', 'ohos.sceneboard') + AND p.name NOT LIKE 'system_%' + AND p.name NOT LIKE 'com.ohos.%' + AND (c.name LIKE '%RequestBuffer%' + OR c.name LIKE '%FlushBuffer%' + OR c.name LIKE '%DoFlushBuffer%' + OR c.name LIKE '%NativeWindow%') + """ + cursor.execute(nw_query, (min_ts, max_ts)) + nw_records = cursor.fetchall() + + # 构建事件缓存（按线程ID索引） + nw_events_cache = defaultdict(list) + for itid, ts in nw_records: + nw_events_cache[itid].append(ts) + + # 排序以便二分查找 + for itid in nw_events_cache: + nw_events_cache[itid].sort() + + print(f'预加载NativeWindow API事件: {len(nw_records)}条记录, {len(nw_events_cache)}个线程') + + # 统一过滤假阳性 + filtered_empty_frames = [] + false_positive_count = 0 + for frame_data in empty_frames: + rowid, ts, dur, itid, callstack_id, vsync, flag, tid, thread_name, pid, process_name = frame_data + frame_start = ts + frame_end = ts + (dur if dur else 0) # 处理dur可能为None的情况 + + # 检查是否为假阳性 + is_false_positive = False + if itid in nw_events_cache: + event_timestamps = nw_events_cache[itid] + idx = bisect_left(event_timestamps, frame_start) + if idx < len(event_timestamps) and event_timestamps[idx] <= frame_end: + is_false_positive = True + + if is_false_positive: + false_positive_count += 1 + else: + filtered_empty_frames.append(frame_data) + + empty_frames = filtered_empty_frames + total_before_filter = len(empty_frames) + false_positive_count + false_positive_rate = (false_positive_count / total_before_filter * 100) if total_before_filter > 0 else 0 + print( + f'假阳性过滤: 过滤前={total_before_filter}, 过滤后={len(empty_frames)}, 假阳性={false_positive_count} ({false_positive_rate:.1f}%)' + ) + + # 更新假阳性计数器 + if hasattr(analyze_empty_frame_wakeup_chain, '_false_positive_count'): + analyze_empty_frame_wakeup_chain._false_positive_count = false_positive_count + + # 随机采样 + if sample_size is not None and sample_size < len(empty_frames): + empty_frames = random.sample(empty_frames, sample_size) + print(f'随机采样 {sample_size} 个帧进行分析') + + results = [] + + # 性能分析：记录各阶段耗时（保存到字典中） + stage_timings = { + 'preload': 0.0, + 'find_last_event': 0.0, + 'find_wakeup_chain': 0.0, + 'filter_threads': 0.0, + 'map_perf_thread': 0.0, + 'calculate_instructions': 0.0, + 'build_thread_info': 0.0, + } + + # 性能优化：预加载数据库表到内存 + print('\n[性能优化] 开始预加载数据库表到内存...') + preload_start = time.time() + + # 计算所有帧的时间范围 + if empty_frames: + min_ts = min(ts for _, ts, _, _, _, _, _, _, _, _, _ in empty_frames) - 10_000_000 + max_ts = max(ts + (dur if dur else 0) for _, ts, dur, _, _, _, _, _, _, _, _ in empty_frames) + 5_000_000 + + cursor = trace_conn.cursor() + + # 1. 预加载 instant 表（唤醒事件） + print(f' [预加载] 加载 instant 表 (时间范围: {min_ts} - {max_ts})...') + instant_query = """ + SELECT i.ref, i.wakeup_from, i.ts + FROM instant i + WHERE i.name IN ('sched_wakeup', 'sched_waking') + AND i.ts >= ? + AND i.ts <= ? + AND i.ref_type = 'itid' + AND i.wakeup_from IS NOT NULL + AND i.ref IS NOT NULL + """ + cursor.execute(instant_query, (min_ts, max_ts)) + instant_data = cursor.fetchall() + + # 构建 instant 缓存：按 ref 分组，存储 (wakeup_from, ts) 列表 + instant_cache = {} + for ref, wakeup_from, ts in instant_data: + if ref not in instant_cache: + instant_cache[ref] = [] + instant_cache[ref].append((wakeup_from, ts)) + + print(f' [预加载] instant 表: {len(instant_data)} 条记录，{len(instant_cache)} 个线程') + + # 2. 预加载 callstack 表（用于filter_threads和find_frame_last_event） + print(' [预加载] 加载 callstack 表...') + # 2.1: 加载所有callstack记录（用于find_frame_last_event） + callstack_all_query = """ + SELECT c.id, c.ts, c.dur, c.name, c.callid + FROM callstack c + WHERE c.ts >= ? + AND c.ts <= ? + """ + cursor.execute(callstack_all_query, (min_ts, max_ts)) + callstack_all_data = cursor.fetchall() + + # 构建callstack_id缓存 + callstack_id_cache = {row[0]: row for row in callstack_all_data} + + # 构建callstack事件缓存：按itid分组，存储该线程的所有事件 + callstack_events_by_itid = {} + for row in callstack_all_data: + itid = row[4] # callid + if itid not in callstack_events_by_itid: + callstack_events_by_itid[itid] = [] + callstack_events_by_itid[itid].append(row) + + # 构建callstack_cache（用于filter_threads） + callstack_cache = {callid for _, _, _, _, callid in callstack_all_data} + print( + f' [预加载] callstack 表: {len(callstack_all_data)} 条记录，{len(callstack_cache)} 个线程，{len(callstack_id_cache)} 个callstack_id' + ) + + # 3. 预加载 thread_state 表（只加载Running/Runnable状态） + print(' [预加载] 加载 thread_state 表...') + thread_state_query = """ + SELECT DISTINCT itid + FROM thread_state + WHERE state IN ('R', 'Runing') + AND ts <= ? + AND ts + dur >= ? + """ + cursor.execute(thread_state_query, (max_ts, min_ts)) + thread_state_data = cursor.fetchall() + thread_state_cache = {itid for (itid,) in thread_state_data} + print(f' [预加载] thread_state 表: {len(thread_state_cache)} 个线程') + + # 4. 预加载 thread 和 process 表（避免每帧都查询） + print(' [预加载] 加载 thread 和 process 表...') + thread_process_query = """ + SELECT t.itid, t.tid, t.name as thread_name, p.pid, p.name as process_name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + """ + cursor.execute(thread_process_query) + thread_process_data = cursor.fetchall() + + # 构建多种索引以支持不同的查询需求 + # 1. tid -> {thread_name, process_name} (用于calculate_thread_instructions) + tid_to_info_cache = {} + # 2. itid -> tid (用于map_itid_to_perf_thread_id) + itid_to_tid_cache = {} + # 3. itid -> {tid, thread_name, pid, process_name} (用于build_thread_info) + itid_to_full_info_cache = {} + # 4. pid -> [(itid, tid, thread_name, process_name), ...] (用于查找进程的所有线程) + pid_to_threads_cache = {} + + for itid, tid, thread_name, pid, process_name in thread_process_data: + # tid -> {thread_name, process_name} + tid_to_info_cache[tid] = {'thread_name': thread_name, 'process_name': process_name} + # itid -> tid + itid_to_tid_cache[itid] = tid + # itid -> 完整信息 + itid_to_full_info_cache[itid] = { + 'tid': tid, + 'thread_name': thread_name, + 'pid': pid, + 'process_name': process_name, + } + # pid -> 线程列表 + if pid not in pid_to_threads_cache: + pid_to_threads_cache[pid] = [] + pid_to_threads_cache[pid].append( + {'itid': itid, 'tid': tid, 'thread_name': thread_name, 'process_name': process_name} + ) + + print( + f' [预加载] thread/process 表: {len(thread_process_data)} 个线程，{len(pid_to_threads_cache)} 个进程' + ) + + # 5. 预加载 NativeWindow API 事件到内存（用于假阳性检测优化） + print(' [预加载] 加载 NativeWindow API 事件...') + nw_preload_start = time.time() + nativewindow_events_cache = {} # 初始化缓存字典 + + # 查询所有应用进程的 NativeWindow API 事件 + nw_query = """ + SELECT + c.callid as itid, + c.ts + FROM callstack c + INNER JOIN thread t ON c.callid = t.id + INNER JOIN process p ON t.ipid = p.ipid + WHERE c.ts >= ? AND c.ts <= ? + AND p.name NOT IN ('render_service', 'rmrenderservice', 'ohos.sceneboard') + AND p.name NOT LIKE 'system_%' + AND p.name NOT LIKE 'com.ohos.%' + AND (c.name LIKE '%RequestBuffer%' + OR c.name LIKE '%FlushBuffer%' + OR c.name LIKE '%DoFlushBuffer%' + OR c.name LIKE '%NativeWindow%') + """ + cursor.execute(nw_query, (min_ts, max_ts)) + nw_records = cursor.fetchall() + + # 构建缓存: itid -> [ts1, ts2, ...] + for record in nw_records: + itid, ts = record + if itid not in nativewindow_events_cache: + nativewindow_events_cache[itid] = [] + nativewindow_events_cache[itid].append(ts) + + nw_preload_elapsed = time.time() - nw_preload_start + print( + f' [预加载] NativeWindow API 事件: {len(nw_records)} 条记录，{len(nativewindow_events_cache)} 个线程' + ) + print(f' [预加载] NativeWindow API 加载耗时: {nw_preload_elapsed:.3f}秒') + + preload_elapsed = time.time() - preload_start + stage_timings['preload'] = preload_elapsed + print(f'[性能优化] 预加载完成，耗时: {preload_elapsed:.3f}秒\n') + else: + instant_cache = {} + callstack_cache = set() + callstack_id_cache = {} + callstack_events_by_itid = {} + thread_state_cache = set() + tid_to_info_cache = {} + itid_to_tid_cache = {} + itid_to_full_info_cache = {} + pid_to_threads_cache = {} + nativewindow_events_cache = {} # 初始化NativeWindow缓存 + + # 性能优化：缓存perf_sample表的字段名（避免每帧都查询） + perf_timestamp_field = None + if perf_conn: + try: + perf_cursor_temp = perf_conn.cursor() + perf_cursor_temp.execute('PRAGMA table_info(perf_sample)') + columns = [row[1] for row in perf_cursor_temp.fetchall()] + perf_timestamp_field = 'timestamp_trace' if 'timestamp_trace' in columns else 'timeStamp' + perf_cursor_temp.close() + except Exception as e: + logger.warning('无法获取perf_sample表字段名: %s', str(e)) + + # 性能优化：预加载 perf_sample 表数据（如果perf_conn存在） + perf_sample_cache = {} # thread_id -> [(timestamp, event_count), ...] + if perf_conn and empty_frames and perf_timestamp_field: + print(' [预加载] 加载 perf_sample 表...') + preload_perf_start = time.time() + + # 计算所有帧的时间范围（扩展1ms以包含时间戳对齐问题） + min_ts = min(ts for _, ts, _, _, _, _, _, _, _, _, _ in empty_frames) - 1_000_000 + max_ts = max(ts + (dur if dur else 0) for _, ts, dur, _, _, _, _, _, _, _, _ in empty_frames) + 1_000_000 + + # 先快速遍历所有帧，收集所有可能的线程ID（通过itid_to_tid_cache） + # 由于我们还没有完成唤醒链查找，先收集所有线程的tid + all_possible_tids = set(itid_to_tid_cache.values()) if itid_to_tid_cache else set() + + if all_possible_tids: + # 一次性查询所有时间范围内的perf_sample数据 + perf_cursor = perf_conn.cursor() + tid_list = list(all_possible_tids) + placeholders = ','.join('?' * len(tid_list)) + + perf_query = f""" + SELECT ps.thread_id, ps.{perf_timestamp_field} as ts, ps.event_count + FROM perf_sample ps + WHERE ps.thread_id IN ({placeholders}) + AND ps.{perf_timestamp_field} >= ? AND ps.{perf_timestamp_field} <= ? + """ + + perf_cursor.execute(perf_query, tid_list + [min_ts, max_ts]) + perf_data = perf_cursor.fetchall() + + # 构建缓存：按thread_id分组，存储(timestamp, event_count)列表 + for thread_id, ts, event_count in perf_data: + if thread_id not in perf_sample_cache: + perf_sample_cache[thread_id] = [] + perf_sample_cache[thread_id].append((ts, event_count)) + + # 按timestamp排序，便于后续范围查询 + for _thread_id, samples in perf_sample_cache.items(): + samples.sort(key=lambda x: x[0]) + + print(f' [预加载] perf_sample 表: {len(perf_data)} 条记录，{len(perf_sample_cache)} 个线程') + preload_perf_elapsed = time.time() - preload_perf_start + print(f' [预加载] perf_sample 加载耗时: {preload_perf_elapsed:.3f}秒') + else: + print(' [预加载] perf_sample 表: 无可用线程ID，跳过') + + # 步骤2: 对每个空刷帧进行分析 + for frame_row in empty_frames: + frame_id, ts, dur, itid, callstack_id, vsync, flag, tid, thread_name, pid, process_name = frame_row + + frame_info = { + 'frame_id': frame_id, + 'ts': ts, + 'dur': dur if dur else 0, + 'itid': itid, + 'callstack_id': callstack_id, + 'vsync': vsync, + 'flag': flag, + 'tid': tid, + 'thread_name': thread_name, + 'pid': pid, + 'process_name': process_name, + } + + frame_start = ts + frame_end = ts + (dur if dur else 0) + + # 步骤2.1: 找到帧的最后事件（计时，使用缓存） + stage_start = time.time() + frame_itid = frame_info['itid'] + # 从缓存中查找该线程在帧时间范围内的最后一个事件 + last_event = None + if frame_info.get('callstack_id') and frame_info['callstack_id'] in callstack_id_cache: + # 使用callstack_id缓存 + result = callstack_id_cache[frame_info['callstack_id']] + last_event = { + 'id': result[0], + 'ts': result[1], + 'dur': result[2] if result[2] else 0, + 'name': result[3], + 'callid': result[4], + 'itid': result[4], + } + elif frame_itid in callstack_events_by_itid: + # 从该线程的事件中查找帧时间范围内的最后一个 + events = callstack_events_by_itid[frame_itid] + filtered = [ + (row[0], row[1], row[2], row[3], row[4]) for row in events if frame_start <= row[1] <= frame_end + ] + if filtered: + result = max(filtered, key=lambda x: x[1]) # 按ts排序，取最后一个 + last_event = { + 'id': result[0], + 'ts': result[1], + 'dur': result[2] if result[2] else 0, + 'name': result[3], + 'callid': result[4], + 'itid': result[4], + } + + # 如果缓存中没有找到，使用原函数（向后兼容） + if not last_event: + last_event = find_frame_last_event( + trace_conn, frame_info, callstack_id_cache=callstack_id_cache, callstack_events_cache=None + ) + + elapsed = time.time() - stage_start + stage_timings['find_last_event'] += elapsed + # 只在debug模式下输出每个帧的详细耗时 + logger.debug(f' [帧{frame_id}] find_last_event: {elapsed:.3f}秒') + + # 步骤2.2: 找到唤醒链（所有相关的线程）（计时，使用缓存） + # 优先查找应用进程内的线程（除了ArkWeb，其他框架都在应用进程内开新线程） + stage_start = time.time() + + # 处理last_event：如果找不到，使用帧所在线程的itid作为起始点 + if not last_event: + logger.warning('帧 %d 未找到最后事件，使用帧所在线程itid作为起始点', frame_id) + start_itid = itid # 使用帧所在线程的itid + else: + start_itid = last_event.get('itid') or last_event.get('callid') or itid + # 获取应用进程PID（如果帧所在进程是应用进程） + app_pid = None + if not is_system_thread(frame_info.get('process_name'), frame_info.get('thread_name')): + app_pid = frame_info.get('pid') + elif frame_info.get('pid'): + # 即使帧在系统线程，也尝试查找应用进程（通过进程名判断） + # 例如：com.qunar.hos的OS_VSyncThread，应用进程就是com.qunar.hos + app_pid = frame_info.get('pid') + + related_itids_ordered = find_wakeup_chain( + trace_conn, start_itid, frame_start, frame_end, instant_cache=instant_cache, app_pid=app_pid + ) + # related_itids_ordered 是 [(itid, depth), ...] 列表，按唤醒链顺序 + related_itids = {itid for itid, _ in related_itids_ordered} # 转换为集合用于后续查找 + elapsed = time.time() - stage_start + stage_timings['find_wakeup_chain'] += elapsed + logger.debug( + f' [帧{frame_id}] find_wakeup_chain: {elapsed:.3f}秒 (找到{len(related_itids_ordered)}个线程)' + ) + + logger.debug('帧 %d: 找到 %d 个相关线程（唤醒链）', frame_id, len(related_itids)) + + # 步骤2.2.1: 过滤出在帧时间范围内实际在CPU上执行过的线程（计时，使用缓存） + stage_start = time.time() + executed_itids = filter_threads_executed_in_frame( + trace_conn, + related_itids, + frame_start, + frame_end, + callstack_cache=callstack_cache, + thread_state_cache=thread_state_cache, + ) + elapsed = time.time() - stage_start + stage_timings['filter_threads'] += elapsed + logger.debug(f' [帧{frame_id}] filter_threads: {elapsed:.3f}秒 (过滤后{len(executed_itids)}个线程)') + + # 确保起始线程包含在内（即使没有执行记录，因为它是帧的入口） + executed_itids.add(start_itid) + + # 特殊处理：对于应用进程，总是查找应用进程内的所有线程（包括主线程、RNOH_JS等） + # 参考RN实现：对于空刷帧，应该查找应用进程内的所有线程（包括主线程、RNOH_JS等） + # 即使唤醒链中已经找到了应用进程的线程，也要确保包含所有应用线程（如RNOH_JS） + # 对于ArkWeb，还需要查找对应的render进程（如com.jd.hm.mall -> com.jd.hm.mall:render） + if app_pid: + # 性能优化：使用预加载的缓存查找应用进程的线程 + if pid_to_threads_cache and app_pid in pid_to_threads_cache: + # 从缓存中查找应用进程的所有线程 + app_threads = pid_to_threads_cache[app_pid] + # 检查当前executed_itids中是否有应用进程的线程 + {t['itid'] for t in app_threads if t['itid'] in executed_itids} + + # 总是查找应用进程内的所有线程（包括RNOH_JS），确保不遗漏 + logger.debug('帧 %d: 查找应用进程 %d 的所有线程（包括RNOH_JS）', frame_id, app_pid) + for thread_info in app_threads: + app_itid = thread_info['itid'] + app_tid = thread_info['tid'] + app_thread_name = thread_info['thread_name'] + app_process_name = thread_info['process_name'] + if not is_system_thread(app_process_name, app_thread_name): + executed_itids.add(app_itid) + if app_thread_name == 'RNOH_JS': + logger.debug('帧 %d: 找到RNOH_JS线程 (itid=%d, tid=%d)', frame_id, app_itid, app_tid) + else: + logger.debug( + '帧 %d: 找到应用进程线程 %s (itid=%d)', frame_id, app_thread_name, app_itid + ) + else: + # 向后兼容：如果没有缓存，查询数据库 + cursor.execute( + """ + SELECT t.itid + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE t.itid IN ({}) + AND p.pid = ? + """.format(','.join('?' * len(executed_itids))), + list(executed_itids) + [app_pid], + ) + {row[0] for row in cursor.fetchall()} + + # 总是查找应用进程内的所有线程（包括RNOH_JS），确保不遗漏 + logger.debug('帧 %d: 查找应用进程 %d 的所有线程（包括RNOH_JS）', frame_id, app_pid) + cursor.execute( + """ + SELECT t.itid, t.tid, t.name, p.pid, p.name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid = ? + AND t.name NOT LIKE 'OS_%' + AND t.name NOT LIKE 'ohos.%' + AND t.name NOT LIKE 'pp.%' + """, + (app_pid,), + ) + app_threads = cursor.fetchall() + for app_itid, app_tid, app_thread_name, _app_pid_val, app_process_name in app_threads: + if not is_system_thread(app_process_name, app_thread_name): + executed_itids.add(app_itid) + if app_thread_name == 'RNOH_JS': + logger.debug('帧 %d: 找到RNOH_JS线程 (itid=%d, tid=%d)', frame_id, app_itid, app_tid) + else: + logger.debug( + '帧 %d: 找到应用进程线程 %s (itid=%d)', frame_id, app_thread_name, app_itid + ) + + # 特殊处理：对于ArkWeb，查找对应的render进程 + # ArkWeb使用多进程架构：主进程（如com.jd.hm.mall）和render进程（如.hm.mall:render或com.jd.hm.mall:render） + app_process_name = frame_info.get('process_name', '') + if app_process_name and '.' in app_process_name: + # 查找所有包含":render"的进程（ArkWeb的render进程通常以":render"结尾） + # 例如：.hm.mall:render, com.jd.hm.mall:render等 + cursor.execute(""" + SELECT DISTINCT p.pid, p.name + FROM process p + WHERE p.name LIKE '%:render' + OR (p.name LIKE '%render%' AND p.name != 'render_service' AND p.name != 'rmrenderservice') + """) + all_render_processes = cursor.fetchall() + + # 尝试匹配与主进程相关的render进程 + # 可能的匹配模式： + # 1. com.jd.hm.mall -> com.jd.hm.mall:render + # 2. com.jd.hm.mall -> .hm.mall:render (去掉com前缀) + # 3. com.jd.hm.mall -> hm.mall:render + matched_render_processes = [] + for render_pid, render_process_name in all_render_processes: + # 提取主进程名的关键部分（去掉com.前缀） + main_key = app_process_name.split('.', 1)[-1] if '.' in app_process_name else app_process_name + # 检查render进程名是否包含主进程的关键部分 + if main_key in render_process_name or app_process_name.split('.')[-1] in render_process_name: + matched_render_processes.append((render_pid, render_process_name)) + + # 如果没找到精确匹配，查找所有以":render"结尾的进程（可能是共享的render进程） + if not matched_render_processes: + cursor.execute(""" + SELECT DISTINCT p.pid, p.name + FROM process p + WHERE p.name LIKE '%:render' + """) + matched_render_processes = cursor.fetchall() + + for render_pid, render_process_name in matched_render_processes: + logger.info( + '帧 %d: 找到ArkWeb render进程 %s (PID=%d)', frame_id, render_process_name, render_pid + ) + # 查找render进程的所有线程 + cursor.execute( + """ + SELECT t.itid, t.tid, t.name, p.pid, p.name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE p.pid = ? + AND t.name NOT LIKE 'OS_%' + AND t.name NOT LIKE 'ohos.%' + AND t.name NOT LIKE 'pp.%' + """, + (render_pid,), + ) + render_threads = cursor.fetchall() + for ( + render_itid, + _render_tid, + render_thread_name, + _render_pid_val, + render_proc_name, + ) in render_threads: + if not is_system_thread(render_proc_name, render_thread_name): + executed_itids.add(render_itid) + logger.info( + '帧 %d: 找到render进程线程 %s (itid=%d, pid=%d, process=%s)', + frame_id, + render_thread_name, + render_itid, + render_pid, + render_proc_name, + ) + + logger.debug('帧 %d: 其中 %d 个线程在帧时间范围内实际执行过', frame_id, len(executed_itids)) + + # 使用执行过的线程集合 + related_itids = executed_itids + + # 步骤2.3: 将 itid 映射到 perf_thread_id（计时） + stage_start = time.time() + itid_to_perf_thread = {} + if perf_conn: + itid_to_perf_thread = map_itid_to_perf_thread_id( + trace_conn, perf_conn, related_itids, itid_to_tid_cache + ) + elapsed = time.time() - stage_start + stage_timings['map_perf_thread'] += elapsed + logger.debug(f' [帧{frame_id}] map_perf_thread: {elapsed:.3f}秒 (映射{len(itid_to_perf_thread)}个线程)') + + # 步骤2.4: 计算进程级 CPU 指令数（新方法：使用进程级统计，与update命令保持一致） + stage_start = time.time() + app_thread_instructions = {} + system_thread_instructions = {} + total_wasted_instructions = 0 # 包括所有线程（应用+系统）的指令数 + total_system_instructions = 0 + + # 获取应用进程PID（优先使用已获取的app_pid） + app_pid_for_cpu = app_pid + if not app_pid_for_cpu and frame_info.get('pid'): + app_pid_for_cpu = frame_info.get('pid') + + # 使用进程级CPU统计（与update命令保持一致） + if perf_conn and trace_conn and app_pid_for_cpu: + try: + app_instructions, sys_instructions = calculate_process_instructions( + perf_conn=perf_conn, + trace_conn=trace_conn, + app_pid=app_pid_for_cpu, + frame_start=frame_start, + frame_end=frame_end, + perf_sample_cache=perf_sample_cache, + perf_timestamp_field=perf_timestamp_field, + tid_to_info_cache=tid_to_info_cache, + ) + total_wasted_instructions = app_instructions + sys_instructions + total_system_instructions = sys_instructions + # 为了兼容性，构建空的字典（唤醒链分析仍然用于wakeup_threads） + app_thread_instructions = {} + system_thread_instructions = {} + logger.debug( + f' [帧{frame_id}] 进程级CPU统计: {total_wasted_instructions:,} 指令 (进程PID={app_pid_for_cpu})' + ) + except Exception as e: + logger.warning(f' [帧{frame_id}] 进程级CPU计算失败，回退到唤醒链方式: {e}') + # 回退到原来的唤醒链方式 + perf_thread_ids = set(itid_to_perf_thread.values()) + if perf_thread_ids: + app_thread_instructions, system_thread_instructions = calculate_thread_instructions( + perf_conn, + trace_conn, + perf_thread_ids, + frame_start, + frame_end, + tid_to_info_cache, + perf_sample_cache, + perf_timestamp_field, + ) + total_wasted_instructions = sum(app_thread_instructions.values()) + sum( + system_thread_instructions.values() + ) + total_system_instructions = sum(system_thread_instructions.values()) + else: + # 向后兼容：如果没有app_pid，使用原来的唤醒链方式 + perf_thread_ids = set(itid_to_perf_thread.values()) + if perf_conn and perf_thread_ids: + app_thread_instructions, system_thread_instructions = calculate_thread_instructions( + perf_conn, + trace_conn, + perf_thread_ids, + frame_start, + frame_end, + tid_to_info_cache, + perf_sample_cache, + perf_timestamp_field, + ) + total_wasted_instructions = sum(app_thread_instructions.values()) + sum( + system_thread_instructions.values() + ) + total_system_instructions = sum(system_thread_instructions.values()) + + elapsed = time.time() - stage_start + stage_timings['calculate_instructions'] += elapsed + logger.debug(f' [帧{frame_id}] calculate_instructions: {elapsed:.3f}秒') + + # 步骤2.5: 构建线程详细信息（使用预加载缓存）（计时） + stage_start = time.time() + related_threads_info = [] + if related_itids: + # 性能优化：使用预加载的缓存，避免数据库查询 + thread_info_map = {} + if itid_to_full_info_cache: + # 从缓存中查找 + for itid in related_itids: + if itid in itid_to_full_info_cache: + thread_info_map[itid] = itid_to_full_info_cache[itid] + else: + # 向后兼容：如果没有缓存，查询数据库 + itids_list = list(related_itids) + placeholders = ','.join('?' * len(itids_list)) + batch_thread_info_query = f""" + SELECT t.itid, t.tid, t.name, p.pid, p.name + FROM thread t + INNER JOIN process p ON t.ipid = p.ipid + WHERE t.itid IN ({placeholders}) + """ + cursor.execute(batch_thread_info_query, itids_list) + thread_results = cursor.fetchall() + + for itid, tid, thread_name, pid, process_name in thread_results: + thread_info_map[itid] = { + 'tid': tid, + 'thread_name': thread_name, + 'pid': pid, + 'process_name': process_name, + } + + # 构建线程详细信息列表（按唤醒链顺序） + for itid, depth in related_itids_ordered: + thread_info = thread_info_map.get(itid) + if thread_info: + # 优先使用itid_to_perf_thread映射，如果没有则直接使用thread_info中的tid + perf_thread_id = itid_to_perf_thread.get(itid) + if perf_thread_id is None: + # 如果映射中没有，直接使用thread_info中的tid（因为tid就是perf_sample.thread_id） + perf_thread_id = thread_info.get('tid') + + process_name = thread_info['process_name'] + thread_name = thread_info['thread_name'] + + # 判断是否为系统线程 + is_system = is_system_thread(process_name, thread_name) + + # 根据线程类型获取指令数 + # 注意：如果使用进程级CPU统计，app_thread_instructions和system_thread_instructions为空 + # 此时instruction_count为0，但total_wasted_instructions已经包含了所有线程的CPU + instruction_count = 0 + if perf_thread_id: + # 先尝试从应用线程字典查找（如果使用唤醒链方式） + if perf_thread_id in app_thread_instructions: + instruction_count = app_thread_instructions[perf_thread_id] + # 再尝试从系统线程字典查找（如果使用唤醒链方式） + elif perf_thread_id in system_thread_instructions: + instruction_count = system_thread_instructions[perf_thread_id] + # 如果使用进程级统计，app_thread_instructions和system_thread_instructions为空 + # 此时instruction_count为0，这是正常的，因为CPU已经包含在total_wasted_instructions中 + # 如果需要显示每个线程的CPU，需要额外查询（但为了性能，这里不查询） + else: + # perf_thread_id为None，说明该线程在trace数据库中没有对应的tid + # 这可能是数据问题，但指令数确实为0 + logger.debug(f'线程 {thread_name} (itid={itid}) 没有找到perf_thread_id，指令数为0') + + related_threads_info.append( + { + 'itid': itid, + 'tid': thread_info['tid'], + 'thread_name': thread_name, + 'pid': thread_info['pid'], + 'process_name': process_name, + 'perf_thread_id': perf_thread_id, + 'instruction_count': instruction_count, + 'is_system_thread': is_system, + 'wakeup_depth': depth, # 添加唤醒链深度信息 + } + ) + elapsed = time.time() - stage_start + stage_timings['build_thread_info'] += elapsed + logger.debug( + f' [帧{frame_id}] build_thread_info: {elapsed:.3f}秒 (构建{len(related_threads_info)}个线程信息)' + ) + + # 为每帧计算Top 5浪费线程（按指令数排序，便于开发者快速定位问题） + top_5_threads = [] + if related_threads_info: + # 按指令数排序，取前5个 + sorted_threads = sorted(related_threads_info, key=lambda x: x.get('instruction_count', 0), reverse=True) + top_5_threads = sorted_threads[:5] + # 只保留关键信息，减少数据量 + top_5_threads = [ + { + 'process_name': t.get('process_name', 'N/A'), + 'thread_name': t.get('thread_name', 'N/A'), + 'pid': t.get('pid', 'N/A'), + 'tid': t.get('tid', 'N/A'), + 'instruction_count': t.get('instruction_count', 0), + 'is_system_thread': t.get('is_system_thread', False), + 'percentage': (t.get('instruction_count', 0) / total_wasted_instructions * 100) + if total_wasted_instructions > 0 + else 0, + } + for t in top_5_threads + ] + + # 注意：假阳性已经在加载空刷帧后统一过滤了，这里不再需要检查 + # 所有到达这里的帧都是真阳性（真正的空刷帧） + results.append( + { + 'frame_id': frame_id, + 'frame_info': frame_info, + 'last_event': last_event, + 'wakeup_threads': related_threads_info, # 通过唤醒链分析找到的线程 + 'top_5_waste_threads': top_5_threads, # 新增：Top 5浪费线程（便于开发者快速定位问题） + 'total_wasted_instructions': total_wasted_instructions, # 只包含应用线程 + 'total_system_instructions': total_system_instructions, # 系统线程指令数（用于统计） + 'app_thread_instructions': app_thread_instructions, + 'system_thread_instructions': system_thread_instructions, + } + ) + + # 输出假阳性统计 + false_positive_count = getattr(analyze_empty_frame_wakeup_chain, '_false_positive_count', 0) + if false_positive_count > 0: + logger.info('过滤了 %d 个假阳性帧（flag=2但通过NativeWindow API成功提交了帧）', false_positive_count) + + # 计算总event_count（从perf_sample表中SUM所有event_count）作为分母 + total_event_count = 0 + if perf_conn: + try: + perf_cursor = perf_conn.cursor() + perf_cursor.execute('SELECT SUM(event_count) FROM perf_sample') + result = perf_cursor.fetchone() + if result and result[0] is not None: + total_event_count = result[0] + logger.info('perf_sample表总event_count: %d', total_event_count) + else: + logger.warning('无法获取perf_sample表总event_count，可能表为空') + except Exception as e: + logger.warning('查询perf_sample表总event_count失败: %s', str(e)) + + # 计算总浪费指令数和占比 + total_wasted_instructions = sum(r['total_wasted_instructions'] for r in results) + wasted_instruction_percentage = 0.0 + if total_event_count > 0: + wasted_instruction_percentage = (total_wasted_instructions / total_event_count) * 100 + logger.info( + '总浪费指令数: %d, 总event_count: %d, 占比: %.4f%%', + total_wasted_instructions, + total_event_count, + wasted_instruction_percentage, + ) + + # 将总event_count和占比添加到每个结果中（用于后续报告） + for result in results: + result['total_event_count'] = total_event_count + result['wasted_instruction_percentage'] = wasted_instruction_percentage + + trace_conn.close() + # 如果 perf_conn 和 trace_conn 是同一个，不要重复关闭 + # perf_in_trace 在函数开始处定义，如果 perf_conn == trace_conn，说明是同一个连接 + if perf_conn and perf_conn != trace_conn: + perf_conn.close() + + # 按浪费的指令数排序 + results.sort(key=lambda x: x['total_wasted_instructions'], reverse=True) + + # 打印性能分析结果（直接输出到控制台） + total_time = sum(stage_timings.values()) + print('\n' + '=' * 80) + print('性能分析 - 各阶段耗时汇总:') + print('=' * 80) + for stage_name, stage_time in sorted(stage_timings.items(), key=lambda x: x[1], reverse=True): + percentage = (stage_time / total_time * 100) if total_time > 0 else 0 + print(f' {stage_name:<25s}: {stage_time:>8.3f}秒 ({percentage:>5.1f}%)') + print(f' {"总计":<25s}: {total_time:>8.3f}秒 (100.0%)') + print('=' * 80 + '\n') + + # 同时记录到日志 + logger.info('=' * 80) + logger.info('性能分析 - 各阶段耗时:') + logger.info('=' * 80) + for stage_name, stage_time in sorted(stage_timings.items(), key=lambda x: x[1], reverse=True): + percentage = (stage_time / total_time * 100) if total_time > 0 else 0 + logger.info(' %-25s: %.2f秒 (%.1f%%)', stage_name, stage_time, percentage) + logger.info(' %-25s: %.2f秒 (100.0%%)', '总计', total_time) + logger.info('=' * 80) + + logger.info('分析完成: 共分析 %d 个空刷帧', len(results)) + + return results + + except Exception as e: + logger.error('分析空刷帧唤醒链失败: %s', str(e)) + logger.error('异常堆栈跟踪:\n%s', traceback.format_exc()) + return None + + +def print_results(results: list[dict[str, Any]], top_n: int = 10, framework_name: str = ''): + """打印分析结果 + + Args: + results: 分析结果列表 + top_n: 打印前 N 个结果 + framework_name: 框架名称（用于标题） + """ + if not results: + print('未找到空刷帧或分析失败') + return + + print('\n' + '=' * 80) + print(f'应用层空刷帧唤醒链分析报告 - {framework_name}') + print('=' * 80 + '\n') + + print(f'共分析 {len(results)} 个空刷帧\n') + + # 计算并显示总浪费指令占比 + if results: + total_wasted = sum(r['total_wasted_instructions'] for r in results) + total_event_count = results[0].get('total_event_count', 0) + wasted_percentage = results[0].get('wasted_instruction_percentage', 0.0) + + if total_event_count > 0: + print(f'总浪费指令数: {total_wasted:,}') + print(f'perf_sample表总event_count: {total_event_count:,}') + print(f'浪费指令占比: {wasted_percentage:.4f}%\n') + else: + print(f'总浪费指令数: {total_wasted:,}') + print('⚠️ 无法计算占比（perf数据库不可用或表为空）\n') + + # 打印前 N 个结果 + print(f'前 {min(top_n, len(results))} 个结果（按浪费的 CPU 指令数排序）:') + print('-' * 80) + + for idx, result in enumerate(results[:top_n], 1): + frame_info = result['frame_info'] + last_event = result['last_event'] + wakeup_threads = result.get('wakeup_threads', []) + total_instructions = result['total_wasted_instructions'] + + print(f'\n[{idx}] 帧 ID: {result["frame_id"]}') + print(f' 进程: {frame_info["process_name"]} (PID: {frame_info["pid"]})') + print(f' 主线程: {frame_info["thread_name"]} (TID: {frame_info["tid"]}, ITID: {frame_info["itid"]})') + print( + f' 时间: {frame_info["ts"]} - {frame_info["ts"] + frame_info["dur"]} ' + f'(持续 {frame_info["dur"] / 1_000_000:.2f}ms)' + ) + print(f' VSync: {frame_info.get("vsync", "N/A")}') + print(f' 最后事件: {last_event.get("name", "N/A")} (ts: {last_event.get("ts", "N/A")})') + print(f' 唤醒链线程数: {len(wakeup_threads)}') + print(f' 浪费的 CPU 指令数（所有线程）: {total_instructions:,}') + if result.get('total_system_instructions', 0) > 0: + print(f' 系统线程 CPU 指令数: {result.get("total_system_instructions", 0):,}') + + # 优先显示Top 5浪费线程（便于开发者快速定位问题） + top_5_threads = result.get('top_5_waste_threads', []) + if top_5_threads: + print('\n Top 5 浪费线程（按CPU指令数排序，便于调试）:') + print(f' {"序号":<6} {"类型":<8} {"进程名":<25} {"线程名":<30} {"指令数":<15} {"占比":<10}') + print(f' {"-" * 100}') + for thread_idx, thread in enumerate(top_5_threads, 1): + thread_type = '系统' if thread.get('is_system_thread', False) else '应用' + process_name = (thread.get('process_name', 'N/A') or 'N/A')[:24] + thread_name = (thread.get('thread_name', 'N/A') or 'N/A')[:29] + instruction_count = thread.get('instruction_count', 0) + percentage = thread.get('percentage', 0) + print( + f' {thread_idx:<6} {thread_type:<8} {process_name:<25} {thread_name:<30} {instruction_count:<15,} {percentage:>6.2f}%' + ) + + if wakeup_threads: + print('\n 唤醒链线程列表（供人工验证）:') + print( + f' {"序号":<6} {"类型":<8} {"进程名":<25} {"PID":<8} {"线程名":<30} {"TID":<8} {"ITID":<8} {"指令数":<15}' + ) + print(f' {"-" * 110}') + for thread_idx, thread_info in enumerate(wakeup_threads, 1): + process_name = (thread_info.get('process_name') or 'N/A')[:24] + thread_name = (thread_info.get('thread_name') or 'N/A')[:29] + pid = thread_info.get('pid', 'N/A') + tid = thread_info.get('tid', 'N/A') + itid = thread_info.get('itid', 'N/A') + instruction_count = thread_info.get('instruction_count', 0) + is_system = thread_info.get('is_system_thread', False) + thread_type = '系统' if is_system else '应用' + + print( + f' {thread_idx:<6} {thread_type:<8} {process_name:<25} {pid:<8} {thread_name:<30} {tid:<8} {itid:<8} {instruction_count:<15,}' + ) + else: + print(' 注意: 未找到相关线程（可能是 instant 表不存在或唤醒链为空）') + + print('\n' + '-' * 80) + + # 统计信息 + total_frames = len(results) + total_instructions = sum(r['total_wasted_instructions'] for r in results) + total_system_instructions = sum(r.get('total_system_instructions', 0) for r in results) + avg_instructions = total_instructions / total_frames if total_frames > 0 else 0 + max_instructions = max((r['total_wasted_instructions'] for r in results), default=0) + + # 统计所有涉及的线程 + all_threads = {} + for result in results: + for thread_info in result.get('wakeup_threads', []): + thread_key = (thread_info.get('pid'), thread_info.get('tid'), thread_info.get('thread_name')) + if thread_key not in all_threads: + all_threads[thread_key] = { + 'process_name': thread_info.get('process_name'), + 'pid': thread_info.get('pid'), + 'thread_name': thread_info.get('thread_name'), + 'tid': thread_info.get('tid'), + 'itid': thread_info.get('itid'), + 'appear_count': 0, + 'total_instructions': 0, + } + all_threads[thread_key]['appear_count'] += 1 + all_threads[thread_key]['total_instructions'] += thread_info.get('instruction_count', 0) + + print('\n统计信息:') + print(f' - 总空刷帧数: {total_frames}') + print(f' - 总浪费指令数（应用线程）: {total_instructions:,}') + if total_system_instructions > 0: + print(f' - 系统线程指令数（统计用）: {total_system_instructions:,}') + print(f' - 平均每帧浪费指令数（应用线程）: {avg_instructions:,.0f}') + print(f' - 单帧最大浪费指令数（应用线程）: {max_instructions:,}') + print(f' - 涉及的唯一线程数: {len(all_threads)}') + + if all_threads: + print('\n所有涉及的线程汇总（按出现次数排序）:') + print( + f'{"序号":<6} {"类型":<8} {"进程名":<25} {"PID":<8} {"线程名":<30} {"TID":<8} {"ITID":<8} {"出现次数":<12} {"总指令数":<15}' + ) + print(f'{"-" * 130}') + sorted_threads = sorted(all_threads.items(), key=lambda x: x[1]['appear_count'], reverse=True) + for thread_idx, (_key, thread_data) in enumerate(sorted_threads, 1): + process_name = (thread_data['process_name'] or 'N/A')[:24] + thread_name = (thread_data['thread_name'] or 'N/A')[:29] + is_system = is_system_thread(thread_data.get('process_name'), thread_data.get('thread_name')) + thread_type = '系统' if is_system else '应用' + print( + f'{thread_idx:<6} {thread_type:<8} {process_name:<25} {thread_data["pid"]:<8} {thread_name:<30} ' + f'{thread_data["tid"]:<8} {thread_data["itid"]:<8} {thread_data["appear_count"]:<12} ' + f'{thread_data["total_instructions"]:<15,}' + ) + print() + + +def main(): + """主函数""" + if len(sys.argv) < 3: + print( + '使用方法: python app_empty_frame_wakeup_chain.py [app_pids...] [framework_name]' + ) + print('示例: python app_empty_frame_wakeup_chain.py trace.db perf.db 12345 12346 Flutter') + sys.exit(1) + + trace_db_path = sys.argv[1] + perf_db_path = sys.argv[2] + + # 解析参数：app_pids 和 framework_name + app_pids = None + framework_name = '' + + if len(sys.argv) > 3: + # 尝试解析为 PID 或框架名称 + remaining_args = sys.argv[3:] + pids = [] + for arg in remaining_args: + try: + pids.append(int(arg)) + except ValueError: + framework_name = arg + if pids: + app_pids = pids + + # 执行分析 + results = analyze_empty_frame_wakeup_chain(trace_db_path, perf_db_path, app_pids) + + if results is None: + print('分析失败，请检查日志') + sys.exit(1) + + # 打印结果 + print_results(results, top_n=10, framework_name=framework_name) + + # 保存结果到 JSON 文件 + output_file = os.path.join( + os.path.dirname(__file__), + f'app_empty_frame_wakeup_chain_results_{framework_name.lower() if framework_name else "all"}.json', + ) + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(results, f, ensure_ascii=False, indent=2) + print(f'结果已保存到: {output_file}') + + +if __name__ == '__main__': + main() diff --git a/perf_testing/hapray/core/config/config.yaml b/perf_testing/hapray/core/config/config.yaml index 6d6f7ea3..c97af178 100644 --- a/perf_testing/hapray/core/config/config.yaml +++ b/perf_testing/hapray/core/config/config.yaml @@ -9,6 +9,8 @@ trace: memory: enable: False # Memory采集开关（通过命令行参数控制） max_stack_depth: 100 # Native Memory采集的最大调用栈深度 +ui: + capture_enable: True # UI数据采集开关（截图和组件树） # 帧分析配置 frame_analysis: diff --git a/perf_testing/hapray/core/data_collector.py b/perf_testing/hapray/core/data_collector.py index 8efd73d2..f365066f 100644 --- a/perf_testing/hapray/core/data_collector.py +++ b/perf_testing/hapray/core/data_collector.py @@ -91,7 +91,11 @@ def collect_step_start(self, step_id: int, report_path: str): perf_step_dir, trace_step_dir = self._get_step_directories(step_id, report_path) self._ensure_directories_exist(perf_step_dir, trace_step_dir) self.process_manager.save_process_info(perf_step_dir) - self.capture_ui_handler.capture_ui(step_id, report_path, 'start') + + # 检查UI采集开关 + if Config.get('ui.capture_enable', False): + self.capture_ui_handler.capture_ui(step_id, report_path, 'start') + self.collect_memory_data(step_id, report_path) Log.info(f'步骤 {step_id} 开始采集准备完成') @@ -121,7 +125,11 @@ def collect_step_end(self, device_file: str, step_id: int, report_path: str, red self.data_transfer.transfer_trace_data(device_file, local_trace_path) self.data_transfer.transfer_redundant_data(trace_step_dir, redundant_mode_status) self.data_transfer.collect_coverage_data(perf_step_dir) - self.capture_ui_handler.capture_ui(step_id, report_path, 'end') + + # 检查UI采集开关 + if Config.get('ui.capture_enable', False): + self.capture_ui_handler.capture_ui(step_id, report_path, 'end') + self.collect_memory_data(step_id, report_path) Log.info(f'步骤 {step_id} 数据采集结束处理完成') diff --git a/perf_testing/hapray/core/report.py b/perf_testing/hapray/core/report.py index 39ed8dd2..fdc69baf 100644 --- a/perf_testing/hapray/core/report.py +++ b/perf_testing/hapray/core/report.py @@ -52,7 +52,8 @@ def __init__(self, scene_dir: str, result: dict): 'type': DataType.BASE64_GZIP_JSON.value, 'versionCode': 1, 'basicInfo': {}, - 'perf': {'steps': []}, # 默认空步骤 + 'steps': [], # 步骤基本信息（step_id, step_name） + 'perf': {'steps': []}, # 性能数据步骤（不包含step_id和step_name） }, **result, } @@ -67,7 +68,10 @@ def from_paths(cls, scene_dir: str, result: dict): """ perf_data_path = os.path.join(scene_dir, 'hiperf', 'hiperf_info.json') data = cls(scene_dir, result) - data.load_perf_data(perf_data_path) + steps_path = os.path.join(scene_dir, 'hiperf', 'steps.json') + data._load_steps_data(steps_path) + # 当perf.data不存在时，hiperf_info.json可能不存在，设为非必需 + data.load_perf_data(perf_data_path, required=False) data.load_trace_data(scene_dir) return data @@ -294,6 +298,26 @@ def _compress_trace_data(self, data): return data + def _load_steps_data(self, path: str): + """加载步骤基本信息 + + Args: + path: steps.json 文件路径 + """ + steps_data = self._load_json_safe(path, default=[]) + if len(steps_data) == 0: + raise FileNotFoundError(f'steps.json not found: {path}') + + # 提取 step_id 和 step_name 到 self.result['steps'] + self.result['steps'] = [ + { + 'step_id': step.get('stepIdx'), + 'step_name': step.get('name'), + } + for step in steps_data + if 'stepIdx' in step and 'name' in step + ] + def load_perf_data(self, path, required: bool = True): """加载性能数据 @@ -371,6 +395,13 @@ def load_trace_data(self, scene_dir: str, required: bool = False): self.result['ui']['animate'] = ui_animate_data logging.info(f'Loaded UI Animate data: {len(ui_animate_data)} steps') + # 加载 UI 原始数据（用于对比） + ui_raw_path = os.path.join(report_dir, 'ui_raw.json') + ui_raw_data = self._load_json_safe(ui_raw_path, default={}) + if ui_raw_data: + self.result['ui']['raw'] = ui_raw_data + logging.info(f'Loaded UI Raw data: {len(ui_raw_data)} steps') + def _load_json_safe(self, path, default): """安全加载JSON文件，处理异常情况""" if not os.path.exists(path): @@ -810,8 +841,9 @@ def create_perf_summary_excel(input_path: str) -> bool: merged_data = merge_summary_info(input_path) if not merged_data: - logging.error('错误: 没有找到任何summary_info.json文件或文件内容为空') - return False + # summary_info.json不存在时不影响报告生成，只记录警告 + logging.warning('警告: 没有找到任何summary_info.json文件或文件内容为空，跳过Excel汇总报告生成') + return True # 转为DataFrame df = pd.DataFrame(merged_data) diff --git a/perf_testing/hapray/mode/simple_mode.py b/perf_testing/hapray/mode/simple_mode.py index 6af2f073..69dc98fa 100644 --- a/perf_testing/hapray/mode/simple_mode.py +++ b/perf_testing/hapray/mode/simple_mode.py @@ -62,6 +62,8 @@ def create_simple_mode_structure(report_dir, perf_paths, trace_paths, package_na # 处理perf文件 if i < len(perf_paths): _process_perf_file(perf_paths[i], hiperf_step_dir, target_db_files, package_name, pids) + else: + _create_pids_json(None, hiperf_step_dir, package_name, pids) # 处理trace文件（仅当提供了trace文件时） if trace_paths and i < len(trace_paths): @@ -112,8 +114,8 @@ def create_simple_mode_structure(report_dir, perf_paths, trace_paths, package_na def parse_processes(target_db_file: str, file_path: str, package_name: str, pids: list): """ 解析进程文件，返回包含目标包名的进程pid和进程名列表。 - :param target_db_file: 性能数据库文件路径 - :param file_path: 进程信息文件路径 + :param target_db_file: 性能数据库文件路径（perf.db） + :param file_path: 进程信息文件路径（ps_ef.txt） :param package_name: 目标包名 :param pids: 用户提供的进程ID列表 :return: dict { 'pids': List[int], 'process_names': List[str] } @@ -121,8 +123,9 @@ def parse_processes(target_db_file: str, file_path: str, package_name: str, pids if not package_name: raise ValueError('包名不能为空') result = {'pids': [], 'process_names': []} - if os.path.exists(target_db_file) and target_db_file: - # 连接trace数据库 + + # 优先从 perf.db 的 perf_thread 表中查询 + if target_db_file and os.path.exists(target_db_file): perf_conn = sqlite3.connect(target_db_file) try: # 获取所有perf样本 @@ -133,10 +136,39 @@ def parse_processes(target_db_file: str, file_path: str, package_name: str, pids result['pids'].append(row['process_id']) result['process_names'].append(row['thread_name']) except Exception as e: - logging.error('从db中获取pids时发生异常: %s', str(e)) + logging.error('从perf.db中获取pids时发生异常: %s', str(e)) finally: perf_conn.close() - if os.path.exists(file_path): + + # 如果从 perf.db 没有获取到数据，尝试从 trace.db 的 process 表中查询 + if not result['pids']: + # 尝试查找同目录下的 trace.db + if target_db_file: + trace_db_file = target_db_file.replace('perf.db', '../htrace/step1/trace.db') + trace_db_file = os.path.normpath( + os.path.join(os.path.dirname(target_db_file), '../../htrace/step1/trace.db') + ) + else: + trace_db_file = None + + if trace_db_file and os.path.exists(trace_db_file): + trace_conn = sqlite3.connect(trace_db_file) + try: + # 从 trace.db 的 process 表中查询 + trace_query = 'SELECT DISTINCT pid, name FROM process WHERE name LIKE ?' + params = (f'%{package_name}%',) + trace_pids = pd.read_sql_query(trace_query, trace_conn, params=params) + for _, row in trace_pids.iterrows(): + result['pids'].append(row['pid']) + result['process_names'].append(row['name']) + logging.info('从trace.db中获取到%d个进程', len(result['pids'])) + except Exception as e: + logging.error('从trace.db中获取pids时发生异常: %s', str(e)) + finally: + trace_conn.close() + + # 如果还是没有数据，尝试从 ps_ef.txt 文件中解析 + if not result['pids'] and os.path.exists(file_path): result = {'pids': [], 'process_names': []} try: with open(file_path, encoding='utf-8') as f: @@ -154,6 +186,8 @@ def parse_processes(target_db_file: str, file_path: str, package_name: str, pids result['process_names'].append(process_name) except Exception as err: logging.error('处理文件失败: %s', err) + + # 如果用户提供了 pids，使用用户提供的 if pids != []: process_names = [] for _ in pids: @@ -198,15 +232,25 @@ def _process_perf_file(perf_path, hiperf_step_dir, target_db_files, package_name _copy_ps_ef_file(perf_path, hiperf_step_dir) # 创建pids.json - if current_db_file: - _create_pids_json(current_db_file, hiperf_step_dir, package_name, pids) + _create_pids_json(current_db_file, hiperf_step_dir, package_name, pids) def _process_trace_file(trace_path, htrace_step_dir): - """处理单个trace文件""" - target_htrace_file = os.path.join(htrace_step_dir, 'trace.htrace') - shutil.copy2(trace_path, target_htrace_file) - logging.info('Copied %s to %s', trace_path, target_htrace_file) + """处理单个trace文件 + + 如果输入是.db文件，直接复制为trace.db + 如果输入是.htrace文件，复制为trace.htrace（后续会转换为trace.db） + """ + if trace_path.endswith('.db'): + # 如果输入是.db文件，直接复制为trace.db + target_db_file = os.path.join(htrace_step_dir, 'trace.db') + shutil.copy2(trace_path, target_db_file) + logging.info('Copied %s to %s', trace_path, target_db_file) + else: + # 如果输入是.htrace文件，复制为trace.htrace + target_htrace_file = os.path.join(htrace_step_dir, 'trace.htrace') + shutil.copy2(trace_path, target_htrace_file) + logging.info('Copied %s to %s', trace_path, target_htrace_file) def _handle_perf_db_file(perf_path, hiperf_step_dir, target_data_file, target_db_files): diff --git a/perf_testing/hapray/testcases/com.kuaishou.hmapp/PerfLoad_kuaishou_0010.py b/perf_testing/hapray/testcases/com.kuaishou.hmapp/PerfLoad_kuaishou_0010.py index 4acda3ac..362b8850 100644 --- a/perf_testing/hapray/testcases/com.kuaishou.hmapp/PerfLoad_kuaishou_0010.py +++ b/perf_testing/hapray/testcases/com.kuaishou.hmapp/PerfLoad_kuaishou_0010.py @@ -47,7 +47,7 @@ def step1(): time.sleep(2) # 点击收藏的第一个视频 - self.driver.touch(self.convert_coordinate(195, 1489)) + self.driver.touch(self.convert_coordinate(195, 1767)) time.sleep(2) # 点击收藏的第一个视频 self.driver.touch(self.convert_coordinate(997, 1468)) diff --git a/perf_testing/hapray/testcases/com.kuaishou.hmapp/PerfLoad_kuaishou_0050.py b/perf_testing/hapray/testcases/com.kuaishou.hmapp/PerfLoad_kuaishou_0050.py index b5012978..b8820546 100644 --- a/perf_testing/hapray/testcases/com.kuaishou.hmapp/PerfLoad_kuaishou_0050.py +++ b/perf_testing/hapray/testcases/com.kuaishou.hmapp/PerfLoad_kuaishou_0050.py @@ -36,6 +36,15 @@ def process(self): self.driver.touch(self.convert_coordinate(540, 2253)) time.sleep(2) + self.touch_by_text('翻转', 2) + self.touch_by_text('美化', 2) + self.touch_by_text('美颜', 2) + self.touch_by_text('磨皮', 2) + self.touch_by_text('滤镜', 2) + self.touch_by_text('自然', 2) + # 点击空白退出美化 + self.driver.touch(self.convert_coordinate(649, 911)) + def step1(): # 点击拍摄按钮 self.driver.touch(self.convert_coordinate(541, 2000)) diff --git a/perf_testing/hapray/testcases/com.tencent.wechat/PerfLoad_Wechat_0010.py b/perf_testing/hapray/testcases/com.tencent.wechat/PerfLoad_Wechat_0010.py index 3577b2ae..33dd0513 100644 --- a/perf_testing/hapray/testcases/com.tencent.wechat/PerfLoad_Wechat_0010.py +++ b/perf_testing/hapray/testcases/com.tencent.wechat/PerfLoad_Wechat_0010.py @@ -1,7 +1,6 @@ import time from hypium import BY -from tensorflow import double from hapray.core.perf_testcase import PerfTestCase @@ -65,9 +64,9 @@ def step4(): self.driver.touch(comps[5]) self.driver.wait(0.5) for _i in range(10): - self.driver.touch(self.convert_coordinate(485, 499), double) + self.driver.touch(self.convert_coordinate(485, 499), 'double') time.sleep(2) - self.driver.touch(self.convert_coordinate(485, 499), double) + self.driver.touch(self.convert_coordinate(485, 499), 'double') time.sleep(2) def step5(): diff --git a/perf_testing/hapray/testcases/manual/PerfLoad_UIAnalyzer.py b/perf_testing/hapray/testcases/manual/PerfLoad_UIAnalyzer.py index 630f0557..ada1b969 100644 --- a/perf_testing/hapray/testcases/manual/PerfLoad_UIAnalyzer.py +++ b/perf_testing/hapray/testcases/manual/PerfLoad_UIAnalyzer.py @@ -92,6 +92,6 @@ def app_name(self) -> str: def process(self): def step1(): self.swipes_up(1, 2) - time.sleep(3) + time.sleep(8) - self.execute_performance_step('UI测试', 5, step1) + self.execute_performance_step('UI测试', 10, step1) diff --git a/perf_testing/hapray/ui_detector/ui_tree_comparator.py b/perf_testing/hapray/ui_detector/ui_tree_comparator.py new file mode 100644 index 00000000..b2a4d881 --- /dev/null +++ b/perf_testing/hapray/ui_detector/ui_tree_comparator.py @@ -0,0 +1,170 @@ +""" +Copyright (c) 2025 Huawei Device Co., Ltd. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +from typing import Any + +from PIL import Image, ImageDraw + +from hapray.ui_detector.arkui_tree_parser import compare_arkui_trees, parse_arkui_tree + + +class UITreeComparator: + """UI组件树对比工具""" + + # 默认忽略的属性（系统内部状态，不影响UI） + DEFAULT_IGNORE_ATTRS = { + 'id', + 'accessibilityId', # 系统自动生成的ID + 'rsNode', + 'frameProxy', + 'frameRecord', # 渲染引擎内部状态 + 'contentConstraint', + 'parentLayoutConstraint', + 'user defined constraint', # 布局约束 + } + + def compare_ui_trees( + self, + tree1_path: str, + screenshot1_path: str, + tree2_path: str, + screenshot2_path: str, + output_dir: str, + ignore_attrs: set = None, + filter_minor_changes: bool = False, + ) -> dict[str, Any]: + """对比两个UI组件树并标记差异 + + Args: + tree1_path: 组件树1文件路径 + screenshot1_path: 截图1文件路径 + tree2_path: 组件树2文件路径 + screenshot2_path: 截图2文件路径 + output_dir: 输出目录 + ignore_attrs: 要忽略的属性集合（默认使用DEFAULT_IGNORE_ATTRS） + filter_minor_changes: 是否过滤微小变化（如<5px的位置差异） + + Returns: + 包含差异信息和输出文件路径的字典 + """ + # 使用默认忽略属性 + if ignore_attrs is None: + ignore_attrs = self.DEFAULT_IGNORE_ATTRS + + # 读取并解析组件树 + with open(tree1_path, encoding='utf-8') as f: + tree1 = parse_arkui_tree(f.read()) + with open(tree2_path, encoding='utf-8') as f: + tree2 = parse_arkui_tree(f.read()) + + # 对比组件树 + all_differences = compare_arkui_trees(tree1, tree2) + + # 过滤差异 + differences = self._filter_differences(all_differences, ignore_attrs, filter_minor_changes) + + # 提取差异区域的bounds + diff_regions = self._extract_diff_regions(differences) + + # 生成标记后的对比图 + os.makedirs(output_dir, exist_ok=True) + marked_img1 = os.path.join(output_dir, 'diff_marked_1.png') + marked_img2 = os.path.join(output_dir, 'diff_marked_2.png') + + self._mark_differences(screenshot1_path, diff_regions, marked_img1) + self._mark_differences(screenshot2_path, diff_regions, marked_img2) + + return { + 'differences': differences, + 'diff_count': len(differences), + 'diff_regions': diff_regions, + 'marked_images': [marked_img1, marked_img2], + 'total_differences': len(all_differences), + 'filtered_count': len(all_differences) - len(differences), + } + + def _filter_differences( + self, differences: list[dict[str, Any]], ignore_attrs: set, filter_minor: bool + ) -> list[dict[str, Any]]: + """过滤差异""" + filtered = [] + for diff in differences: + # 过滤属性差异 + filtered_attrs = [] + for attr_diff in diff.get('comparison_result', []): + attr_name = attr_diff.get('attribute', '') + + # 跳过忽略的属性 + if attr_name in ignore_attrs: + continue + + # 过滤微小变化 + if filter_minor and self._is_minor_change(attr_name, attr_diff): + continue + + filtered_attrs.append(attr_diff) + + # 如果还有差异，保留该组件 + if filtered_attrs: + diff_copy = diff.copy() + diff_copy['comparison_result'] = filtered_attrs + filtered.append(diff_copy) + + return filtered + + def _is_minor_change(self, attr_name: str, attr_diff: dict) -> bool: + """判断是否为微小变化""" + # 位置和尺寸的微小变化（<5px） + if attr_name in ['top', 'left', 'width', 'height']: + try: + v1 = float(attr_diff.get('value1', 0)) + v2 = float(attr_diff.get('value2', 0)) + return abs(v1 - v2) < 5 + except (ValueError, TypeError): + return False + return False + + def _extract_diff_regions(self, differences: list[dict[str, Any]]) -> list[tuple[int, int, int, int]]: + """从差异列表中提取bounds区域""" + regions = [] + for diff in differences: + component = diff.get('component', {}) + bounds_rect = component.get('bounds_rect') + if bounds_rect and len(bounds_rect) == 4: + x1, y1, x2, y2 = bounds_rect + # 确保坐标有效（x2 >= x1, y2 >= y1） + if x2 >= x1 and y2 >= y1: + regions.append((x1, y1, x2, y2)) + return regions + + def _mark_differences(self, screenshot_path: str, regions: list[tuple], output_path: str): + """在截图上标记差异区域""" + if not os.path.exists(screenshot_path): + return + + img = Image.open(screenshot_path).convert('RGB') + draw = ImageDraw.Draw(img) + + for i, region in enumerate(regions, 1): + if len(region) != 4: + continue + x1, y1, x2, y2 = region + # 再次验证坐标有效性 + if x2 >= x1 and y2 >= y1: + draw.rectangle([x1, y1, x2, y2], outline='red', width=3) + draw.text((x1 + 5, y1 + 5), f'D{i}', fill='red') + + img.save(output_path) diff --git a/perf_testing/plugin.json b/perf_testing/plugin.json index 57a03951..26674e5d 100644 --- a/perf_testing/plugin.json +++ b/perf_testing/plugin.json @@ -231,24 +231,22 @@ "help": "选择模式：0 COMMUNITY, 1 SIMPLE" }, "perfs": { - "type": "str", + "type": "file", "label": "Perf数据路径", "required": false, - "nargs": "+", - "help": "SIMPLE模式需要的perf数据路径（支持多个文件）" + "help": "SIMPLE模式需要的perf数据文件（支持多个文件）" }, "traces": { - "type": "str", + "type": "file", "label": "Trace文件路径", "required": false, - "nargs": "+", - "help": "SIMPLE模式可选的trace文件路径（支持多个文件）" + "help": "SIMPLE模式可选的trace文件（支持多个文件）" }, "package-name": { "type": "str", "label": "应用包名", "required": false, - "help": "SIMPLE模式需要的应用包名" + "help": "SIMPLE模式需要的应用包名（单个包名字符串）" }, "pids": { "type": "int", @@ -311,6 +309,35 @@ "help": "输出Excel文件路径（默认：当前目录下的compare_result.xlsx）" } } + }, + "ui-compare": { + "name": "UI组件树对比", + "description": "对比两个报告的UI组件树", + "action_mapping": { + "type": "position" + }, + "parameters": { + "base_dir": { + "type": "dir", + "label": "基准报告目录", + "required": true, + "help": "基准报告根目录（如：PerfLoad_meituan_0010）" + }, + "compare_dir": { + "type": "dir", + "label": "对比报告目录", + "required": true, + "help": "对比报告根目录（如：PerfLoad_meituan_0010）" + }, + "output": { + "type": "dir", + "label": "输出目录", + "required": false, + "default": "./ui_compare_output", + "short": "o", + "help": "输出目录（默认：./ui_compare_output）" + } + } } } } diff --git a/perf_testing/scripts/main.py b/perf_testing/scripts/main.py index 4b009268..68bc931a 100644 --- a/perf_testing/scripts/main.py +++ b/perf_testing/scripts/main.py @@ -26,6 +26,7 @@ from hapray.actions.prepare_action import PrepareAction from hapray.actions.static_action import StaticAction from hapray.actions.ui_action import UIAction +from hapray.actions.ui_compare_action import UICompareAction from hapray.actions.update_action import UpdateAction from hapray.core.config.config import Config @@ -92,6 +93,7 @@ def __init__(self): 'compare': CompareAction, 'prepare': PrepareAction, 'ui': UIAction, + 'ui-compare': UICompareAction, } parser = argparse.ArgumentParser( @@ -105,7 +107,7 @@ def __init__(self): choices=list(actions.keys()), nargs='?', default='perf', - help='Action to perform (perf: performance testing, static: HAP static analysis, update: update reports, compare: compare reports, prepare: simplified test execution, ui: UI analysis)', + help='Action to perform (perf: performance testing, static: HAP static analysis, update: update reports, compare: compare reports, prepare: simplified test execution, ui: UI analysis, ui-compare: UI tree comparison)', ) # Parse action action_args = [] diff --git a/scripts/download_test_products.js b/scripts/download_test_products.js index 2a907e83..f30ab8f7 100644 --- a/scripts/download_test_products.js +++ b/scripts/download_test_products.js @@ -10,18 +10,14 @@ const path = require('path'); const { execSync } = require('child_process'); const TEST_PRODUCTS_REPO = 'https://gitcode.com/B1A2/HapRayTestProducts.git'; -const TESTS_DIR = path.join(__dirname, '..', 'tests'); +// 从命令行参数读取下载目录，如果未提供则使用默认值 +const DIST_DIR = process.argv[2]; +const TESTS_DIR = path.join(DIST_DIR, 'tests'); function downloadTestProducts() { try { console.log('正在下载测试工程资源...'); - // 确保 tests 目录存在 - if (!fs.existsSync(TESTS_DIR)) { - fs.mkdirSync(TESTS_DIR, { recursive: true }); - console.log(`创建目录: ${TESTS_DIR}`); - } - // 检查是否已经存在 .git 目录（表示已经是一个 git 仓库） const gitDir = path.join(TESTS_DIR, '.git'); if (fs.existsSync(gitDir)) { @@ -32,7 +28,7 @@ function downloadTestProducts() { console.log('正在克隆测试工程仓库...'); // 如果不存在，则克隆仓库 execSync(`git clone ${TEST_PRODUCTS_REPO} tests`, { - cwd: path.join(__dirname, '..'), + cwd: DIST_DIR, stdio: 'inherit' }); } diff --git a/scripts/e2e_test.js b/scripts/e2e_test.js index 82554559..9f82d60d 100644 --- a/scripts/e2e_test.js +++ b/scripts/e2e_test.js @@ -10,11 +10,30 @@ const path = require('path'); const { execSync } = require('child_process'); const XLSX = require('xlsx'); -const DIST_DIR = path.join(__dirname, '..', 'dist'); +// dist 目录： +// - 读取命令行入参：node e2e_test.js +// - 如果未提供参数，默认使用当前工作目录下的 dist 目录 +const DIST_DIR = path.resolve(process.argv[2]); +console.log(`DIST_DIR: ${DIST_DIR}`); + +// 验证 DIST_DIR 是否存在 +if (!fs.existsSync(DIST_DIR)) { + console.error(`❌ 错误: dist 目录不存在: ${DIST_DIR}`); + console.error(' 用法: node e2e_test.js '); + process.exit(1); +} + +// 验证 DIST_DIR 是否为目录 +const distStat = fs.statSync(DIST_DIR); +if (!distStat.isDirectory()) { + console.error(`❌ 错误: 指定的路径不是目录: ${DIST_DIR}`); + process.exit(1); +} + const TOOLS_DIR = path.join(DIST_DIR, 'tools'); -// 测试资源目录（优先使用外部测试资源） -const TEST_PRODUCTS_DIR = path.join(__dirname, '..', 'tests'); +// 测试资源目录（优先使用外部测试资源），位于打包根目录下 +const TEST_PRODUCTS_DIR = path.join(DIST_DIR, 'tests'); const USE_EXTERNAL_RESOURCES = true; // 输出目录 @@ -112,11 +131,16 @@ function checkFileExists(filePath, description) { function runCommand(command, description, options = {}) { console.log(`执行 ${description}...`); try { - const result = execSync(command, { + const execOptions = { stdio: options.silent ? 'pipe' : 'inherit', env: { ...process.env, ...options.env }, + cwd: DIST_DIR, ...options - }); + }; + // 不需要传递给 execSync 的自定义字段 + delete execOptions.silent; + + const result = execSync(command, execOptions); console.log(`✓ ${description} 成功`); return result; } catch (error) { @@ -198,13 +222,14 @@ function testStaticModule() { runCommand(`${EXECUTABLE} static -i "${testFile}" -o "${outputDir}"`, 'static 模块实际功能测试', { silent: false }); - const files = fs.readdirSync(outputDir); + const files = fs.readdirSync(path.join(outputDir, 'meituan')); if (files.length >= 3) { console.log(`✓ static 模块实际功能测试成功 (生成${files.length}个文件)`); console.log(`输出文件保存在: ${outputDir}`); return { success: true }; } else { console.log(`✗ static 模块输出文件不足 (需要>=3个，实际${files.length}个)`); + console.log(`输出文件: ${files.join(', ')}`); return { success: false, error: `输出文件不足: ${files.length} < 3` }; } } catch (error) { @@ -234,7 +259,7 @@ function getLatestReportFolder(reportsDir) { * 移动perf命令生成的reports目录到tests/output目录下 */ function moveReportsDirectory() { - const reportsDir = path.join(__dirname, '..', 'reports'); + const reportsDir = path.join(DIST_DIR, 'reports'); const targetDir = path.join(OUTPUT_DIR, 'reports'); try { @@ -267,18 +292,14 @@ function testPerfModule() { try { const distTestCaseDir = path.join(DIST_DIR, 'tools', 'perf-testing', '_internal', 'hapray', 'testcases', 'com.sankuai.hmeituan'); const distTestCaseFile = path.join(distTestCaseDir, 'PerfLoad_meituan_0010.json'); - const sourceTestCaseDir = path.join(__dirname, '..', 'perf_testing', 'hapray', 'testcases', 'com.sankuai.hmeituan'); - const sourceTestCaseFile = path.join(sourceTestCaseDir, 'PerfLoad_meituan_0010.json'); - const testCaseFile = fs.existsSync(distTestCaseFile) ? distTestCaseFile : sourceTestCaseFile; - - if (!fs.existsSync(testCaseFile)) { + if (!fs.existsSync(distTestCaseFile)) { return { success: false, error: 'meituan_0010测试用例不存在' }; } console.log(`发现meituan_0010测试用例，尝试执行perf命令...`); // 检查reports目录是否已存在，如果存在则删除 - const oldReportsDir = path.join(__dirname, '..', 'reports'); + const oldReportsDir = path.join(DIST_DIR, 'reports'); if (fs.existsSync(oldReportsDir)) { fs.rmSync(oldReportsDir, { recursive: true, force: true }); } @@ -384,7 +405,7 @@ function testSymbolRecoveryModule() { console.log(`输出结果保存在: ${outputDir}`); // 校验 cache/llm_analysis_cache.json 中的对象数 - const cacheFile = path.join(__dirname, '..', 'cache', 'llm_analysis_cache.json'); + const cacheFile = path.join(DIST_DIR, 'cache', 'llm_analysis_cache.json'); if (fs.existsSync(cacheFile)) { const cacheData = JSON.parse(fs.readFileSync(cacheFile, 'utf8')); const objectCount = Object.keys(cacheData).length; @@ -558,17 +579,17 @@ function verifyArtifactsHash(results) { let skipRules = {}; if (key === 'update') { skipRules = { - 'ecol_load_hiperf_detail': { cols: [9] }, // 第10列（索引9） + 'ecol_load_hiperf_detail': { cols: [1, 4, 9] }, // 第10列（索引9） 'ecol_load_step': { cols: [2, 4, 8] } // 第3,5,9列（索引2,4,8） }; } else if (key === 'static') { skipRules = { - '分析摘要': { rows: [0, 6, 7] }, // 第1,7,8行（索引0,6,7） - '技术栈信息': { cols: [6] } // 第7列（索引6） + '分析摘要': { rows: [0, 1, 6, 7] }, // 第1,7,8行（索引0,6,7） + '技术栈信息': { cols: [6, 7, 12, 13] } // 第7列（索引6） }; } else if (key === 'opt') { skipRules = { - 'optimization': { cols: [1] } // 第2列（索引1） + 'optimization': { cols: [1, 13] } // 第2列（索引1） }; } @@ -594,6 +615,9 @@ function verifyArtifactsHash(results) { async function runE2ETests() { console.log('🚀 开始 ArkAnalyzer-HapRay 端到端测试\n'); + // 先下载/更新测试资源 + runCommand(`node ${path.join(__dirname, 'download_test_products.js')} "${DIST_DIR}"`, '下载测试资源', { silent: false }); + // 配置 LLM 环境变量用于符号恢复模块测试 console.log('🤖 配置 LLM 环境变量...'); process.env.LLM_API_KEY = 'sk-14ccee5142d04e7fbbcda3418b715390'; @@ -697,8 +721,7 @@ async function runE2ETests() { process.exit(1); } finally { // 清理缓存目录 - const ROOT_DIR = path.join(__dirname, '..'); - const cacheDir = path.join(ROOT_DIR, 'files_results_cache'); + const cacheDir = path.join(DIST_DIR, 'files_results_cache'); if (fs.existsSync(cacheDir)) { fs.rmSync(cacheDir, { recursive: true, force: true }); console.log('\n🧹 已清理缓存目录: files_results_cache'); diff --git a/scripts/test_release.js b/scripts/test_release.js new file mode 100644 index 00000000..ed18a92d --- /dev/null +++ b/scripts/test_release.js @@ -0,0 +1,165 @@ +#!/usr/bin/env node + +/** + * 测试发布包脚本 + * 接收一个 HapRay 的 zip 包，解压后执行端到端测试 + */ + +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); +const AdmZip = require('adm-zip'); + +// 读取命令行入参：node test_release.js +const ZIP_FILE = process.argv[2]; + +if (!ZIP_FILE) { + console.error('❌ 错误: 未提供 zip 文件路径'); + console.error(' 用法: node test_release.js '); + process.exit(1); +} + +// 验证 zip 文件是否存在 +const zipPath = path.resolve(ZIP_FILE); +if (!fs.existsSync(zipPath)) { + console.error(`❌ 错误: zip 文件不存在: ${zipPath}`); + process.exit(1); +} + +// 验证是否为文件 +const zipStat = fs.statSync(zipPath); +if (!zipStat.isFile()) { + console.error(`❌ 错误: 指定的路径不是文件: ${zipPath}`); + process.exit(1); +} + +// 创建临时解压目录 +const tempDir = path.join(__dirname, '../temp_release_test'); +const extractDir = path.join(tempDir, path.basename(zipPath, path.extname(zipPath))); + +/** + * 解压 zip 文件（跨平台支持） + */ +function unzipFile(zipPath, extractPath) { + console.log(`📦 开始解压 zip 文件...`); + console.log(` 源文件: ${zipPath}`); + console.log(` 目标目录: ${extractPath}`); + + try { + // 如果目标目录已存在，先删除 + if (fs.existsSync(extractPath)) { + console.log(` 清理已存在的目录: ${extractPath}`); + fs.rmSync(extractPath, { recursive: true, force: true }); + } + + // 确保父目录存在 + const parentDir = path.dirname(extractPath); + if (!fs.existsSync(parentDir)) { + fs.mkdirSync(parentDir, { recursive: true }); + } + + // 根据平台选择解压方式 + const platform = process.platform; + if (platform === 'win32') { + // Windows 使用 AdmZip（不支持软链接，但至少能解压） + console.log(' 使用 tar 命令解压 (Windows)'); + if (!fs.existsSync(extractPath)) { + fs.mkdirSync(extractPath, { recursive: true }); + } + execSync(`tar -xf "${zipPath}" -C "${extractPath}"`, { + stdio: 'inherit', + }); + } else { + // macOS/Linux 使用系统 unzip 命令（保留软链接） + console.log(` 使用系统 unzip 命令解压（${platform} 平台，保留软链接）`); + execSync(`unzip -q "${zipPath}" -d "${extractPath}"`, { + stdio: 'inherit', + }); + } + + console.log(`✓ 解压成功`); + return extractPath; + } catch (error) { + console.error(`✗ 解压失败: ${error.message}`); + throw error; + } +} + +/** + * 清理临时目录 + */ +function cleanup() { + try { + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + console.log(`🧹 已清理临时目录: ${tempDir}`); + } + } catch (error) { + console.warn(`⚠️ 清理临时目录失败: ${error.message}`); + } +} + +/** + * 执行端到端测试 + */ +function runE2ETest(distDir) { + console.log(`\n🧪 开始执行端到端测试...`); + console.log(` 测试目录: ${distDir}\n`); + + try { + const e2eTestScript = path.join(__dirname, 'e2e_test.js'); + execSync(`node "${e2eTestScript}" "${distDir}"`, { + stdio: 'inherit', + cwd: __dirname + }); + console.log(`\n✓ 端到端测试完成`); + } catch (error) { + console.error(`\n✗ 端到端测试失败`); + throw error; + } +} + +/** + * 主函数 + */ +async function main() { + console.log('🚀 开始测试发布包\n'); + console.log(`📁 Zip 文件: ${zipPath}\n`); + + try { + // 1. 解压 zip 文件 + const extractedDir = unzipFile(zipPath, extractDir); + + // 验证解压后的目录是否存在 + if (!fs.existsSync(extractedDir)) { + throw new Error(`解压后的目录不存在: ${extractedDir}`); + } + + // 验证是否为目录 + const extractStat = fs.statSync(extractedDir); + if (!extractStat.isDirectory()) { + throw new Error(`解压后的路径不是目录: ${extractedDir}`); + } + + console.log(`\n✓ 解压验证通过\n`); + + // 2. 执行端到端测试 + runE2ETest(extractedDir); + + console.log('\n🎉 测试完成！'); + process.exit(0); + } catch (error) { + console.error('\n❌ 测试失败:', error.message); + process.exit(1); + } finally { + // 清理临时目录 + cleanup(); + } +} + +// 如果直接运行此脚本 +if (require.main === module) { + main(); +} + +module.exports = { main, unzipFile }; diff --git a/tools/optimization_detector/README.md b/tools/optimization_detector/README.md index fc92bc99..661e8e93 100644 --- a/tools/optimization_detector/README.md +++ b/tools/optimization_detector/README.md @@ -100,7 +100,7 @@ ONEDIR_MODE=true ./build.sh opt-detector -i libexample.so -o report.xlsx # 检测目录中的所有二进制文件 -opt-detector -i /path/to/binaries -o report.xlsx --workers 4 +opt-detector -i /path/to/binaries -o report.xlsx --jobs 4 # 只检测优化级别，不检测LTO opt-detector -i libexample.so -o report.xlsx --no-lto @@ -109,7 +109,7 @@ opt-detector -i libexample.so -o report.xlsx --no-lto opt-detector -i libexample.so -o report.xlsx --no-opt # 使用4个并行工作线程 -opt-detector -i /path/to/binaries -o report.xlsx -w 4 +opt-detector -i /path/to/binaries -o report.xlsx -j 4 # 设置文件分析超时时间（秒） opt-detector -i libexample.so -o report.xlsx --timeout 300 diff --git a/tools/optimization_detector/optimization_detector/file_info.py b/tools/optimization_detector/optimization_detector/file_info.py index c33b5483..b8eabccc 100644 --- a/tools/optimization_detector/optimization_detector/file_info.py +++ b/tools/optimization_detector/optimization_detector/file_info.py @@ -30,6 +30,7 @@ FILE_STATUS_MAPPING = { 'analyzed': 'Successfully Analyzed', 'skipped': 'Skipped (System Library)', + 'skipped_few_chunks': 'Skipped (Too Few Chunks)', 'failed': 'Analysis Failed', } @@ -46,9 +47,35 @@ class FileInfo: TEXT_SECTION = '.text' CACHE_DIR = 'files_results_cache' + @staticmethod + def _is_elf_file(file_path: str) -> bool: + """Check if file is an ELF file by reading magic number + + ELF files start with 0x7f followed by 'ELF' (bytes: 0x7f 0x45 0x4c 0x46) + This method directly checks the file content rather than relying on file extension. + + Args: + file_path: Path to the file to check + + Returns: + bool: True if file is an ELF file, False otherwise + """ + try: + with open(file_path, 'rb') as f: + magic = f.read(4) + # ELF magic number: 0x7f 'E' 'L' 'F' + return magic == b'\x7fELF' + except Exception: + return False + @staticmethod def _is_so_file(file_path: str) -> bool: - """Check if file is a so file: ends with .so or contains .so. in filename""" + """Check if file is a so file: ends with .so or contains .so. in filename + + Note: This is a fallback check. The actual ELF detection should use _is_elf_file() + for more accurate results, as it can detect ELF files regardless of extension + and avoid false positives (e.g., linkerscript files with .so extension). + """ filename = os.path.basename(file_path) return file_path.endswith('.so') or '.so.' in filename @@ -58,9 +85,17 @@ def __init__(self, absolute_path: str, logical_path: Optional[str] = None): self.file_size = self._get_file_size() self.file_hash = self._calculate_file_hash() self.file_id = self._generate_file_id() + + # 优先通过ELF magic number检测，更准确 if absolute_path.endswith('.a'): self.file_type = FileType.AR + elif self._is_elf_file(absolute_path): + # 直接检测ELF文件，不依赖文件扩展名 + # 这样可以匹配 xx.so.653 这样的文件，也能避免误判 linkerscript 文件 + self.file_type = FileType.SO elif self._is_so_file(absolute_path): + # 回退到扩展名检查（用于向后兼容，但可能误判） + # 如果文件扩展名看起来像so文件，但实际不是ELF，会在后续分析时失败 self.file_type = FileType.SO else: self.file_type = FileType.NOT_SUPPORT @@ -137,9 +172,27 @@ def _is_symlink(file_path: str) -> bool: @staticmethod def _is_binary_file(file_path: str) -> bool: - """Check if file is a binary file (.so, .so.*, or .a)""" + """Check if file is a binary file by detecting ELF magic number or checking extension + + Priority: + 1. Check if it's an ELF file by reading magic number (most accurate) + 2. Check if it's an .a archive file + 3. Fallback to extension check (.so, .so.*) + + This ensures files like xx.so.653 are detected correctly, and non-ELF files + with .so extension (like linkerscript files) are filtered out early. + """ + # 优先检测ELF文件（最准确） + if FileInfo._is_elf_file(file_path): + return True + + # 检查是否是 .a 归档文件 + if file_path.endswith('.a'): + return True + + # 回退到扩展名检查（向后兼容） filename = os.path.basename(file_path) - return file_path.endswith('.a') or file_path.endswith('.so') or '.so.' in filename + return file_path.endswith('.so') or '.so.' in filename def collect_binary_files(self, input_path: str) -> list[FileInfo]: """Collect binary files for analysis""" @@ -191,17 +244,22 @@ def _extract_hap_file(self, hap_path: str) -> list[FileInfo]: # Handle both 'package/' prefix and direct paths if file_path.startswith('package/'): file_path = file_path[8:] # Remove 'package/' prefix - if ( - file_path.startswith('libs/arm64') or file_path.startswith('lib/arm64') - ) and file_path.endswith('.so'): + if (file_path.startswith('libs/arm64') or file_path.startswith('lib/arm64')) and ( + file_path.endswith('.so') or '.so.' in file_path + ): output_path = os.path.join(temp_dir, file_path) os.makedirs(os.path.dirname(output_path), exist_ok=True) with tar_ref.extractfile(member) as src, open(output_path, 'wb') as dest: dest.write(src.read()) - file_info = FileInfo( - absolute_path=output_path, logical_path=f'{hap_path}/{member.name}' - ) - extracted_files.append(file_info) + + # 提取后再次检查是否是有效的ELF文件，过滤掉无效文件 + if FileInfo._is_elf_file(output_path): + file_info = FileInfo( + absolute_path=output_path, logical_path=f'{hap_path}/{member.name}' + ) + extracted_files.append(file_info) + else: + logging.debug('Skipping non-ELF file from archive: %s', file_path) except Exception as e: logging.error('Failed to extract tar.gz file %s: %s', hap_path, e) else: @@ -209,13 +267,20 @@ def _extract_hap_file(self, hap_path: str) -> list[FileInfo]: try: with zipfile.ZipFile(hap_path, 'r') as zip_ref: for file in zip_ref.namelist(): - if (file.startswith('libs/arm64') or file.startswith('lib/arm64')) and file.endswith('.so'): + if (file.startswith('libs/arm64') or file.startswith('lib/arm64')) and ( + file.endswith('.so') or '.so.' in file + ): output_path = os.path.join(temp_dir, file[5:]) os.makedirs(os.path.dirname(output_path), exist_ok=True) with zip_ref.open(file) as src, open(output_path, 'wb') as dest: dest.write(src.read()) - file_info = FileInfo(absolute_path=output_path, logical_path=f'{hap_path}/{file}') - extracted_files.append(file_info) + + # 提取后再次检查是否是有效的ELF文件，过滤掉无效文件 + if FileInfo._is_elf_file(output_path): + file_info = FileInfo(absolute_path=output_path, logical_path=f'{hap_path}/{file}') + extracted_files.append(file_info) + else: + logging.debug('Skipping non-ELF file from archive: %s', file) except Exception as e: logging.error('Failed to extract HAP file %s: %s', hap_path, e) return extracted_files diff --git a/tools/optimization_detector/optimization_detector/lto_detector.py b/tools/optimization_detector/optimization_detector/lto_detector.py index c5ad60e0..f9aae2a1 100644 --- a/tools/optimization_detector/optimization_detector/lto_detector.py +++ b/tools/optimization_detector/optimization_detector/lto_detector.py @@ -1,11 +1,11 @@ """ LTO (Link-Time Optimization) Detector -基于SVM的LTO检测器，封装lto_demo的功能供opt指令使用 +基于统一SVM模型的LTO检测器，模型文件位于 package 内的 models/lto/ 目录 """ import json import logging -import sys +import math import traceback from pathlib import Path from typing import Optional @@ -26,195 +26,138 @@ class LtoDetector: 功能： - 检测SO文件是否使用了LTO优化 - - 根据优化级别自动选择合适的模型 + - 使用统一的SVM模型（不区分优化级别） - 返回LTO分数和判定结果 """ - # 模型映射：优化级别 -> 使用的模型 - MODEL_MAPPING = { - 'O0': 'O2', # O0使用O2模型 - 'O1': 'O2', # O1使用O2模型 - 'O2': 'O2', # O2使用专用模型 - 'O3': 'O3', # O3使用专用模型 - 'Os': 'Os', # Os使用专用模型 - 'Mixed': 'O2', # Mixed使用O2模型 - None: 'O2', # 未知级别使用O2模型 - } - - def __init__(self, model_base_dir: Optional[Path] = None): + def __init__(self, model_dir: Optional[Path] = None): """ 初始化LTO检测器 Args: - model_base_dir: 模型目录，默认为package内的models/lto/ + model_dir: 统一模型目录，默认为 package 内的 models/lto/ """ - if model_base_dir is None: - # 默认模型目录 + if model_dir is None: + # 默认模型目录（在 package 内） try: from importlib.resources import files # noqa: PLC0415 - model_base_dir = files('optimization_detector').joinpath('models/lto') + model_dir = files('optimization_detector').joinpath('models/lto') except Exception: # noqa: S110 # 备选：相对路径 - model_base_dir = Path(__file__).parent / 'models' / 'lto' + model_dir = Path(__file__).parent / 'models' / 'lto' - self.model_base_dir = Path(model_base_dir) - self.models = {} # 缓存已加载的模型 - self.feature_extractors = {} # 缓存特征提取器 + self.model_dir = Path(model_dir) + self.model = None # 统一模型（延迟加载） + self.feature_names = None # 特征名称列表 + self.feature_extractor = None # 特征提取器（延迟创建） # 导入特征提取器 self._import_feature_extractor() def _import_feature_extractor(self): - """导入特征提取器和SVM类（pickle反序列化需要）""" + """导入AllFeatureExtractor（统一模型使用的特征提取器）""" try: - # 直接导入同一目录下的lto_feature_pipeline模块 - from .lto_feature_pipeline import ( # noqa: PLC0415 - CompilerProvenanceRF, # 新的RF模型 - CompilerProvenanceSVM, # 旧的SVM模型 - HybridFeatureExtractor, # 混合特征提取（RF用） - LegacyFeatureExtractor, # Legacy特征提取（SVM用） - ) - - self.LegacyFeatureExtractor = LegacyFeatureExtractor - self.HybridFeatureExtractor = HybridFeatureExtractor - self.CompilerProvenanceSVM = CompilerProvenanceSVM - self.CompilerProvenanceRF = CompilerProvenanceRF - logging.debug('Successfully imported from lto_feature_pipeline') + # 从同一目录下的 lto_feature_pipeline 导入（git 仓库内） + from .lto_feature_pipeline import AllFeatureExtractor # noqa: PLC0415 + + self.AllFeatureExtractor = AllFeatureExtractor + logging.debug('Successfully imported AllFeatureExtractor from local lto_feature_pipeline') except Exception as e: # noqa: BLE001 # 捕获所有异常，包括 ImportError 和其他可能的错误（如依赖缺失） - error_msg = f'Failed to import lto_feature_pipeline: {e}\n{traceback.format_exc()}' + error_msg = f'Failed to import AllFeatureExtractor: {e}\n{traceback.format_exc()}' logging.error(error_msg) - self.LegacyFeatureExtractor = None - self.HybridFeatureExtractor = None - self.CompilerProvenanceSVM = None - self.CompilerProvenanceRF = None + self.AllFeatureExtractor = None - def _load_model(self, model_name: str) -> tuple[Optional[object], Optional[list], Optional[str]]: + def _load_model(self): """ - 加载指定的模型（SVM或RF） - - Args: - model_name: 模型名称 (O2/O3/Os) + 加载统一的SVM模型（延迟加载） Returns: - (model, feature_names, model_type) 或 (None, None, None) + True 如果成功加载，False 否则 """ - if model_name in self.models: - return self.models[model_name] - - model_dir = self.model_base_dir / model_name - - # 优先使用RF模型（最新），回退到SVM模型 - rf_path = model_dir / 'rf_model.joblib' - svm_path = model_dir / 'svm_model.joblib' - feature_path = model_dir / 'feature_names.json' - - # 确定使用哪个模型 - if rf_path.exists(): - model_path = rf_path - model_type = 'RF' - model_class = self.CompilerProvenanceRF - elif svm_path.exists(): - model_path = svm_path - model_type = 'SVM' - model_class = self.CompilerProvenanceSVM - else: - logging.warning('LTO model not found: %s', model_dir) - return None, None + if self.model is not None: + return True # 已经加载 + + model_path = self.model_dir / 'model.pkl' + feature_path = self.model_dir / 'feature_names.json' + + if not model_path.exists(): + logging.warning('LTO model not found: %s', model_path) + return False if not feature_path.exists(): logging.warning('Feature names not found: %s', feature_path) - return None, None, None + return False try: - # 确保模型类已导入（pickle反序列化需要） - if model_class is None: - logging.error('%s model class not imported, cannot load model', model_type) - return None, None, None - - # 将模型类添加到全局命名空间（pickle需要） - import builtins # noqa: PLC0415 - - if model_type == 'RF': - if not hasattr(builtins, 'CompilerProvenanceRF'): - builtins.CompilerProvenanceRF = self.CompilerProvenanceRF - # 同时添加到主模块，因为pickle可能从那里查找 - if hasattr(sys.modules.get('__main__'), '__dict__'): - sys.modules['__main__'].CompilerProvenanceRF = self.CompilerProvenanceRF - if hasattr(sys.modules.get('scripts.main'), '__dict__'): - sys.modules['scripts.main'].CompilerProvenanceRF = self.CompilerProvenanceRF - elif model_type == 'SVM': - if not hasattr(builtins, 'CompilerProvenanceSVM'): - builtins.CompilerProvenanceSVM = self.CompilerProvenanceSVM - if hasattr(sys.modules.get('__main__'), '__dict__'): - sys.modules['__main__'].CompilerProvenanceSVM = self.CompilerProvenanceSVM - if hasattr(sys.modules.get('scripts.main'), '__dict__'): - sys.modules['scripts.main'].CompilerProvenanceSVM = self.CompilerProvenanceSVM - - # 加载模型 - model = joblib.load(model_path) - logging.info('Loaded %s model for %s', model_type, model_name) + # 加载模型（Pipeline，包含 StandardScaler 和 SVC） + self.model = joblib.load(model_path) + logging.info('Loaded unified LTO SVM model from %s', model_path) # 加载特征名称 with open(feature_path, encoding='utf-8') as f: - feature_names = json.load(f).get('feature_names', []) + self.feature_names = json.load(f).get('feature_names', []) - # 缓存模型（包含类型信息） - self.models[model_name] = (model, feature_names, model_type) - logging.debug('Loaded %s model %s: %d features', model_type, model_name, len(feature_names)) + if not self.feature_names: + logging.warning('No feature names found in %s', feature_path) + return False - return model, feature_names, model_type + logging.debug('Loaded unified LTO model: %d features', len(self.feature_names)) + return True except Exception as e: - logging.error('Failed to load LTO model %s: %s', model_name, e) - return None, None, None + logging.error('Failed to load LTO model: %s', e) + self.model = None + self.feature_names = None + return False - def _extract_features(self, so_path: str, feature_names: list, use_hybrid: bool = False) -> Optional[np.ndarray]: + def _extract_features(self, so_path: str) -> Optional[np.ndarray]: """ - 提取SO文件的特征 + 提取SO文件的特征（使用AllFeatureExtractor） Args: so_path: SO文件路径 - feature_names: 训练时的特征名称列表 - use_hybrid: 是否使用混合特征提取器（RF模型需要） Returns: 特征向量 或 None """ - # 根据模型类型选择特征提取器 - if use_hybrid: - if self.HybridFeatureExtractor is None: - logging.error('HybridFeatureExtractor not available') - return None - extractor_key = 'hybrid' - ExtractorClass = self.HybridFeatureExtractor - else: - if self.LegacyFeatureExtractor is None: - logging.error('LegacyFeatureExtractor not available') - return None - extractor_key = 'legacy' - ExtractorClass = self.LegacyFeatureExtractor + if self.AllFeatureExtractor is None: + logging.error('AllFeatureExtractor not available') + return None + + if self.feature_names is None: + logging.error('Feature names not loaded') + return None try: # 创建或获取特征提取器 - if extractor_key not in self.feature_extractors: - self.feature_extractors[extractor_key] = ExtractorClass() - - extractor = self.feature_extractors[extractor_key] + if self.feature_extractor is None: + self.feature_extractor = self.AllFeatureExtractor() # 提取特征 - feat, names, _ = extractor.extract(so_path) + feat, names, _ = self.feature_extractor.extract(so_path) # 对齐到训练时的特征名称 - name_to_index = {n: i for i, n in enumerate(feature_names)} - vec = np.zeros(len(feature_names), dtype=np.float32) + name_to_index = {n: i for i, n in enumerate(self.feature_names)} + vec = np.zeros(len(self.feature_names), dtype=np.float32) + matched_count = 0 for i, n in enumerate(names): j = name_to_index.get(n) if j is not None: vec[j] = float(feat[i]) + matched_count += 1 + + if matched_count == 0: + logging.warning( + 'No features matched for %s. Extracted %d features, expected %d', + so_path, + len(names), + len(self.feature_names), + ) + return None return vec @@ -224,18 +167,17 @@ def _extract_features(self, so_path: str, feature_names: list, use_hybrid: bool def detect(self, file_info: FileInfo, opt_level: Optional[str] = None) -> dict: """ - 检测单个SO文件是否使用LTO + 检测单个SO文件是否使用LTO（使用统一模型，opt_level参数已废弃但保留以兼容） Args: file_info: FileInfo对象，包含文件路径、ID和哈希等信息 - opt_level: 优化级别 (O0/O1/O2/O3/Os/Mixed/None) - 如果为None，将使用集成预测（所有模型） + opt_level: 优化级别（已废弃，统一模型不区分优化级别，保留以兼容旧代码） Returns: { 'score': float, # LTO分数 [0-1] 'prediction': str, # 'LTO' 或 'No LTO' - 'model_used': str # 使用的模型名称 + 'model_used': str # 使用的模型名称（统一SVM） } """ so_path = file_info.absolute_path @@ -246,119 +188,51 @@ def detect(self, file_info: FileInfo, opt_level: Optional[str] = None) -> dict: logging.debug('LTO cache hit for %s', so_path) return cache_result - # 如果优化级别未知，使用集成预测 - if opt_level is None or opt_level not in self.MODEL_MAPPING: - result = self._detect_with_ensemble(so_path) - else: - result = self._detect_with_model(so_path, opt_level) - - # 保存到缓存 - self._save_to_cache(file_info, result) - - return result - - def _detect_with_model(self, so_path: str, opt_level: str) -> dict: - """ - 使用指定模型进行检测（内部方法，从detect中提取） - - Args: - so_path: SO文件路径 - opt_level: 优化级别 - - Returns: - 检测结果字典 - """ - - # 选择模型 - model_name = self.MODEL_MAPPING.get(opt_level, 'O2') - model, feature_names, model_type = self._load_model(model_name) - - if model is None or feature_names is None: + # 延迟加载模型 + if not self._load_model(): return {'score': None, 'prediction': 'N/A', 'model_used': 'N/A'} - # 根据模型类型选择特征提取器 - use_hybrid = model_type == 'RF' - # 提取特征 - features = self._extract_features(so_path, feature_names, use_hybrid=use_hybrid) - + features = self._extract_features(so_path) if features is None: - return {'score': None, 'prediction': 'Failed', 'model_used': model_name} + return {'score': None, 'prediction': 'Failed', 'model_used': 'Unified SVM'} try: - # 预测 + # 预测（模型是 Pipeline，包含 StandardScaler 和 SVC） features_2d = features.reshape(1, -1) # 转为2D数组 - proba = model.predict_proba(features_2d) - lto_score = float(proba[0, 1]) # 第二列是LTO的概率 - lto_label = 1 if lto_score >= 0.5 else 0 - - return {'score': lto_score, 'prediction': 'LTO' if lto_label == 1 else 'No LTO', 'model_used': model_name} - - except Exception as e: - logging.debug('LTO prediction failed for %s: %s', so_path, e) - return {'score': None, 'prediction': 'Error', 'model_used': model_name} - - def _detect_with_ensemble(self, so_path: str) -> dict: - """ - 使用集成预测（当优化级别未知时） - - 策略：用O2/O3/Os三个模型都预测，取加权平均 - 权重：O2=0.5, O3=0.3, Os=0.2（基于实际使用频率） - - Args: - so_path: SO文件路径 - Returns: - 集成预测结果 - """ - logging.debug('Using ensemble prediction for %s', so_path) - - models_to_use = ['O2', 'O3', 'Os'] - weights = {'O2': 0.5, 'O3': 0.3, 'Os': 0.2} # 根据实际使用频率 - - scores = [] - valid_models = [] - - for model_name in models_to_use: - model, feature_names, model_type = self._load_model(model_name) - if model is None or feature_names is None: - continue - - # 根据模型类型选择特征提取器 - use_hybrid = model_type == 'RF' - features = self._extract_features(so_path, feature_names, use_hybrid=use_hybrid) - if features is None: - continue + # 注意：SVM Pipeline 在训练时设置了 probability=False，所以没有 predict_proba + # 使用 decision_function 并转换为概率 + decision = self.model.decision_function(features_2d) + decision_value = float(decision[0]) + # 将 decision_function 转换为概率 [0, 1] + # 使用 sigmoid 函数：1 / (1 + exp(-decision)) + # 如果 scipy 可用，使用 expit；否则手动计算 try: - features_2d = features.reshape(1, -1) - proba = model.predict_proba(features_2d) - score = float(proba[0, 1]) - scores.append((model_name, score)) - valid_models.append(model_name) - except Exception as e: - logging.debug('Ensemble prediction failed for model %s: %s', model_name, e) - continue + from scipy.special import expit # noqa: PLC0415 - if not scores: - return {'score': None, 'prediction': 'Failed', 'model_used': 'Ensemble(failed)'} + lto_score = float(expit(decision_value)) + except ImportError: + # 如果没有 scipy，手动计算 sigmoid + lto_score = 1.0 / (1.0 + math.exp(-decision_value)) - # 计算加权平均 - weighted_score = sum(weights[m] * s for m, s in scores) / sum(weights[m] for m in valid_models) - - # 取最高分（作为参考） - max_score = max(s for _, s in scores) - max_model = [m for m, s in scores if s == max_score][0] + lto_label = 1 if lto_score >= 0.5 else 0 - # 使用加权平均作为最终分数 - final_score = weighted_score - lto_label = 1 if final_score >= 0.5 else 0 + result = { + 'score': lto_score, + 'prediction': 'LTO' if lto_label == 1 else 'No LTO', + 'model_used': 'Unified SVM', + } - model_str = f'Ensemble({"+".join(valid_models)})' + # 保存到缓存 + self._save_to_cache(file_info, result) - logging.debug('Ensemble scores: %s, weighted=%.4f, max=%.4f(%s)', scores, weighted_score, max_score, max_model) + return result - return {'score': final_score, 'prediction': 'LTO' if lto_label == 1 else 'No LTO', 'model_used': model_str} + except Exception as e: + logging.debug('LTO prediction failed for %s: %s', so_path, e) + return {'score': None, 'prediction': 'Error', 'model_used': 'Unified SVM'} def _get_cache_path(self, file_info: FileInfo) -> Path: """ @@ -392,6 +266,9 @@ def _load_from_cache(self, file_info: FileInfo) -> Optional[dict]: try: with open(cache_path, encoding='utf-8') as f: result = json.load(f) + # 统一模型：确保 model_used 始终为 "Unified SVM"（兼容旧缓存） + if result.get('model_used') != 'Unified SVM': + result['model_used'] = 'Unified SVM' logging.debug('Loaded LTO cache from %s', cache_path) return result except Exception as e: diff --git a/tools/optimization_detector/optimization_detector/lto_feature_pipeline.py b/tools/optimization_detector/optimization_detector/lto_feature_pipeline.py index 2a7bfaa2..da01aad7 100644 --- a/tools/optimization_detector/optimization_detector/lto_feature_pipeline.py +++ b/tools/optimization_detector/optimization_detector/lto_feature_pipeline.py @@ -1867,6 +1867,11 @@ def extract(self, path: str) -> tuple[np.ndarray, list[str], str]: return feats, names, k +# AllFeatureExtractor 是 HybridFeatureExtractor 的别名（用于统一模型） +# 两者实现相同，只是命名不同以保持与 lto_demo 的兼容性 +AllFeatureExtractor = HybridFeatureExtractor + + class HybridFeatureTrainer(_BaseTrainer): """ 混合特征训练器： diff --git a/tools/optimization_detector/optimization_detector/models/lto/O2/feature_names.json b/tools/optimization_detector/optimization_detector/models/lto/O2/feature_names.json deleted file mode 100644 index c1366c89..00000000 --- a/tools/optimization_detector/optimization_detector/models/lto/O2/feature_names.json +++ /dev/null @@ -1,278 +0,0 @@ -{ - "feature_names": [ - "LEG_absolute_symbols", - "LEG_artificial_ratio", - "LEG_artificial_symbols", - "LEG_ascii_ratio", - "LEG_bss_sections", - "LEG_bss_size_ratio", - "LEG_build_id_present", - "LEG_common_symbols", - "LEG_compiler_info", - "LEG_compiler_string_ratio", - "LEG_compiler_strings", - "LEG_data_ratio", - "LEG_data_relocations", - "LEG_data_sections", - "LEG_data_size_ratio", - "LEG_data_symbols", - "LEG_debug_sections", - "LEG_debug_symbols", - "LEG_dynamic_ratio", - "LEG_dynamic_relocations", - "LEG_elf_header_present", - "LEG_elf_type", - "LEG_entropy", - "LEG_entry_point", - "LEG_file_size", - "LEG_fini_functions", - "LEG_function_ratio", - "LEG_function_symbols", - "LEG_gcc_strings", - "LEG_global_ratio", - "LEG_global_symbols", - "LEG_gnu_strings", - "LEG_got_ratio", - "LEG_got_relocations", - "LEG_has_comment_section", - "LEG_has_debug_sections", - "LEG_has_inline_functions", - "LEG_has_note_section", - "LEG_has_tail_calls", - "LEG_has_unrolled_loops", - "LEG_has_vectorized_code", - "LEG_header_hex", - "LEG_init_functions", - "LEG_is_elf", - "LEG_linker_strings", - "LEG_local_ratio", - "LEG_local_symbols", - "LEG_lto_patterns", - "LEG_lto_ratio", - "LEG_lto_string_ratio", - "LEG_lto_strings", - "LEG_lto_symbols", - "LEG_machine_type", - "LEG_needed_libraries", - "LEG_optimization_level", - "LEG_optimization_string_ratio", - "LEG_optimization_strings", - "LEG_plt_ratio", - "LEG_plt_relocations", - "LEG_program_header_count", - "LEG_rpath_entries", - "LEG_runpath_entries", - "LEG_section_header_count", - "LEG_symbol_versions", - "LEG_text_relocations", - "LEG_text_sections", - "LEG_text_size_ratio", - "LEG_total_bss_size", - "LEG_total_data_size", - "LEG_total_debug_size", - "LEG_total_dynamic_entries", - "LEG_total_relocations", - "LEG_total_sections", - "LEG_total_strings", - "LEG_total_symbols", - "LEG_total_text_size", - "LEG_undefined_symbols", - "LEG_weak_symbols", - "LEG_zero_byte_ratio", - "HIER_byte_freq_7", - "HIER_byte_freq_11", - "HIER_byte_freq_20", - "HIER_byte_freq_21", - "HIER_byte_freq_26", - "HIER_byte_freq_33", - "HIER_byte_freq_37", - "HIER_byte_freq_38", - "HIER_byte_freq_40", - "HIER_byte_freq_42", - "HIER_byte_freq_43", - "HIER_byte_freq_55", - "HIER_byte_freq_57", - "HIER_byte_freq_62", - "HIER_byte_freq_65", - "HIER_byte_freq_67", - "HIER_byte_freq_68", - "HIER_byte_freq_72", - "HIER_byte_freq_77", - "HIER_byte_freq_78", - "HIER_byte_freq_79", - "HIER_byte_freq_90", - "HIER_byte_freq_92", - "HIER_byte_freq_94", - "HIER_byte_freq_98", - "HIER_byte_freq_99", - "HIER_byte_freq_100", - "HIER_byte_freq_104", - "HIER_byte_freq_105", - "HIER_byte_freq_106", - "HIER_byte_freq_110", - "HIER_byte_freq_117", - "HIER_byte_freq_119", - "HIER_byte_freq_121", - "HIER_byte_freq_124", - "HIER_byte_freq_131", - "HIER_byte_freq_134", - "HIER_byte_freq_142", - "HIER_byte_freq_144", - "HIER_byte_freq_150", - "HIER_byte_freq_151", - "HIER_byte_freq_152", - "HIER_byte_freq_153", - "HIER_byte_freq_155", - "HIER_byte_freq_156", - "HIER_byte_freq_157", - "HIER_byte_freq_159", - "HIER_byte_freq_161", - "HIER_byte_freq_162", - "HIER_byte_freq_163", - "HIER_byte_freq_164", - "HIER_byte_freq_178", - "HIER_byte_freq_179", - "HIER_byte_freq_181", - "HIER_byte_freq_186", - "HIER_byte_freq_187", - "HIER_byte_freq_190", - "HIER_byte_freq_191", - "HIER_byte_freq_194", - "HIER_byte_freq_197", - "HIER_byte_freq_199", - "HIER_byte_freq_200", - "HIER_byte_freq_201", - "HIER_byte_freq_205", - "HIER_byte_freq_215", - "HIER_byte_freq_221", - "HIER_byte_freq_224", - "HIER_byte_freq_225", - "HIER_byte_freq_226", - "HIER_byte_freq_227", - "HIER_byte_freq_231", - "HIER_byte_freq_235", - "HIER_byte_freq_238", - "HIER_byte_freq_241", - "HIER_byte_freq_244", - "HIER_byte_freq_246", - "HIER_byte_freq_249", - "HIER_byte_freq_251", - "HIER_byte_freq_252", - "HIER_byte_freq_253", - "HIER_byte_freq_255", - "HIER_byte_pair_11", - "HIER_byte_pair_12", - "HIER_byte_pair_13", - "HIER_byte_pair_14", - "HIER_byte_pair_15", - "HIER_byte_pair_33", - "HIER_byte_pair_34", - "HIER_byte_pair_36", - "HIER_byte_pair_37", - "HIER_byte_pair_38", - "HIER_byte_pair_39", - "HIER_byte_pair_40", - "HIER_byte_pair_41", - "HIER_byte_pair_42", - "HIER_byte_pair_43", - "HIER_byte_pair_48", - "HIER_byte_pair_49", - "HIER_byte_pair_50", - "HIER_byte_pair_53", - "HIER_byte_pair_55", - "HIER_byte_pair_56", - "HIER_byte_pair_60", - "HIER_byte_pair_61", - "HIER_byte_pair_67", - "HIER_byte_pair_68", - "HIER_byte_pair_69", - "HIER_byte_pair_70", - "HIER_byte_pair_71", - "HIER_byte_pair_101", - "HIER_byte_pair_102", - "HIER_byte_pair_107", - "HIER_byte_pair_109", - "HIER_byte_pair_110", - "HIER_byte_pair_111", - "HIER_byte_pair_112", - "HIER_byte_pair_119", - "HIER_byte_pair_120", - "HIER_byte_pair_121", - "HIER_entropy", - "HIER_byte_diversity", - "HIER_avg_byte_norm", - "HIER_file_size", - "HIER_percentile_25", - "HIER_percentile_75", - "HIER_percentile_90", - "HIER_percentile_99", - "HIER_percentile_range_99_5", - "O3_sz_text", - "O3_sz_rodata", - "O3_sz_data", - "O3_sz_bss", - "O3_sz_plt", - "O3_sz_plt_sec", - "O3_sz_got", - "O3_sz_gotplt", - "O3_sz_eh_frame", - "O3_sz_gcc_except", - "O3_sz_text_hot", - "O3_sz_text_unlikely", - "O3_r_text_total", - "O3_r_ro_total", - "O3_r_data_total", - "O3_r_bss_total", - "O3_r_gotplt_text", - "O3_r_plt_text", - "O3_r_pltsec_text", - "O3_log_sz_text", - "O3_log_sz_plt", - "O3_log_sz_gotplt", - "O3_log_sz_eh_frame", - "O3_log_sz_except", - "O3_rel_total", - "O3_rel_js_ratio", - "O3_rel_gd_ratio", - "O3_rel_rel_ratio", - "O3_rel_irel_ratio", - "O3_rel_entropy", - "O3_log_rel_total", - "O3_func_total", - "O3_func_global", - "O3_func_local", - "O3_func_weak", - "O3_r_func_global_total", - "O3_r_func_local_total", - "O3_r_hidden_default", - "O3_r_funcs_total", - "O3_export_per_kb", - "O3_log_func_total", - "O3_log_func_global", - "O3_d_bl_per_kb", - "O3_d_blr_per_kb", - "O3_d_br_per_kb", - "O3_d_b_per_kb", - "O3_d_bcond_per_kb", - "O3_d_ret_per_kb", - "O3_d_cbz_per_kb", - "O3_d_cbnz_per_kb", - "O3_d_tbz_per_kb", - "O3_d_tbnz_per_kb", - "O3_d_adrp_per_kb", - "O3_d_ldr_per_kb", - "O3_d_add_per_kb", - "O3_d_vec_per_kb", - "O3_r_call_ret", - "O3_r_blr_total", - "O3_r_cond_uncond", - "O3_r_bl_ret", - "O3_r_vec_total", - "O3_got_triplet_per_kb", - "O3_plt_entries", - "O3_d_plt_entries_per_kb", - "O3_r_plt_calls", - "O3_has_symver", - "O3_has_comdat_group" - ] -} \ No newline at end of file diff --git a/tools/optimization_detector/optimization_detector/models/lto/O2/rf_model.joblib b/tools/optimization_detector/optimization_detector/models/lto/O2/rf_model.joblib deleted file mode 100644 index 11a2b448..00000000 Binary files a/tools/optimization_detector/optimization_detector/models/lto/O2/rf_model.joblib and /dev/null differ diff --git a/tools/optimization_detector/optimization_detector/models/lto/O2/svm_model.joblib b/tools/optimization_detector/optimization_detector/models/lto/O2/svm_model.joblib deleted file mode 100644 index 573ee488..00000000 Binary files a/tools/optimization_detector/optimization_detector/models/lto/O2/svm_model.joblib and /dev/null differ diff --git a/tools/optimization_detector/optimization_detector/models/lto/O3/feature_names.json b/tools/optimization_detector/optimization_detector/models/lto/O3/feature_names.json deleted file mode 100644 index bb137d36..00000000 --- a/tools/optimization_detector/optimization_detector/models/lto/O3/feature_names.json +++ /dev/null @@ -1,278 +0,0 @@ -{ - "feature_names": [ - "LEG_absolute_symbols", - "LEG_artificial_ratio", - "LEG_artificial_symbols", - "LEG_ascii_ratio", - "LEG_bss_sections", - "LEG_bss_size_ratio", - "LEG_build_id_present", - "LEG_common_symbols", - "LEG_compiler_info", - "LEG_compiler_string_ratio", - "LEG_compiler_strings", - "LEG_data_ratio", - "LEG_data_relocations", - "LEG_data_sections", - "LEG_data_size_ratio", - "LEG_data_symbols", - "LEG_debug_sections", - "LEG_debug_symbols", - "LEG_dynamic_ratio", - "LEG_dynamic_relocations", - "LEG_elf_header_present", - "LEG_elf_type", - "LEG_entropy", - "LEG_entry_point", - "LEG_file_size", - "LEG_fini_functions", - "LEG_function_ratio", - "LEG_function_symbols", - "LEG_gcc_strings", - "LEG_global_ratio", - "LEG_global_symbols", - "LEG_gnu_strings", - "LEG_got_ratio", - "LEG_got_relocations", - "LEG_has_comment_section", - "LEG_has_debug_sections", - "LEG_has_inline_functions", - "LEG_has_note_section", - "LEG_has_tail_calls", - "LEG_has_unrolled_loops", - "LEG_has_vectorized_code", - "LEG_header_hex", - "LEG_init_functions", - "LEG_is_elf", - "LEG_linker_strings", - "LEG_local_ratio", - "LEG_local_symbols", - "LEG_lto_patterns", - "LEG_lto_ratio", - "LEG_lto_string_ratio", - "LEG_lto_strings", - "LEG_lto_symbols", - "LEG_machine_type", - "LEG_needed_libraries", - "LEG_optimization_level", - "LEG_optimization_string_ratio", - "LEG_optimization_strings", - "LEG_plt_ratio", - "LEG_plt_relocations", - "LEG_program_header_count", - "LEG_rpath_entries", - "LEG_runpath_entries", - "LEG_section_header_count", - "LEG_symbol_versions", - "LEG_text_relocations", - "LEG_text_sections", - "LEG_text_size_ratio", - "LEG_total_bss_size", - "LEG_total_data_size", - "LEG_total_debug_size", - "LEG_total_dynamic_entries", - "LEG_total_relocations", - "LEG_total_sections", - "LEG_total_strings", - "LEG_total_symbols", - "LEG_total_text_size", - "LEG_undefined_symbols", - "LEG_weak_symbols", - "LEG_zero_byte_ratio", - "HIER_byte_freq_0", - "HIER_byte_freq_4", - "HIER_byte_freq_6", - "HIER_byte_freq_7", - "HIER_byte_freq_16", - "HIER_byte_freq_18", - "HIER_byte_freq_19", - "HIER_byte_freq_21", - "HIER_byte_freq_22", - "HIER_byte_freq_23", - "HIER_byte_freq_27", - "HIER_byte_freq_29", - "HIER_byte_freq_36", - "HIER_byte_freq_38", - "HIER_byte_freq_44", - "HIER_byte_freq_47", - "HIER_byte_freq_51", - "HIER_byte_freq_57", - "HIER_byte_freq_58", - "HIER_byte_freq_60", - "HIER_byte_freq_61", - "HIER_byte_freq_62", - "HIER_byte_freq_65", - "HIER_byte_freq_66", - "HIER_byte_freq_69", - "HIER_byte_freq_70", - "HIER_byte_freq_73", - "HIER_byte_freq_75", - "HIER_byte_freq_78", - "HIER_byte_freq_80", - "HIER_byte_freq_89", - "HIER_byte_freq_91", - "HIER_byte_freq_93", - "HIER_byte_freq_96", - "HIER_byte_freq_100", - "HIER_byte_freq_103", - "HIER_byte_freq_104", - "HIER_byte_freq_116", - "HIER_byte_freq_119", - "HIER_byte_freq_121", - "HIER_byte_freq_123", - "HIER_byte_freq_127", - "HIER_byte_freq_128", - "HIER_byte_freq_134", - "HIER_byte_freq_137", - "HIER_byte_freq_140", - "HIER_byte_freq_142", - "HIER_byte_freq_146", - "HIER_byte_freq_151", - "HIER_byte_freq_152", - "HIER_byte_freq_154", - "HIER_byte_freq_157", - "HIER_byte_freq_158", - "HIER_byte_freq_164", - "HIER_byte_freq_166", - "HIER_byte_freq_171", - "HIER_byte_freq_172", - "HIER_byte_freq_177", - "HIER_byte_freq_178", - "HIER_byte_freq_182", - "HIER_byte_freq_184", - "HIER_byte_freq_187", - "HIER_byte_freq_188", - "HIER_byte_freq_190", - "HIER_byte_freq_191", - "HIER_byte_freq_200", - "HIER_byte_freq_204", - "HIER_byte_freq_207", - "HIER_byte_freq_208", - "HIER_byte_freq_221", - "HIER_byte_freq_222", - "HIER_byte_freq_223", - "HIER_byte_freq_225", - "HIER_byte_freq_226", - "HIER_byte_freq_228", - "HIER_byte_freq_232", - "HIER_byte_freq_233", - "HIER_byte_freq_235", - "HIER_byte_freq_236", - "HIER_byte_freq_238", - "HIER_byte_freq_249", - "HIER_byte_freq_251", - "HIER_byte_freq_252", - "HIER_byte_freq_253", - "HIER_byte_pair_11", - "HIER_byte_pair_48", - "HIER_byte_pair_52", - "HIER_byte_pair_54", - "HIER_byte_pair_55", - "HIER_byte_pair_57", - "HIER_byte_pair_60", - "HIER_byte_pair_61", - "HIER_byte_pair_63", - "HIER_byte_pair_69", - "HIER_byte_pair_71", - "HIER_byte_pair_73", - "HIER_byte_pair_74", - "HIER_byte_pair_78", - "HIER_byte_pair_80", - "HIER_byte_pair_81", - "HIER_byte_pair_84", - "HIER_byte_pair_85", - "HIER_byte_pair_86", - "HIER_byte_pair_94", - "HIER_byte_pair_95", - "HIER_byte_pair_101", - "HIER_byte_pair_102", - "HIER_byte_pair_103", - "HIER_byte_pair_104", - "HIER_byte_pair_105", - "HIER_byte_pair_106", - "HIER_byte_pair_108", - "HIER_byte_pair_109", - "HIER_byte_pair_110", - "HIER_byte_pair_111", - "HIER_byte_pair_115", - "HIER_byte_pair_117", - "HIER_byte_pair_119", - "HIER_byte_pair_120", - "HIER_byte_pair_122", - "HIER_byte_pair_124", - "HIER_entropy", - "HIER_zero_ratio", - "HIER_byte_diversity", - "HIER_avg_byte_norm", - "HIER_byte_std_norm", - "HIER_file_size", - "HIER_percentile_99", - "O3_sz_text", - "O3_sz_rodata", - "O3_sz_data", - "O3_sz_bss", - "O3_sz_plt", - "O3_sz_plt_sec", - "O3_sz_got", - "O3_sz_gotplt", - "O3_sz_eh_frame", - "O3_sz_gcc_except", - "O3_sz_text_hot", - "O3_sz_text_unlikely", - "O3_r_text_total", - "O3_r_ro_total", - "O3_r_data_total", - "O3_r_bss_total", - "O3_r_gotplt_text", - "O3_r_plt_text", - "O3_r_pltsec_text", - "O3_log_sz_text", - "O3_log_sz_plt", - "O3_log_sz_gotplt", - "O3_log_sz_eh_frame", - "O3_log_sz_except", - "O3_rel_total", - "O3_rel_js_ratio", - "O3_rel_gd_ratio", - "O3_rel_rel_ratio", - "O3_rel_irel_ratio", - "O3_rel_entropy", - "O3_log_rel_total", - "O3_func_total", - "O3_func_global", - "O3_func_local", - "O3_func_weak", - "O3_r_func_global_total", - "O3_r_func_local_total", - "O3_r_hidden_default", - "O3_r_funcs_total", - "O3_export_per_kb", - "O3_log_func_total", - "O3_log_func_global", - "O3_d_bl_per_kb", - "O3_d_blr_per_kb", - "O3_d_br_per_kb", - "O3_d_b_per_kb", - "O3_d_bcond_per_kb", - "O3_d_ret_per_kb", - "O3_d_cbz_per_kb", - "O3_d_cbnz_per_kb", - "O3_d_tbz_per_kb", - "O3_d_tbnz_per_kb", - "O3_d_adrp_per_kb", - "O3_d_ldr_per_kb", - "O3_d_add_per_kb", - "O3_d_vec_per_kb", - "O3_r_call_ret", - "O3_r_blr_total", - "O3_r_cond_uncond", - "O3_r_bl_ret", - "O3_r_vec_total", - "O3_got_triplet_per_kb", - "O3_plt_entries", - "O3_d_plt_entries_per_kb", - "O3_r_plt_calls", - "O3_has_symver", - "O3_has_comdat_group" - ] -} \ No newline at end of file diff --git a/tools/optimization_detector/optimization_detector/models/lto/O3/rf_model.joblib b/tools/optimization_detector/optimization_detector/models/lto/O3/rf_model.joblib deleted file mode 100644 index 041deeab..00000000 Binary files a/tools/optimization_detector/optimization_detector/models/lto/O3/rf_model.joblib and /dev/null differ diff --git a/tools/optimization_detector/optimization_detector/models/lto/O3/svm_model.joblib b/tools/optimization_detector/optimization_detector/models/lto/O3/svm_model.joblib deleted file mode 100644 index ecc75c3a..00000000 Binary files a/tools/optimization_detector/optimization_detector/models/lto/O3/svm_model.joblib and /dev/null differ diff --git a/tools/optimization_detector/optimization_detector/models/lto/Os/feature_names.json b/tools/optimization_detector/optimization_detector/models/lto/Os/feature_names.json deleted file mode 100644 index 65264887..00000000 --- a/tools/optimization_detector/optimization_detector/models/lto/Os/feature_names.json +++ /dev/null @@ -1,278 +0,0 @@ -{ - "feature_names": [ - "LEG_absolute_symbols", - "LEG_artificial_ratio", - "LEG_artificial_symbols", - "LEG_ascii_ratio", - "LEG_bss_sections", - "LEG_bss_size_ratio", - "LEG_build_id_present", - "LEG_common_symbols", - "LEG_compiler_info", - "LEG_compiler_string_ratio", - "LEG_compiler_strings", - "LEG_data_ratio", - "LEG_data_relocations", - "LEG_data_sections", - "LEG_data_size_ratio", - "LEG_data_symbols", - "LEG_debug_sections", - "LEG_debug_symbols", - "LEG_dynamic_ratio", - "LEG_dynamic_relocations", - "LEG_elf_header_present", - "LEG_elf_type", - "LEG_entropy", - "LEG_entry_point", - "LEG_file_size", - "LEG_fini_functions", - "LEG_function_ratio", - "LEG_function_symbols", - "LEG_gcc_strings", - "LEG_global_ratio", - "LEG_global_symbols", - "LEG_gnu_strings", - "LEG_got_ratio", - "LEG_got_relocations", - "LEG_has_comment_section", - "LEG_has_debug_sections", - "LEG_has_inline_functions", - "LEG_has_note_section", - "LEG_has_tail_calls", - "LEG_has_unrolled_loops", - "LEG_has_vectorized_code", - "LEG_header_hex", - "LEG_init_functions", - "LEG_is_elf", - "LEG_linker_strings", - "LEG_local_ratio", - "LEG_local_symbols", - "LEG_lto_patterns", - "LEG_lto_ratio", - "LEG_lto_string_ratio", - "LEG_lto_strings", - "LEG_lto_symbols", - "LEG_machine_type", - "LEG_needed_libraries", - "LEG_optimization_level", - "LEG_optimization_string_ratio", - "LEG_optimization_strings", - "LEG_plt_ratio", - "LEG_plt_relocations", - "LEG_program_header_count", - "LEG_rpath_entries", - "LEG_runpath_entries", - "LEG_section_header_count", - "LEG_symbol_versions", - "LEG_text_relocations", - "LEG_text_sections", - "LEG_text_size_ratio", - "LEG_total_bss_size", - "LEG_total_data_size", - "LEG_total_debug_size", - "LEG_total_dynamic_entries", - "LEG_total_relocations", - "LEG_total_sections", - "LEG_total_strings", - "LEG_total_symbols", - "LEG_total_text_size", - "LEG_undefined_symbols", - "LEG_weak_symbols", - "LEG_zero_byte_ratio", - "HIER_byte_freq_0", - "HIER_byte_freq_3", - "HIER_byte_freq_7", - "HIER_byte_freq_9", - "HIER_byte_freq_14", - "HIER_byte_freq_15", - "HIER_byte_freq_19", - "HIER_byte_freq_21", - "HIER_byte_freq_23", - "HIER_byte_freq_33", - "HIER_byte_freq_34", - "HIER_byte_freq_36", - "HIER_byte_freq_37", - "HIER_byte_freq_39", - "HIER_byte_freq_42", - "HIER_byte_freq_43", - "HIER_byte_freq_45", - "HIER_byte_freq_47", - "HIER_byte_freq_49", - "HIER_byte_freq_51", - "HIER_byte_freq_53", - "HIER_byte_freq_55", - "HIER_byte_freq_61", - "HIER_byte_freq_68", - "HIER_byte_freq_76", - "HIER_byte_freq_78", - "HIER_byte_freq_82", - "HIER_byte_freq_85", - "HIER_byte_freq_93", - "HIER_byte_freq_99", - "HIER_byte_freq_104", - "HIER_byte_freq_110", - "HIER_byte_freq_114", - "HIER_byte_freq_117", - "HIER_byte_freq_118", - "HIER_byte_freq_119", - "HIER_byte_freq_122", - "HIER_byte_freq_123", - "HIER_byte_freq_130", - "HIER_byte_freq_134", - "HIER_byte_freq_135", - "HIER_byte_freq_140", - "HIER_byte_freq_141", - "HIER_byte_freq_142", - "HIER_byte_freq_145", - "HIER_byte_freq_146", - "HIER_byte_freq_148", - "HIER_byte_freq_150", - "HIER_byte_freq_151", - "HIER_byte_freq_153", - "HIER_byte_freq_157", - "HIER_byte_freq_159", - "HIER_byte_freq_163", - "HIER_byte_freq_164", - "HIER_byte_freq_165", - "HIER_byte_freq_166", - "HIER_byte_freq_168", - "HIER_byte_freq_172", - "HIER_byte_freq_174", - "HIER_byte_freq_175", - "HIER_byte_freq_176", - "HIER_byte_freq_180", - "HIER_byte_freq_182", - "HIER_byte_freq_189", - "HIER_byte_freq_190", - "HIER_byte_freq_191", - "HIER_byte_freq_193", - "HIER_byte_freq_194", - "HIER_byte_freq_197", - "HIER_byte_freq_198", - "HIER_byte_freq_205", - "HIER_byte_freq_210", - "HIER_byte_freq_211", - "HIER_byte_freq_219", - "HIER_byte_freq_224", - "HIER_byte_freq_235", - "HIER_byte_freq_236", - "HIER_byte_freq_237", - "HIER_byte_freq_238", - "HIER_byte_freq_241", - "HIER_byte_freq_243", - "HIER_byte_freq_246", - "HIER_byte_freq_247", - "HIER_byte_freq_251", - "HIER_byte_freq_254", - "HIER_byte_freq_255", - "HIER_byte_pair_43", - "HIER_byte_pair_44", - "HIER_byte_pair_46", - "HIER_byte_pair_47", - "HIER_byte_pair_48", - "HIER_byte_pair_49", - "HIER_byte_pair_50", - "HIER_byte_pair_51", - "HIER_byte_pair_52", - "HIER_byte_pair_53", - "HIER_byte_pair_54", - "HIER_byte_pair_55", - "HIER_byte_pair_59", - "HIER_byte_pair_60", - "HIER_byte_pair_68", - "HIER_byte_pair_70", - "HIER_byte_pair_83", - "HIER_byte_pair_97", - "HIER_byte_pair_100", - "HIER_byte_pair_102", - "HIER_byte_pair_103", - "HIER_byte_pair_104", - "HIER_byte_pair_105", - "HIER_byte_pair_106", - "HIER_byte_pair_107", - "HIER_byte_pair_108", - "HIER_byte_pair_109", - "HIER_byte_pair_110", - "HIER_byte_pair_111", - "HIER_byte_pair_112", - "HIER_byte_pair_113", - "HIER_byte_pair_114", - "HIER_byte_pair_121", - "HIER_byte_pair_122", - "HIER_byte_pair_124", - "HIER_byte_pair_125", - "HIER_entropy", - "HIER_zero_ratio", - "HIER_byte_diversity", - "HIER_percentile_25", - "HIER_percentile_95", - "HIER_percentile_range_95_10", - "O3_sz_text", - "O3_sz_rodata", - "O3_sz_data", - "O3_sz_bss", - "O3_sz_plt", - "O3_sz_plt_sec", - "O3_sz_got", - "O3_sz_gotplt", - "O3_sz_eh_frame", - "O3_sz_gcc_except", - "O3_sz_text_hot", - "O3_sz_text_unlikely", - "O3_r_text_total", - "O3_r_ro_total", - "O3_r_data_total", - "O3_r_bss_total", - "O3_r_gotplt_text", - "O3_r_plt_text", - "O3_r_pltsec_text", - "O3_log_sz_text", - "O3_log_sz_plt", - "O3_log_sz_gotplt", - "O3_log_sz_eh_frame", - "O3_log_sz_except", - "O3_rel_total", - "O3_rel_js_ratio", - "O3_rel_gd_ratio", - "O3_rel_rel_ratio", - "O3_rel_irel_ratio", - "O3_rel_entropy", - "O3_log_rel_total", - "O3_func_total", - "O3_func_global", - "O3_func_local", - "O3_func_weak", - "O3_r_func_global_total", - "O3_r_func_local_total", - "O3_r_hidden_default", - "O3_r_funcs_total", - "O3_export_per_kb", - "O3_log_func_total", - "O3_log_func_global", - "O3_d_bl_per_kb", - "O3_d_blr_per_kb", - "O3_d_br_per_kb", - "O3_d_b_per_kb", - "O3_d_bcond_per_kb", - "O3_d_ret_per_kb", - "O3_d_cbz_per_kb", - "O3_d_cbnz_per_kb", - "O3_d_tbz_per_kb", - "O3_d_tbnz_per_kb", - "O3_d_adrp_per_kb", - "O3_d_ldr_per_kb", - "O3_d_add_per_kb", - "O3_d_vec_per_kb", - "O3_r_call_ret", - "O3_r_blr_total", - "O3_r_cond_uncond", - "O3_r_bl_ret", - "O3_r_vec_total", - "O3_got_triplet_per_kb", - "O3_plt_entries", - "O3_d_plt_entries_per_kb", - "O3_r_plt_calls", - "O3_has_symver", - "O3_has_comdat_group" - ] -} \ No newline at end of file diff --git a/tools/optimization_detector/optimization_detector/models/lto/Os/rf_model.joblib b/tools/optimization_detector/optimization_detector/models/lto/Os/rf_model.joblib deleted file mode 100644 index fdd13285..00000000 Binary files a/tools/optimization_detector/optimization_detector/models/lto/Os/rf_model.joblib and /dev/null differ diff --git a/tools/optimization_detector/optimization_detector/models/lto/Os/svm_model.joblib b/tools/optimization_detector/optimization_detector/models/lto/Os/svm_model.joblib deleted file mode 100644 index 1ec96869..00000000 Binary files a/tools/optimization_detector/optimization_detector/models/lto/Os/svm_model.joblib and /dev/null differ diff --git a/tools/optimization_detector/optimization_detector/models/lto/feature_names.json b/tools/optimization_detector/optimization_detector/models/lto/feature_names.json new file mode 100644 index 00000000..535af09e --- /dev/null +++ b/tools/optimization_detector/optimization_detector/models/lto/feature_names.json @@ -0,0 +1,555 @@ +{ + "feature_names": [ + "LEG_absolute_symbols", + "LEG_artificial_ratio", + "LEG_artificial_symbols", + "LEG_ascii_ratio", + "LEG_bss_sections", + "LEG_bss_size_ratio", + "LEG_build_id_present", + "LEG_common_symbols", + "LEG_compiler_info", + "LEG_compiler_string_ratio", + "LEG_compiler_strings", + "LEG_data_ratio", + "LEG_data_relocations", + "LEG_data_sections", + "LEG_data_size_ratio", + "LEG_data_symbols", + "LEG_debug_sections", + "LEG_debug_symbols", + "LEG_dynamic_ratio", + "LEG_dynamic_relocations", + "LEG_elf_header_present", + "LEG_elf_type", + "LEG_entropy", + "LEG_entry_point", + "LEG_file_size", + "LEG_fini_functions", + "LEG_function_ratio", + "LEG_function_symbols", + "LEG_gcc_strings", + "LEG_global_ratio", + "LEG_global_symbols", + "LEG_gnu_strings", + "LEG_got_ratio", + "LEG_got_relocations", + "LEG_has_comment_section", + "LEG_has_debug_sections", + "LEG_has_inline_functions", + "LEG_has_note_section", + "LEG_has_tail_calls", + "LEG_has_unrolled_loops", + "LEG_has_vectorized_code", + "LEG_header_hex", + "LEG_init_functions", + "LEG_is_elf", + "LEG_linker_strings", + "LEG_local_ratio", + "LEG_local_symbols", + "LEG_lto_patterns", + "LEG_lto_ratio", + "LEG_lto_string_ratio", + "LEG_lto_strings", + "LEG_lto_symbols", + "LEG_machine_type", + "LEG_needed_libraries", + "LEG_optimization_level", + "LEG_optimization_string_ratio", + "LEG_optimization_strings", + "LEG_plt_ratio", + "LEG_plt_relocations", + "LEG_program_header_count", + "LEG_rpath_entries", + "LEG_runpath_entries", + "LEG_section_header_count", + "LEG_symbol_versions", + "LEG_text_relocations", + "LEG_text_sections", + "LEG_text_size_ratio", + "LEG_total_bss_size", + "LEG_total_data_size", + "LEG_total_debug_size", + "LEG_total_dynamic_entries", + "LEG_total_relocations", + "LEG_total_sections", + "LEG_total_strings", + "LEG_total_symbols", + "LEG_total_text_size", + "LEG_undefined_symbols", + "LEG_weak_symbols", + "LEG_zero_byte_ratio", + "HIER_byte_freq_0", + "HIER_byte_freq_1", + "HIER_byte_freq_2", + "HIER_byte_freq_3", + "HIER_byte_freq_4", + "HIER_byte_freq_5", + "HIER_byte_freq_6", + "HIER_byte_freq_7", + "HIER_byte_freq_8", + "HIER_byte_freq_9", + "HIER_byte_freq_10", + "HIER_byte_freq_11", + "HIER_byte_freq_12", + "HIER_byte_freq_13", + "HIER_byte_freq_14", + "HIER_byte_freq_15", + "HIER_byte_freq_16", + "HIER_byte_freq_17", + "HIER_byte_freq_18", + "HIER_byte_freq_19", + "HIER_byte_freq_20", + "HIER_byte_freq_21", + "HIER_byte_freq_22", + "HIER_byte_freq_23", + "HIER_byte_freq_24", + "HIER_byte_freq_25", + "HIER_byte_freq_26", + "HIER_byte_freq_27", + "HIER_byte_freq_28", + "HIER_byte_freq_29", + "HIER_byte_freq_30", + "HIER_byte_freq_31", + "HIER_byte_freq_32", + "HIER_byte_freq_33", + "HIER_byte_freq_34", + "HIER_byte_freq_35", + "HIER_byte_freq_36", + "HIER_byte_freq_37", + "HIER_byte_freq_38", + "HIER_byte_freq_39", + "HIER_byte_freq_40", + "HIER_byte_freq_41", + "HIER_byte_freq_42", + "HIER_byte_freq_43", + "HIER_byte_freq_44", + "HIER_byte_freq_45", + "HIER_byte_freq_46", + "HIER_byte_freq_47", + "HIER_byte_freq_48", + "HIER_byte_freq_49", + "HIER_byte_freq_50", + "HIER_byte_freq_51", + "HIER_byte_freq_52", + "HIER_byte_freq_53", + "HIER_byte_freq_54", + "HIER_byte_freq_55", + "HIER_byte_freq_56", + "HIER_byte_freq_57", + "HIER_byte_freq_58", + "HIER_byte_freq_59", + "HIER_byte_freq_60", + "HIER_byte_freq_61", + "HIER_byte_freq_62", + "HIER_byte_freq_63", + "HIER_byte_freq_64", + "HIER_byte_freq_65", + "HIER_byte_freq_66", + "HIER_byte_freq_67", + "HIER_byte_freq_68", + "HIER_byte_freq_69", + "HIER_byte_freq_70", + "HIER_byte_freq_71", + "HIER_byte_freq_72", + "HIER_byte_freq_73", + "HIER_byte_freq_74", + "HIER_byte_freq_75", + "HIER_byte_freq_76", + "HIER_byte_freq_77", + "HIER_byte_freq_78", + "HIER_byte_freq_79", + "HIER_byte_freq_80", + "HIER_byte_freq_81", + "HIER_byte_freq_82", + "HIER_byte_freq_83", + "HIER_byte_freq_84", + "HIER_byte_freq_85", + "HIER_byte_freq_86", + "HIER_byte_freq_87", + "HIER_byte_freq_88", + "HIER_byte_freq_89", + "HIER_byte_freq_90", + "HIER_byte_freq_91", + "HIER_byte_freq_92", + "HIER_byte_freq_93", + "HIER_byte_freq_94", + "HIER_byte_freq_95", + "HIER_byte_freq_96", + "HIER_byte_freq_97", + "HIER_byte_freq_98", + "HIER_byte_freq_99", + "HIER_byte_freq_100", + "HIER_byte_freq_101", + "HIER_byte_freq_102", + "HIER_byte_freq_103", + "HIER_byte_freq_104", + "HIER_byte_freq_105", + "HIER_byte_freq_106", + "HIER_byte_freq_107", + "HIER_byte_freq_108", + "HIER_byte_freq_109", + "HIER_byte_freq_110", + "HIER_byte_freq_111", + "HIER_byte_freq_112", + "HIER_byte_freq_113", + "HIER_byte_freq_114", + "HIER_byte_freq_115", + "HIER_byte_freq_116", + "HIER_byte_freq_117", + "HIER_byte_freq_118", + "HIER_byte_freq_119", + "HIER_byte_freq_120", + "HIER_byte_freq_121", + "HIER_byte_freq_122", + "HIER_byte_freq_123", + "HIER_byte_freq_124", + "HIER_byte_freq_125", + "HIER_byte_freq_126", + "HIER_byte_freq_127", + "HIER_byte_freq_128", + "HIER_byte_freq_129", + "HIER_byte_freq_130", + "HIER_byte_freq_131", + "HIER_byte_freq_132", + "HIER_byte_freq_133", + "HIER_byte_freq_134", + "HIER_byte_freq_135", + "HIER_byte_freq_136", + "HIER_byte_freq_137", + "HIER_byte_freq_138", + "HIER_byte_freq_139", + "HIER_byte_freq_140", + "HIER_byte_freq_141", + "HIER_byte_freq_142", + "HIER_byte_freq_143", + "HIER_byte_freq_144", + "HIER_byte_freq_145", + "HIER_byte_freq_146", + "HIER_byte_freq_147", + "HIER_byte_freq_148", + "HIER_byte_freq_149", + "HIER_byte_freq_150", + "HIER_byte_freq_151", + "HIER_byte_freq_152", + "HIER_byte_freq_153", + "HIER_byte_freq_154", + "HIER_byte_freq_155", + "HIER_byte_freq_156", + "HIER_byte_freq_157", + "HIER_byte_freq_158", + "HIER_byte_freq_159", + "HIER_byte_freq_160", + "HIER_byte_freq_161", + "HIER_byte_freq_162", + "HIER_byte_freq_163", + "HIER_byte_freq_164", + "HIER_byte_freq_165", + "HIER_byte_freq_166", + "HIER_byte_freq_167", + "HIER_byte_freq_168", + "HIER_byte_freq_169", + "HIER_byte_freq_170", + "HIER_byte_freq_171", + "HIER_byte_freq_172", + "HIER_byte_freq_173", + "HIER_byte_freq_174", + "HIER_byte_freq_175", + "HIER_byte_freq_176", + "HIER_byte_freq_177", + "HIER_byte_freq_178", + "HIER_byte_freq_179", + "HIER_byte_freq_180", + "HIER_byte_freq_181", + "HIER_byte_freq_182", + "HIER_byte_freq_183", + "HIER_byte_freq_184", + "HIER_byte_freq_185", + "HIER_byte_freq_186", + "HIER_byte_freq_187", + "HIER_byte_freq_188", + "HIER_byte_freq_189", + "HIER_byte_freq_190", + "HIER_byte_freq_191", + "HIER_byte_freq_192", + "HIER_byte_freq_193", + "HIER_byte_freq_194", + "HIER_byte_freq_195", + "HIER_byte_freq_196", + "HIER_byte_freq_197", + "HIER_byte_freq_198", + "HIER_byte_freq_199", + "HIER_byte_freq_200", + "HIER_byte_freq_201", + "HIER_byte_freq_202", + "HIER_byte_freq_203", + "HIER_byte_freq_204", + "HIER_byte_freq_205", + "HIER_byte_freq_206", + "HIER_byte_freq_207", + "HIER_byte_freq_208", + "HIER_byte_freq_209", + "HIER_byte_freq_210", + "HIER_byte_freq_211", + "HIER_byte_freq_212", + "HIER_byte_freq_213", + "HIER_byte_freq_214", + "HIER_byte_freq_215", + "HIER_byte_freq_216", + "HIER_byte_freq_217", + "HIER_byte_freq_218", + "HIER_byte_freq_219", + "HIER_byte_freq_220", + "HIER_byte_freq_221", + "HIER_byte_freq_222", + "HIER_byte_freq_223", + "HIER_byte_freq_224", + "HIER_byte_freq_225", + "HIER_byte_freq_226", + "HIER_byte_freq_227", + "HIER_byte_freq_228", + "HIER_byte_freq_229", + "HIER_byte_freq_230", + "HIER_byte_freq_231", + "HIER_byte_freq_232", + "HIER_byte_freq_233", + "HIER_byte_freq_234", + "HIER_byte_freq_235", + "HIER_byte_freq_236", + "HIER_byte_freq_237", + "HIER_byte_freq_238", + "HIER_byte_freq_239", + "HIER_byte_freq_240", + "HIER_byte_freq_241", + "HIER_byte_freq_242", + "HIER_byte_freq_243", + "HIER_byte_freq_244", + "HIER_byte_freq_245", + "HIER_byte_freq_246", + "HIER_byte_freq_247", + "HIER_byte_freq_248", + "HIER_byte_freq_249", + "HIER_byte_freq_250", + "HIER_byte_freq_251", + "HIER_byte_freq_252", + "HIER_byte_freq_253", + "HIER_byte_freq_254", + "HIER_byte_freq_255", + "HIER_byte_pair_0", + "HIER_byte_pair_1", + "HIER_byte_pair_2", + "HIER_byte_pair_3", + "HIER_byte_pair_4", + "HIER_byte_pair_5", + "HIER_byte_pair_6", + "HIER_byte_pair_7", + "HIER_byte_pair_8", + "HIER_byte_pair_9", + "HIER_byte_pair_10", + "HIER_byte_pair_11", + "HIER_byte_pair_12", + "HIER_byte_pair_13", + "HIER_byte_pair_14", + "HIER_byte_pair_15", + "HIER_byte_pair_16", + "HIER_byte_pair_17", + "HIER_byte_pair_18", + "HIER_byte_pair_19", + "HIER_byte_pair_20", + "HIER_byte_pair_21", + "HIER_byte_pair_22", + "HIER_byte_pair_23", + "HIER_byte_pair_24", + "HIER_byte_pair_25", + "HIER_byte_pair_26", + "HIER_byte_pair_27", + "HIER_byte_pair_28", + "HIER_byte_pair_29", + "HIER_byte_pair_30", + "HIER_byte_pair_31", + "HIER_byte_pair_32", + "HIER_byte_pair_33", + "HIER_byte_pair_34", + "HIER_byte_pair_35", + "HIER_byte_pair_36", + "HIER_byte_pair_37", + "HIER_byte_pair_38", + "HIER_byte_pair_39", + "HIER_byte_pair_40", + "HIER_byte_pair_41", + "HIER_byte_pair_42", + "HIER_byte_pair_43", + "HIER_byte_pair_44", + "HIER_byte_pair_45", + "HIER_byte_pair_46", + "HIER_byte_pair_47", + "HIER_byte_pair_48", + "HIER_byte_pair_49", + "HIER_byte_pair_50", + "HIER_byte_pair_51", + "HIER_byte_pair_52", + "HIER_byte_pair_53", + "HIER_byte_pair_54", + "HIER_byte_pair_55", + "HIER_byte_pair_56", + "HIER_byte_pair_57", + "HIER_byte_pair_58", + "HIER_byte_pair_59", + "HIER_byte_pair_60", + "HIER_byte_pair_61", + "HIER_byte_pair_62", + "HIER_byte_pair_63", + "HIER_byte_pair_64", + "HIER_byte_pair_65", + "HIER_byte_pair_66", + "HIER_byte_pair_67", + "HIER_byte_pair_68", + "HIER_byte_pair_69", + "HIER_byte_pair_70", + "HIER_byte_pair_71", + "HIER_byte_pair_72", + "HIER_byte_pair_73", + "HIER_byte_pair_74", + "HIER_byte_pair_75", + "HIER_byte_pair_76", + "HIER_byte_pair_77", + "HIER_byte_pair_78", + "HIER_byte_pair_79", + "HIER_byte_pair_80", + "HIER_byte_pair_81", + "HIER_byte_pair_82", + "HIER_byte_pair_83", + "HIER_byte_pair_84", + "HIER_byte_pair_85", + "HIER_byte_pair_86", + "HIER_byte_pair_87", + "HIER_byte_pair_88", + "HIER_byte_pair_89", + "HIER_byte_pair_90", + "HIER_byte_pair_91", + "HIER_byte_pair_92", + "HIER_byte_pair_93", + "HIER_byte_pair_94", + "HIER_byte_pair_95", + "HIER_byte_pair_96", + "HIER_byte_pair_97", + "HIER_byte_pair_98", + "HIER_byte_pair_99", + "HIER_byte_pair_100", + "HIER_byte_pair_101", + "HIER_byte_pair_102", + "HIER_byte_pair_103", + "HIER_byte_pair_104", + "HIER_byte_pair_105", + "HIER_byte_pair_106", + "HIER_byte_pair_107", + "HIER_byte_pair_108", + "HIER_byte_pair_109", + "HIER_byte_pair_110", + "HIER_byte_pair_111", + "HIER_byte_pair_112", + "HIER_byte_pair_113", + "HIER_byte_pair_114", + "HIER_byte_pair_115", + "HIER_byte_pair_116", + "HIER_byte_pair_117", + "HIER_byte_pair_118", + "HIER_byte_pair_119", + "HIER_byte_pair_120", + "HIER_byte_pair_121", + "HIER_byte_pair_122", + "HIER_byte_pair_123", + "HIER_byte_pair_124", + "HIER_byte_pair_125", + "HIER_byte_pair_126", + "HIER_byte_pair_127", + "HIER_entropy", + "HIER_zero_ratio", + "HIER_ascii_ratio", + "HIER_byte_diversity", + "HIER_avg_byte_norm", + "HIER_byte_std_norm", + "HIER_file_size", + "HIER_log_file_size", + "HIER_file_size_kb", + "HIER_file_size_mb", + "HIER_percentile_5", + "HIER_percentile_10", + "HIER_percentile_25", + "HIER_percentile_50", + "HIER_percentile_75", + "HIER_percentile_90", + "HIER_percentile_95", + "HIER_percentile_99", + "HIER_percentile_range_99_5", + "HIER_percentile_range_95_10", + "HIER_percentile_range_90_25", + "O3_sz_text", + "O3_sz_rodata", + "O3_sz_data", + "O3_sz_bss", + "O3_sz_plt", + "O3_sz_plt_sec", + "O3_sz_got", + "O3_sz_gotplt", + "O3_sz_eh_frame", + "O3_sz_gcc_except_table", + "O3_sz_text_hot", + "O3_sz_text_unlikely", + "O3_log_sz_text", + "O3_log_sz_plt", + "O3_log_sz_gotplt", + "O3_log_sz_eh_frame", + "O3_log_sz_except", + "O3_r_text_total", + "O3_r_ro_total", + "O3_r_data_total", + "O3_r_bss_total", + "O3_r_gotplt_text", + "O3_r_plt_text", + "O3_r_pltsec_text", + "O3_rel_total", + "O3_rel_js_ratio", + "O3_rel_gd_ratio", + "O3_rel_rel_ratio", + "O3_rel_irel_ratio", + "O3_rel_entropy", + "O3_log_rel_total", + "O3_func_total", + "O3_func_global", + "O3_func_local", + "O3_func_weak", + "O3_r_func_global_total", + "O3_r_func_local_total", + "O3_r_hidden_default", + "O3_r_funcs_total", + "O3_export_per_kb", + "O3_log_func_total", + "O3_log_func_global", + "O3_d_bl_per_kb", + "O3_d_blr_per_kb", + "O3_d_br_per_kb", + "O3_d_b_per_kb", + "O3_d_bcond_per_kb", + "O3_d_ret_per_kb", + "O3_d_cbz_per_kb", + "O3_d_cbnz_per_kb", + "O3_d_tbz_per_kb", + "O3_d_tbnz_per_kb", + "O3_d_adrp_per_kb", + "O3_d_ldr_per_kb", + "O3_d_add_per_kb", + "O3_d_vec_per_kb", + "O3_r_call_ret", + "O3_r_blr_total", + "O3_r_cond_uncond", + "O3_r_bl_ret", + "O3_r_vec_total", + "O3_got_triplet_per_kb", + "O3_plt_entries", + "O3_d_plt_entries_per_kb", + "O3_r_plt_calls", + "O3_has_symver", + "O3_has_comdat_group" + ] +} \ No newline at end of file diff --git a/tools/optimization_detector/optimization_detector/models/lto/metrics.json b/tools/optimization_detector/optimization_detector/models/lto/metrics.json new file mode 100644 index 00000000..35622b91 --- /dev/null +++ b/tools/optimization_detector/optimization_detector/models/lto/metrics.json @@ -0,0 +1,22 @@ +{ + "train": { + "accuracy": 0.776519052523172, + "precision": 0.8852772466539197, + "recall": 0.7467741935483871, + "f1": 0.810148731408574, + "tp": 463, + "tn": 291, + "fp": 60, + "fn": 157 + }, + "test": { + "accuracy": 0.7418032786885246, + "precision": 0.8916666666666667, + "recall": 0.6815286624203821, + "f1": 0.7725631768953067, + "tp": 107, + "tn": 74, + "fp": 13, + "fn": 50 + } +} \ No newline at end of file diff --git a/tools/optimization_detector/optimization_detector/models/lto/model.pkl b/tools/optimization_detector/optimization_detector/models/lto/model.pkl new file mode 100644 index 00000000..7bf8357a Binary files /dev/null and b/tools/optimization_detector/optimization_detector/models/lto/model.pkl differ diff --git a/tools/optimization_detector/optimization_detector/models/lto/report.txt b/tools/optimization_detector/optimization_detector/models/lto/report.txt new file mode 100644 index 00000000..304260c1 --- /dev/null +++ b/tools/optimization_detector/optimization_detector/models/lto/report.txt @@ -0,0 +1,5 @@ +======================================================================== +LTO Detection Report (tag=ALL, model=SVM) +======================================================================== +TRAIN : Prec=0.8853 Rec=0.7468 F1=0.8101 Acc=0.7765 TP=463 TN=291 FP=60 FN=157 +TEST : Prec=0.8917 Rec=0.6815 F1=0.7726 Acc=0.7418 TP=107 TN=74 FP=13 FN=50 \ No newline at end of file diff --git a/tools/optimization_detector/optimization_detector/optimization_detector.py b/tools/optimization_detector/optimization_detector/optimization_detector.py index 9f865f74..e677c4cf 100644 --- a/tools/optimization_detector/optimization_detector/optimization_detector.py +++ b/tools/optimization_detector/optimization_detector/optimization_detector.py @@ -9,6 +9,7 @@ import numpy as np import pandas as pd import tensorflow as tf +from elftools.elf.elffile import ELFFile from tqdm import tqdm from optimization_detector.file_info import FILE_STATUS_MAPPING, FileInfo @@ -140,18 +141,23 @@ def detect_optimization(self, file_infos: list[FileInfo]) -> list[tuple[str, pd. @staticmethod def _extract_features(file_info: FileInfo, features: int = 2048) -> Optional[list[int]]: - data = file_info.extract_dot_text() - if not data or len(data) == 0: + """Extract features from file, returns None if no data, empty list if error, list if success""" + try: + data = file_info.extract_dot_text() + if not data or len(data) == 0: + return None + + sequences = [] + for counter in range(0, len(data), features): + seq = data[counter : counter + features] + if len(seq) < features: + seq = np.pad(seq, (0, features - len(seq)), 'constant') + sequences.append(seq) + return sequences + except Exception: + # 如果提取失败，返回 None（会在 _run_analysis 中捕获异常） return None - sequences = [] - for counter in range(0, len(data), features): - seq = data[counter : counter + features] - if len(seq) < features: - seq = np.pad(seq, (0, features - len(seq)), 'constant') - sequences.append(seq) - return sequences - def apply_model(self, data, model): if not data.size: return None @@ -208,8 +214,57 @@ def _detect_lto(self, file_infos: list[FileInfo], flags_results: dict) -> dict: return lto_results - def _run_analysis(self, file_info: FileInfo) -> tuple[FileInfo, Optional[list[tuple[int, float]]]]: - """Run optimization flag detection on a single file with optional timeout""" + def _run_analysis(self, file_info: FileInfo) -> tuple[FileInfo, Optional[list[tuple[int, float]]], Optional[str]]: + """Run optimization flag detection on a single file with optional timeout + Returns: (file_info, flags, error_reason) + - flags: None means skip (too few chunks), [] means no data, list means success + - error_reason: None means no error, str means failure reason + """ + try: + # 检查chunk数量，如果小于10个chunk，跳过预测 + MIN_CHUNKS = 10 + # 先尝试直接打开 ELF 文件，以便捕获更详细的错误信息 + try: + with open(file_info.absolute_path, 'rb') as f: + elf = ELFFile(f) + section = elf.get_section_by_name('.text') + if not section: + error_msg = 'No .text section found in ELF file' + return file_info, [], error_msg + except Exception as e: + # 捕获 ELF 文件读取异常（如 Magic number does not match） + error_msg = f'Failed to read ELF file: {str(e)}' + logging.error('Error reading ELF file %s: %s', file_info.absolute_path, e) + return file_info, [], error_msg + + # 如果 ELF 文件读取成功，再尝试提取 .text 段数据 + text_data = file_info.extract_dot_text() + if not text_data: + # 检查文件类型 + if file_info.file_type.value == 0xFF: # NOT_SUPPORT + error_msg = 'File type not supported (not .so or .a)' + else: + error_msg = 'No .text section data extracted (section exists but data is empty)' + return file_info, [], error_msg + + # 提取特征 + features_array = self._extract_features(file_info, features=2048) + # 只有当有数据但chunk数量不足时才跳过预测 + # 如果没有数据（features_array is None），继续执行让_run_inference返回[]，最终标记为failed + if features_array is not None and len(features_array) < MIN_CHUNKS: + logging.debug( + 'Skipping file with too few chunks (%d < %d): %s', + len(features_array), + MIN_CHUNKS, + file_info.absolute_path, + ) + # 返回特殊标记，表示跳过（使用None表示skip，与"没有数据"区分） + return file_info, None, None + except Exception as e: + error_msg = f'Failed to extract features: {str(e)}' + logging.error('Error extracting features from %s: %s', file_info.absolute_path, e) + return file_info, [], error_msg + # Lazy load model if self.model is None: # 使用 GPU 策略（如果可用） @@ -232,24 +287,30 @@ def _run_analysis(self, file_info: FileInfo) -> tuple[FileInfo, Optional[list[tu self.model.compile(optimizer=self.model.optimizer, loss=self.model.loss, metrics=['accuracy']) logging.info('Model loaded with CPU') - if self.timeout is None: - # No timeout, run normally - return file_info, self._run_inference(file_info) - - # Use ThreadPoolExecutor with timeout for cross-platform compatibility - from concurrent.futures import ThreadPoolExecutor # noqa: PLC0415 - from concurrent.futures import TimeoutError as FutureTimeoutError # noqa: PLC0415 - - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(self._run_inference, file_info) - try: - result = future.result(timeout=self.timeout) - return file_info, result - except FutureTimeoutError: - logging.warning('File analysis timeout after %d seconds: %s', self.timeout, file_info.absolute_path) - # Cancel the future if possible - future.cancel() - return file_info, None + try: + if self.timeout is None: + # No timeout, run normally + result = self._run_inference(file_info) + return file_info, result, None + # Use ThreadPoolExecutor with timeout for cross-platform compatibility + from concurrent.futures import ThreadPoolExecutor # noqa: PLC0415 + from concurrent.futures import TimeoutError as FutureTimeoutError # noqa: PLC0415 + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(self._run_inference, file_info) + try: + result = future.result(timeout=self.timeout) + return file_info, result, None + except FutureTimeoutError: + error_msg = f'Analysis timeout after {self.timeout} seconds' + logging.warning('File analysis timeout after %d seconds: %s', self.timeout, file_info.absolute_path) + # Cancel the future if possible + future.cancel() + return file_info, [], error_msg + except Exception as e: + error_msg = f'Analysis error: {str(e)}' + logging.error('Error analyzing file %s: %s', file_info.absolute_path, e) + return file_info, [], error_msg def _analyze_files(self, file_infos: list[FileInfo]) -> tuple[int, int, dict]: # Filter out analyzed files @@ -281,27 +342,101 @@ def _analyze_files(self, file_infos: list[FileInfo]) -> tuple[int, int, dict]: results = [process_func(fi) for fi in tqdm(remaining_files, desc='Analyzing binaries optimization')] # Save intermediate results - for file_info, flags in results: - if flags: - flags_path = os.path.join(FileInfo.CACHE_DIR, f'flags_{file_info.file_id}.csv') + for file_info, flags, error_reason in results: + flags_path = os.path.join(FileInfo.CACHE_DIR, f'flags_{file_info.file_id}.csv') + error_path = os.path.join(FileInfo.CACHE_DIR, f'error_{file_info.file_id}.txt') + + if flags is None: + # 跳过预测（chunk数量太少），创建skip标记文件 + with open(flags_path, 'w', encoding='UTF-8') as f: + f.write('file,prediction,confidence\n') + f.write(f'{file_info.file_id},skip,0.0\n') + elif flags: + # 正常预测结果 with open(flags_path, 'w', encoding='UTF-8') as f: f.write('file,prediction,confidence\n') for pred, conf in flags: f.write(f'{file_info.file_id},{pred},{conf}\n') + # 如果flags是空列表[]，表示没有数据或失败 + if error_reason: + # 保存失败原因 + with open(error_path, 'w', encoding='UTF-8') as f: + f.write(error_reason) + elif not flags and flags is not None and not os.path.exists(error_path): + # 没有数据但没有错误信息，这种情况不应该发生（应该在 _run_analysis 中已经捕获） + # 但为了安全起见，仍然尝试提取失败原因 + try: + text_data = file_info.extract_dot_text() + if not text_data: + # 检查文件类型 + if file_info.file_type.value == 0xFF: # NOT_SUPPORT + fallback_error_reason = 'File type not supported (not .so or .a)' + else: + fallback_error_reason = 'No .text section found in ELF file' + else: + fallback_error_reason = 'Unknown error: extracted data is empty' + with open(error_path, 'w', encoding='UTF-8') as f: + f.write(fallback_error_reason) + except Exception as e: + fallback_error_reason = f'Failed to extract .text section: {str(e)}' + with open(error_path, 'w', encoding='UTF-8') as f: + f.write(fallback_error_reason) flags_results = {} files_with_results = 0 for file_info in file_infos: flags_path = os.path.join(FileInfo.CACHE_DIR, f'flags_{file_info.file_id}.csv') + error_path = os.path.join(FileInfo.CACHE_DIR, f'error_{file_info.file_id}.txt') + + # 读取失败原因（如果存在） + error_reason = None + if os.path.exists(error_path): + try: + with open(error_path, encoding='UTF-8') as f: + error_reason = f.read().strip() + except Exception: + pass + if os.path.exists(flags_path): try: flags_df = pd.read_csv(flags_path) if not flags_df.empty: - file_results = self._merge_chunk_results(flags_df) - flags_results.update(file_results) - files_with_results += 1 + # 检查是否是skip标记 + if len(flags_df) == 1 and flags_df.iloc[0]['prediction'] == 'skip': + # 跳过预测的文件，标记为skip + flags_results[file_info.file_id] = { + 'prediction': 'skip', + 'confidence': 0.0, + 'distribution': {}, + 'opt_score': None, + 'opt_category': None, + 'total_chunks': 0, + 'error_reason': None, + } + files_with_results += 1 + else: + # 正常预测结果 + file_results = self._merge_chunk_results(flags_df) + for _file_id, result in file_results.items(): + result['error_reason'] = None + flags_results.update(file_results) + files_with_results += 1 except Exception as e: logging.error('Error loading results for %s: %s', file_info.absolute_path, e) + if not error_reason: + error_reason = f'Error loading results: {str(e)}' + + # 如果没有结果但有错误原因，保存错误信息 + if file_info.file_id not in flags_results and error_reason: + flags_results[file_info.file_id] = { + 'prediction': None, + 'confidence': None, + 'distribution': {}, + 'opt_score': None, + 'opt_category': None, + 'total_chunks': 0, + 'error_reason': error_reason, + } return files_with_results, len(file_infos) - files_with_results, flags_results @@ -314,6 +449,7 @@ def _collect_results( report_data = [] for file_info in sorted(file_infos, key=lambda x: x.logical_path): result = flags_results.get(file_info.file_id) + error_reason = None if result is None: status = FILE_STATUS_MAPPING['failed'] opt_category = 'N/A' @@ -321,6 +457,25 @@ def _collect_results( distribution = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0} total_chunks = 0 size_optimized = 'N/A' + error_reason = 'No analysis result found' + elif result.get('prediction') == 'skip': + # 跳过预测的文件（chunk数量太少） + status = FILE_STATUS_MAPPING['skipped_few_chunks'] + opt_category = 'N/A' + opt_score = 'N/A' + distribution = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0} + total_chunks = 0 + size_optimized = 'N/A' + error_reason = None + elif result.get('error_reason'): + # 有错误原因 + status = FILE_STATUS_MAPPING['failed'] + opt_category = 'N/A' + opt_score = 'N/A' + distribution = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0} + total_chunks = 0 + size_optimized = 'N/A' + error_reason = result.get('error_reason') else: status = FILE_STATUS_MAPPING['analyzed'] opt_category = result['opt_category'] @@ -330,6 +485,7 @@ def _collect_results( os_chunks = distribution.get(4, 0) os_ratio = os_chunks / total_chunks if total_chunks > 0 else 0 size_optimized = f'{"Yes" if os_ratio >= 0.5 else "No"} ({os_ratio:.1%})' + error_reason = None row = { 'File': file_info.logical_path, @@ -344,6 +500,7 @@ def _collect_results( 'Total Chunks': total_chunks, 'File Size (bytes)': file_info.file_size, 'Size Optimized': size_optimized, + 'Failure Reason': error_reason if error_reason else 'N/A', } # 添加LTO检测列 @@ -351,11 +508,9 @@ def _collect_results( lto_result = lto_results[file_info.file_id] row['LTO Score'] = f'{lto_result["score"]:.4f}' if lto_result['score'] is not None else 'N/A' row['LTO Prediction'] = lto_result['prediction'] - row['LTO Model Used'] = lto_result['model_used'] elif self.enable_lto: row['LTO Score'] = 'N/A' row['LTO Prediction'] = 'N/A' - row['LTO Model Used'] = 'N/A' report_data.append(row) return pd.DataFrame(report_data) diff --git a/tools/optimization_detector/webpack.config.js b/tools/optimization_detector/webpack.config.js index 1a19d3ea..f0db051d 100644 --- a/tools/optimization_detector/webpack.config.js +++ b/tools/optimization_detector/webpack.config.js @@ -1,8 +1,30 @@ const path = require('path'); +const fs = require('fs'); const CopyPlugin = require('copy-webpack-plugin'); const { PackPlugin } = require('../../scripts/webpack_plugin'); const version = require('./package.json').version; +// 构建拷贝模式数组 +const copyPatterns = [ + { + from: path.resolve(__dirname, 'README.md'), + to: path.resolve(__dirname, '../../dist/tools/opt-detector/README.md'), + noErrorOnMissing: true, + }, + { + from: path.resolve(__dirname, 'plugin.json'), + to: path.resolve(__dirname, '../../dist/tools/opt-detector/plugin.json') + }, +]; + +// Mac 平台时拷贝 run_macos.sh 文件 +if (process.platform === 'darwin') { + copyPatterns.push({ + from: path.resolve(__dirname, '../../scripts/run_macos.sh'), + to: path.resolve(__dirname, '../../dist/tools/opt-detector/run_macos.sh'), + noErrorOnMissing: true, + }); +} module.exports = { target: 'node', @@ -15,15 +37,24 @@ module.exports = { plugins: [ // 拷贝 README.md 和 plugin.json 到目标目录 new CopyPlugin({ - patterns: [ - { - from: path.resolve(__dirname, 'README.md'), - to: path.resolve(__dirname, '../../dist/tools/opt-detector/README.md'), - noErrorOnMissing: true, - }, - { from: path.resolve(__dirname, 'plugin.json'), to: path.resolve(__dirname, '../../dist/tools/opt-detector/plugin.json') }, - ], + patterns: copyPatterns, }), + // 确保 macOS 下 run_macos.sh 拷贝后仍然是可执行文件 + { + apply: (compiler) => { + compiler.hooks.afterEmit.tap('MakeRunMacosExecutable', () => { + if (process.platform !== 'darwin') { + return; + } + const dest = path.resolve(__dirname, '../../dist/tools/opt-detector/run_macos.sh'); + try { + fs.chmodSync(dest, 0o755); + } catch (e) { + // 这里静默失败即可，不影响构建 + } + }); + }, + }, // 打包插件：创建 zip 文件 new PackPlugin({ zipName: 'opt-detector', diff --git a/tools/static_analyzer/res/techstack/techstack-config.yaml b/tools/static_analyzer/res/techstack/techstack-config.yaml index a475f19b..0dea57a3 100644 --- a/tools/static_analyzer/res/techstack/techstack-config.yaml +++ b/tools/static_analyzer/res/techstack/techstack-config.yaml @@ -228,3 +228,251 @@ detections: fileRules: - type: "filename" patterns: ["libweex_ability\\.so$", "libweex_framework\\.so$"] + + # ==================== Cangjie (仓颉) ==================== + - id: "Cangjie_Framework" + name: "Cangjie Framework" + type: "Cangjie" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libcj_.*\\.so$", "libcjhybridview\\.so$", "libcontact_cj.*\\.so$"] + + # ==================== @duke/logan ==================== + - id: "@duke/logan" + name: "@duke/logan" + type: "@duke/logan" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["liblogan\\.so$"] + + # ==================== LevelDB ==================== + - id: "@devzeng/leveldb" + name: "@devzeng/leveldb" + type: "@devzeng/leveldb" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libleveldb\\.so$"] + + # ==================== WebSocket/Crypto ==================== + - id: "@duke/websocket-client" + name: "@duke/websocket-client" + type: "@duke/websocket-client" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libwebsocketClient\\.so$"] + + # ==================== @ohos/flate2 ==================== + - id: "@ohos/flate2" + name: "@ohos/flate2" + type: "@ohos/flate2" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libflate2\\.so$"] + + # ==================== @ohos/gpu_transform ==================== + - id: "GPU_Transform" + name: "@ohos/gpu_transform" + type: "@ohos/gpu_transform" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libnativeGpu\\.so$"] + + # ==================== @ohos/imageknifepro ==================== + - id: "@ohos/imageknifepro" + name: "@ohos/imageknifepro" + type: "@ohos/imageknifepro" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libimageknifepro\\.so$"] + + # ==================== @tencent/libpag ==================== + - id: "PAG_Framework" + name: "@tencent/libpag" + type: "@tencent/libpag" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libpag\\.so$", "libnative_libpag\\.so$"] + + # ==================== @ohos/linphone ==================== + - id: "@ohos/linphone" + name: "@ohos/linphone" + type: "@ohos/linphone" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["liblinphone\\.so$", "libbelle-sip\\.so$", "libmediastreamer2\\.so$", "libohos_linphone\\.so$", "libbctoolbox\\.so$", "libbelr\\.so$", "libortp\\.so$"] + + # ==================== @ohos/lottie-turbo ==================== + - id: "@ohos/lottie-turbo" + name: "@ohos/lottie-turbo" + type: "@ohos/lottie-turbo" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["liblottie-turbo\\.so$"] + + # ==================== @tencent/mmkv ==================== + - id: "@tencent/mmkv" + name: "@tencent/mmkv" + type: "@tencent/mmkv" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libmmkv\\.so$"] + + # ==================== @ohos/mp4parser ==================== + - id: "@ohos/mp4parser" + name: "@ohos/mp4parser" + type: "@ohos/mp4parser" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libmp4parser_napi\\.so$"] + + # ==================== @ohos/vap ==================== + - id: "@ohos/vap" + name: "@ohos/vap" + type: "@ohos/vap" + confidence: 0.95 + fileRules: + - type: "filename" + patterns: ["libvap\\.so$"] + + - id: '@hw-agconnect/hmcore' + name: '@hw-agconnect/hmcore' + type: '@hw-agconnect/hmcore' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libagccrypto\\.so$" + - "libagcmbedtls\\.so$" + + - id: '@kuiklybase/knoi' + name: '@kuiklybase/knoi' + type: '@kuiklybase/knoi' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libknoi\\.so$" + + - id: '@mysoft/archive' + name: '@mysoft/archive' + type: '@mysoft/archive' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libarchive\\.so$" + - "libmyarchive\\.so$" + + - id: '@tencentyun/libiotvideo' + name: '@tencentyun/libiotvideo' + type: '@tencentyun/libiotvideo' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "liblibiotvideo\\.so$" + + - id: '@volcengine/apmplus' + name: '@volcengine/apmplus' + type: '@volcengine/apmplus' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libnpth\\.so$" + + - id: '@wolfx/untar' + name: '@wolfx/untar' + type: '@wolfx/untar' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libboundscheck\\.so$" + - "libctar\\.so$" + - "libtar\\.so$" + + - id: '@ohos/ijkplayer' + name: '@ohos/ijkplayer' + type: '@ohos/ijkplayer' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libijkplayer\\.so$" + - "libijkplayer_audio_napi\\.so$" + - "libijkplayer_napi\\.so$" + - "libijksdl\\.so$" + + + - id: '@zaiohos/ijkplayer-surface' + name: '@zaiohos/ijkplayer-surface' + type: '@zaiohos/ijkplayer-surface' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libijkplayer_surface_napi\\.so$" + + - id: jlreader + name: jlreader + type: jlreader + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libjlreader\\.so$" + + - id: ksadsdk + name: ksadsdk + type: ksadsdk + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libksadsdk\\.so$" + - "libkwai-v8-lite\\.so$" + + - id: '@ohos/aki' + name: '@ohos/aki' + type: '@ohos/aki' + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libaki\\.so$" + - "libaki_jsbind\\.so$" + + - id: rapid_kit + name: rapid_kit + type: rapid_kit + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libPPCS\\.so$" + - "libRapidCore\\.so$" + - "libRapidMedia\\.so$" + - "libRapidSDL\\.so$" + - "libnative_rapid_kit\\.so$" + + - id: shiply + name: shiply + type: shiply + confidence: 0.95 + fileRules: + - type: filename + patterns: + - "libshiply\\.so$" \ No newline at end of file diff --git a/tools/static_analyzer/src/core/elf/elf_analyzer.ts b/tools/static_analyzer/src/core/elf/elf_analyzer.ts index 76dd4a83..ca7ef8d6 100644 --- a/tools/static_analyzer/src/core/elf/elf_analyzer.ts +++ b/tools/static_analyzer/src/core/elf/elf_analyzer.ts @@ -64,28 +64,29 @@ export class ElfAnalyzer { const exportedSymbols: Array = []; const importedSymbols: Array = []; - // Extract exported symbols from .dynsym + // Extract exported symbols from .dynsym (与 nm -D 一致) + // 只提取 GLOBAL FUNC 类型的符号（与 grep " T " 一致，T 表示全局函数） if (elf.body.symbols) { for (const sym of elf.body.symbols) { - if (sym.section === 'SHN_UNDEF') { - importedSymbols.push((await demangle(sym.name)) || sym.name); - } else { - exportedSymbols.push((await demangle(sym.name)) || sym.name); + // 跳过空符号 + if (!sym.name || sym.name.length === 0) { + continue; } - } - } - - // Extract exported symbols from .symtab - if (elf.body.symtabSymbols) { - for (const sym of elf.body.symtabSymbols) { + if (sym.section === 'SHN_UNDEF') { importedSymbols.push((await demangle(sym.name)) || sym.name); } else { - exportedSymbols.push((await demangle(sym.name)) || sym.name); + // 只提取 GLOBAL FUNC 类型的导出符号（与 nm -D | grep " T " 一致） + // T 表示全局函数（GLOBAL FUNC），t 表示局部函数（LOCAL FUNC） + if (sym.type === 'FUNC' && sym.binding === 'GLOBAL') { + exportedSymbols.push((await demangle(sym.name)) || sym.name); + } } } } + // 注意：不提取 .symtab 中的符号，因为 nm -D 只显示 .dynsym + // 提取依赖库 const dependencies = this.extractDependencies(elf); diff --git a/tools/static_analyzer/src/core/elf/elfy.ts b/tools/static_analyzer/src/core/elf/elfy.ts index 8f823001..b976c49c 100644 --- a/tools/static_analyzer/src/core/elf/elfy.ts +++ b/tools/static_analyzer/src/core/elf/elfy.ts @@ -1368,6 +1368,7 @@ class Parser { if (!symtabSection) { throw new Error('No .symtab section found in ELF file'); } + logger.warn('Found .dynsym section, may no strip this section'); // Find the associated string table (.strtab) const strtabSection = elf.body.sections.find((sec) => sec.name === '.strtab' && sec.type === 'strtab'); @@ -1448,7 +1449,7 @@ export function parseELF(buf: Buffer): ELF { elfHeader.body.symtabSymbols = symtabSymbols; } catch (error) { // Handle cases where .symtab or .strtab might not exist - logger.warn('Symbol parsing for .symtab failed:', (error as Error).message); + logger.debug('Symbol parsing for .symtab failed:', (error as Error).message); } // Optionally parse PLT diff --git a/tools/static_analyzer/src/core/perf/perf_database.ts b/tools/static_analyzer/src/core/perf/perf_database.ts index e829dc45..5158e1ed 100644 --- a/tools/static_analyzer/src/core/perf/perf_database.ts +++ b/tools/static_analyzer/src/core/perf/perf_database.ts @@ -14,6 +14,7 @@ */ import fs from 'fs'; +import path from 'path'; import sqlJs from 'sql.js'; import type { PerfSymbolDetailData, TestStep } from './perf_analyzer_base'; import { PerfEvent } from './perf_analyzer_base'; @@ -55,7 +56,7 @@ export class PerfDatabase { symbol_events INTEGER, symbol_total_events INTEGER, sub_category_name TEXT, - component_category INTEGER, + component_category INTEGER ); CREATE TABLE IF NOT EXISTS perf_test_step ( @@ -78,6 +79,11 @@ export class PerfDatabase { db = new SQL.Database(); await this.initializeDatabase(db); const data = db.export(); + // 确保目录存在 + const dir = path.dirname(this.dbpath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } fs.writeFileSync(this.dbpath, Buffer.from(data)); } diff --git a/tools/static_analyzer/src/core/techstack/report/base-formatter.ts b/tools/static_analyzer/src/core/techstack/report/base-formatter.ts new file mode 100644 index 00000000..77108995 --- /dev/null +++ b/tools/static_analyzer/src/core/techstack/report/base-formatter.ts @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Hap } from '../../hap/hap_parser'; +import type { FormatOptions, FormatResult } from './index'; + +/** + * 抽象格式化器基类 + */ +export abstract class BaseFormatter { + protected options: FormatOptions; + + constructor(options: FormatOptions) { + this.options = options; + } + + /** + * 格式化分析结果 + * @param result 分析结果 + * @returns 格式化结果 + */ + abstract format(result: Hap): Promise; + + /** + * 获取输出文件扩展名 + */ + abstract getFileExtension(): string; + + /** + * 验证格式化选项 + */ + protected validateOptions(): void { + if (!this.options.outputPath) { + throw new Error('Output path is required'); + } + } + + /** + * 格式化文件大小 + */ + protected formatFileSize(bytes: number): string { + if (bytes === 0) {return '0 B';} + + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; + } + + /** + * 格式化日期时间 + */ + protected formatDateTime(date: Date): string { + return date.toLocaleString('zh-CN', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit' + }); + } + + /** + * 计算百分比 + */ + protected calculatePercentage(value: number, total: number): string { + if (total === 0) {return '0%';} + return ((value / total) * 100).toFixed(1) + '%'; + } + + /** + * 获取文件类型统计 + */ + protected getFileTypeStats(result: Hap): Array<{type: string, count: number, percentage: string, barWidth: number}> { + // 简化实现，只返回技术栈统计 + const stats: Array<{type: string, count: number, percentage: string, barWidth: number}> = []; + const total = result.techStackDetections.length; + + if (total === 0) {return stats;} + + // 按技术栈分组统计 + const techStackCount = new Map(); + for (const detection of result.techStackDetections) { + const count = techStackCount.get(detection.techStack) ?? 0; + techStackCount.set(detection.techStack, count + 1); + } + + for (const [techStack, count] of techStackCount) { + const percentage = ((count / total) * 100).toFixed(1); + const barWidth = Math.max(5, (count / total) * 100); + + stats.push({ + type: techStack, + count: count, + percentage: `${percentage}%`, + barWidth: barWidth + }); + } + + return stats.sort((a, b) => b.count - a.count); + } + + /** + * 获取技术栈统计 + */ + protected getTechStackStats(result: Hap): Array<{techStack: string, count: number, percentage: string}> { + const techStackCount = new Map(); + + result.techStackDetections.forEach(techStackDetection => { + // 过滤掉 Unknown 技术栈 + if (techStackDetection.techStack !== 'Unknown') { + techStackCount.set(techStackDetection.techStack, (techStackCount.get(techStackDetection.techStack) ?? 0) + 1); + } + }); + + const stats: Array<{techStack: string, count: number, percentage: string}> = []; + const total = result.techStackDetections.length; + + for (const [techStack, count] of techStackCount) { + stats.push({ + techStack, + count, + percentage: this.calculatePercentage(count, total) + }); + } + + return stats.sort((a, b) => b.count - a.count); + } +} + diff --git a/tools/static_analyzer/src/core/techstack/report/excel-report.ts b/tools/static_analyzer/src/core/techstack/report/excel-report.ts index 2e0c2f82..12c5c846 100644 --- a/tools/static_analyzer/src/core/techstack/report/excel-report.ts +++ b/tools/static_analyzer/src/core/techstack/report/excel-report.ts @@ -19,7 +19,7 @@ import os from 'os'; import writeXlsxFile from 'write-excel-file/node'; import type { SheetData } from 'write-excel-file'; import type { FormatResult } from './index'; -import { BaseFormatter } from './index'; +import { BaseFormatter } from './base-formatter'; import type { Hap } from '../../hap/hap_parser'; /** @@ -183,7 +183,8 @@ export class ExcelFormatter extends BaseFormatter { ...sortedMetadataColumns.map(column => ({ value: column, fontWeight: 'bold' as const - })) + })), + { value: 'soExports数量', fontWeight: 'bold' as const } ]; sheetData.push(headerRow); @@ -231,6 +232,11 @@ export class ExcelFormatter extends BaseFormatter { row.push({ value: cellValue, type: String }); } + // 添加 soExports 符号数量统计 + const soExports = detection.metadata.soExports; + const soExportsCount = Array.isArray(soExports) ? soExports.length : 0; + row.push({ value: soExportsCount.toString(), type: String }); + sheetData.push(row); }); diff --git a/tools/static_analyzer/src/core/techstack/report/html-report.ts b/tools/static_analyzer/src/core/techstack/report/html-report.ts index 01f04ac3..d24f7b9c 100644 --- a/tools/static_analyzer/src/core/techstack/report/html-report.ts +++ b/tools/static_analyzer/src/core/techstack/report/html-report.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import Handlebars from 'handlebars'; import type { FormatResult } from './index'; -import { BaseFormatter } from './index'; +import { BaseFormatter } from './base-formatter'; import type { Hap, TechStackDetection } from '../../hap/hap_parser'; /** @@ -293,6 +293,10 @@ export class HtmlFormatter extends BaseFormatter { // 构建数据项 for (const detection of detections) { + // 统计 soExports 符号数量 + const soExports = detection.metadata.soExports; + const soExportsCount = Array.isArray(soExports) ? soExports.length : 0; + const item: Record = { fileName: detection.file, filePath: `${detection.folder}/${detection.file}`, @@ -303,7 +307,8 @@ export class HtmlFormatter extends BaseFormatter { sourceHapPath: detection.sourceHapPath ?? '', sourceBundleName: detection.sourceBundleName ?? '', sourceVersionCode: detection.sourceVersionCode?.toString() ?? '', - sourceVersionName: detection.sourceVersionName ?? '' + sourceVersionName: detection.sourceVersionName ?? '', + soExportsCount }; // 添加 metadata 字段（已过滤掉 soDependencies、soExports 和 soImports） @@ -325,7 +330,7 @@ export class HtmlFormatter extends BaseFormatter { return { items, - metadataColumns: sortedMetadataColumns + metadataColumns: ['soExportsCount', ...sortedMetadataColumns] }; } diff --git a/tools/static_analyzer/src/core/techstack/report/index.ts b/tools/static_analyzer/src/core/techstack/report/index.ts index f332579f..ce776f10 100644 --- a/tools/static_analyzer/src/core/techstack/report/index.ts +++ b/tools/static_analyzer/src/core/techstack/report/index.ts @@ -13,7 +13,11 @@ * limitations under the License. */ -import type { Hap } from '../../hap/hap_parser'; +// 导入具体的格式化器实现 +import { JsonFormatter } from './json-report'; +import { HtmlFormatter } from './html-report'; +import { ExcelFormatter } from './excel-report'; +import type { BaseFormatter } from './base-formatter'; /** * 支持的输出格式 @@ -56,131 +60,8 @@ export interface FormatResult { error?: string; } -/** - * 抽象格式化器基类 - */ -export abstract class BaseFormatter { - protected options: FormatOptions; - - constructor(options: FormatOptions) { - this.options = options; - } - - /** - * 格式化分析结果 - * @param result 分析结果 - * @returns 格式化结果 - */ - abstract format(result: Hap): Promise; - - /** - * 获取输出文件扩展名 - */ - abstract getFileExtension(): string; - - /** - * 验证格式化选项 - */ - protected validateOptions(): void { - if (!this.options.outputPath) { - throw new Error('Output path is required'); - } - } - - /** - * 格式化文件大小 - */ - protected formatFileSize(bytes: number): string { - if (bytes === 0) {return '0 B';} - - const k = 1024; - const sizes = ['B', 'KB', 'MB', 'GB']; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - - return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; - } - - /** - * 格式化日期时间 - */ - protected formatDateTime(date: Date): string { - return date.toLocaleString('zh-CN', { - year: 'numeric', - month: '2-digit', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - second: '2-digit' - }); - } - - /** - * 计算百分比 - */ - protected calculatePercentage(value: number, total: number): string { - if (total === 0) {return '0%';} - return ((value / total) * 100).toFixed(1) + '%'; - } - - /** - * 获取文件类型统计 - */ - protected getFileTypeStats(result: Hap): Array<{type: string, count: number, percentage: string, barWidth: number}> { - // 简化实现，只返回技术栈统计 - const stats: Array<{type: string, count: number, percentage: string, barWidth: number}> = []; - const total = result.techStackDetections.length; - - if (total === 0) {return stats;} - - // 按技术栈分组统计 - const techStackCount = new Map(); - for (const detection of result.techStackDetections) { - const count = techStackCount.get(detection.techStack) ?? 0; - techStackCount.set(detection.techStack, count + 1); - } - - for (const [techStack, count] of techStackCount) { - const percentage = ((count / total) * 100).toFixed(1); - const barWidth = Math.max(5, (count / total) * 100); - - stats.push({ - type: techStack, - count: count, - percentage: `${percentage}%`, - barWidth: barWidth - }); - } - - return stats.sort((a, b) => b.count - a.count); - } - - /** - * 获取技术栈统计 - */ - protected getTechStackStats(result: Hap): Array<{techStack: string, count: number, percentage: string}> { - const techStackCount = new Map(); - - result.techStackDetections.forEach(techStackDetection => { - // 过滤掉 Unknown 技术栈 - if (techStackDetection.techStack !== 'Unknown') { - techStackCount.set(techStackDetection.techStack, (techStackCount.get(techStackDetection.techStack) ?? 0) + 1); - } - }); - - const stats: Array<{techStack: string, count: number, percentage: string}> = []; - const total = result.techStackDetections.length; - - for (const [techStack, count] of techStackCount) { - stats.push({ - techStack, - count, - percentage: this.calculatePercentage(count, total) - }); - } - - return stats.sort((a, b) => b.count - a.count); - } -} +// 导出 BaseFormatter（从单独的文件导入以避免循环依赖） +export { BaseFormatter } from './base-formatter'; /** * 格式化器工厂 @@ -219,9 +100,5 @@ export class FormatterFactory { } } -// 导入具体的格式化器实现 -import { JsonFormatter } from './json-report'; -import { HtmlFormatter } from './html-report'; -import { ExcelFormatter } from './excel-report'; - +// 重新导出格式化器实现 export { JsonFormatter, HtmlFormatter, ExcelFormatter }; diff --git a/tools/static_analyzer/src/core/techstack/report/json-report.ts b/tools/static_analyzer/src/core/techstack/report/json-report.ts index de6fac54..5693914c 100644 --- a/tools/static_analyzer/src/core/techstack/report/json-report.ts +++ b/tools/static_analyzer/src/core/techstack/report/json-report.ts @@ -16,7 +16,7 @@ import fs from 'fs'; import path from 'path'; import type { FormatResult } from './index'; -import { BaseFormatter } from './index'; +import { BaseFormatter } from './base-formatter'; import type { Hap, TechStackDetection } from '../../hap/hap_parser'; /** diff --git a/tools/static_analyzer/src/core/techstack/rules/custom-extractors.ts b/tools/static_analyzer/src/core/techstack/rules/custom-extractors.ts index bb4655a8..f89ce72a 100644 --- a/tools/static_analyzer/src/core/techstack/rules/custom-extractors.ts +++ b/tools/static_analyzer/src/core/techstack/rules/custom-extractors.ts @@ -5,6 +5,7 @@ import type { CustomExtractor, FileInfo, MetadataPattern } from '../types'; import fs from 'fs'; import path from 'path'; +import { createHash } from 'crypto'; import { ElfAnalyzer } from '../../elf/elf_analyzer'; /** @@ -410,7 +411,7 @@ async function extractLastModified(fileInfo: FileInfo, _pattern?: MetadataPatter } /** - * SO文件ELF信息缓存（基于文件路径） + * SO文件ELF信息缓存（基于文件hash值） * 避免同一个文件被多次解析 */ const soElfInfoCache = new Map; exports: Array; imports: Array } | null>>(); @@ -429,8 +430,8 @@ async function getSoElfInfo(fileInfo: FileInfo): Promise<{ dependencies: Array { @@ -9,4 +10,47 @@ describe('ElfAnalyzerTest', () => { expect(strings[0]).eq('2317265cd53ac33c71f76ecc22cc06d97309b6db'); expect(strings[1]).eq('274d4c13286757c366db077bdd73bf8d44bba863'); }); + + it('getElfInfo - should extract exports, imports and dependencies', async () => { + const elfAnalyzer = ElfAnalyzer.getInstance(); + const elfInfo = await elfAnalyzer.getElfInfo(path.join(__dirname, '../resources/libflutter.so')); + + // 验证返回的数据结构 + expect(elfInfo).toBeDefined(); + // nm -D libflutter.so | grep " T " | wc -l 结果为 60 + // 只提取 .dynsym 中的 GLOBAL FUNC 类型符号（与 nm -D | grep " T " 一致） + expect(elfInfo.exports.length).toBe(60); + expect(elfInfo.imports).toBeInstanceOf(Array); + expect(elfInfo.dependencies).toBeInstanceOf(Array); + + // 验证去重功能：exports 和 imports 应该没有重复项 + const exportsSet = new Set(elfInfo.exports); + const importsSet = new Set(elfInfo.imports); + expect(exportsSet.size).toBe(60); + expect(importsSet.size).toBe(elfInfo.imports.length); + }); + + it('getElfInfo - should handle invalid ELF file gracefully', async () => { + const elfAnalyzer = ElfAnalyzer.getInstance(); + + // 创建一个临时的无效 ELF 文件 + const invalidPath = path.join(__dirname, '../resources/invalid_elf.so'); + const invalidContent = Buffer.from('This is not a valid ELF file'); + fs.writeFileSync(invalidPath, invalidContent); + + try { + // 对于无效的 ELF 文件，应该返回空数组 + const elfInfo = await elfAnalyzer.getElfInfo(invalidPath); + + expect(elfInfo).toBeDefined(); + expect(elfInfo.exports).toEqual([]); + expect(elfInfo.imports).toEqual([]); + expect(elfInfo.dependencies).toEqual([]); + } finally { + // 清理临时文件 + if (fs.existsSync(invalidPath)) { + fs.unlinkSync(invalidPath); + } + } + }); }); \ No newline at end of file diff --git a/tools/symbol_recovery/core/analyzers/perf_analyzer.py b/tools/symbol_recovery/core/analyzers/perf_analyzer.py index 2f0f8b2b..116f2e22 100644 --- a/tools/symbol_recovery/core/analyzers/perf_analyzer.py +++ b/tools/symbol_recovery/core/analyzers/perf_analyzer.py @@ -45,13 +45,19 @@ def find_trace_streamer(self): else: # Linux possible_names = ['trace_streamer_linux', 'trace_streamer'] - # 检查常见位置 + # 基于当前文件所在目录检查常见位置 + base_dir = Path(__file__).resolve() + # 打包: symbol-recovery/_internal + # 开发: symbol_recovery + symbol_recovery_root = base_dir.parents[2] + search_paths = [ - Path.cwd(), - Path.cwd() / 'dist' / 'tools' / 'trace_streamer_binary', - Path.cwd() / '..' / 'trace_streamer_binary', - Path.cwd() / '..' / '..' / 'dist' / 'tools' / 'trace_streamer_binary', - Path.cwd() / 'tools' / 'trace_streamer_binary', + # 打包：.../tools/trace_streamer_binary + symbol_recovery_root / '..' / '..' / '..' / 'tools' / 'trace_streamer_binary', + # 开发：.../dist/tools/trace_streamer_binary + symbol_recovery_root / '..' / '..' / 'dist' / 'tools' / 'trace_streamer_binary', + # 可选：插件本目录下放一份 + symbol_recovery_root / '..' / 'trace_streamer_binary', ] for search_path in search_paths: diff --git a/tools/symbol_recovery/core/llm/initializer.py b/tools/symbol_recovery/core/llm/initializer.py index a922105c..af9b5401 100644 --- a/tools/symbol_recovery/core/llm/initializer.py +++ b/tools/symbol_recovery/core/llm/initializer.py @@ -60,9 +60,7 @@ def init_llm_analyzer( output_dir=output_dir, open_source_lib=open_source_lib, ) - logger.info( - f'Using batch LLM analyzer: model={model_name}, batch_size={batch_size}' - ) + logger.info(f'Using batch LLM analyzer: model={model_name}, batch_size={batch_size}') if save_prompts: logger.info('Prompt saving enabled') return analyzer, True, True diff --git a/web/src/components/common/AppNavigation.vue b/web/src/components/common/AppNavigation.vue index ec051d6b..6be25eb0 100644 --- a/web/src/components/common/AppNavigation.vue +++ b/web/src/components/common/AppNavigation.vue @@ -89,7 +89,7 @@ Memory分析 - + @@ -169,9 +169,20 @@ 故障树对比 + + + + + + UI对比 + + @@ -227,7 +238,8 @@ import { Trophy, Upload, Share, - Coin + Coin, + Picture } from '@element-plus/icons-vue'; const props = defineProps<{ @@ -263,19 +275,16 @@ watch(() => props.currentPage, (newPage) => { // 获取步骤数据 const jsonDataStore = useJsonDataStore(); -const perfData = jsonDataStore.perfData; // 存储每个步骤是否有Memory数据的缓存（使用对象而不是Map以保持响应性） const memoryDataCache = ref>({}); +const uiAnimateDataCache = ref>({}); const testSteps = computed(() => { - if (!perfData?.steps) return []; - return perfData.steps.map((step, index) => ({ - id: index + 1, + if (!jsonDataStore?.steps) return []; + return jsonDataStore.steps.map((step) => ({ + id: step.step_id, step_name: step.step_name, - count: step.count, - round: step.round, - perf_data_path: step.perf_data_path, })); }); @@ -288,19 +297,36 @@ const checkMemoryData = async (stepId: number): Promise => { try { const dbApi = getDbApi(); + + // 检查memory_meminfo表 + console.log(`[AppNavigation] Checking meminfo data for step ${stepId}...`); + try { + const meminfoData = await dbApi.queryMemoryMeminfo(stepId); + console.log(`[AppNavigation] Meminfo data for step ${stepId}:`, meminfoData?.length || 0, 'rows'); + if (meminfoData && meminfoData.length > 0) { + console.log(`[AppNavigation] Step ${stepId} has meminfo data, showing Memory menu`); + memoryDataCache.value[stepId] = true; + return; + } + } catch (meminfoError) { + console.log(`[AppNavigation] Error querying meminfo (table may not exist):`, meminfoError); + } + // 检查memory_results表中是否有该步骤的数据 + console.log(`[AppNavigation] Checking memory_results for step ${stepId}...`); const results = await dbApi.queryMemoryResults(stepId); - - // 如果memory_results表中有数据，还需要检查memory_records表中是否有实际的记录 + console.log(`[AppNavigation] Memory results for step ${stepId}:`, results?.length || 0, 'rows'); if (results && results.length > 0) { // 查询该步骤的memory_records数量 const records = await dbApi.queryOverviewTimeline(stepId); + console.log(`[AppNavigation] Memory records for step ${stepId}:`, records?.length || 0, 'rows'); memoryDataCache.value[stepId] = records && records.length > 0; } else { memoryDataCache.value[stepId] = false; } + console.log(`[AppNavigation] Final memory data status for step ${stepId}:`, memoryDataCache.value[stepId]); } catch (error) { - console.warn(`Failed to check memory data for step ${stepId}:`, error); + console.warn(`[AppNavigation] Failed to check memory data for step ${stepId}:`, error); memoryDataCache.value[stepId] = false; } }; @@ -310,10 +336,23 @@ const getHasMemoryData = (stepId: number): boolean => { return memoryDataCache.value[stepId] || false; }; +// 检查UI动画数据 +const checkUIAnimateData = (stepId: number) => { + const uiAnimateData = jsonDataStore.uiAnimateData; + const stepKey = `step${stepId}`; + uiAnimateDataCache.value[stepId] = !!(uiAnimateData && uiAnimateData[stepKey]); +}; + +// 获取步骤是否有UI动画数据 +const getHasUIAnimateData = (stepId: number): boolean => { + return uiAnimateDataCache.value[stepId] || false; +}; + // 当步骤改变时，检查Memory数据 watch(() => testSteps.value, (newSteps) => { newSteps.forEach(step => { void checkMemoryData(step.id); + checkUIAnimateData(step.id); }); }, { immediate: true }); @@ -322,6 +361,15 @@ const hasCompareData = computed(() => { return jsonDataStore.comparePerfData && jsonDataStore.comparePerfData.steps.length > 0; }); +// 检查指定步骤是否有UI对比数据 +const hasUICompareData = (stepId: number): boolean => { + if (!jsonDataStore.uiCompareData) { + return false; + } + const stepKey = `step${stepId}`; + return !!jsonDataStore.uiCompareData[stepKey]; +}; + // 从 jsonDataStore 读取版本号，提供响应式和默认值 const version = computed(() => { return 'v' + jsonDataStore.version || 'v1.0.0'; diff --git a/web/src/components/common/charts/BarChart.vue b/web/src/components/common/charts/BarChart.vue index 52a33d74..665dec56 100644 --- a/web/src/components/common/charts/BarChart.vue +++ b/web/src/components/common/charts/BarChart.vue @@ -21,8 +21,9 @@ function processData(data: PerfData|null) { return { xData: [], yData: [], fullNames: [] } } const { steps } = data; + // step_name 已移到 jsonDataStore.steps 中，这里使用默认值 const stepCounts = steps.map((step, index) => ({ - stepName: step.step_name, + stepName: `步骤${index + 1}`, count: step.count, originalIndex: index // 保存原始索引 })); @@ -40,7 +41,7 @@ function processData(data: PerfData|null) { const processedData = processData(props.chartData); const { xData, yData, fullNames } = processedData; -const title = props.chartData?.steps[0].data[0].eventType==0?'cycles':'instructions'; +const title = props.chartData?.steps?.[0]?.data?.[0]?.eventType == 0 ? 'cycles' : 'instructions'; const option = { title: { diff --git a/web/src/components/common/charts/LineChart.vue b/web/src/components/common/charts/LineChart.vue index 680ebba0..c4b7665d 100644 --- a/web/src/components/common/charts/LineChart.vue +++ b/web/src/components/common/charts/LineChart.vue @@ -33,7 +33,8 @@ const processData = (data: PerfData | null, seriesType: string) => { } const { steps } = data; - const xData = steps.map((step) => step.step_name); + // step_name 已移到 jsonDataStore.steps 中，这里使用默认值 + const xData = steps.map((step, index) => `步骤${index + 1}`); const categoryMap = new Map(); // 初始化categoryMap，为每个x轴位置创建一个数组 @@ -92,7 +93,7 @@ const updateChart = () => { if (!myChart || !chartRef.value) return; const { xData, legendData, series } = processData(props.chartData, props.seriesType); - const title = props.chartData?.steps[0].data[0].eventType == 0 ? 'cycles' : 'instructions'; + const title = props.chartData?.steps[0]?.data[0]?.eventType == 0 ? 'cycles' : 'instructions'; const option = { title: { text: '步骤负载：' + title, diff --git a/web/src/components/compare/CompareOverview.vue b/web/src/components/compare/CompareOverview.vue index b34692a0..3126c013 100644 --- a/web/src/components/compare/CompareOverview.vue +++ b/web/src/components/compare/CompareOverview.vue @@ -144,10 +144,10 @@

按步骤对比分析

-
+
步骤{{ step.id }} - {{ formatNumber(step.count) }} + {{ formatNumber(getStepPerfData(index).count) }}
{{ step.step_name }}
@@ -226,16 +226,27 @@ const comparePerformanceData = computed(() => } ); -// 测试步骤数据 -const testSteps = computed(() => - perfData.steps.map((step, index) => ({ - id: index + 1, +// testSteps 只从 jsonDataStore.steps 生成，与 perfData 解耦 +const testSteps = computed(() => { + const steps = jsonDataStore.steps || []; + return steps.map((step, index) => ({ + id: step.step_id ?? (index + 1), step_name: step.step_name, + })); +}); + +// 获取步骤的性能数据（从 perfData 中通过索引获取） +const getStepPerfData = (stepIndex: number) => { + if (!perfData || !perfData.steps || stepIndex < 0 || stepIndex >= perfData.steps.length) { + return { count: 0, round: 0, perf_data_path: '' }; + } + const step = perfData.steps[stepIndex]; + return { count: step.count, round: step.round, perf_data_path: step.perf_data_path, - })) -); + }; +}; // 图表数据 const pieChartTitle = computed(() => diff --git a/web/src/components/compare/CompareStepLoad.vue b/web/src/components/compare/CompareStepLoad.vue index 550dea47..aee00aec 100644 --- a/web/src/components/compare/CompareStepLoad.vue +++ b/web/src/components/compare/CompareStepLoad.vue @@ -240,25 +240,31 @@ function mergeJSONData(baselineData: PerfData, compareData: PerfData, cur_step_i const mergedData: PerfData = { steps: [] }; // 合并基线数据 + // step_id 和 step_name 已移到 jsonDataStore.steps 中 const baselineStep = { - step_name: "基线", - step_id: 0, count: baselineData.steps.reduce((sum, step) => sum + step.count, 0), round: baselineData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: baselineData.steps.map(s => s.perf_data_path).join(";"), - data: baselineData.steps.filter(step => step.step_id === cur_step_id).flatMap(step => + data: baselineData.steps.filter((step, index) => { + // step_id 已移到 jsonDataStore.steps 中，这里使用索引 + 1 作为 step_id + const stepId = index + 1; + return stepId === cur_step_id; + }).flatMap(step => step.data.map(item => ({ ...item })) ) }; // 合并对比数据 + // step_id 和 step_name 已移到 jsonDataStore.steps 中 const comparisonStep = { - step_name: "迭代", - step_id: 1, count: compareData.steps.reduce((sum, step) => sum + step.count, 0), round: compareData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: compareData.steps.map(s => s.perf_data_path).join(";"), - data: compareData.steps.filter(step => step.step_id === cur_step_id).flatMap(step => + data: compareData.steps.filter((step, index) => { + // step_id 已移到 jsonDataStore.steps 中，这里使用索引 + 1 作为 step_id + const stepId = index + 1; + return stepId === cur_step_id; + }).flatMap(step => step.data.map(item => ({ ...item })) ) }; diff --git a/web/src/components/compare/DetailDataCompare.vue b/web/src/components/compare/DetailDataCompare.vue index 13e0db68..a65e66e1 100644 --- a/web/src/components/compare/DetailDataCompare.vue +++ b/web/src/components/compare/DetailDataCompare.vue @@ -202,16 +202,14 @@ watch(() => props.step, (newStep) => { currentStepIndex.value = newStep || 0; }, { immediate: true }); -// 测试步骤数据 -const testSteps = computed(() => - perfData.steps.map((step, index) => ({ - id: index + 1, +// testSteps 只从 jsonDataStore.steps 生成，与 perfData 解耦 +const testSteps = computed(() => { + const steps = jsonDataStore.steps || []; + return steps.map((step, index) => ({ + id: step.step_id ?? (index + 1), step_name: step.step_name, - count: step.count, - round: step.round, - perf_data_path: step.perf_data_path, - })) -); + })); +}); // 合并数据 const mergedProcessPerformanceData = computed(() => diff --git a/web/src/components/compare/NewDataAnalysis.vue b/web/src/components/compare/NewDataAnalysis.vue index 47002a55..0985df7c 100644 --- a/web/src/components/compare/NewDataAnalysis.vue +++ b/web/src/components/compare/NewDataAnalysis.vue @@ -190,16 +190,14 @@ watch(() => props.step, (newStep) => { currentStepIndex.value = newStep || 0; }, { immediate: true }); -// 测试步骤数据 -const testSteps = computed(() => - perfData.steps.map((step, index) => ({ - id: index + 1, +// testSteps 只从 jsonDataStore.steps 生成，与 perfData 解耦 +const testSteps = computed(() => { + const steps = jsonDataStore.steps || []; + return steps.map((step, index) => ({ + id: step.step_id ?? (index + 1), step_name: step.step_name, - count: step.count, - round: step.round, - perf_data_path: step.perf_data_path, - })) -); + })); +}); // 合并数据 const mergedFilePerformanceData = computed(() => diff --git a/web/src/components/compare/PerfCompare.vue b/web/src/components/compare/PerfCompare.vue index 408b4ea0..f6bf9e31 100644 --- a/web/src/components/compare/PerfCompare.vue +++ b/web/src/components/compare/PerfCompare.vue @@ -142,7 +142,7 @@ ]" @click="handleStepClick(0)">
STEP 0 - {{ getTotalTestStepsCount(testSteps) }} + {{ getTotalTestStepsCount() }}
全部步骤
@@ -155,7 +155,7 @@ v-for="(step, index) in testSteps" :key="index" :class="[ ]" @click="handleStepClick(step.id)">
STEP {{ step.id }} - {{ formatDuration(step.count) }} + {{ formatDuration(getStepPerfData(index).count) }}
{{ step.step_name }}
@@ -500,9 +500,8 @@ function mergeJSONData(baselineData: PerfData, compareData: PerfData, cur_step_i }; // 合并第一个 JSON 的所有 steps 为"基线" + // step_id 和 step_name 已移到 jsonDataStore.steps 中，这里使用索引 + 1 作为 step_id const baselineStep = cur_step_id === 0 ? { - step_name: "基线", - step_id: 0, count: baselineData.steps.reduce((sum, step) => sum + step.count, 0), round: baselineData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: baselineData.steps.map(s => s.perf_data_path).join(";"), @@ -512,12 +511,13 @@ function mergeJSONData(baselineData: PerfData, compareData: PerfData, cur_step_i })) ) } : { - step_name: "基线", - step_id: 0, count: baselineData.steps.reduce((sum, step) => sum + step.count, 0), round: baselineData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: baselineData.steps.map(s => s.perf_data_path).join(";"), - data: baselineData.steps.filter(step => step.step_id === cur_step_id).flatMap(step => + data: baselineData.steps.filter((step, index) => { + const stepId = index + 1; + return stepId === cur_step_id; + }).flatMap(step => step.data.map(item => ({ ...item })) @@ -525,9 +525,8 @@ function mergeJSONData(baselineData: PerfData, compareData: PerfData, cur_step_i }; // 合并第二个 JSON 的所有 steps 为"迭代" + // step_id 和 step_name 已移到 jsonDataStore.steps 中，这里使用索引 + 1 作为 step_id const comparisonStep = cur_step_id === 0 ? { - step_name: "迭代", - step_id: 1, count: compareData.steps.reduce((sum, step) => sum + step.count, 0), round: compareData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: compareData.steps.map(s => s.perf_data_path).join(";"), @@ -537,12 +536,13 @@ function mergeJSONData(baselineData: PerfData, compareData: PerfData, cur_step_i })) ) } : { - step_name: "迭代", - step_id: 1, count: compareData.steps.reduce((sum, step) => sum + step.count, 0), round: compareData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: compareData.steps.map(s => s.perf_data_path).join(";"), - data: compareData.steps.filter(step => step.step_id === cur_step_id).flatMap(step => + data: compareData.steps.filter((step, index) => { + const stepId = index + 1; + return stepId === cur_step_id; + }).flatMap(step => step.data.map(item => ({ ...item })) @@ -567,28 +567,36 @@ sceneDiff.value = compareSceneLineChartData.value && compareSceneLineChartData.v -//测试步骤导航卡片 -const testSteps = ref( - perfData.steps.map((step, index) => ({ - //从1开始 - id: index + 1, +// testSteps 只从 jsonDataStore.steps 生成，与 perfData 解耦 +const testSteps = computed(() => { + const steps = jsonDataStore.steps || []; + return steps.map((step, index) => ({ + id: step.step_id ?? (index + 1), step_name: step.step_name, + })); +}); + +// 获取步骤的性能数据（从 perfData 中通过索引获取） +const getStepPerfData = (stepIndex: number) => { + if (!perfData || !perfData.steps || stepIndex < 0 || stepIndex >= perfData.steps.length) { + return { count: 0, round: 0, perf_data_path: '' }; + } + const step = perfData.steps[stepIndex]; + return { count: step.count, round: step.round, perf_data_path: step.perf_data_path, - })) -); - -// 全部步骤负载总数 -const getTotalTestStepsCount = (testSteps: {count: number}[]) => { - let total = 0; + }; +}; - testSteps.forEach((step) => { - total += step.count; - }); - return total; +// 获取所有步骤的总计数 +const getTotalTestStepsCount = () => { + if (!perfData || !perfData.steps) return 0; + return perfData.steps.reduce((total, step) => total + step.count, 0); }; +// 全部步骤负载总数 + // 格式化持续时间的方法 const formatDuration = (milliseconds: number) => { return `指令数：${milliseconds}`; diff --git a/web/src/components/compare/StepLoadCompare.vue b/web/src/components/compare/StepLoadCompare.vue index aa72a505..c4a4f088 100644 --- a/web/src/components/compare/StepLoadCompare.vue +++ b/web/src/components/compare/StepLoadCompare.vue @@ -26,7 +26,7 @@ @click="handleStepClick(0)">
STEP 0 - {{ getTotalTestStepsCount(testSteps) }} + {{ getTotalTestStepsCount() }}
全部步骤
@@ -37,7 +37,7 @@ @click="handleStepClick(step.id)">
STEP {{ step.id }} - {{ formatDuration(step.count) }} + {{ formatDuration(getStepPerfData(index).count) }}
{{ step.step_name }}
@@ -207,16 +207,33 @@ const hasCompareData = computed(() => { // 当前步骤索引 const currentStepIndex = ref(0); -// 测试步骤数据 -const testSteps = computed(() => - perfData.steps.map((step, index) => ({ - id: index + 1, +// testSteps 只从 jsonDataStore.steps 生成，与 perfData 解耦 +const testSteps = computed(() => { + const steps = jsonDataStore.steps || []; + return steps.map((step, index) => ({ + id: step.step_id ?? (index + 1), step_name: step.step_name, + })); +}); + +// 获取步骤的性能数据（从 perfData 中通过索引获取） +const getStepPerfData = (stepIndex: number) => { + if (!perfData || !perfData.steps || stepIndex < 0 || stepIndex >= perfData.steps.length) { + return { count: 0, round: 0, perf_data_path: '' }; + } + const step = perfData.steps[stepIndex]; + return { count: step.count, round: step.round, perf_data_path: step.perf_data_path, - })) -); + }; +}; + +// 获取所有步骤的总计数 +const getTotalTestStepsCount = () => { + if (!perfData || !perfData.steps) return 0; + return perfData.steps.reduce((total, step) => total + step.count, 0); +}; // 图表数据 const pieChartTitle = computed(() => @@ -239,10 +256,6 @@ watch(currentStepIndex, (stepId) => { stepDiff.value = compareLineChartData.value && compareLineChartData.value.steps.length >= 2 ? calculateCategoryCountDifference(compareLineChartData.value) : []; }); -// 工具函数 -const getTotalTestStepsCount = (testSteps: {count: number}[]) => { - return testSteps.reduce((total, step) => total + step.count, 0); -}; const formatDuration = (milliseconds: number) => { return `指令数：${milliseconds}`; @@ -261,9 +274,8 @@ function mergeJSONData(baselineData: PerfData, compareData: PerfData, cur_step_i const mergedData: PerfData = { steps: [] }; // 合并基线数据 + // step_id 和 step_name 已移到 jsonDataStore.steps 中，这里使用索引 + 1 作为 step_id const baselineStep = cur_step_id === 0 ? { - step_name: "基线", - step_id: 0, count: baselineData.steps.reduce((sum, step) => sum + step.count, 0), round: baselineData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: baselineData.steps.map(s => s.perf_data_path).join(";"), @@ -271,20 +283,20 @@ function mergeJSONData(baselineData: PerfData, compareData: PerfData, cur_step_i step.data.map(item => ({ ...item })) ) } : { - step_name: "基线", - step_id: 0, count: baselineData.steps.reduce((sum, step) => sum + step.count, 0), round: baselineData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: baselineData.steps.map(s => s.perf_data_path).join(";"), - data: baselineData.steps.filter(step => step.step_id === cur_step_id).flatMap(step => + data: baselineData.steps.filter((step, index) => { + const stepId = index + 1; + return stepId === cur_step_id; + }).flatMap(step => step.data.map(item => ({ ...item })) ) }; // 合并对比数据 + // step_id 和 step_name 已移到 jsonDataStore.steps 中，这里使用索引 + 1 作为 step_id const comparisonStep = cur_step_id === 0 ? { - step_name: "迭代", - step_id: 1, count: compareData.steps.reduce((sum, step) => sum + step.count, 0), round: compareData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: compareData.steps.map(s => s.perf_data_path).join(";"), @@ -292,12 +304,13 @@ function mergeJSONData(baselineData: PerfData, compareData: PerfData, cur_step_i step.data.map(item => ({ ...item })) ) } : { - step_name: "迭代", - step_id: 1, count: compareData.steps.reduce((sum, step) => sum + step.count, 0), round: compareData.steps.reduce((sum, step) => sum + step.round, 0), perf_data_path: compareData.steps.map(s => s.perf_data_path).join(";"), - data: compareData.steps.filter(step => step.step_id === cur_step_id).flatMap(step => + data: compareData.steps.filter((step, index) => { + const stepId = index + 1; + return stepId === cur_step_id; + }).flatMap(step => step.data.map(item => ({ ...item })) ) }; diff --git a/web/src/components/compare/Top10DataCompare.vue b/web/src/components/compare/Top10DataCompare.vue index 21634146..550d3a20 100644 --- a/web/src/components/compare/Top10DataCompare.vue +++ b/web/src/components/compare/Top10DataCompare.vue @@ -214,16 +214,14 @@ watch(() => props.step, (newStep) => { currentStepIndex.value = newStep || 0; }, { immediate: true }); -// 测试步骤数据 -const testSteps = computed(() => - perfData.steps.map((step, index) => ({ - id: index + 1, +// testSteps 只从 jsonDataStore.steps 生成，与 perfData 解耦 +const testSteps = computed(() => { + const steps = jsonDataStore.steps || []; + return steps.map((step, index) => ({ + id: step.step_id ?? (index + 1), step_name: step.step_name, - count: step.count, - round: step.round, - perf_data_path: step.perf_data_path, - })) -); + })); +}); // 基线和对比数据 - 使用空的对比数据来获取单独的数据 const emptyPerfData = { steps: [] }; diff --git a/web/src/components/compare/UICompare.vue b/web/src/components/compare/UICompare.vue new file mode 100644 index 00000000..39e84021 --- /dev/null +++ b/web/src/components/compare/UICompare.vue @@ -0,0 +1,314 @@ + + + + + + diff --git a/web/src/components/index.ts b/web/src/components/index.ts index 676f5727..6f71abd0 100644 --- a/web/src/components/index.ts +++ b/web/src/components/index.ts @@ -63,6 +63,7 @@ export { default as NewDataAnalysis } from './compare/NewDataAnalysis.vue'; export { default as Top10DataCompare } from './compare/Top10DataCompare.vue'; export { default as FaultTreeCompare } from './compare/FaultTreeCompare.vue'; export { default as SceneLoadCompare } from './compare/SceneLoadCompare.vue'; +export { default as UICompare } from './compare/UICompare.vue'; // ==================== 多版本趋势 ==================== export { default as PerfMulti } from './multi-version/PerfMulti.vue'; diff --git a/web/src/components/multi-version/PerfMulti.vue b/web/src/components/multi-version/PerfMulti.vue index 7d767d9f..9d2a1e6c 100644 --- a/web/src/components/multi-version/PerfMulti.vue +++ b/web/src/components/multi-version/PerfMulti.vue @@ -394,9 +394,10 @@ const processJsonData = async (jsonData: JSONData, fileName: string): Promise ({ - stepId: step.step_id, - stepName: step.step_name, + const steps = jsonData.steps || []; + stepData = jsonData.perf.steps.map((step, index) => ({ + stepId: steps[index]?.step_id ?? (index + 1), + stepName: steps[index]?.step_name ?? `步骤${index + 1}`, count: step.count })); } catch (error) { diff --git a/web/src/components/single-analysis/overview/PerfLoadOverview.vue b/web/src/components/single-analysis/overview/PerfLoadOverview.vue index 8b81a606..b1f995f5 100644 --- a/web/src/components/single-analysis/overview/PerfLoadOverview.vue +++ b/web/src/components/single-analysis/overview/PerfLoadOverview.vue @@ -37,7 +37,7 @@
⚡

总指令数

-
{{ formatNumber(getTotalTestStepsCount(testSteps)) }}
+
{{ formatNumber(getTotalTestStepsCount()) }}

所有步骤的指令数总和

@@ -46,7 +46,7 @@
🔋

总功耗

-
{{ formatEnergy(getTotalTestStepsCount(testSteps)) }}
+
{{ formatEnergy(getTotalTestStepsCount()) }}

预估的总功耗消耗

@@ -55,7 +55,7 @@
📈

平均负载

-
{{ formatNumber(Math.round(getTotalTestStepsCount(testSteps) / testSteps.length)) }}
+
{{ formatNumber(Math.round(getTotalTestStepsCount() / testSteps.length)) }}

每个步骤的平均指令数

--> @@ -120,7 +120,7 @@ @@ -136,7 +136,7 @@ + + + + + diff --git a/web/src/components/single-analysis/step/frame/FrameAnalysis.vue b/web/src/components/single-analysis/step/frame/FrameAnalysis.vue index 530554e8..dd870fa9 100644 --- a/web/src/components/single-analysis/step/frame/FrameAnalysis.vue +++ b/web/src/components/single-analysis/step/frame/FrameAnalysis.vue @@ -254,34 +254,22 @@ width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="#409EFF" + + +
- - 调用栈信息 -
-
-
-
-
- 调用栈 {{ idx + 1 }} -
-
- 负载: {{ chain.load_percentage.toFixed(2) }}% -
-
-
-
- -
[{{ call.depth }}] {{ call.path }} - {{ call.symbol }}
-
-
-
+ + 调用栈火焰图
+

未找到调用栈信息

@@ -564,6 +552,7 @@ class="filter-item" :class="{ active: fileUsageFilter === 'unused' }" import { ref, onMounted, computed, watch } from 'vue'; import * as echarts from 'echarts'; import { useJsonDataStore, getDefaultEmptyFrameData, getDefaultColdStartData, safeProcessColdStartData, getDefaultGcThreadStepData, getDefaultFrameStepData, getDefaultEmptyFrameStepData, getDefaultComponentResuStepData, getDefaultColdStartStepData, safeProcessGcThreadData, getDefaultGcThreadData, getDefaultVSyncAnomalyData, getDefaultVSyncAnomalyStepData, safeProcessVSyncAnomalyData } from '../../../../stores/jsonDataStore.ts'; +import EmptyFrameFlameGraph from './EmptyFrameFlameGraph.vue'; // 获取存储实例 const jsonDataStore = useJsonDataStore(); @@ -903,61 +892,39 @@ const initCharts = () => { // 收集空刷帧点 const emptyFramePoints = []; - // 主线程空刷帧 - emptyFrameData.value.top_frames.main_thread_empty_frames.forEach(frame => { - const timeMs = frame.ts / 1000000; // 转换为毫秒 - if (timeMs !== 0) { - allTimestamps.push(timeMs); - emptyFramePoints.push({ - time: timeMs, - frame: frame, - type: 'main_thread' - }); - } - - }); - // 后台线程空刷帧 - emptyFrameData.value.top_frames.background_thread.forEach(frame => { + // 统一处理所有空刷帧（不再区分主线程和后台线程） + emptyFrameData.value.top_frames.forEach(frame => { const timeMs = frame.ts / 1000000; // 转换为毫秒 if (timeMs !== 0) { allTimestamps.push(timeMs); + // 根据 is_main_thread 字段判断类型 + const frameType = frame.is_main_thread === 1 ? 'main_thread' : 'background_thread'; emptyFramePoints.push({ time: timeMs, frame: frame, - type: 'background_thread' + type: frameType }); } - }); // 收集空刷负载（用于柱状图） const frameLoadData = []; const loadData = []; - // 主线程空刷帧 - emptyFrameData.value.top_frames.main_thread_empty_frames.forEach(frame => { + // 统一处理所有空刷帧（不再区分主线程和后台线程） + emptyFrameData.value.top_frames.forEach(frame => { const timeMs = frame.ts / 1000000; // 转换为毫秒 + // 根据 is_main_thread 字段判断类型 + const frameType = frame.is_main_thread === 1 ? 'main_thread' : 'background_thread'; frameLoadData.push({ time: timeMs, load: frame.frame_load, frame: frame, // 添加完整的帧对象 - type: 'main_thread' + type: frameType }); loadData.push(frame.frame_load); }); - // 后台线程空刷帧 - //emptyFrameData.value.top_frames.background_thread.forEach(frame => { - // const timeMs = frame.ts / 1000000; // 转换为毫秒 - // frameLoadData.push({ - // time: timeMs, - // load: frame.frame_load, - // frame: frame, // 添加完整的帧对象 - // type: 'background_thread' - // }); - // loadData.push(frame.frame_load); - //}); - // 收集VSync异常数据 @@ -1946,9 +1913,12 @@ body { } .detail-content { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 25px; + display: block; + margin-bottom: 25px; +} + +.flamegraph-section { + margin-top: 25px; } diff --git a/web/src/components/single-analysis/step/load/PerfLoadAnalysis.vue b/web/src/components/single-analysis/step/load/PerfLoadAnalysis.vue index 16752a1d..17b2750b 100644 --- a/web/src/components/single-analysis/step/load/PerfLoadAnalysis.vue +++ b/web/src/components/single-analysis/step/load/PerfLoadAnalysis.vue @@ -66,8 +66,8 @@ ]" @click="handleStepClick(0)">
STEP 0 - {{ getTotalTestStepsCount(testSteps) }} - {{ formatEnergy(getTotalTestStepsCount(testSteps)) }} + {{ getTotalTestStepsCount() }} + {{ formatEnergy(getTotalTestStepsCount()) }}
全部步骤
@@ -80,8 +80,8 @@ ]" @click="handleStepClick(step.id)">
STEP {{ step.id }} - {{ formatDuration(step.count) }} - {{ formatEnergy(step.count) }} + {{ formatDuration(getStepPerfData(index).count) }} + {{ formatEnergy(getStepPerfData(index).count) }}
{{ step.step_name }}