diff --git a/copilot_agent_client/mcp_agent_loop.py b/copilot_agent_client/mcp_agent_loop.py index d9f93b4..d40af6c 100644 --- a/copilot_agent_client/mcp_agent_loop.py +++ b/copilot_agent_client/mcp_agent_loop.py @@ -5,13 +5,14 @@ sys.path.append(".") import json +import subprocess from PIL import Image import io from tools.image_tools import make_b64_url -from copilot_front_end.mobile_action_helper import capture_screenshot, dectect_screen_on, press_home_key +from copilot_front_end.mobile_action_helper import capture_screenshot, dectect_screen_on, press_home_key, _get_adb_command from copilot_front_end.mobile_action_helper import init_device, open_screen from copilot_front_end.pu_frontend_executor import act_on_device, uiTars_to_frontend_action @@ -27,6 +28,26 @@ import threading + +def _check_yadb_installed(device_id, print_command=False): + """ + 只检查 yadb 是否安装，不按 HOME 键。 + """ + adb_command = _get_adb_command(device_id) + + command = f"{adb_command} shell md5sum /data/local/tmp/yadb" + if print_command: + print(f"Executing command: {command}") + + result = subprocess.run(command, shell=True, capture_output=True, text=True) + if "29a0cd3b3adea92350dd5a25594593df" not in result.stdout: + command = f"{adb_command} push yadb /data/local/tmp" + print(f"YADB is not installed on the device. Installing now...") + if print_command: + print(f"Executing command: {command}") + subprocess.run(command, shell=True, capture_output=True, text=True) + + def auto_reply(current_image_url, task, info_action, model_provider, model_name): """ Reply with information action. @@ -193,11 +214,15 @@ def gui_agent_loop( # init device for the first time open_screen(device_id) - init_device(device_id) - - # if reset_environment, press home key before starting the task + + # 只在需要重置环境时调用 init_device（它内部会按 HOME 键） + # 否则只检查 yadb 是否安装，不按 HOME 键 if reset_environment and session_id is None and task is not None: - press_home_key(device_id, print_command=True) + init_device(device_id, reset_environment=True) # init_device 内部会按 HOME 键 + else: + # 只检查 yadb，不按 HOME 键 + init_device(device_id, reset_environment=False) + # task, task_type = task, rollout_config['task_type'] task_type = agent_loop_config['task_type'] @@ -265,6 +290,9 @@ def gui_agent_loop( global_step_idx = 0 # restart the steps from 0, even continuing an existing session for step_idx in range(max_steps): + # 打印步骤开始分隔符 + step_label = f" Step {step_idx+1} start " + print(f"\n{step_label:-^50}") if not dectect_screen_on(device_id): print("Screen is off, turn on the screen first") @@ -371,7 +399,17 @@ def gui_agent_loop( history_actions.append(action) - print(f"Step {step_idx+1}/{max_steps} done.\nAction Type: {action['action_type']}, cot: {action.get('cot', '')}\nSession ID: {session_id}\n") + # 清理 cot 中的 标签用于打印 + cot_display = action.get('cot', '') + if cot_display: + import re + cot_display = re.sub(r'<\s*/?THINK\s*>', '', cot_display, flags=re.IGNORECASE).strip() + + print(f"Action: {action['action_type']}") + if cot_display: + print(f"cot: {cot_display}") + step_end_label = f" Step {step_idx+1} end " + print(f"{step_end_label:-^50}") # print(f"local:{step_idx+1}/global:{global_step_idx}/{max_steps} done. Action: {action}") @@ -446,5 +484,3 @@ def gui_agent_loop( # print(f"Task {task} done in {len(history_actions)} steps. Session ID: {session_id}") return return_log - - diff --git a/copilot_agent_client/pu_client.py b/copilot_agent_client/pu_client.py index b4c4ec0..a97cbba 100644 --- a/copilot_agent_client/pu_client.py +++ b/copilot_agent_client/pu_client.py @@ -5,23 +5,19 @@ sys.path.append(".") import json - from PIL import Image import io +from collections import OrderedDict from tools.image_tools import draw_points, make_b64_url - -from copilot_front_end.mobile_action_helper import capture_screenshot, dectect_screen_on, press_home_key - +from copilot_front_end.mobile_action_helper import capture_screenshot, dectect_screen_on from copilot_front_end.mobile_action_helper import init_device, open_screen from copilot_front_end.pu_frontend_executor import act_on_device, uiTars_to_frontend_action - from megfile import smart_remove - import time - from tools.ask_llm_v2 import ask_llm_anything + def reply_info_action(current_image_url, task, info_action, model_provider, model_name): """ Reply with information action. @@ -32,22 +28,18 @@ def reply_info_action(current_image_url, task, info_action, model_provider, mode "content": [ { "type": "text", - "text": f"""# 角色 + "text": f"""# 角色 你将扮演一个正在使用GUI Agent完成任务的用户。 - # 任务 阅读下方提供的所有背景信息，针对[Agent的澄清问题]，生成一个提供关键信息的、简短直接的回答。 - # 背景信息 - **任务目标:** {task} - **agent 问的问题:** {json.dumps(info_action, ensure_ascii=False)} - # 输出要求 - 你的回答必须极其简短和明确。 - 你的回答应直接命中问题的核心，解决Agent的疑惑。 - 不要进行任何额外的解释、对话或使用礼貌用语。 - 只输出回答本身，不要添加任何引号或其他修饰。 - 以下是当前页面内容: """, }, @@ -64,7 +56,6 @@ def reply_info_action(current_image_url, task, info_action, model_provider, mode ] } ] - response = ask_llm_anything( model_provider=model_provider, model_name=model_name, @@ -76,43 +67,55 @@ def reply_info_action(current_image_url, task, info_action, model_provider, mode "frequency_penalty": 0.0, } ) - if "" in response: response = response.split("")[-1].strip() - return response -# delay after act on device -# rollout config -# device info -# def evaluate_task_on_device(agent_server, device_info, task, frontend_action_converter, ask_action_function_func, max_steps = 40, delay_after_capture = 2): -def evaluate_task_on_device(agent_server, device_info, task, rollout_config, extra_info = {}, reflush_app=True, auto_reply = False, reset_environment=True): + +def evaluate_task_on_device(agent_server, device_info, task, rollout_config, extra_info={}, reflush_app=True, auto_reply=False, reset_environment=False): """ Evaluate a task on a device using the provided frontend action converter and action function. - """ + # ===== 新增：本地美化函数 ===== + def _pretty_format_action(act): + if not isinstance(act, (dict, OrderedDict)): + return str(act) + lines = [] + # 不再打印 标签 + if 'cot' in act and act['cot']: + cot_clean = str(act['cot']).replace('\n', ' ').replace('\r', ' ') + # 移除 和 标签 + import re + cot_clean = re.sub(r'<\s*/?THINK\s*>', '', cot_clean, flags=re.IGNORECASE).strip() + if cot_clean: + lines.append(f"cot: {cot_clean}") + # Define field order for readability + field_order = ['explain', 'action', 'value', 'point', 'point1', 'point2', 'return', 'summary'] + for key in field_order: + if key in act: + val = act[key] + if isinstance(val, list): + val_str = ",".join(str(x) for x in val) + else: + val_str = str(val).replace('\n', ' ').strip() + lines.append(f"{key}: {val_str}") + return "\n".join(lines) + # ============================ + # init device for the first time device_id = device_info['device_id'] open_screen(device_id) - init_device(device_id) - - - if reset_environment: - press_home_key(device_id, print_command=True) + init_device(device_id, reset_environment=reset_environment) task, task_type = task, rollout_config['task_type'] - session_id = agent_server.get_session({ "task": task, "task_type": task_type, "model_config": rollout_config['model_config'], "extra_info": extra_info - }) - print(f"Session ID: {session_id}") - return_log = { "session_id": session_id, "device_info": device_info, @@ -120,25 +123,24 @@ def evaluate_task_on_device(agent_server, device_info, task, rollout_config, ext "rollout_config": rollout_config, "extra_info": extra_info } - device_id, device_wm_size = device_info['device_id'], device_info['device_wm_size'] - max_steps = rollout_config.get('max_steps', 40) delay_after_capture = rollout_config.get('delay_after_capture', 2) - history_actions = [] for step_idx in range(max_steps): - + # 打印步骤开始分隔符 + step_label = f" Step {step_idx+1} start " + print(f"\n{step_label:-^50}") + if not dectect_screen_on(device_id): print("Screen is off, turn on the screen first") break image_path = capture_screenshot(device_id, "tmp_screenshot", print_command=False) - image_b64_url = make_b64_url(image_path, resize_config=rollout_config['model_config'].get("resize_config", None)) smart_remove(image_path) - + payload = { "session_id": session_id, "observation": { @@ -150,43 +152,42 @@ def evaluate_task_on_device(agent_server, device_info, task, rollout_config, ext }, } } - if history_actions[-1]['action_type'] == "INFO" if len(history_actions) > 0 else False: - info_action = history_actions[-1] + if history_actions and history_actions[-1]['action_type'] == "INFO": + info_action = history_actions[-1] if auto_reply: print(f"AUTO REPLY INFO FROM MODEL!") - reply_info = reply_info_action(image_b64_url, task, info_action, model_provider=rollout_config['model_config']['model_provider'], model_name=rollout_config['model_config']['model_name']) + reply_info = reply_info_action( + image_b64_url, task, info_action, + model_provider=rollout_config['model_config']['model_provider'], + model_name=rollout_config['model_config']['model_name'] + ) print(f"info: {reply_info}") - else: print(f"EN: Agent asks: {history_actions[-1]['value']} Please Reply: ") print(f"ZH: Agent 问你: {history_actions[-1]['value']} 回复一下：") - reply_info = input("Your reply:") - print(f"Replied info action: {reply_info}") - payload['observation']['query'] = reply_info - action = agent_server.automate_step(payload)['action'] - - #TODO: to replace with the new function action = uiTars_to_frontend_action(action) - - act_on_device(action, device_id, device_wm_size, print_command=True, reflush_app=reflush_app) - + act_on_device(action, device_id, device_wm_size, print_command=True, reflush_app=reflush_app, print_executing_command=True) history_actions.append(action) - - print(f"Step {step_idx+1}/{max_steps} done. Action: {action}") + # ===== 替换原始打印：使用美观格式 ===== + print(f"Action: {action['action_type']}") + print(_pretty_format_action(action)) + step_end_label = f" Step {step_idx+1} end " + print(f"{step_end_label:-^50}") + # =================================== if action['action_type'].upper() in ['COMPLETE', "ABORT"]: stop_reason = action['action_type'].upper() break time.sleep(delay_after_capture) - + if action['action_type'] in ['COMPLETE', "ABORT"]: stop_reason = action['action_type'] elif step_idx == max_steps - 1: @@ -194,13 +195,7 @@ def evaluate_task_on_device(agent_server, device_info, task, rollout_config, ext else: stop_reason = "MANUAL_STOP" - # return_log['session_id'] = session_id return_log['stop_reason'] = stop_reason - return_log['stop_steps'] = step_idx + 1 - - print(f"Task {task} done in {len(history_actions)} steps. Session ID: {session_id}") - + print(f"\ndone in {len(history_actions)} steps.\nSession ID: {session_id}") return return_log - - diff --git a/copilot_front_end/mobile_action_helper.py b/copilot_front_end/mobile_action_helper.py index a8efae6..584b835 100644 --- a/copilot_front_end/mobile_action_helper.py +++ b/copilot_front_end/mobile_action_helper.py @@ -75,7 +75,7 @@ def press_home_key(device_id, print_command = False): subprocess.run(command, shell=True, capture_output=True, text=True) -def init_device(device_id, print_command = False): +def init_device(device_id, reset_environment=False, print_command = False): """ Initialize the device by checking if yadb is installed. """ @@ -99,9 +99,12 @@ def init_device(device_id, print_command = False): subprocess.run(command, shell=True, capture_output=True, text=True) else: - print("yadb is already installed on the device.") + # print("yadb is already installed on the device.") + pass + + if reset_environment: + press_home_key(device_id, print_command=print_command) - # press_home_key(device_id, print_command=print_command) def init_all_devices(): """ @@ -109,9 +112,10 @@ def init_all_devices(): """ devices = list_devices() for device_id in tqdm(devices): - init_device(device_id) + init_device(device_id, reset_environment=True) print(f"Initialized device: {device_id}") + def dectect_screen_on(device_id, print_command = False): """ Detect whether the screen is on for the specified device. @@ -636,9 +640,10 @@ def __init__(self, device_id = None): self.device_id = device_id self.wm_size = get_device_wm_size(self.device_id) if self.device_id is not None: - init_device(self.device_id, print_command=True) + init_device(self.device_id, reset_environment=True, print_command=True) # _open_screen(self.device_id, print_command=True) + pass def set_device_id(self, device_id): @@ -730,4 +735,4 @@ def step_interaction(self, action, capture_duration = 0.5, image_full_path = Non print(get_device_wm_size("bc23727a")) open_screen(None, print_command=True) - pass \ No newline at end of file + pass diff --git a/copilot_front_end/package_map.py b/copilot_front_end/package_map.py index 99401aa..d648cc8 100644 --- a/copilot_front_end/package_map.py +++ b/copilot_front_end/package_map.py @@ -209,6 +209,7 @@ "osmAnd": "net.osmand", "给到": "com.guanaitong", "百词斩": "com.jiongji.andriod.card", + "象棋": "com.tencent.qqgame.xq", } diff --git a/copilot_tools/scrcpy/linux/icon.png b/copilot_tools/scrcpy/linux/icon.png new file mode 100644 index 0000000..b96a1af Binary files /dev/null and b/copilot_tools/scrcpy/linux/icon.png differ diff --git a/copilot_tools/scrcpy/linux/scrcpy b/copilot_tools/scrcpy/linux/scrcpy new file mode 100644 index 0000000..48c4b75 Binary files /dev/null and b/copilot_tools/scrcpy/linux/scrcpy differ diff --git a/copilot_tools/scrcpy/linux/scrcpy-server b/copilot_tools/scrcpy/linux/scrcpy-server new file mode 100644 index 0000000..b36f14d Binary files /dev/null and b/copilot_tools/scrcpy/linux/scrcpy-server differ diff --git a/copilot_tools/scrcpy/linux/scrcpy.1 b/copilot_tools/scrcpy/linux/scrcpy.1 new file mode 100644 index 0000000..d72fda1 --- /dev/null +++ b/copilot_tools/scrcpy/linux/scrcpy.1 @@ -0,0 +1,860 @@ +.TH "scrcpy" "1" +.SH NAME +scrcpy \- Display and control your Android device + + +.SH SYNOPSIS +.B scrcpy +.RI [ options ] + + +.SH DESCRIPTION +.B scrcpy +provides display and control of Android devices connected on USB (or over TCP/IP). It does not require any root access. + + +.SH OPTIONS + +.TP +.B \-\-always\-on\-top +Make scrcpy window always on top (above other windows). + +.TP +.BI "\-\-angle " degrees +Rotate the video content by a custom angle, in degrees (clockwise). + +.TP +.BI "\-\-audio\-bit\-rate " value +Encode the audio at the given bit rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). + +Default is 128K (128000). + +.TP +.BI "\-\-audio\-buffer " ms +Configure the audio buffering delay (in milliseconds). + +Lower values decrease the latency, but increase the likelihood of buffer underrun (causing audio glitches). + +Default is 50. + +.TP +.BI "\-\-audio\-codec " name +Select an audio codec (opus, aac, flac or raw). + +Default is opus. + +.TP +.BI "\-\-audio\-codec\-options " key\fR[:\fItype\fR]=\fIvalue\fR[,...] +Set a list of comma-separated key:type=value options for the device audio encoder. + +The possible values for 'type' are 'int' (default), 'long', 'float' and 'string'. + +The list of possible codec options is available in the Android documentation: + + + +.TP +.B \-\-audio\-dup +Duplicate audio (capture and keep playing on the device). + +This feature is only available with --audio-source=playback. + +.TP +.BI "\-\-audio\-encoder " name +Use a specific MediaCodec audio encoder (depending on the codec provided by \fB\-\-audio\-codec\fR). + +The available encoders can be listed by \fB\-\-list\-encoders\fR. + +.TP +.BI "\-\-audio\-source " source +Select the audio source. Possible values are: + + - "output": forwards the whole audio output, and disables playback on the device. + - "playback": captures the audio playback (Android apps can opt-out, so the whole output is not necessarily captured). + - "mic": captures the microphone. + - "mic-unprocessed": captures the microphone unprocessed (raw) sound. + - "mic-camcorder": captures the microphone tuned for video recording, with the same orientation as the camera if available. + - "mic-voice-recognition": captures the microphone tuned for voice recognition. + - "mic-voice-communication": captures the microphone tuned for voice communications (it will for instance take advantage of echo cancellation or automatic gain control if available). + - "voice-call": captures voice call. + - "voice-call-uplink": captures voice call uplink only. + - "voice-call-downlink": captures voice call downlink only. + - "voice-performance": captures audio meant to be processed for live performance (karaoke), includes both the microphone and the device playback. + +Default is output. + +.TP +.BI "\-\-audio\-output\-buffer " ms +Configure the size of the SDL audio output buffer (in milliseconds). + +If you get "robotic" audio playback, you should test with a higher value (10). Do not change this setting otherwise. + +Default is 5. + +.TP +.BI "\-b, \-\-video\-bit\-rate " value +Encode the video at the given bit rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). + +Default is 8M (8000000). + +.TP +.BI "\-\-camera\-ar " ar +Select the camera size by its aspect ratio (+/- 10%). + +Possible values are "sensor" (use the camera sensor aspect ratio), "\fInum\fR:\fIden\fR" (e.g. "4:3") and "\fIvalue\fR" (e.g. "1.6"). + +.TP +.BI "\-\-camera\-facing " facing +Select the device camera by its facing direction. + +Possible values are "front", "back" and "external". + +.TP +.BI "\-\-camera\-fps " fps +Specify the camera capture frame rate. + +If not specified, Android's default frame rate (30 fps) is used. + +.TP +.B \-\-camera\-high\-speed +Enable high-speed camera capture mode. + +This mode is restricted to specific resolutions and frame rates, listed by \fB\-\-list\-camera\-sizes\fR. + +.TP +.BI "\-\-camera\-id " id +Specify the device camera id to mirror. + +The available camera ids can be listed by \fB\-\-list\-cameras\fR. + +.TP +.BI "\-\-camera\-size " width\fRx\fIheight +Specify an explicit camera capture size. + +.TP +.BI "\-\-capture\-orientation " value +Possible values are 0, 90, 180, 270, flip0, flip90, flip180 and flip270, possibly prefixed by '@'. + +The number represents the clockwise rotation in degrees; the "flip" keyword applies a horizontal flip before the rotation. + +If a leading '@' is passed (@90) for display capture, then the rotation is locked, and is relative to the natural device orientation. + +If '@' is passed alone, then the rotation is locked to the initial device orientation. + +Default is 0. + +.TP +.BI "\-\-crop " width\fR:\fIheight\fR:\fIx\fR:\fIy +Crop the device screen on the server. + +The values are expressed in the device natural orientation (typically, portrait for a phone, landscape for a tablet). + +.TP +.B \-d, \-\-select\-usb +Use USB device (if there is exactly one, like adb -d). + +Also see \fB\-e\fR (\fB\-\-select\-tcpip\fR). + +.TP +.BI "\-\-disable\-screensaver" +Disable screensaver while scrcpy is running. + +.TP +.BI "\-\-display\-id " id +Specify the device display id to mirror. + +The available display ids can be listed by \fB\-\-list\-displays\fR. + +Default is 0. + +.TP +.BI "\-\-display\-ime\-policy " value +Set the policy for selecting where the IME should be displayed. + +Possible values are "local", "fallback" and "hide": + + - "local" means that the IME should appear on the local display. + - "fallback" means that the IME should appear on a fallback display (the default display). + - "hide" means that the IME should be hidden. + +By default, the IME policy is left unchanged. + + +.TP +.BI "\-\-display\-orientation " value +Set the initial display orientation. + +Possible values are 0, 90, 180, 270, flip0, flip90, flip180 and flip270. The number represents the clockwise rotation in degrees; the "flip" keyword applies a horizontal flip before the rotation. + +Default is 0. + +.TP +.B \-e, \-\-select\-tcpip +Use TCP/IP device (if there is exactly one, like adb -e). + +Also see \fB\-d\fR (\fB\-\-select\-usb\fR). + +.TP +.B \-f, \-\-fullscreen +Start in fullscreen. + +.TP +.B \-\-force\-adb\-forward +Do not attempt to use "adb reverse" to connect to the device. + +.TP +.B \-G +Same as \fB\-\-gamepad=uhid\fR, or \fB\-\-keyboard=aoa\fR if \fB\-\-otg\fR is set. + +.TP +.BI "\-\-gamepad " mode +Select how to send gamepad inputs to the device. + +Possible values are "disabled", "uhid" and "aoa": + + - "disabled" does not send gamepad inputs to the device. + - "uhid" simulates physical HID gamepads using the Linux HID kernel module on the device. + - "aoa" simulates physical HID gamepads using the AOAv2 protocol. It may only work over USB. + +Also see \fB\-\-keyboard\f and R\fB\-\-mouse\fR. +.TP +.B \-h, \-\-help +Print this help. + +.TP +.B \-K +Same as \fB\-\-keyboard=uhid\fR, or \fB\-\-keyboard=aoa\fR if \fB\-\-otg\fR is set. + +.TP +.BI "\-\-keyboard " mode +Select how to send keyboard inputs to the device. + +Possible values are "disabled", "sdk", "uhid" and "aoa": + + - "disabled" does not send keyboard inputs to the device. + - "sdk" uses the Android system API to deliver keyboard events to applications. + - "uhid" simulates a physical HID keyboard using the Linux HID kernel module on the device. + - "aoa" simulates a physical HID keyboard using the AOAv2 protocol. It may only work over USB. + +For "uhid" and "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly using the shortcut MOD+k (except in OTG mode), or by executing: + + adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS + +This option is only available when the HID keyboard is enabled (or a physical keyboard is connected). + +Also see \fB\-\-mouse\fR and \fB\-\-gamepad\fR. + +.TP +.B \-\-kill\-adb\-on\-close +Kill adb when scrcpy terminates. + +.TP +.B \-\-legacy\-paste +Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v). + +This is a workaround for some devices not behaving as expected when setting the device clipboard programmatically. + +.TP +.B \-\-list\-apps +List Android apps installed on the device. + +.TP +.B \-\-list\-camera\-sizes +List the valid camera capture sizes. + +.TP +.B \-\-list\-cameras +List cameras available on the device. + +.TP +.B \-\-list\-encoders +List video and audio encoders available on the device. + +.TP +.B \-\-list\-displays +List displays available on the device. + +.TP +.BI "\-m, \-\-max\-size " value +Limit both the width and height of the video to \fIvalue\fR. The other dimension is computed so that the device aspect\-ratio is preserved. + +Default is 0 (unlimited). + +.TP +.B \-M +Same as \fB\-\-mouse=uhid\fR, or \fB\-\-mouse=aoa\fR if \fB\-\-otg\fR is set. + +.TP +.BI "\-\-max\-fps " value +Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions). + +.TP +.BI "\-\-mouse " mode +Select how to send mouse inputs to the device. + +Possible values are "disabled", "sdk", "uhid" and "aoa": + + - "disabled" does not send mouse inputs to the device. + - "sdk" uses the Android system API to deliver mouse events to applications. + - "uhid" simulates a physical HID mouse using the Linux HID kernel module on the device. + - "aoa" simulates a physical mouse using the AOAv2 protocol. It may only work over USB. + +In "uhid" and "aoa" modes, the computer mouse is captured to control the device directly (relative mouse mode). + +LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer. + +Also see \fB\-\-keyboard\fR and \fB\-\-gamepad\fR. + +.TP +.BI "\-\-mouse\-bind " xxxx[:xxxx] +Configure bindings of secondary clicks. + +The argument must be one or two sequences (separated by ':') of exactly 4 characters, one for each secondary click (in order: right click, middle click, 4th click, 5th click). + +The first sequence defines the primary bindings, used when a mouse button is pressed alone. The second sequence defines the secondary bindings, used when a mouse button is pressed while the Shift key is held. + +If the second sequence of bindings is omitted, then it is the same as the first one. + +Each character must be one of the following: + + - '+': forward the click to the device + - '-': ignore the click + - 'b': trigger shortcut BACK (or turn screen on if off) + - 'h': trigger shortcut HOME + - 's': trigger shortcut APP_SWITCH + - 'n': trigger shortcut "expand notification panel" + +Default is 'bhsn:++++' for SDK mouse, and '++++:bhsn' for AOA and UHID. + + +.TP +.B \-n, \-\-no\-control +Disable device control (mirror the device in read\-only). + +.TP +.B \-N, \-\-no\-playback +Disable video and audio playback on the computer (equivalent to \fB\-\-no\-video\-playback \-\-no\-audio\-playback\fR). + +.TP +\fB\-\-new\-display\fR[=[\fIwidth\fRx\fIheight\fR][/\fIdpi\fR]] +Create a new display with the specified resolution and density. If not provided, they default to the main display dimensions and DPI. + +Examples: + + \-\-new\-display=1920x1080 + \-\-new\-display=1920x1080/420 + \-\-new\-display # main display size and density + \-\-new\-display=/240 # main display size and 240 dpi + +.TP +.B \-\-no\-audio +Disable audio forwarding. + +.TP +.B \-\-no\-audio\-playback +Disable audio playback on the computer. + +.TP +.B \-\-no\-cleanup +By default, scrcpy removes the server binary from the device and restores the device state (show touches, stay awake and power mode) on exit. + +This option disables this cleanup. + +.TP +.B \-\-no\-clipboard\-autosync +By default, scrcpy automatically synchronizes the computer clipboard to the device clipboard before injecting Ctrl+v, and the device clipboard to the computer clipboard whenever it changes. + +This option disables this automatic synchronization. + +.TP +.B \-\-no\-downsize\-on\-error +By default, on MediaCodec error, scrcpy automatically tries again with a lower definition. + +This option disables this behavior. + +.TP +.B \-\-no\-key\-repeat +Do not forward repeated key events when a key is held down. + +.TP +.B \-\-no\-mipmaps +If the renderer is OpenGL 3.0+ or OpenGL ES 2.0+, then mipmaps are automatically generated to improve downscaling quality. This option disables the generation of mipmaps. + +.TP +.B \-\-no\-mouse\-hover +Do not forward mouse hover (mouse motion without any clicks) events. + +.TP +.B \-\-no\-power\-on +Do not power on the device on start. + +.TP +.B \-\-no\-vd\-destroy\-content +Disable virtual display "destroy content on removal" flag. + +With this option, when the virtual display is closed, the running apps are moved to the main display rather than being destroyed. + +.TP +.B \-\-no\-vd\-system\-decorations +Disable virtual display system decorations flag. + +.TP +.B \-\-no\-video +Disable video forwarding. + +.TP +.B \-\-no\-video\-playback +Disable video playback on the computer. + +.TP +.B \-\-no\-window +Disable scrcpy window. Implies --no-video-playback. + +.TP +.BI "\-\-orientation " value +Same as --display-orientation=value --record-orientation=value. + +.TP +.B \-\-otg +Run in OTG mode: simulate physical keyboard and mouse, as if the computer keyboard and mouse were plugged directly to the device via an OTG cable. + +In this mode, adb (USB debugging) is not necessary, and mirroring is disabled. + +LAlt, LSuper or RSuper toggle the mouse capture mode, to give control of the mouse back to the computer. + +If any of \fB\-\-hid\-keyboard\fR or \fB\-\-hid\-mouse\fR is set, only enable keyboard or mouse respectively, otherwise enable both. + +It may only work over USB. + +See \fB\-\-keyboard\fR, \fB\-\-mouse\fR and \fB\-\-gamepad\fR. + +.TP +.BI "\-p, \-\-port " port\fR[:\fIport\fR] +Set the TCP port (range) used by the client to listen. + +Default is 27183:27199. + +.TP +\fB\-\-pause\-on\-exit\fR[=\fImode\fR] +Configure pause on exit. Possible values are "true" (always pause on exit), "false" (never pause on exit) and "if-error" (pause only if an error occurred). + +This is useful to prevent the terminal window from automatically closing, so that error messages can be read. + +Default is "false". + +Passing the option without argument is equivalent to passing "true". + +.TP +.B \-\-power\-off\-on\-close +Turn the device screen off when closing scrcpy. + +.TP +.B \-\-prefer\-text +Inject alpha characters and space as text events instead of key events. + +This avoids issues when combining multiple keys to enter special characters, +but breaks the expected behavior of alpha keys in games (typically WASD). + +.TP +.B "\-\-print\-fps +Start FPS counter, to print framerate logs to the console. It can be started or stopped at any time with MOD+i. + +.TP +.BI "\-\-push\-target " path +Set the target directory for pushing files to the device by drag & drop. It is passed as\-is to "adb push". + +Default is "/sdcard/Download/". + +.TP +.BI "\-r, \-\-record " file +Record screen to +.IR file . + +The format is determined by the +.B \-\-record\-format +option if set, or by the file extension. + +.TP +.B \-\-raw\-key\-events +Inject key events for all input keys, and ignore text events. + +.TP +.BI "\-\-record\-format " format +Force recording format (mp4, mkv, m4a, mka, opus, aac, flac or wav). + +.TP +.BI "\-\-record\-orientation " value +Set the record orientation. + +Possible values are 0, 90, 180 and 270. The number represents the clockwise rotation in degrees. + +Default is 0. + +.TP +.BI "\-\-render\-driver " name +Request SDL to use the given render driver (this is just a hint). + +Supported names are currently "direct3d", "opengl", "opengles2", "opengles", "metal" and "software". + + + +.TP +.B \-\-require\-audio +By default, scrcpy mirrors only the video if audio capture fails on the device. This option makes scrcpy fail if audio is enabled but does not work. + +.TP +.BI "\-s, \-\-serial " number +The device serial number. Mandatory only if several devices are connected to adb. + +.TP +.B \-S, \-\-turn\-screen\-off +Turn the device screen off immediately. + +.TP +.B "\-\-screen\-off\-timeout " seconds +Set the screen off timeout while scrcpy is running (restore the initial value on exit). + +.TP +.BI "\-\-shortcut\-mod " key\fR[+...]][,...] +Specify the modifiers to use for scrcpy shortcuts. Possible keys are "lctrl", "rctrl", "lalt", "ralt", "lsuper" and "rsuper". + +Several shortcut modifiers can be specified, separated by ','. + +For example, to use either LCtrl or LSuper for scrcpy shortcuts, pass "lctrl,lsuper". + +Default is "lalt,lsuper" (left-Alt or left-Super). + +.TP +.BI "\-\-start\-app " name +Start an Android app, by its exact package name. + +Add a '?' prefix to select an app whose name starts with the given name, case-insensitive (retrieving app names on the device may take some time): + + scrcpy --start-app=?firefox + +Add a '+' prefix to force-stop before starting the app: + + scrcpy --new-display --start-app=+org.mozilla.firefox + +Both prefixes can be used, in that order: + + scrcpy --start-app=+?firefox + +.TP +.B \-t, \-\-show\-touches +Enable "show touches" on start, restore the initial value on exit. + +It only shows physical touches (not clicks from scrcpy). + +.TP +.BI "\-\-tcpip\fR[=[+]\fIip\fR[:\fIport\fR]] +Configure and connect the device over TCP/IP. + +If a destination address is provided, then scrcpy connects to this address before starting. The device must listen on the given TCP port (default is 5555). + +If no destination address is provided, then scrcpy attempts to find the IP address and adb port of the current device (typically connected over USB), enables TCP/IP mode if necessary, then connects to this address before starting. + +Prefix the address with a '+' to force a reconnection. + +.TP +.BI "\-\-time\-limit " seconds +Set the maximum mirroring time, in seconds. + +.TP +.BI "\-\-tunnel\-host " ip +Set the IP address of the adb tunnel to reach the scrcpy server. This option automatically enables \fB\-\-force\-adb\-forward\fR. + +Default is localhost. + +.TP +.BI "\-\-tunnel\-port " port +Set the TCP port of the adb tunnel to reach the scrcpy server. This option automatically enables \fB\-\-force\-adb\-forward\fR. + +Default is 0 (not forced): the local port used for establishing the tunnel will be used. + +.TP +.B \-v, \-\-version +Print the version of scrcpy. + +.TP +.BI "\-V, \-\-verbosity " value +Set the log level ("verbose", "debug", "info", "warn" or "error"). + +Default is "info" for release builds, "debug" for debug builds. + +.TP +.BI "\-\-v4l2-sink " /dev/videoN +Output to v4l2loopback device. + +.TP +.BI "\-\-v4l2-buffer " ms +Add a buffering delay (in milliseconds) before pushing frames. This increases latency to compensate for jitter. + +This option is similar to \fB\-\-video\-buffer\fR, but specific to V4L2 sink. + +Default is 0 (no buffering). + +.TP +.BI "\-\-video\-buffer " ms +Add a buffering delay (in milliseconds) before displaying video frames. + +This increases latency to compensate for jitter. + +Default is 0 (no buffering). + +.TP +.BI "\-\-video\-codec " name +Select a video codec (h264, h265 or av1). + +Default is h264. + +.TP +.BI "\-\-video\-codec\-options " key\fR[:\fItype\fR]=\fIvalue\fR[,...] +Set a list of comma-separated key:type=value options for the device video encoder. + +The possible values for 'type' are 'int' (default), 'long', 'float' and 'string'. + +The list of possible codec options is available in the Android documentation: + + + +.TP +.BI "\-\-video\-encoder " name +Use a specific MediaCodec video encoder (depending on the codec provided by \fB\-\-video\-codec\fR). + +The available encoders can be listed by \fB\-\-list\-encoders\fR. + +.TP +.BI "\-\-video\-source " source +Select the video source (display or camera). + +Camera mirroring requires Android 12+. + +Default is display. + +.TP +.B \-w, \-\-stay-awake +Keep the device on while scrcpy is running, when the device is plugged in. + +.TP +.B \-\-window\-borderless +Disable window decorations (display borderless window). + +.TP +.BI "\-\-window\-title " text +Set a custom window title. + +.TP +.BI "\-\-window\-x " value +Set the initial window horizontal position. + +Default is "auto". + +.TP +.BI "\-\-window\-y " value +Set the initial window vertical position. + +Default is "auto". + +.TP +.BI "\-\-window\-width " value +Set the initial window width. + +Default is 0 (automatic). + +.TP +.BI "\-\-window\-height " value +Set the initial window height. + +Default is 0 (automatic). + +.SH EXIT STATUS +.B scrcpy +will exit with code 0 on normal program termination. If an initial +connection cannot be established, the exit code 1 will be returned. If the +device disconnects while a session is active, exit code 2 will be returned. + +.SH SHORTCUTS + +In the following list, MOD is the shortcut modifier. By default, it's (left) +Alt or (left) Super, but it can be configured by \fB\-\-shortcut\-mod\fR (see above). + +.TP +.B MOD+f +Switch fullscreen mode + +.TP +.B MOD+Left +Rotate display left + +.TP +.B MOD+Right +Rotate display right + +.TP +.B MOD+Shift+Left, MOD+Shift+Right +Flip display horizontally + +.TP +.B MOD+Shift+Up, MOD+Shift+Down +Flip display vertically + +.TP +.B MOD+z +Pause or re-pause display + +.TP +.B MOD+Shift+z +Unpause display + +.TP +.B MOD+Shift+r +Reset video capture/encoding + +.TP +.B MOD+g +Resize window to 1:1 (pixel\-perfect) + +.TP +.B MOD+w, Double\-click on black borders +Resize window to remove black borders + +.TP +.B MOD+h, Home, Middle\-click +Click on HOME + +.TP +.B MOD+b, MOD+Backspace, Right\-click (when screen is on) +Click on BACK + +.TP +.B MOD+s +Click on APP_SWITCH + +.TP +.B MOD+m +Click on MENU + +.TP +.B MOD+Up +Click on VOLUME_UP + +.TP +.B MOD+Down +Click on VOLUME_DOWN + +.TP +.B MOD+p +Click on POWER (turn screen on/off) + +.TP +.B Right\-click (when screen is off) +Turn screen on + +.TP +.B MOD+o +Turn device screen off (keep mirroring) + +.TP +.B MOD+Shift+o +Turn device screen on + +.TP +.B MOD+r +Rotate device screen + +.TP +.B MOD+n +Expand notification panel + +.TP +.B MOD+Shift+n +Collapse notification panel + +.TP +.B Mod+c +Copy to clipboard (inject COPY keycode, Android >= 7 only) + +.TP +.B Mod+x +Cut to clipboard (inject CUT keycode, Android >= 7 only) + +.TP +.B MOD+v +Copy computer clipboard to device, then paste (inject PASTE keycode, Android >= 7 only) + +.TP +.B MOD+Shift+v +Inject computer clipboard text as a sequence of key events + +.TP +.B MOD+k +Open keyboard settings on the device (for HID keyboard only) + +.TP +.B MOD+i +Enable/disable FPS counter (print frames/second in logs) + +.TP +.B Ctrl+click-and-move +Pinch-to-zoom and rotate from the center of the screen + +.TP +.B Shift+click-and-move +Tilt vertically (slide with 2 fingers) + +.TP +.B Ctrl+Shift+click-and-move +Tilt horizontally (slide with 2 fingers) + +.TP +.B Drag & drop APK file +Install APK from computer + +.TP +.B Drag & drop non-APK file +Push file to device (see \fB\-\-push\-target\fR) + + +.SH Environment variables + +.TP +.B ADB +Path to adb. + +.TP +.B ANDROID_SERIAL +Device serial to use if no selector (\fB-s\fR, \fB-d\fR, \fB-e\fR or \fB\-\-tcpip=\fIaddr\fR) is specified. + +.TP +.B SCRCPY_ICON_PATH +Path to the program icon. + +.TP +.B SCRCPY_SERVER_PATH +Path to the server binary. + + +.SH AUTHORS +.B scrcpy +is written by Romain Vimont. + +This manual page was written by +.MT mmyangfl@gmail.com +Yangfl +.ME +for the Debian Project (and may be used by others). + + +.SH "REPORTING BUGS" +Report bugs to . + +.SH COPYRIGHT +Copyright \(co 2018 Genymobile + +Copyright \(co 2018\-2025 Romain Vimont + +Licensed under the Apache License, Version 2.0. + +.SH WWW + diff --git a/copilot_tools/scrcpy/mac/icon.png b/copilot_tools/scrcpy/mac/icon.png new file mode 100644 index 0000000..b96a1af Binary files /dev/null and b/copilot_tools/scrcpy/mac/icon.png differ diff --git a/copilot_tools/scrcpy/mac/scrcpy b/copilot_tools/scrcpy/mac/scrcpy new file mode 100644 index 0000000..550b588 Binary files /dev/null and b/copilot_tools/scrcpy/mac/scrcpy differ diff --git a/copilot_tools/scrcpy/mac/scrcpy-server b/copilot_tools/scrcpy/mac/scrcpy-server new file mode 100644 index 0000000..b36f14d Binary files /dev/null and b/copilot_tools/scrcpy/mac/scrcpy-server differ diff --git a/copilot_tools/scrcpy/mac/scrcpy.1 b/copilot_tools/scrcpy/mac/scrcpy.1 new file mode 100644 index 0000000..d72fda1 --- /dev/null +++ b/copilot_tools/scrcpy/mac/scrcpy.1 @@ -0,0 +1,860 @@ +.TH "scrcpy" "1" +.SH NAME +scrcpy \- Display and control your Android device + + +.SH SYNOPSIS +.B scrcpy +.RI [ options ] + + +.SH DESCRIPTION +.B scrcpy +provides display and control of Android devices connected on USB (or over TCP/IP). It does not require any root access. + + +.SH OPTIONS + +.TP +.B \-\-always\-on\-top +Make scrcpy window always on top (above other windows). + +.TP +.BI "\-\-angle " degrees +Rotate the video content by a custom angle, in degrees (clockwise). + +.TP +.BI "\-\-audio\-bit\-rate " value +Encode the audio at the given bit rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). + +Default is 128K (128000). + +.TP +.BI "\-\-audio\-buffer " ms +Configure the audio buffering delay (in milliseconds). + +Lower values decrease the latency, but increase the likelihood of buffer underrun (causing audio glitches). + +Default is 50. + +.TP +.BI "\-\-audio\-codec " name +Select an audio codec (opus, aac, flac or raw). + +Default is opus. + +.TP +.BI "\-\-audio\-codec\-options " key\fR[:\fItype\fR]=\fIvalue\fR[,...] +Set a list of comma-separated key:type=value options for the device audio encoder. + +The possible values for 'type' are 'int' (default), 'long', 'float' and 'string'. + +The list of possible codec options is available in the Android documentation: + + + +.TP +.B \-\-audio\-dup +Duplicate audio (capture and keep playing on the device). + +This feature is only available with --audio-source=playback. + +.TP +.BI "\-\-audio\-encoder " name +Use a specific MediaCodec audio encoder (depending on the codec provided by \fB\-\-audio\-codec\fR). + +The available encoders can be listed by \fB\-\-list\-encoders\fR. + +.TP +.BI "\-\-audio\-source " source +Select the audio source. Possible values are: + + - "output": forwards the whole audio output, and disables playback on the device. + - "playback": captures the audio playback (Android apps can opt-out, so the whole output is not necessarily captured). + - "mic": captures the microphone. + - "mic-unprocessed": captures the microphone unprocessed (raw) sound. + - "mic-camcorder": captures the microphone tuned for video recording, with the same orientation as the camera if available. + - "mic-voice-recognition": captures the microphone tuned for voice recognition. + - "mic-voice-communication": captures the microphone tuned for voice communications (it will for instance take advantage of echo cancellation or automatic gain control if available). + - "voice-call": captures voice call. + - "voice-call-uplink": captures voice call uplink only. + - "voice-call-downlink": captures voice call downlink only. + - "voice-performance": captures audio meant to be processed for live performance (karaoke), includes both the microphone and the device playback. + +Default is output. + +.TP +.BI "\-\-audio\-output\-buffer " ms +Configure the size of the SDL audio output buffer (in milliseconds). + +If you get "robotic" audio playback, you should test with a higher value (10). Do not change this setting otherwise. + +Default is 5. + +.TP +.BI "\-b, \-\-video\-bit\-rate " value +Encode the video at the given bit rate, expressed in bits/s. Unit suffixes are supported: '\fBK\fR' (x1000) and '\fBM\fR' (x1000000). + +Default is 8M (8000000). + +.TP +.BI "\-\-camera\-ar " ar +Select the camera size by its aspect ratio (+/- 10%). + +Possible values are "sensor" (use the camera sensor aspect ratio), "\fInum\fR:\fIden\fR" (e.g. "4:3") and "\fIvalue\fR" (e.g. "1.6"). + +.TP +.BI "\-\-camera\-facing " facing +Select the device camera by its facing direction. + +Possible values are "front", "back" and "external". + +.TP +.BI "\-\-camera\-fps " fps +Specify the camera capture frame rate. + +If not specified, Android's default frame rate (30 fps) is used. + +.TP +.B \-\-camera\-high\-speed +Enable high-speed camera capture mode. + +This mode is restricted to specific resolutions and frame rates, listed by \fB\-\-list\-camera\-sizes\fR. + +.TP +.BI "\-\-camera\-id " id +Specify the device camera id to mirror. + +The available camera ids can be listed by \fB\-\-list\-cameras\fR. + +.TP +.BI "\-\-camera\-size " width\fRx\fIheight +Specify an explicit camera capture size. + +.TP +.BI "\-\-capture\-orientation " value +Possible values are 0, 90, 180, 270, flip0, flip90, flip180 and flip270, possibly prefixed by '@'. + +The number represents the clockwise rotation in degrees; the "flip" keyword applies a horizontal flip before the rotation. + +If a leading '@' is passed (@90) for display capture, then the rotation is locked, and is relative to the natural device orientation. + +If '@' is passed alone, then the rotation is locked to the initial device orientation. + +Default is 0. + +.TP +.BI "\-\-crop " width\fR:\fIheight\fR:\fIx\fR:\fIy +Crop the device screen on the server. + +The values are expressed in the device natural orientation (typically, portrait for a phone, landscape for a tablet). + +.TP +.B \-d, \-\-select\-usb +Use USB device (if there is exactly one, like adb -d). + +Also see \fB\-e\fR (\fB\-\-select\-tcpip\fR). + +.TP +.BI "\-\-disable\-screensaver" +Disable screensaver while scrcpy is running. + +.TP +.BI "\-\-display\-id " id +Specify the device display id to mirror. + +The available display ids can be listed by \fB\-\-list\-displays\fR. + +Default is 0. + +.TP +.BI "\-\-display\-ime\-policy " value +Set the policy for selecting where the IME should be displayed. + +Possible values are "local", "fallback" and "hide": + + - "local" means that the IME should appear on the local display. + - "fallback" means that the IME should appear on a fallback display (the default display). + - "hide" means that the IME should be hidden. + +By default, the IME policy is left unchanged. + + +.TP +.BI "\-\-display\-orientation " value +Set the initial display orientation. + +Possible values are 0, 90, 180, 270, flip0, flip90, flip180 and flip270. The number represents the clockwise rotation in degrees; the "flip" keyword applies a horizontal flip before the rotation. + +Default is 0. + +.TP +.B \-e, \-\-select\-tcpip +Use TCP/IP device (if there is exactly one, like adb -e). + +Also see \fB\-d\fR (\fB\-\-select\-usb\fR). + +.TP +.B \-f, \-\-fullscreen +Start in fullscreen. + +.TP +.B \-\-force\-adb\-forward +Do not attempt to use "adb reverse" to connect to the device. + +.TP +.B \-G +Same as \fB\-\-gamepad=uhid\fR, or \fB\-\-keyboard=aoa\fR if \fB\-\-otg\fR is set. + +.TP +.BI "\-\-gamepad " mode +Select how to send gamepad inputs to the device. + +Possible values are "disabled", "uhid" and "aoa": + + - "disabled" does not send gamepad inputs to the device. + - "uhid" simulates physical HID gamepads using the Linux HID kernel module on the device. + - "aoa" simulates physical HID gamepads using the AOAv2 protocol. It may only work over USB. + +Also see \fB\-\-keyboard\f and R\fB\-\-mouse\fR. +.TP +.B \-h, \-\-help +Print this help. + +.TP +.B \-K +Same as \fB\-\-keyboard=uhid\fR, or \fB\-\-keyboard=aoa\fR if \fB\-\-otg\fR is set. + +.TP +.BI "\-\-keyboard " mode +Select how to send keyboard inputs to the device. + +Possible values are "disabled", "sdk", "uhid" and "aoa": + + - "disabled" does not send keyboard inputs to the device. + - "sdk" uses the Android system API to deliver keyboard events to applications. + - "uhid" simulates a physical HID keyboard using the Linux HID kernel module on the device. + - "aoa" simulates a physical HID keyboard using the AOAv2 protocol. It may only work over USB. + +For "uhid" and "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly using the shortcut MOD+k (except in OTG mode), or by executing: + + adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS + +This option is only available when the HID keyboard is enabled (or a physical keyboard is connected). + +Also see \fB\-\-mouse\fR and \fB\-\-gamepad\fR. + +.TP +.B \-\-kill\-adb\-on\-close +Kill adb when scrcpy terminates. + +.TP +.B \-\-legacy\-paste +Inject computer clipboard text as a sequence of key events on Ctrl+v (like MOD+Shift+v). + +This is a workaround for some devices not behaving as expected when setting the device clipboard programmatically. + +.TP +.B \-\-list\-apps +List Android apps installed on the device. + +.TP +.B \-\-list\-camera\-sizes +List the valid camera capture sizes. + +.TP +.B \-\-list\-cameras +List cameras available on the device. + +.TP +.B \-\-list\-encoders +List video and audio encoders available on the device. + +.TP +.B \-\-list\-displays +List displays available on the device. + +.TP +.BI "\-m, \-\-max\-size " value +Limit both the width and height of the video to \fIvalue\fR. The other dimension is computed so that the device aspect\-ratio is preserved. + +Default is 0 (unlimited). + +.TP +.B \-M +Same as \fB\-\-mouse=uhid\fR, or \fB\-\-mouse=aoa\fR if \fB\-\-otg\fR is set. + +.TP +.BI "\-\-max\-fps " value +Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions). + +.TP +.BI "\-\-mouse " mode +Select how to send mouse inputs to the device. + +Possible values are "disabled", "sdk", "uhid" and "aoa": + + - "disabled" does not send mouse inputs to the device. + - "sdk" uses the Android system API to deliver mouse events to applications. + - "uhid" simulates a physical HID mouse using the Linux HID kernel module on the device. + - "aoa" simulates a physical mouse using the AOAv2 protocol. It may only work over USB. + +In "uhid" and "aoa" modes, the computer mouse is captured to control the device directly (relative mouse mode). + +LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer. + +Also see \fB\-\-keyboard\fR and \fB\-\-gamepad\fR. + +.TP +.BI "\-\-mouse\-bind " xxxx[:xxxx] +Configure bindings of secondary clicks. + +The argument must be one or two sequences (separated by ':') of exactly 4 characters, one for each secondary click (in order: right click, middle click, 4th click, 5th click). + +The first sequence defines the primary bindings, used when a mouse button is pressed alone. The second sequence defines the secondary bindings, used when a mouse button is pressed while the Shift key is held. + +If the second sequence of bindings is omitted, then it is the same as the first one. + +Each character must be one of the following: + + - '+': forward the click to the device + - '-': ignore the click + - 'b': trigger shortcut BACK (or turn screen on if off) + - 'h': trigger shortcut HOME + - 's': trigger shortcut APP_SWITCH + - 'n': trigger shortcut "expand notification panel" + +Default is 'bhsn:++++' for SDK mouse, and '++++:bhsn' for AOA and UHID. + + +.TP +.B \-n, \-\-no\-control +Disable device control (mirror the device in read\-only). + +.TP +.B \-N, \-\-no\-playback +Disable video and audio playback on the computer (equivalent to \fB\-\-no\-video\-playback \-\-no\-audio\-playback\fR). + +.TP +\fB\-\-new\-display\fR[=[\fIwidth\fRx\fIheight\fR][/\fIdpi\fR]] +Create a new display with the specified resolution and density. If not provided, they default to the main display dimensions and DPI. + +Examples: + + \-\-new\-display=1920x1080 + \-\-new\-display=1920x1080/420 + \-\-new\-display # main display size and density + \-\-new\-display=/240 # main display size and 240 dpi + +.TP +.B \-\-no\-audio +Disable audio forwarding. + +.TP +.B \-\-no\-audio\-playback +Disable audio playback on the computer. + +.TP +.B \-\-no\-cleanup +By default, scrcpy removes the server binary from the device and restores the device state (show touches, stay awake and power mode) on exit. + +This option disables this cleanup. + +.TP +.B \-\-no\-clipboard\-autosync +By default, scrcpy automatically synchronizes the computer clipboard to the device clipboard before injecting Ctrl+v, and the device clipboard to the computer clipboard whenever it changes. + +This option disables this automatic synchronization. + +.TP +.B \-\-no\-downsize\-on\-error +By default, on MediaCodec error, scrcpy automatically tries again with a lower definition. + +This option disables this behavior. + +.TP +.B \-\-no\-key\-repeat +Do not forward repeated key events when a key is held down. + +.TP +.B \-\-no\-mipmaps +If the renderer is OpenGL 3.0+ or OpenGL ES 2.0+, then mipmaps are automatically generated to improve downscaling quality. This option disables the generation of mipmaps. + +.TP +.B \-\-no\-mouse\-hover +Do not forward mouse hover (mouse motion without any clicks) events. + +.TP +.B \-\-no\-power\-on +Do not power on the device on start. + +.TP +.B \-\-no\-vd\-destroy\-content +Disable virtual display "destroy content on removal" flag. + +With this option, when the virtual display is closed, the running apps are moved to the main display rather than being destroyed. + +.TP +.B \-\-no\-vd\-system\-decorations +Disable virtual display system decorations flag. + +.TP +.B \-\-no\-video +Disable video forwarding. + +.TP +.B \-\-no\-video\-playback +Disable video playback on the computer. + +.TP +.B \-\-no\-window +Disable scrcpy window. Implies --no-video-playback. + +.TP +.BI "\-\-orientation " value +Same as --display-orientation=value --record-orientation=value. + +.TP +.B \-\-otg +Run in OTG mode: simulate physical keyboard and mouse, as if the computer keyboard and mouse were plugged directly to the device via an OTG cable. + +In this mode, adb (USB debugging) is not necessary, and mirroring is disabled. + +LAlt, LSuper or RSuper toggle the mouse capture mode, to give control of the mouse back to the computer. + +If any of \fB\-\-hid\-keyboard\fR or \fB\-\-hid\-mouse\fR is set, only enable keyboard or mouse respectively, otherwise enable both. + +It may only work over USB. + +See \fB\-\-keyboard\fR, \fB\-\-mouse\fR and \fB\-\-gamepad\fR. + +.TP +.BI "\-p, \-\-port " port\fR[:\fIport\fR] +Set the TCP port (range) used by the client to listen. + +Default is 27183:27199. + +.TP +\fB\-\-pause\-on\-exit\fR[=\fImode\fR] +Configure pause on exit. Possible values are "true" (always pause on exit), "false" (never pause on exit) and "if-error" (pause only if an error occurred). + +This is useful to prevent the terminal window from automatically closing, so that error messages can be read. + +Default is "false". + +Passing the option without argument is equivalent to passing "true". + +.TP +.B \-\-power\-off\-on\-close +Turn the device screen off when closing scrcpy. + +.TP +.B \-\-prefer\-text +Inject alpha characters and space as text events instead of key events. + +This avoids issues when combining multiple keys to enter special characters, +but breaks the expected behavior of alpha keys in games (typically WASD). + +.TP +.B "\-\-print\-fps +Start FPS counter, to print framerate logs to the console. It can be started or stopped at any time with MOD+i. + +.TP +.BI "\-\-push\-target " path +Set the target directory for pushing files to the device by drag & drop. It is passed as\-is to "adb push". + +Default is "/sdcard/Download/". + +.TP +.BI "\-r, \-\-record " file +Record screen to +.IR file . + +The format is determined by the +.B \-\-record\-format +option if set, or by the file extension. + +.TP +.B \-\-raw\-key\-events +Inject key events for all input keys, and ignore text events. + +.TP +.BI "\-\-record\-format " format +Force recording format (mp4, mkv, m4a, mka, opus, aac, flac or wav). + +.TP +.BI "\-\-record\-orientation " value +Set the record orientation. + +Possible values are 0, 90, 180 and 270. The number represents the clockwise rotation in degrees. + +Default is 0. + +.TP +.BI "\-\-render\-driver " name +Request SDL to use the given render driver (this is just a hint). + +Supported names are currently "direct3d", "opengl", "opengles2", "opengles", "metal" and "software". + + + +.TP +.B \-\-require\-audio +By default, scrcpy mirrors only the video if audio capture fails on the device. This option makes scrcpy fail if audio is enabled but does not work. + +.TP +.BI "\-s, \-\-serial " number +The device serial number. Mandatory only if several devices are connected to adb. + +.TP +.B \-S, \-\-turn\-screen\-off +Turn the device screen off immediately. + +.TP +.B "\-\-screen\-off\-timeout " seconds +Set the screen off timeout while scrcpy is running (restore the initial value on exit). + +.TP +.BI "\-\-shortcut\-mod " key\fR[+...]][,...] +Specify the modifiers to use for scrcpy shortcuts. Possible keys are "lctrl", "rctrl", "lalt", "ralt", "lsuper" and "rsuper". + +Several shortcut modifiers can be specified, separated by ','. + +For example, to use either LCtrl or LSuper for scrcpy shortcuts, pass "lctrl,lsuper". + +Default is "lalt,lsuper" (left-Alt or left-Super). + +.TP +.BI "\-\-start\-app " name +Start an Android app, by its exact package name. + +Add a '?' prefix to select an app whose name starts with the given name, case-insensitive (retrieving app names on the device may take some time): + + scrcpy --start-app=?firefox + +Add a '+' prefix to force-stop before starting the app: + + scrcpy --new-display --start-app=+org.mozilla.firefox + +Both prefixes can be used, in that order: + + scrcpy --start-app=+?firefox + +.TP +.B \-t, \-\-show\-touches +Enable "show touches" on start, restore the initial value on exit. + +It only shows physical touches (not clicks from scrcpy). + +.TP +.BI "\-\-tcpip\fR[=[+]\fIip\fR[:\fIport\fR]] +Configure and connect the device over TCP/IP. + +If a destination address is provided, then scrcpy connects to this address before starting. The device must listen on the given TCP port (default is 5555). + +If no destination address is provided, then scrcpy attempts to find the IP address and adb port of the current device (typically connected over USB), enables TCP/IP mode if necessary, then connects to this address before starting. + +Prefix the address with a '+' to force a reconnection. + +.TP +.BI "\-\-time\-limit " seconds +Set the maximum mirroring time, in seconds. + +.TP +.BI "\-\-tunnel\-host " ip +Set the IP address of the adb tunnel to reach the scrcpy server. This option automatically enables \fB\-\-force\-adb\-forward\fR. + +Default is localhost. + +.TP +.BI "\-\-tunnel\-port " port +Set the TCP port of the adb tunnel to reach the scrcpy server. This option automatically enables \fB\-\-force\-adb\-forward\fR. + +Default is 0 (not forced): the local port used for establishing the tunnel will be used. + +.TP +.B \-v, \-\-version +Print the version of scrcpy. + +.TP +.BI "\-V, \-\-verbosity " value +Set the log level ("verbose", "debug", "info", "warn" or "error"). + +Default is "info" for release builds, "debug" for debug builds. + +.TP +.BI "\-\-v4l2-sink " /dev/videoN +Output to v4l2loopback device. + +.TP +.BI "\-\-v4l2-buffer " ms +Add a buffering delay (in milliseconds) before pushing frames. This increases latency to compensate for jitter. + +This option is similar to \fB\-\-video\-buffer\fR, but specific to V4L2 sink. + +Default is 0 (no buffering). + +.TP +.BI "\-\-video\-buffer " ms +Add a buffering delay (in milliseconds) before displaying video frames. + +This increases latency to compensate for jitter. + +Default is 0 (no buffering). + +.TP +.BI "\-\-video\-codec " name +Select a video codec (h264, h265 or av1). + +Default is h264. + +.TP +.BI "\-\-video\-codec\-options " key\fR[:\fItype\fR]=\fIvalue\fR[,...] +Set a list of comma-separated key:type=value options for the device video encoder. + +The possible values for 'type' are 'int' (default), 'long', 'float' and 'string'. + +The list of possible codec options is available in the Android documentation: + + + +.TP +.BI "\-\-video\-encoder " name +Use a specific MediaCodec video encoder (depending on the codec provided by \fB\-\-video\-codec\fR). + +The available encoders can be listed by \fB\-\-list\-encoders\fR. + +.TP +.BI "\-\-video\-source " source +Select the video source (display or camera). + +Camera mirroring requires Android 12+. + +Default is display. + +.TP +.B \-w, \-\-stay-awake +Keep the device on while scrcpy is running, when the device is plugged in. + +.TP +.B \-\-window\-borderless +Disable window decorations (display borderless window). + +.TP +.BI "\-\-window\-title " text +Set a custom window title. + +.TP +.BI "\-\-window\-x " value +Set the initial window horizontal position. + +Default is "auto". + +.TP +.BI "\-\-window\-y " value +Set the initial window vertical position. + +Default is "auto". + +.TP +.BI "\-\-window\-width " value +Set the initial window width. + +Default is 0 (automatic). + +.TP +.BI "\-\-window\-height " value +Set the initial window height. + +Default is 0 (automatic). + +.SH EXIT STATUS +.B scrcpy +will exit with code 0 on normal program termination. If an initial +connection cannot be established, the exit code 1 will be returned. If the +device disconnects while a session is active, exit code 2 will be returned. + +.SH SHORTCUTS + +In the following list, MOD is the shortcut modifier. By default, it's (left) +Alt or (left) Super, but it can be configured by \fB\-\-shortcut\-mod\fR (see above). + +.TP +.B MOD+f +Switch fullscreen mode + +.TP +.B MOD+Left +Rotate display left + +.TP +.B MOD+Right +Rotate display right + +.TP +.B MOD+Shift+Left, MOD+Shift+Right +Flip display horizontally + +.TP +.B MOD+Shift+Up, MOD+Shift+Down +Flip display vertically + +.TP +.B MOD+z +Pause or re-pause display + +.TP +.B MOD+Shift+z +Unpause display + +.TP +.B MOD+Shift+r +Reset video capture/encoding + +.TP +.B MOD+g +Resize window to 1:1 (pixel\-perfect) + +.TP +.B MOD+w, Double\-click on black borders +Resize window to remove black borders + +.TP +.B MOD+h, Home, Middle\-click +Click on HOME + +.TP +.B MOD+b, MOD+Backspace, Right\-click (when screen is on) +Click on BACK + +.TP +.B MOD+s +Click on APP_SWITCH + +.TP +.B MOD+m +Click on MENU + +.TP +.B MOD+Up +Click on VOLUME_UP + +.TP +.B MOD+Down +Click on VOLUME_DOWN + +.TP +.B MOD+p +Click on POWER (turn screen on/off) + +.TP +.B Right\-click (when screen is off) +Turn screen on + +.TP +.B MOD+o +Turn device screen off (keep mirroring) + +.TP +.B MOD+Shift+o +Turn device screen on + +.TP +.B MOD+r +Rotate device screen + +.TP +.B MOD+n +Expand notification panel + +.TP +.B MOD+Shift+n +Collapse notification panel + +.TP +.B Mod+c +Copy to clipboard (inject COPY keycode, Android >= 7 only) + +.TP +.B Mod+x +Cut to clipboard (inject CUT keycode, Android >= 7 only) + +.TP +.B MOD+v +Copy computer clipboard to device, then paste (inject PASTE keycode, Android >= 7 only) + +.TP +.B MOD+Shift+v +Inject computer clipboard text as a sequence of key events + +.TP +.B MOD+k +Open keyboard settings on the device (for HID keyboard only) + +.TP +.B MOD+i +Enable/disable FPS counter (print frames/second in logs) + +.TP +.B Ctrl+click-and-move +Pinch-to-zoom and rotate from the center of the screen + +.TP +.B Shift+click-and-move +Tilt vertically (slide with 2 fingers) + +.TP +.B Ctrl+Shift+click-and-move +Tilt horizontally (slide with 2 fingers) + +.TP +.B Drag & drop APK file +Install APK from computer + +.TP +.B Drag & drop non-APK file +Push file to device (see \fB\-\-push\-target\fR) + + +.SH Environment variables + +.TP +.B ADB +Path to adb. + +.TP +.B ANDROID_SERIAL +Device serial to use if no selector (\fB-s\fR, \fB-d\fR, \fB-e\fR or \fB\-\-tcpip=\fIaddr\fR) is specified. + +.TP +.B SCRCPY_ICON_PATH +Path to the program icon. + +.TP +.B SCRCPY_SERVER_PATH +Path to the server binary. + + +.SH AUTHORS +.B scrcpy +is written by Romain Vimont. + +This manual page was written by +.MT mmyangfl@gmail.com +Yangfl +.ME +for the Debian Project (and may be used by others). + + +.SH "REPORTING BUGS" +Report bugs to . + +.SH COPYRIGHT +Copyright \(co 2018 Genymobile + +Copyright \(co 2018\-2025 Romain Vimont + +Licensed under the Apache License, Version 2.0. + +.SH WWW + diff --git a/copilot_tools/scrcpy/win/SDL2.dll b/copilot_tools/scrcpy/win/SDL2.dll new file mode 100644 index 0000000..82d2b1a Binary files /dev/null and b/copilot_tools/scrcpy/win/SDL2.dll differ diff --git a/copilot_tools/scrcpy/win/avcodec-61.dll b/copilot_tools/scrcpy/win/avcodec-61.dll new file mode 100644 index 0000000..a44b6c6 Binary files /dev/null and b/copilot_tools/scrcpy/win/avcodec-61.dll differ diff --git a/copilot_tools/scrcpy/win/avformat-61.dll b/copilot_tools/scrcpy/win/avformat-61.dll new file mode 100644 index 0000000..347e657 Binary files /dev/null and b/copilot_tools/scrcpy/win/avformat-61.dll differ diff --git a/copilot_tools/scrcpy/win/avutil-59.dll b/copilot_tools/scrcpy/win/avutil-59.dll new file mode 100644 index 0000000..9b847e3 Binary files /dev/null and b/copilot_tools/scrcpy/win/avutil-59.dll differ diff --git a/copilot_tools/scrcpy/win/icon.png b/copilot_tools/scrcpy/win/icon.png new file mode 100644 index 0000000..b96a1af Binary files /dev/null and b/copilot_tools/scrcpy/win/icon.png differ diff --git a/copilot_tools/scrcpy/win/libusb-1.0.dll b/copilot_tools/scrcpy/win/libusb-1.0.dll new file mode 100644 index 0000000..b36c945 Binary files /dev/null and b/copilot_tools/scrcpy/win/libusb-1.0.dll differ diff --git a/copilot_tools/scrcpy/win/open_a_terminal_here.bat b/copilot_tools/scrcpy/win/open_a_terminal_here.bat new file mode 100644 index 0000000..24d557f --- /dev/null +++ b/copilot_tools/scrcpy/win/open_a_terminal_here.bat @@ -0,0 +1 @@ +@cmd diff --git a/copilot_tools/scrcpy/win/scrcpy-console.bat b/copilot_tools/scrcpy/win/scrcpy-console.bat new file mode 100644 index 0000000..0ea7619 --- /dev/null +++ b/copilot_tools/scrcpy/win/scrcpy-console.bat @@ -0,0 +1,2 @@ +@echo off +scrcpy.exe --pause-on-exit=if-error %* diff --git a/copilot_tools/scrcpy/win/scrcpy-noconsole.vbs b/copilot_tools/scrcpy/win/scrcpy-noconsole.vbs new file mode 100644 index 0000000..d509ad7 --- /dev/null +++ b/copilot_tools/scrcpy/win/scrcpy-noconsole.vbs @@ -0,0 +1,7 @@ +strCommand = "cmd /c scrcpy.exe" + +For Each Arg In WScript.Arguments + strCommand = strCommand & " """ & replace(Arg, """", """""""""") & """" +Next + +CreateObject("Wscript.Shell").Run strCommand, 0, false diff --git a/copilot_tools/scrcpy/win/scrcpy-server b/copilot_tools/scrcpy/win/scrcpy-server new file mode 100644 index 0000000..b36f14d Binary files /dev/null and b/copilot_tools/scrcpy/win/scrcpy-server differ diff --git a/copilot_tools/scrcpy/win/scrcpy.exe b/copilot_tools/scrcpy/win/scrcpy.exe new file mode 100644 index 0000000..496ec24 Binary files /dev/null and b/copilot_tools/scrcpy/win/scrcpy.exe differ diff --git a/copilot_tools/scrcpy/win/swresample-5.dll b/copilot_tools/scrcpy/win/swresample-5.dll new file mode 100644 index 0000000..49d9cb0 Binary files /dev/null and b/copilot_tools/scrcpy/win/swresample-5.dll differ diff --git a/examples/visualization_group_control/run_single_task_group_control_Job_Scheduler.py b/examples/visualization_group_control/run_single_task_group_control_Job_Scheduler.py new file mode 100644 index 0000000..16c9153 --- /dev/null +++ b/examples/visualization_group_control/run_single_task_group_control_Job_Scheduler.py @@ -0,0 +1,424 @@ +import os +import sys +import time +import subprocess +import platform +import re +import threading + +if "." not in sys.path: + sys.path.append(".") + +from copilot_agent_client.pu_client import evaluate_task_on_device +from copilot_front_end.mobile_action_helper import list_devices, get_device_wm_size +from copilot_agent_server.local_server import LocalServer + +tmp_server_config = { + "log_dir": "running_log/server_log/os-copilot-local-eval-logs/traces", + "image_dir": "running_log/server_server/os-copilot-local-eval-logs/images", + "debug": False +} + + +local_model_config = { + "task_type": "parser_0922_summary", + "model_config": { + "model_name": "gelab-zero-4b-preview", + "model_provider": "local", + "args": { + "temperature": 0.1, + "top_p": 0.95, + "frequency_penalty": 0.0, + "max_tokens": 4096, + }, + }, + + "max_steps": 400, + "delay_after_capture": 2, + "debug": False +} + +# ===== 配置变量定义 (全局作用域) ===== +tmp_rollout_config = local_model_config + +# ===== 中央配置：设备能力与任务队列===== +DEVICE_CAPABILITIES = { + "FIRST_DEVICE": {"tag": "SHOPPING_CONSUMPTION"}, + "SECOND_DEVICE": {"tag": "VIDEO_STREAMING"}, +} + +JOB_QUEUE = [ + {"task": "在淘宝上搜索并下单一个键盘", "required_tag": "SHOPPING_CONSUMPTION"}, + {"task": "打开爱奇艺，播放《疯狂动物城》", "required_tag": "VIDEO_STREAMING"}, + {"task": "检查微信是否有未读消息", "required_tag": "SOCIAL_MEDIA"}, +] + +# ===== 用于记录每步耗时 ===== +_step_times = [] + +# 全局字典，用于存储 scrcpy 的实际 PID (而非 Shell/CMD 的 PID) +scrcpy_pids_to_kill = {} + + +# ===== 包装 automate_step 方法 ===== +def wrap_automate_step_with_timing(server_instance): + original_method = server_instance.automate_step + + def timed_automate_step(payload): + step_start = time.time() + try: + result = original_method(payload) + finally: + duration = time.time() - step_start + _step_times.append(duration) + print(f"[Thread {threading.get_ident()}] Step {len(_step_times)} took: {duration:.2f} seconds") + return result + + server_instance.automate_step = timed_automate_step + +# ===== 等待目标设备重新稳定上线 ===== +def wait_for_device_stability(target_device_ids, timeout=10, interval=0.5): + print(f"\n等待 {len(target_device_ids)} 个设备在 ADB 中稳定...") + start_time = time.time() + + devices_to_wait_for = set(target_device_ids) + + while time.time() - start_time < timeout and devices_to_wait_for: + try: + connected_devices = list_devices() + + stable_now = devices_to_wait_for.intersection(set(connected_devices)) + if stable_now: + print(f" -> 设备 {list(stable_now)} 已稳定连接。") + devices_to_wait_for -= stable_now + + except Exception: + pass + + if devices_to_wait_for: + time.sleep(interval) + + if devices_to_wait_for: + print(f"错误：等待设备 {list(devices_to_wait_for)} 超时 ({timeout}秒)，任务终止。") + return False + + print("所有目标设备已稳定连接。") + return True + +# ===== 在单个设备上运行任务的封装（移除线程内清理，仅执行任务） ===== +def run_task_on_device(device_id, tmp_server_config, task): + global tmp_rollout_config + + l2_server = LocalServer(tmp_server_config) + wrap_automate_step_with_timing(l2_server) + + try: + device_wm_size = get_device_wm_size(device_id) + device_info = { + "device_id": device_id, + "device_wm_size": device_wm_size + } + + print(f"\n>>>> 任务开始在设备: {device_id} 上执行 (任务: {task[:10]}...) <<<<") + evaluate_task_on_device(l2_server, device_info, task, tmp_rollout_config, reflush_app=True) + print(f">>>> 任务执行完毕 (设备: {device_id}) <<<<\n") + + except Exception as e: + print(f"任务执行过程中发生错误 (设备: {device_id}): {e}") + + finally: + pass + + +# ===== scrcpy 基础路径定义 ===== +SCRCPY_BASE_DIR = os.path.join("copilot_tools", "scrcpy") + + +# ===== 动态获取 scrcpy 路径 ===== +def get_scrcpy_path(): + """根据操作系统，返回 SCRCPY_BASE_DIR 文件夹中的 scrcpy 可执行文件路径""" + + system = platform.system() + + if system == "Windows": + path = os.path.join(SCRCPY_BASE_DIR, "win", "scrcpy.exe") + elif system == "Darwin": # macOS + path = os.path.join(SCRCPY_BASE_DIR, "mac", "scrcpy") + elif system == "Linux": + path = os.path.join(SCRCPY_BASE_DIR, "linux", "scrcpy") + else: + raise OSError(f"不支持的操作系统: {system}") + + + if not os.path.exists(path): + if system == "Linux": + try: + subprocess.run(["scrcpy", "-h"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return "scrcpy" + except (subprocess.CalledProcessError, FileNotFoundError): + pass + + raise FileNotFoundError(f"scrcpy 可执行文件未找到，请检查路径: {os.path.abspath(path)}") + + return os.path.abspath(path) + +# ===== 【新增】获取当前所有 scrcpy 进程的 PID 集合 (跨平台) ===== +def get_scrcpy_pids(scrcpy_path): + system = platform.system() + pids = set() + + if system == "Windows": + try: + # 查找所有名为 scrcpy.exe 的进程 + result = subprocess.run('tasklist /FI "IMAGENAME eq scrcpy.exe" /NH', capture_output=True, text=True, shell=True, check=False) + + for line in result.stdout.strip().split('\n'): + if line.strip(): + parts = line.split() + try: + pids.add(int(parts[1])) + except (ValueError, IndexError): + continue + except Exception: + pass + + elif system in ["Darwin", "Linux"]: + try: + # 查找所有包含 scrcpy 路径的进程 + result = subprocess.run(f'pgrep -f "{scrcpy_path}"', capture_output=True, text=True, shell=True, check=False) + + for pid_str in result.stdout.strip().split('\n'): + if pid_str.strip(): + try: + pids.add(int(pid_str)) + except ValueError: + continue + except Exception: + pass + + return pids + + +# ===== 通过端口查找并终止 PID (跨平台兼容) ===== +def terminate_process_by_port(port): + """使用 Windows 的 netstat/taskkill 或 macOS/Linux 的 lsof/kill 终止占用指定端口的进程""" + system = platform.system() + + if system == "Windows": + # Windows 逻辑 (使用 netstat 和 taskkill) + try: + command = f'netstat -ano | findstr LISTEN | findstr :{port}' + result = subprocess.run(command, capture_output=True, text=True, shell=True, check=False) + + if result.returncode != 0 or not result.stdout.strip(): + return + + pid_match = re.search(r'\s+(\d+)\s*$', result.stdout.strip().split('\n')[0]) + + if pid_match: + pid = pid_match.group(1) + print(f" -> [保险清理] 终止 PID {pid} (端口 {port})...") + # 使用 taskkill /F /T 终止进程树 + subprocess.run(f"taskkill /PID {pid} /F /T", shell=True, check=False, capture_output=True) + return + + except Exception as e: + print(f" -> [保险清理] 终止端口 {port} 失败 (Windows): {e}") + + elif system in ["Darwin", "Linux"]: + # macOS/Linux 逻辑 (使用 lsof 和 kill) + try: + command = f'lsof -i tcp:{port} | grep LISTEN' + result = subprocess.run(command, capture_output=True, text=True, shell=True, check=False) + + if not result.stdout.strip(): + return + + pid_match = re.search(r'\s+(\d+)\s+', result.stdout.strip().split('\n')[0]) + + if pid_match: + pid = pid_match.group(1) + print(f" -> [保险清理] 终止 PID {pid} (端口 {port})...") + subprocess.run(f"kill -9 {pid}", shell=True, check=False, capture_output=True) + return + + except Exception as e: + print(f" -> [保险清理] 终止端口 {port} 失败 (macOS/Linux): {e}") + +# ============================================================= + +if __name__ == "__main__": + # 1. 获取所有设备信息 + all_device_ids = list_devices() + if not all_device_ids: + print("未检测到任何 ADB 设备，请检查设备连接！") + sys.exit(1) + + + # 构建动态设备能力映射，并获取需要启动 scrcpy 的设备列表 + devices_to_start_scrcpy = [] + + temp_device_map = DEVICE_CAPABILITIES.copy() + DEVICE_CAPABILITIES = {} + + # 将配置中的占位符ID替换为实际的在线设备ID + if len(all_device_ids) >= 1: + DEVICE_CAPABILITIES[all_device_ids[0]] = temp_device_map.pop("FIRST_DEVICE") + devices_to_start_scrcpy.append(all_device_ids[0]) + if len(all_device_ids) >= 2: + DEVICE_CAPABILITIES[all_device_ids[1]] = temp_device_map.pop("SECOND_DEVICE") + devices_to_start_scrcpy.append(all_device_ids[1]) + + + scrcpy_ports = {} + # 使用全局的 scrcpy_pids_to_kill 字典 + + total_start = time.time() + task_threads = [] + + START_X = 50 + + try: + # 2. 动态获取 scrcpy 路径 + SCRCPY_PATH = get_scrcpy_path() + SCRCPY_DIR = os.path.dirname(SCRCPY_PATH) + print(f"scrcpy 路径确定为: {SCRCPY_PATH}") + + system = platform.system() + + # 3. 循环启动所有设备的 scrcpy 进程 + print(f"检测到 {len(devices_to_start_scrcpy)} 个配置的设备，正在启动 scrcpy 窗口...") + + #在启动前记录所有 scrcpy 进程的 PID + initial_scrcpy_pids = get_scrcpy_pids(SCRCPY_PATH) + + start_port = 27183 + offset_y = 50 + OFFSET_STEP = 65 + + for i, device_id in enumerate(devices_to_start_scrcpy): + window_position_arg = f"--window-x {START_X} --window-y {offset_y}" + port_arg = f"-p {start_port}" + + # 跨平台启动命令的构建 + command_args = f'"{SCRCPY_PATH}" -s {device_id} {port_arg} {window_position_arg}' + + if system == "Windows": + # 使用 start "" 来分离子进程 + scrcpy_command_str = f'start "" {command_args}' + else: + # 使用 nohup ... & 来分离子进程 + scrcpy_command_str = f'nohup {command_args} > /dev/null 2>&1 &' + + # 启动 Shell/CMD 进程 + subprocess.Popen( + scrcpy_command_str, + shell=True, + ) + + # 等待 scrcpy 窗口启动，并找到其真实的 PID + time.sleep(1.5) # 给予 scrcpy 足够的启动时间 + + current_scrcpy_pids = get_scrcpy_pids(SCRCPY_PATH) + new_pids = current_scrcpy_pids - initial_scrcpy_pids + + if new_pids: + # 找到新启动的 PID (通常只有一个) + actual_pid = list(new_pids)[0] + # 记录实际 PID 用于后续终止 + scrcpy_pids_to_kill[device_id] = actual_pid + initial_scrcpy_pids = current_scrcpy_pids # 更新初始列表供下一个设备使用 + print(f"设备 {device_id} 的 scrcpy 进程已启动 (端口: {start_port}, 实际PID: {actual_pid})...") + else: + print(f"警告: 未能找到设备 {device_id} 的 scrcpy 进程的实际 PID!") + + scrcpy_ports[device_id] = start_port + + offset_y += OFFSET_STEP + start_port += 1 + + if len(devices_to_start_scrcpy) > 1: + time.sleep(0.5) + + # 4. 在执行任务前，等待 ADB 稳定 + if not wait_for_device_stability(devices_to_start_scrcpy): + raise Exception("部分 ADB 连接未能稳定，任务终止。") + + # 5. 任务分配与并发执行 + assignment_batch = [] + available_devices = set(devices_to_start_scrcpy) + + for job in JOB_QUEUE: + required_tag = job['required_tag'] + assigned_now = None + + for device_id in available_devices: + if DEVICE_CAPABILITIES.get(device_id, {}).get('tag') == required_tag: + + assignment_batch.append({ + "device_id": device_id, + "task": job['task'], + "port": scrcpy_ports[device_id] + }) + assigned_now = device_id + break + + if assigned_now: + available_devices.remove(assigned_now) + + + if not assignment_batch: + print("\n没有找到匹配的任务和设备上下文，跳过任务执行。") + else: + print(f"\n>>>> 正在 {len(assignment_batch)} 个设备上并发执行任务 (精确匹配模式) <<<<") + + for assignment in assignment_batch: + device_id = assignment['device_id'] + task = assignment['task'] + + thread = threading.Thread( + target=run_task_on_device, + args=(device_id, tmp_server_config, task) + ) + task_threads.append(thread) + thread.start() + + for thread in task_threads: + thread.join() + + print("\n>>>> 所有并发任务执行完毕 <<<<") + + except FileNotFoundError as e: + print(f"致命错误: {e}") + + except Exception as e: + print(f"任务执行过程中发生错误: {e}") + + finally: + # 6. 终止所有 scrcpy 进程和计时 + total_time = time.time() - total_start + print(f"总计执行时间为 {total_time:.2f} 秒") + + print("\n正在执行最终清理...") + + # 通过真实的 scrcpy PID 终止进程 (最可靠的方式) + for device_id, pid in scrcpy_pids_to_kill.items(): + print(f" -> 尝试通过实际 PID 终止设备 {device_id} 的 scrcpy 进程 (PID: {pid})...") + try: + if platform.system() == "Windows": + # Windows 上使用 taskkill /F /T 来终止进程树 + subprocess.run(f"taskkill /PID {pid} /F /T", shell=True, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + # macOS/Linux 上使用 kill -9 终止进程 + subprocess.run(f"kill -9 {pid}", shell=True, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + print(f" -> 设备 {device_id} 的 scrcpy 进程已终止。") + except Exception as e: + print(f" -> 终止 scrcpy 进程失败 ({device_id}): {e}") + + # 保险措施：通过端口终止残留进程 + print("\n -> 执行端口进程的二次保险清理...") + for port in scrcpy_ports.values(): + terminate_process_by_port(port) + + pass diff --git a/mcp_server/Chinese_chess/__init__.py b/mcp_server/Chinese_chess/__init__.py new file mode 100644 index 0000000..07eb4d9 --- /dev/null +++ b/mcp_server/Chinese_chess/__init__.py @@ -0,0 +1 @@ +# Chinese Chess MCP Server diff --git a/mcp_server/Chinese_chess/chess_mcp_server.py b/mcp_server/Chinese_chess/chess_mcp_server.py new file mode 100644 index 0000000..681fc01 --- /dev/null +++ b/mcp_server/Chinese_chess/chess_mcp_server.py @@ -0,0 +1,676 @@ +""" +中国象棋 MCP 服务器 + +架构设计： +1. prepare_game: GeLab 导航到棋盘界面 +2. get_board_state: 视觉模型解析棋盘，返回可视化棋盘图（给外部 GLM 深度思考） +3. execute_chess_move: 外部 GLM 深度思考后，GeLab 执行点击操作 + +关键：外部 GLM 必须先深度思考分析，得出明确结论后，再调用执行工具 +""" + +from fastmcp import FastMCP +import sys +import os +import time +import re +import json + +if "." not in sys.path: + sys.path.append(".") + +project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +from copilot_front_end.mobile_action_helper import ( + list_devices, + get_device_wm_size, + capture_screenshot, + open_screen, +) +from copilot_agent_client.pu_client import evaluate_task_on_device +from tools.image_tools import make_b64_url +from tools.ask_llm_v2 import ask_llm_anything + +from typing import Annotated +from pydantic import Field +from megfile import smart_remove + +# ============== 绝杀规则 ============== +CHECKMATE_RULES = """ +【绝杀技巧】 +1. 白脸将：将帅同列无子阻挡 +2. 马后炮：马将军+炮在后 +3. 卧槽马：马在九宫角将军 +4. 重炮：双炮同列前炮将军 +5. 闷宫：堵死将的退路 + +【优先级】绝杀 > 吃大子(车>马=炮) > 改善局势 +""" + +# ============== 棋盘解析提示词 ============== +BOARD_ANALYSIS_PROMPT = """请分析这张中国象棋棋盘截图。 + +## 重要：中国象棋的棋子放在交叉点上！ + +中国象棋棋盘是由横线和竖线组成的网格，棋子放在**线的交叉点**上，不是格子里面！ +- 横线：共10条（从下到上编号 0-9） +- 竖线：共9条（从左到右编号 A-I） +- 棋子位置 = 某条横线和某条竖线的交叉点 + +## 坐标系统 +- 列：A-I（从左到右，共9条竖线） +- 行：0-9（从下到上，共10条横线） +- 坐标格式：列+行，如 E0、A9 +- E0 = 第E列与第0行的交叉点（红方底线正中间） +- E9 = 第E列与第9行的交叉点（黑方底线正中间） + +## 标准初始位置参考 +红方（第0-4行）： +- 帅: E0（底线正中） +- 仕: D0, F0 +- 相: C0, G0 +- 马: B0, H0 +- 车: A0, I0 +- 炮: B2, H2 +- 兵: A3, C3, E3, G3, I3 + +黑方（第5-9行）： +- 将: E9（顶线正中） +- 士: D9, F9 +- 象: C9, G9 +- 马: B9, H9 +- 车: A9, I9 +- 炮: B7, H7 +- 卒: A6, C6, E6, G6, I6 + +## 棋子识别 +- 红方：帅(K)、仕(A)、相(B)、马(N)、车(R)、炮(C)、兵(P) +- 黑方：将(k)、士(a)、象(b)、马(n)、车(r)、炮(c)、卒(p) + +## 输出格式（严格JSON） +```json +{ + "pieces": [ + {"type": "K", "pos": "E0", "name": "帅"}, + {"type": "R", "pos": "A0", "name": "车"}, + {"type": "k", "pos": "E9", "name": "将"}, + {"type": "r", "pos": "I9", "name": "车"} + ], + "game_status": "playing" +} +``` + +## game_status +- "playing": 对局中 +- "victory": 看到胜利/成功/恭喜提示 +- "defeat": 看到失败提示 + +只输出JSON。 +""" + +# ============== MCP 服务器 ============== +mcp = FastMCP( + name="Chinese-Chess-MCP-Server", + instructions=""" +你是一个中国象棋AI，通过 MCP 工具指挥 GeLab 在手机上下棋。 + +## ⚠️ 核心规则：必须先深度思考！ + +当你收到棋盘状态后，**禁止立即调用工具**！你必须： + +### 第一步：深度思考（在你的回复中完成，不调用任何工具） + +请按以下格式进行分析： + +``` +【棋局分析】 +1. 红方棋子：... +2. 黑方棋子：... +3. 当前局势：... + +【绝杀检查】 +- 白脸将：是否可行？ +- 马后炮：是否可行？ +- 其他绝杀：... + +【可选走法】 +1. 走法A：... 优点/缺点 +2. 走法B：... 优点/缺点 +3. 走法C：... 优点/缺点 + +【最终决定】 +我选择：[具体走法] +理由：... +``` + +### 第二步：确定走法后，调用 execute_chess_move() + +只有在完成上述深度思考，并明确写出"我选择：xxx"后，才能调用工具。 + +## 工作流程 + +1. `prepare_game(task)` - GeLab 导航到棋盘 +2. `get_board_state()` - 获取棋盘（返回可视化棋盘图） +3. **你深度思考分析**（按上述格式） +4. **写出明确结论**："我选择：xxx" +5. `execute_chess_move(instruction)` - GeLab 执行 +6. 重复 2-5 + +## execute_chess_move 指令格式 + +使用坐标描述走法： +- "把 E0 的帅移动到 E1" +- "把 A0 的车移动到 A9 吃掉黑车" +- "把 B2 的炮移动到 E2" + +或用自然语言： +- "红车从左下角向上直走到顶" +- "红炮平到中路" +""" +) + +# 全局状态 +_current_device_id = None +_device_wm_size = None +_local_server = None +_last_board_state = None + +# GeLab 配置 +GELAB_CONFIG = { + "task_type": "parser_0922_summary", + "model_config": { + "model_name": "step-gui", + "model_provider": "stepfun", + "args": {"temperature": 0.1, "top_p": 0.95, "max_tokens": 4096}, + }, + "max_steps": 30, + "delay_after_capture": 3, +} + +# 视觉模型配置 +VISION_MODEL_CONFIG = { + "model_provider": "stepfun", + "model_name": "step-1v-8k", +} + +SERVER_CONFIG = { + "log_dir": "running_log/server_log/chess-mcp-logs/traces", + "image_dir": "running_log/server_log/chess-mcp-logs/images", + "debug": False +} + + +def _init_device_info(device_id: str = None): + """初始化设备信息""" + global _current_device_id, _device_wm_size + if device_id is None: + devices = list_devices() + if not devices: + raise RuntimeError("未连接设备") + device_id = devices[0] + + open_screen(device_id) + _current_device_id = device_id + _device_wm_size = get_device_wm_size(device_id) + return {"device_id": device_id, "device_wm_size": _device_wm_size} + + +def _get_local_server(): + """获取 LocalServer 实例""" + global _local_server + if _local_server is None: + from copilot_agent_server.local_server import LocalServer + _local_server = LocalServer(SERVER_CONFIG) + return _local_server + + +def _analyze_board(device_info) -> dict: + """截图并用视觉模型解析棋盘""" + image_path = capture_screenshot(device_info['device_id'], "tmp_screenshot", print_command=False) + image_b64_url = make_b64_url(image_path) + smart_remove(image_path) + + messages = [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": image_b64_url}}, + {"type": "text", "text": BOARD_ANALYSIS_PROMPT} + ] + }] + + response = ask_llm_anything( + model_provider=VISION_MODEL_CONFIG["model_provider"], + model_name=VISION_MODEL_CONFIG["model_name"], + messages=messages, + args={"max_tokens": 2048, "temperature": 0.1} + ) + + try: + if "" in response: + response = response.split("")[-1].strip() + + json_match = re.search(r'\{[\s\S]*\}', response) + if json_match: + data = json.loads(json_match.group()) + if "pieces" in data: + return data + except Exception as e: + print(f"解析失败: {e}") + + return {"error": "解析失败", "raw": response} + + +def _render_board(data: dict) -> str: + """渲染可视化棋盘""" + if "error" in data: + return f"解析失败: {data.get('raw', '')}" + + # 初始化空棋盘 (10行 x 9列) + board = [[' · ' for _ in range(9)] for _ in range(10)] + + # 棋子符号映射 + piece_symbols = { + # 红方（大写） + 'K': '帅', 'A': '仕', 'B': '相', 'N': '马', 'R': '车', 'C': '炮', 'P': '兵', + # 黑方（小写） + 'k': '将', 'a': '士', 'b': '象', 'n': '馬', 'r': '車', 'c': '砲', 'p': '卒', + } + + # 列名到索引 + col_map = {'A': 0, 'B': 1, 'C': 2, 'D': 3, 'E': 4, 'F': 5, 'G': 6, 'H': 7, 'I': 8} + + # 放置棋子 + pieces_list = data.get("pieces", []) + for piece in pieces_list: + pos = piece.get("pos", "") + ptype = piece.get("type", "") + if len(pos) >= 2 and pos[0] in col_map: + col = col_map[pos[0]] + try: + row = int(pos[1]) + if 0 <= row <= 9 and ptype in piece_symbols: + symbol = piece_symbols[ptype] + # 红方用 [] 包围，黑方用 () 包围 + if ptype.isupper(): + board[row][col] = f'[{symbol}]' + else: + board[row][col] = f'({symbol})' + except ValueError: + pass + + # 渲染棋盘（从上到下：9到0） + lines = [ + "", + "=" * 50, + " 当前棋盘 [红方] vs (黑方) 你执红棋", + "=" * 50, + "", + " A B C D E F G H I", + " ┌───┬───┬───┬───┬───┬───┬───┬───┬───┐" + ] + + for row in range(9, -1, -1): # 从9到0 + row_str = f"{row} │" + for col in range(9): + row_str += board[row][col] + "│" + lines.append(row_str) + + if row == 5: # 楚河汉界 + lines.append(" ├───┴───┴───┴───┴───┴───┴───┴───┴───┤") + lines.append(" │ 楚 河 汉 界 │") + lines.append(" ├───┬───┬───┬───┬───┬───┬───┬───┬───┤") + elif row > 0: + lines.append(" ├───┼───┼───┼───┼───┼───┼───┼───┼───┤") + + lines.append(" └───┴───┴───┴───┴───┴───┴───┴───┴───┘") + lines.append("") + lines.append("坐标说明：列 A-I（左到右），行 0-9（下到上）") + lines.append("红方底线=0行，黑方底线=9行，红方九宫=D0-F2，黑方九宫=D7-F9") + lines.append("=" * 50) + + return "\n".join(lines) + + +def _get_pieces_summary(data: dict) -> str: + """获取棋子列表摘要""" + if "error" in data: + return "" + + red_pieces = [] + black_pieces = [] + + for piece in data.get("pieces", []): + ptype = piece.get("type", "") + pos = piece.get("pos", "") + name = piece.get("name", "") + + if ptype.isupper(): + red_pieces.append(f"{name}({pos})") + else: + black_pieces.append(f"{name}({pos})") + + lines = [ + "", + "【红方棋子】" + "、".join(red_pieces), + "【黑方棋子】" + "、".join(black_pieces), + "" + ] + return "\n".join(lines) + + +def _pos_to_screen_desc(pos: str) -> str: + """将坐标转换为屏幕位置描述""" + if len(pos) < 2: + return "" + + col = pos[0].upper() + try: + row = int(pos[1]) + except ValueError: + return "" + + # 列的描述（A-I 对应屏幕左到右） + col_desc = { + 'A': '最左边', 'B': '左边偏左', 'C': '左边', + 'D': '中间偏左', 'E': '正中间', 'F': '中间偏右', + 'G': '右边', 'H': '右边偏右', 'I': '最右边' + }.get(col, '') + + # 行的描述（0-9 对应屏幕下到上） + if row <= 1: + row_desc = '最下方（红方底线附近）' + elif row <= 3: + row_desc = '下方（红方区域）' + elif row <= 5: + row_desc = '中间（楚河汉界附近）' + elif row <= 7: + row_desc = '上方（黑方区域）' + else: + row_desc = '最上方（黑方底线附近）' + + return f"屏幕{col_desc}、{row_desc}" + + +def _generate_screen_position_hint(instruction: str) -> str: + """从走棋指令中提取坐标并生成屏幕位置提示""" + import re + + # 匹配坐标模式：字母+数字，如 F5, C5, E0 + positions = re.findall(r'[A-Ia-i][0-9]', instruction) + + if len(positions) >= 2: + from_pos = positions[0].upper() + to_pos = positions[1].upper() + + from_desc = _pos_to_screen_desc(from_pos) + to_desc = _pos_to_screen_desc(to_pos) + + return f""" +【屏幕位置提示】 +- 起始位置 {from_pos}：{from_desc} +- 目标位置 {to_pos}：{to_desc} +""" + elif len(positions) == 1: + pos = positions[0].upper() + desc = _pos_to_screen_desc(pos) + return f"\n【屏幕位置提示】位置 {pos}：{desc}\n" + + return "" + + +# ============== MCP 工具 ============== + +@mcp.tool +def prepare_game( + task: Annotated[str, Field(description="导航任务，如'打开象棋APP，进入残局第一关'")], + device_id: Annotated[str | None, Field(description="设备ID")] = None, +) -> dict: + """ + 让 GeLab 导航到棋盘界面。看到棋盘后停止，不会下棋。 + """ + try: + device_info = _init_device_info(device_id) + server = _get_local_server() + + nav_task = f"""{task} + +【重要】你只负责导航到棋盘界面： +1. 打开象棋APP +2. 按要求进入对应模式/关卡 +3. 当你看到棋盘界面（显示红黑棋子）时，立即 COMPLETE +4. 绝对不要点击任何棋子！不要下棋！""" + + print(f"GeLab 导航: {task}") + result = evaluate_task_on_device( + agent_server=server, + device_info=device_info, + task=nav_task, + rollout_config=GELAB_CONFIG, + reflush_app=False, + reset_environment=True + ) + + success = result.get("stop_reason") == "COMPLETE" + return { + "success": success, + "message": f"导航{'成功' if success else '失败'}", + "next_step": "请调用 get_board_state() 获取棋盘状态" + } + except Exception as e: + return {"success": False, "error": str(e)} + + +@mcp.tool +def get_board_state( + device_id: Annotated[str | None, Field(description="设备ID")] = None, +) -> dict: + """ + 获取当前棋盘状态，返回可视化棋盘图。 + + ⚠️ 重要：收到棋盘后，请先进行深度思考分析，不要立即调用执行工具！ + + 请按以下格式分析： + 1. 【棋局分析】列出双方棋子和局势 + 2. 【绝杀检查】检查各种绝杀可能 + 3. 【可选走法】列出几个候选走法 + 4. 【最终决定】明确写出"我选择：xxx" + + 只有写出明确结论后，才能调用 execute_chess_move() + """ + global _last_board_state + + try: + device_info = _init_device_info(device_id) + + print("解析棋盘...") + data = _analyze_board(device_info) + + if data.get("game_status") == "victory": + return { + "success": True, + "game_over": True, + "victory": True, + "message": "🎉 恭喜获胜！游戏结束。" + } + + if data.get("game_status") == "defeat": + return { + "success": True, + "game_over": True, + "victory": False, + "message": "游戏失败。" + } + + _last_board_state = data + board_visual = _render_board(data) + pieces_summary = _get_pieces_summary(data) + + return { + "success": True, + "game_over": False, + "board": board_visual, + "pieces": pieces_summary, + "raw_data": data, + "thinking_guide": f""" +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +⚠️ 请先深度思考，不要立即调用工具！ + +请按以下格式分析后再决定走法： + +【棋局分析】 +- 红方棋子：... +- 黑方棋子：... +- 局势判断：... + +【绝杀检查】 +{CHECKMATE_RULES} + +【可选走法】 +1. ... +2. ... +3. ... + +【最终决定】 +我选择：[写出具体走法，如"把E0的帅移动到E1"] +理由：... + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +只有写出"我选择：xxx"后，才能调用 execute_chess_move() +""" + } + except Exception as e: + return {"success": False, "error": str(e)} + + +@mcp.tool +def execute_chess_move( + move_instruction: Annotated[str, Field(description="走棋指令，如'把F5的红车移动到C5'")], + device_id: Annotated[str | None, Field(description="设备ID")] = None, +) -> dict: + """ + 让 GeLab 执行一步走棋。 + + ⚠️ 调用前请确保你已经： + 1. 完成了深度思考分析 + 2. 明确写出了"我选择：xxx" + + 指令格式： + - 坐标格式："把 F5 的红车移动到 C5" + - 自然语言："红车从右边平移到左边" + """ + try: + device_info = _init_device_info(device_id) + server = _get_local_server() + + # 解析坐标并生成屏幕位置描述 + screen_hint = _generate_screen_position_hint(move_instruction) + + gelab_task = f"""象棋走棋 - 必须完成两次点击！ + +【任务】{move_instruction} +{screen_hint} + +【坐标说明】 +- 列 A-I = 屏幕左到右 +- 行 0-9 = 屏幕下到上（0=红方底线，9=黑方底线） + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +⚠️ 这个任务需要点击两次才能完成！ +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +【第一次点击】选中棋子 +→ 点击要移动的棋子 +→ 棋子会高亮，出现绿色可走位置提示 +→ 此时任务还没完成！必须继续！ + +【第二次点击】落子 +→ 点击目标位置（绿色提示点） +→ 棋子移动到新位置 +→ 现在才算完成！ + +【完成标准】 +❌ 只看到棋子高亮 = 没完成，继续点击目标！ +✅ 棋子已经移动到新位置 = 完成，COMPLETE + +【特别注意】 +- 第一次点击后如果看到绿色提示点，说明选中成功 +- 此时必须立即进行第二次点击！ +- 不要在选中后就停止！ +- 只有棋子真正移动了才算完成！""" + + move_config = GELAB_CONFIG.copy() + move_config["max_steps"] = 10 # 增加更多步数余量 + + print(f"GeLab 执行: {move_instruction}") + result = evaluate_task_on_device( + agent_server=server, + device_info=device_info, + task=gelab_task, + rollout_config=move_config, + reflush_app=False, + reset_environment=False + ) + + success = result.get("stop_reason") == "COMPLETE" + + if success: + print("等待对手...") + time.sleep(2) + + print("获取新棋盘...") + new_state = _analyze_board(device_info) + + if new_state.get("game_status") == "victory": + return { + "success": True, + "game_over": True, + "victory": True, + "message": "🎉 恭喜获胜！" + } + + board_visual = _render_board(new_state) + pieces_summary = _get_pieces_summary(new_state) + global _last_board_state + _last_board_state = new_state + + return { + "success": True, + "game_over": False, + "move_executed": True, + "new_board": board_visual, + "pieces": pieces_summary, + "raw_data": new_state, + "next_step": "请对新棋盘进行深度思考分析，然后决定下一步" + } + else: + return { + "success": False, + "message": f"走棋失败: {result.get('stop_reason', '')}", + "hint": "请检查指令是否清晰，或调用 get_board_state() 重新确认" + } + except Exception as e: + return {"success": False, "error": str(e)} + + +@mcp.tool +def reset_session() -> dict: + """重置会话""" + global _local_server, _last_board_state + _local_server = None + _last_board_state = None + return {"success": True, "message": "已重置"} + + +if __name__ == "__main__": + import yaml + config_path = os.path.join(project_root, "mcp_server_config.yaml") + port = 8705 + if os.path.exists(config_path): + with open(config_path, "r") as f: + config = yaml.safe_load(f) + port = config.get('server_config', {}).get('chess_mcp_port', port) + + print(f"中国象棋 MCP 服务器启动在端口: {port}") + mcp.run(transport="http", port=port)