import argparse from ast import arg import datetime import io import logging import os import platform import sys import time from pathlib import Path from dotenv import load_dotenv from PIL import Image from gui_agents.maestro.controller.main_controller import MainController # Import analyze_display functionality from gui_agents.utils.analyze_display import analyze_display_json, aggregate_results, format_output_line from gui_agents.utils.common_utils import show_task_completion_notification from desktop_env.desktop_env import DesktopEnv from gui_agents.utils.common_utils import ImageDataFilter, SafeLoggingFilter env_path = Path(os.path.dirname(os.path.abspath(__file__))) / '.env' if env_path.exists(): load_dotenv(dotenv_path=env_path) else: parent_env_path = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) / '.env' if parent_env_path.exists(): load_dotenv(dotenv_path=parent_env_path) logger = logging.getLogger() logger.setLevel(logging.DEBUG) datetime_str: str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") log_dir = "runtime" os.makedirs(os.path.join(log_dir, datetime_str), exist_ok=True) file_handler = logging.FileHandler( os.path.join(log_dir, datetime_str, "agents3.log"), encoding="utf-8" ) debug_handler = logging.FileHandler( os.path.join(log_dir, datetime_str, "agents3_debug.log"), encoding="utf-8" ) stdout_handler = logging.StreamHandler(sys.stdout) # Add dedicated doubao API log handler doubao_handler = logging.FileHandler( os.path.join(log_dir, datetime_str, "doubao_api.log"), encoding="utf-8" ) # Create dedicated doubao API logger doubao_logger = logging.getLogger("doubao_api") doubao_logger.setLevel(logging.DEBUG) doubao_logger.addHandler(doubao_handler) file_handler.setLevel(logging.INFO) debug_handler.setLevel(logging.DEBUG) stdout_handler.setLevel(logging.INFO) doubao_handler.setLevel(logging.DEBUG) # Add SafeLoggingFilter to prevent format errors from third-party libraries (like OpenAI) safe_filter = SafeLoggingFilter() debug_handler.addFilter(safe_filter) # Also apply SafeLoggingFilter to OpenAI library loggers try: import openai openai_logger = logging.getLogger('openai') openai_logger.addFilter(safe_filter) httpx_logger = logging.getLogger('httpx') httpx_logger.addFilter(safe_filter) logger.info("SafeLoggingFilter applied to third-party libraries (OpenAI, HTTPX)") except ImportError: logger.info("SafeLoggingFilter applied to main handlers only (OpenAI not available)") pass if os.getenv('KEEP_IMAGE_LOGS', 'false').lower() != 'true': image_filter = ImageDataFilter() debug_handler.addFilter(image_filter) logger.info("Image data filtering enabled - image data in debug logs will be filtered") else: logger.info("Image data filtering disabled - debug logs will contain complete image data") formatter = logging.Formatter( fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s" ) file_handler.setFormatter(formatter) debug_handler.setFormatter(formatter) stdout_handler.setFormatter(formatter) doubao_handler.setFormatter(formatter) logger.addHandler(file_handler) logger.addHandler(debug_handler) logger.addHandler(stdout_handler) def auto_analyze_execution(timestamp_dir: str): """ Automatically analyze execution statistics from display.json files after task completion Args: timestamp_dir: Directory containing the execution logs and display.json """ import time try: # Analyze the display.json file for this execution display_json_path = os.path.join(timestamp_dir, "display.json") # Wait for file to be fully written max_wait_time = 10 # Maximum wait time in seconds wait_interval = 0.5 # Check every 0.5 seconds waited_time = 0 while waited_time < max_wait_time: if os.path.exists(display_json_path): # Check if file is still being written by monitoring its size try: size1 = os.path.getsize(display_json_path) time.sleep(wait_interval) size2 = os.path.getsize(display_json_path) # If file size hasn't changed in the last 0.5 seconds, it's likely complete if size1 == size2: logger.info(f"Display.json file appears to be complete (size: {size1} bytes)") break else: logger.info(f"Display.json file still being written (size changed from {size1} to {size2} bytes)") waited_time += wait_interval continue except OSError: # File might be temporarily inaccessible time.sleep(wait_interval) waited_time += wait_interval continue else: logger.info(f"Waiting for display.json file to be created... ({waited_time:.1f}s)") time.sleep(wait_interval) waited_time += wait_interval if os.path.exists(display_json_path): logger.info(f"Auto-analyzing execution statistics from: {display_json_path}") # Analyze the single display.json file result = analyze_display_json(display_json_path) if result: # Format and log the statistics output_line = format_output_line(result) logger.info("=" * 80) logger.info("EXECUTION STATISTICS:") logger.info("Steps, Duration (seconds), (Input Tokens, Output Tokens, Total Tokens), Cost") logger.info("=" * 80) logger.info(output_line) logger.info("=" * 80) # Also print to console for immediate visibility print("\n" + "=" * 80) print("EXECUTION STATISTICS:") print("Steps, Duration (seconds), (Input Tokens, Output Tokens, Total Tokens), Cost") print("=" * 80) print(output_line) print("=" * 80) else: logger.warning("No valid data found in display.json for analysis") else: logger.warning(f"Display.json file not found at: {display_json_path} after waiting {max_wait_time} seconds") except Exception as e: logger.error(f"Error during auto-analysis: {e}") def run_agent_maestro(params: dict): """ Run the maestro controller with the given instruction Args: controller: The NewController instance to run instruction: The instruction/task to execute max_steps: Maximum number of steps to execute """ backend = params["backend"] user_query = params["query"] max_steps = params["max_steps"] current_platform = params["current_platform"] env = params["env"] env_password = params["env_password"] import time logger.info(f"Starting maestro execution with instruction: {user_query}") total_start_time = time.time() # Ensure necessary directory structure exists timestamp_dir = os.path.join(log_dir, datetime_str) cache_dir = os.path.join(timestamp_dir, "cache", "screens") state_dir = os.path.join(timestamp_dir, "state") os.makedirs(cache_dir, exist_ok=True) os.makedirs(state_dir, exist_ok=True) # registry = Registry(global_state) # Initialize NewController (which includes all other components) controller = MainController( platform=current_platform, backend=backend, user_query=user_query, max_steps=max_steps, env=env, env_password=env_password, log_dir=log_dir, datetime_str=datetime_str ) try: # Set the user query in the controller controller.execute_main_loop() # Check task status after execution to determine if task was successful task = controller.global_state.get_task() if task and task.status == "fulfilled": # Task completed successfully logger.info("Task completed successfully") show_task_completion_notification("success") elif task and task.status == "rejected": # Task was rejected/failed logger.info("Task was rejected/failed") show_task_completion_notification("failed") else: # Task status unknown or incomplete logger.info("Task execution completed with unknown status") show_task_completion_notification("completed") except Exception as e: logger.error(f"Error during maestro execution: {e}") # Show error notification show_task_completion_notification("error", str(e)) raise finally: total_end_time = time.time() total_duration = total_end_time - total_start_time logger.info(f"Total execution time: {total_duration:.2f} seconds") # Auto-analyze execution statistics after task completion auto_analyze_execution(timestamp_dir) def main(): parser = argparse.ArgumentParser(description='Maestro CLI Application') parser.add_argument( '--backend', type=str, default='lybic', help='Backend to use (e.g., lybic, pyautogui, pyautogui_vmware)') parser.add_argument('--query', type=str, default='', help='Initial query to execute') parser.add_argument('--max-steps', type=int, default=50, help='Maximum number of steps to execute (default: 50)') parser.add_argument( '--lybic-sid', type=str, default=None, help='Lybic precreated sandbox ID (if not provided, will use LYBIC_PRECREATE_SID environment variable)') args = parser.parse_args() env = None env_password = "" # Set platform to Windows if backend is lybic if args.backend == 'lybic': current_platform = 'Windows' # Initialize hardware interface backend_kwargs = {"platform": current_platform} if args.lybic_sid is not None: backend_kwargs["precreate_sid"] = args.lybic_sid logger.info(f"Using Lybic SID from command line: {args.lybic_sid}") else: logger.info("Using Lybic SID from environment variable LYBIC_PRECREATE_SID") elif args.backend == 'pyautogui_vmware': env_password = "password" current_platform = os.getenv("USE_PRECREATE_VM", "Windows") if current_platform == "Ubuntu": path_to_vm = os.path.join("vmware_vm_data", "Ubuntu0", "Ubuntu0.vmx") elif current_platform == "Windows": path_to_vm = os.path.join("vmware_vm_data", "Windows0", "Windows0.vmx") else: raise ValueError(f"USE_PRECREATE_VM={current_platform} is not supported. Please use Ubuntu or Windows.") env = DesktopEnv( path_to_vm=path_to_vm, provider_name="vmware", os_type=current_platform, action_space="pyautogui", require_a11y_tree=False ) env.reset() else: current_platform = platform.system() logger.info(f"Running maestro on platform: {current_platform}") logger.info(f"Using backend: {args.backend}") logger.info("Maestro components initialized successfully") params = { "backend": args.backend, "query": '', "max_steps": args.max_steps, "current_platform": current_platform, "env": env, "env_password": env_password } # if query is provided, run the agent on the query if args.query: logger.info(f"Executing query: {args.query}") params["query"] = args.query run_agent_maestro(params) else: while True: query = input("Query: ") params["query"] = query # Run the agent on your own device run_agent_maestro(params) response = input("Would you like to provide another query? (y/n): ") if response.lower() != "y": break if __name__ == "__main__": """ python gui_agents/cli_app_maestro.py --backend lybic python gui_agents/cli_app_maestro.py --backend pyautogui --max-steps 1 python gui_agents/cli_app_maestro.py --backend pyautogui_vmware --max-steps 1 python gui_agents/cli_app_maestro.py --backend lybic --max-steps 15 python gui_agents/cli_app_maestro.py --backend lybic --lybic-sid SBX-01K1X6ZKAERXAN73KTJ1XXJXAF """ main()