from typing import List, Dict, Any def get_replay(env, trajectory: List[Dict[str, Any]]) -> None: # fixme: need to be combined with the accessibility tree to activate the selection of the target window def parse(action): if action["type"] == "hotkey": keys = "', '".join(action["param"]) return f"pyautogui.hotkey('{keys}')" if action["type"] == "typewrite": text = action["param"] return f"pyautogui.typewrite('{text}')" if action["type"] == "press": key = action["param"] return f"pyautogui.press('{key}')" for action in trajectory: env.controller.execute_python_command(parse(action))