mirrored 12 minutes ago
0
Tianbao XieRobust Evaluation, Blocking File Open, Grader Sensitivity, and LibreOffice Writer Fixes (#217) * Refactor evaluator structure in LibreOffice Writer example JSON to support multiple expected and result files, enhancing evaluation flexibility. * Update instance type to t3.large and add VNC access URL logging for allocated VMs, enhancing remote access capabilities. * Update instance type to t3.large and add VNC access URL logging for allocated VMs, enhancing remote access capabilities. * Update time format in get_vm_file function to include hours, minutes, and seconds for more precise file naming with time suffix. * More delay for 936321ce-5236-426a-9a20-e0e3c5dc536f; support one more potential solutions. * Enhance SetupController with configurable retry limit and improved error handling for file opening requests. Introduce new function to compare unique training records, and update logging for better debugging. Adjust JSON examples for evaluation to support multiple expected and result files. * Clean debug code --------- Co-authored-by: yuanmengqi <yuanmengqi@mail.ustc.edu.cn>4e11eaf
{
  "id": "6f81754e-285d-4ce0-b59e-af7edb02d108",
  "snapshot": "libreoffice_writer",
  "instruction": "A certain railway company in Hong Kong uses a signaling system to keep track of trains in its railway system. Each line in the docx file represents a train calling at a station from 0600 to 1200 on 2022-09-22, and has the following format: time_HH:MM:SS, train_id, station_id, platform_no.. I want to remove duplicated train ids in order to know how many different trains are running from 0600 to 1200. Could you help me on this? I am doing it manually and it is very inefficient.",
  "source": "https://superuser.com/questions/789473/remove-duplicate-lines-in-libreoffice-openoffice-writer",
  "config": [
    {
      "type": "download",
      "parameters": {
        "files": [
          {
            "url": "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108/HK_train_record.docx",
            "path": "/home/user/Desktop/HK_train_record.docx"
          }
        ]
      }
    },
    {
      "type": "open",
      "parameters": {
        "path": "/home/user/Desktop/HK_train_record.docx"
      }
    }
  ],
  "trajectory": "trajectories/",
  "related_apps": [
    "libreoffice_writer"
  ],
  "evaluator": {
    "postconfig": [
      {
        "type": "activate_window",
        "parameters": {
          "window_name": "HK_train_record.docx - LibreOffice Writer",
          "strict": true
        }
      },
      {
        "type": "sleep",
        "parameters": {
          "seconds": 0.5
        }
      },
      {
        "type": "execute",
        "parameters": {
          "command": [
            "python",
            "-c",
            "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
          ]
        }
      }
    ],
    "func": "compare_unique_train_records",
    "result": {
      "type": "vm_file",
      "path": "/home/user/Desktop/HK_train_record.docx",
      "dest": "HK_train_record.docx"
    },
    "expected": {
      "type": "cloud_file",
      "path": [
        "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108/HK_train_record_Gold.docx",
        "https://huggingface.co/datasets/xlangai/ubuntu_osworld_file_cache/resolve/main/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108/HK_train_record.docx"
      ],
      "dest": [
        "HK_train_record_Gold.docx",
        "HK_train_record_Original.docx"
      ],
      "multi": true,
      "gives": [0, 1]
    }
  },
  "proxy": false
}