73 lines
2.5 KiB
JSON
73 lines
2.5 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"role": "system",
|
||
|
|
"content": "You are a helpful assistant.",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "user",
|
||
|
|
"content": "You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n\nAction: ...\n\n\n## Action Space\nclick(start_box='<|box_start|>(x1,y1)<|box_end|>')\nselect(start_box='<|box_start|>(x1,y1)<|box_end|>') # To expand a dropdown menu or select an item from it.\ntype(content='', start_box='<|box_start|>(x1,y1)<|box_end|>') # First click on the textbox, and then type the content.\n\n## User Instruction\nBuy a pop rock album CD from the United Kingdom that was released in 2016, is between £15 and £20 and in perfect condition.\n\n ",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "assistant",
|
||
|
|
"content": "Action: click(start_box='<|box_start|>(495,30)<|box_end|>')",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "assistant",
|
||
|
|
"content": "Action: click(start_box='<|box_start|>(381,153)<|box_end|>')",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "assistant",
|
||
|
|
"content": "Action: click(start_box='<|box_start|>(66,263)<|box_end|>')",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "assistant",
|
||
|
|
"content": "Action: click(start_box='<|box_start|>(66,851)<|box_end|>')",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "user",
|
||
|
|
"content": "<|vision_start|><|image_pad|>...<|image_pad|><|vision_end|>",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "assistant",
|
||
|
|
"content": "Action: click(start_box='<|box_start|>(66,638)<|box_end|>')",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "user",
|
||
|
|
"content": "<|vision_start|><|image_pad|>...<|image_pad|><|vision_end|>",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "assistant",
|
||
|
|
"content": "Action: click(start_box='<|box_start|>(66,86)<|box_end|>')",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "user",
|
||
|
|
"content": "<|vision_start|><|image_pad|>...<|image_pad|><|vision_end|>",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "assistant",
|
||
|
|
"content": "Action: click(start_box='<|box_start|>(66,5)<|box_end|>')",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "user",
|
||
|
|
"content": "<|vision_start|><|image_pad|>...<|image_pad|><|vision_end|>",
|
||
|
|
"loss_mask": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"role": "assistant",
|
||
|
|
"content": "Action: click(start_box='<|box_start|>(293,218)<|box_end|>')",
|
||
|
|
"loss_mask": 1
|
||
|
|
}
|
||
|
|
]
|